<commit_before>from typing import (AnyStr, ByteString, Callable, Counter, Deque, Dict, FrozenSet,
                    GenericMeta, List, Optional, Set, Tuple, Any, TypeVar)

import os

import ast

import collections

_NO_DEFAULT = object()
TYPES_THAT_NEED_TO_BE_PARSED = [bool, list, set, tuple, dict]
TYPING_TO_REGULAR_TYPE = {
    AnyStr: str,
    ByteString: bytes,
    Counter: collections.Counter,
    Deque: collections.deque,
    Dict: dict,
    FrozenSet: frozenset,
    List: list,
    Set: set,
    Tuple: tuple
}

WantedType = TypeVar('WantedType')

def cast(representation: str, wanted_type: type):
    """
    Casts the string ``representation`` to the ``wanted type``.
    This function also supports some ``typing``types by mapping them to 'real' types.

    Some types need to be parsed with ast.
    """
    # If it's a typing meta replace it with the real type
    wanted_type = TYPING_TO_REGULAR_TYPE.get(wanted_type, wanted_type)
    if isinstance(wanted_type, GenericMeta):
        # Fallback to try to replace typing metas to real types
        wanted_type = wanted_type.__bases__[0]
    if wanted_type in TYPES_THAT_NEED_TO_BE_PARSED:
        value = (ast.literal_eval(representation)
                 if isinstance(representation, str)
                 else representation)
        return wanted_type(value)
    else:
        return wanted_type(representation)

class Variable:
    """
    Class to handle specific properties
    """
    def __init__(self, variable_name: str, default=_NO_DEFAULT, *,
                 transform: Callable[[str, type], Any] = cast):
        """
        :param variable_name: Environment variable to get
        :param default: Default value.
        :param transform: function to convert the env string to the wanted type
        """
        self.name = variable_name
        self.default = default
        self.transform = transform

    def get(self, wanted_type: WantedType) -> WantedType:
        """
        Gets ``self.variable_name`` from the environment and tries to cast it to ``wanted_type``.
        If ``self.default`` is ``_NO_DEFAULT`` and the env variable is not set this will raise an ``AttributeError``.
        If casting fails, this function will raise a ``ValueError``.

        :param wanted_type: type to return
        :return: value as wanted_type
        """
        try:
            raw_value = os.environ[self.name]
        except KeyError:
            if self.default is _NO_DEFAULT:
                raise AttributeError(f"Configuration error: '{self.name}' is not set.")
            else:
                return self.default

        try:
            value = self.transform(raw_value, wanted_type)
        except ValueError as e:
            raise ValueError(f"Invalid configuration for '{self.name}': {e}.")

        return value

class ConfigMeta(type):
    def __new__(cls, class_name, super_classes, attribute_dict: Dict[str, Any],
                prefix: Optional[str] = None):
        # TODO document this
        annotations: Dict[str, type] = attribute_dict.get('__annotations__', {})

        # Add attributes without defaults to the the attribute dict
        attribute_dict.update({attribute_name: _NO_DEFAULT
                               for attribute_name in annotations.keys()
                               if attribute_name not in attribute_dict})

        for attribute_name, default_value in attribute_dict.items():
            if attribute_name.startswith('_'):
                # private attributes are not changed
                continue
            default = attribute_dict.get(attribute_name, _NO_DEFAULT)
            if isinstance(default, Variable):
                attribute = default
            elif isinstance(default, ConfigMeta):
                # passthrough for nested configs
                attribute_dict[attribute_name] = default
                continue
            else:
                if prefix:
                    env_variable_name = f"{prefix}_{attribute_name}".upper()
                else:
                    env_variable_name = attribute_name.upper()
                attribute = Variable(env_variable_name, default)

            attribute_type = annotations.get(attribute_name, str)  # by default attributes are strings
            value = attribute.get(attribute_type)
            attribute_dict[attribute_name] = value

        return type.__new__(cls, class_name, super_classes, attribute_dict)

class AutoConfig(metaclass=ConfigMeta):
    # TODO document this
    pass
<commit_msg>Support Python 3.6.0<commit_after>from typing import (AnyStr, ByteString, Callable, Dict, FrozenSet, GenericMeta, List, Optional, Set, Tuple, Any,
                    TypeVar)

import os

import ast

import collections

_NOT_IMPORTED = object()

try:  # Types added in Python 3.6.1
    from typing import Counter, Deque
except ImportError:
    Deque = Counter = _NOT_IMPORTED

_NO_DEFAULT = object()
TYPES_THAT_NEED_TO_BE_PARSED = [bool, list, set, tuple, dict]
TYPING_TO_REGULAR_TYPE = {
    AnyStr: str,
    ByteString: bytes,
    Counter: collections.Counter,
    Deque: collections.deque,
    Dict: dict,
    FrozenSet: frozenset,
    List: list,
    Set: set,
    Tuple: tuple
}

WantedType = TypeVar('WantedType')

def cast(representation: str, wanted_type: type):
    """
    Casts the string ``representation`` to the ``wanted type``.
    This function also supports some ``typing``types by mapping them to 'real' types.

    Some types need to be parsed with ast.
    """
    # If it's a typing meta replace it with the real type
    wanted_type = TYPING_TO_REGULAR_TYPE.get(wanted_type, wanted_type)
    if isinstance(wanted_type, GenericMeta):
        # Fallback to try to replace typing metas to real types
        wanted_type = wanted_type.__bases__[0]
    if wanted_type in TYPES_THAT_NEED_TO_BE_PARSED:
        value = (ast.literal_eval(representation)
                 if isinstance(representation, str)
                 else representation)
        return wanted_type(value)
    else:
        return wanted_type(representation)

class Variable:
    """
    Class to handle specific properties
    """
    def __init__(self, variable_name: str, default=_NO_DEFAULT, *,
                 transform: Callable[[str, type], Any] = cast):
        """
        :param variable_name: Environment variable to get
        :param default: Default value.
        :param transform: function to convert the env string to the wanted type
        """
        self.name = variable_name
        self.default = default
        self.transform = transform

    def get(self, wanted_type: WantedType) -> WantedType:
        """
        Gets ``self.variable_name`` from the environment and tries to cast it to ``wanted_type``.
        If ``self.default`` is ``_NO_DEFAULT`` and the env variable is not set this will raise an ``AttributeError``.
        If casting fails, this function will raise a ``ValueError``.

        :param wanted_type: type to return
        :return: value as wanted_type
        """
        try:
            raw_value = os.environ[self.name]
        except KeyError:
            if self.default is _NO_DEFAULT:
                raise AttributeError(f"Configuration error: '{self.name}' is not set.")
            else:
                return self.default

        try:
            value = self.transform(raw_value, wanted_type)
        except ValueError as e:
            raise ValueError(f"Invalid configuration for '{self.name}': {e}.")

        return value

class ConfigMeta(type):
    def __new__(cls, class_name, super_classes, attribute_dict: Dict[str, Any],
                prefix: Optional[str] = None):
        # TODO document this
        annotations: Dict[str, type] = attribute_dict.get('__annotations__', {})

        # Add attributes without defaults to the the attribute dict
        attribute_dict.update({attribute_name: _NO_DEFAULT
                               for attribute_name in annotations.keys()
                               if attribute_name not in attribute_dict})

        for attribute_name, default_value in attribute_dict.items():
            if attribute_name.startswith('_'):
                # private attributes are not changed
                continue
            default = attribute_dict.get(attribute_name, _NO_DEFAULT)
            if isinstance(default, Variable):
                attribute = default
            elif isinstance(default, ConfigMeta):
                # passthrough for nested configs
                attribute_dict[attribute_name] = default
                continue
            else:
                if prefix:
                    env_variable_name = f"{prefix}_{attribute_name}".upper()
                else:
                    env_variable_name = attribute_name.upper()
                attribute = Variable(env_variable_name, default)

            attribute_type = annotations.get(attribute_name, str)  # by default attributes are strings
            value = attribute.get(attribute_type)
            attribute_dict[attribute_name] = value

        return type.__new__(cls, class_name, super_classes, attribute_dict)

class AutoConfig(metaclass=ConfigMeta):
    # TODO document this
    pass
<|endoftext|>


<commit_before>
import logging

import os
import copy

from time import sleep
from datetime import datetime, timedelta

from airflow.operators import BaseOperator
from airflow.utils import apply_defaults
from airflow.operators.dagrun_operator import DagRunOrder
from airflow.exceptions import AirflowSkipException

def default_is_valid_session_id(session_id):
    sid = session_id.strip().lower()
    return not ('delete' in sid) and not ('phantom' in sid)

def default_look_for_ready_file_marker(daily_folder_date):
    return daily_folder_date.date() == datetime.today().date()

def default_trigger_dagrun(context, dag_run_obj):
    if True:
        session_id = context['params']['session_id']
        start_date = context['params']['start_date']
        logging.info('Trigger DAG run : %s at %s', (str(session_id), start_date.strftime('%Y%m%d-%h%M')))
        # The payload will be available in target dag context as
        # kwargs['dag_run'].conf
        dag_run_obj.payload = context['params']
        dag_run_obj.run_id = session_id + '-' + \
            start_date.strftime('%Y%m%d-%h%M')
        return dag_run_obj

def roundTime(dt=None, dateDelta=timedelta(minutes=1)):
    """Round a datetime object to a multiple of a timedelta
    dt : datetime.datetime object, default now.
    dateDelta : timedelta object, we round to a multiple of this, default 1 minute.
    Author: Thierry Husson 2012 - Use it as you want but don't blame me.
            Stijn Nevens 2014 - Changed to use only datetime objects as variables
    """
    roundTo = dateDelta.total_seconds()

    if dt == None : dt = datetime.now()
    seconds = (dt - dt.min).seconds
    # rounding up, // is a floor division, not a comment on following line:
    rounding = (seconds+roundTo) // roundTo * roundTo
    return dt + timedelta(0,rounding-seconds,-dt.microsecond)

class ScanFolderOperator(BaseOperator):
    """
    Triggers a DAG run for a specified ``dag_id`` for each scan folder discovered
    in a daily folder

    :param trigger_dag_id: the dag_id to trigger
    :type trigger_dag_id: str
    :param python_callable: a reference to a python function that will be
        called while passing it the ``context`` object and a placeholder
        object ``obj`` for your callable to fill and return if you want
        a DagRun created. This ``obj`` object contains a ``run_id`` and
        ``payload`` attribute that you can modify in your function.
        The ``run_id`` should be a unique identifier for that DAG run, and
        the payload has to be a picklable object that will be made available
        to your tasks while executing that DAG run. Your function header
        should look like ``def foo(context, dag_run_obj):``
    :type python_callable: python callable
    """
    template_fields = tuple()
    template_ext = tuple()
    ui_color = '#bbefeb'

    @apply_defaults
    def __init__(
            self,
            folder,
            trigger_dag_id,
            python_callable=default_trigger_dagrun,
            is_valid_session_id=default_is_valid_session_id,
            look_for_ready_file_marker=default_look_for_ready_file_marker,
            ready_file_marker='.ready',
            *args, **kwargs):
        super(ScanFolderOperator, self).__init__(*args, **kwargs)
        self.folder = folder
        self.python_callable = python_callable
        self.trigger_dag_id = trigger_dag_id
        self.is_valid_session_id = is_valid_session_id
        self.look_for_ready_file_marker = look_for_ready_file_marker
        self.ready_file_marker = ready_file_marker

    def execute(self, context):
        self.scan_dirs(self.folder, context)

    def scan_dirs(self, folder, context):
        daily_folder_date = context['execution_date']

        if not os.path.exists(folder):
            raise AirflowSkipException

        daily_folder = os.path.join(folder, daily_folder_date.strftime(
            '%Y'), daily_folder_date.strftime('%Y%m%d'))

        if not os.path.isdir(daily_folder):
            daily_folder = os.path.join(
                folder, '2014', daily_folder_date.strftime('%Y%m%d'))

        if not os.path.isdir(daily_folder):
            raise AirflowSkipException

        for fname in os.listdir(daily_folder):
            path = os.path.join(daily_folder, fname)
            if os.path.isdir(path):

                ready_file_marker = os.path.join(path, self.ready_file_marker)
                if self.is_valid_session_id(fname) and not self.look_for_ready_file_marker(daily_folder_date) or os.access(ready_file_marker, os.R_OK):

                    logging.info(
                        'Prepare trigger for preprocessing : %s', str(fname))

                    self.trigger_dag_run(context, path, fname)

                    # Avoid creating Dags at the same time, overwise may
                    # get 'Duplicate entry pre_process_dicom-2016-06-06
                    # 00:01:00 for key dag_id'
                    sleep(60)

    def trigger_dag_run(self, context, path, session_dir_name):
        context = copy.copy(context)
        context_params = context['params']
        # Folder containing the DICOM files to process
        context_params['folder'] = path
        # Session ID identifies the session for a scan. The
        # last part of the folder path should match session_id
        context_params['session_id'] = session_dir_name

        dr_time = roundTime(datetime.now() + timedelta(minutes=1))
        dro = DagRunOrder(run_id='trig__' + dr_time.isoformat())
        dro = self.python_callable(context, dro)
        if dro:
            session = settings.Session()
            dr = DagRun(
                dag_id=self.trigger_dag_id,
                run_id=dro.run_id,
                conf=dro.payload,
                start_date=dr_time,
                external_trigger=True)
            logging.info("Creating DagRun {}".format(dr))
            session.add(dr)
            session.commit()
            session.close()
        else:
            logging.info("Criteria not met, moving on")
<commit_msg>Fix passing start_date in the context<commit_after>
import logging

import os
import copy

from time import sleep
from datetime import datetime, timedelta

from airflow.operators import BaseOperator
from airflow.utils import apply_defaults
from airflow.operators.dagrun_operator import DagRunOrder
from airflow.exceptions import AirflowSkipException

def default_is_valid_session_id(session_id):
    sid = session_id.strip().lower()
    return not ('delete' in sid) and not ('phantom' in sid)

def default_look_for_ready_file_marker(daily_folder_date):
    return daily_folder_date.date() == datetime.today().date()

def default_trigger_dagrun(context, dag_run_obj):
    if True:
        session_id = context['params']['session_id']
        start_date = context['start_date']
        logging.info('Trigger DAG run : %s at %s', (str(session_id), start_date.strftime('%Y%m%d-%h%M')))
        # The payload will be available in target dag context as
        # kwargs['dag_run'].conf
        dag_run_obj.payload = context['params']
        dag_run_obj.run_id = session_id + '-' + \
            start_date.strftime('%Y%m%d-%h%M')
        return dag_run_obj

def roundTime(dt=None, dateDelta=timedelta(minutes=1)):
    """Round a datetime object to a multiple of a timedelta
    dt : datetime.datetime object, default now.
    dateDelta : timedelta object, we round to a multiple of this, default 1 minute.
    Author: Thierry Husson 2012 - Use it as you want but don't blame me.
            Stijn Nevens 2014 - Changed to use only datetime objects as variables
    """
    roundTo = dateDelta.total_seconds()

    if dt == None : dt = datetime.now()
    seconds = (dt - dt.min).seconds
    # rounding up, // is a floor division, not a comment on following line:
    rounding = (seconds+roundTo) // roundTo * roundTo
    return dt + timedelta(0,rounding-seconds,-dt.microsecond)

class ScanFolderOperator(BaseOperator):
    """
    Triggers a DAG run for a specified ``dag_id`` for each scan folder discovered
    in a daily folder

    :param trigger_dag_id: the dag_id to trigger
    :type trigger_dag_id: str
    :param python_callable: a reference to a python function that will be
        called while passing it the ``context`` object and a placeholder
        object ``obj`` for your callable to fill and return if you want
        a DagRun created. This ``obj`` object contains a ``run_id`` and
        ``payload`` attribute that you can modify in your function.
        The ``run_id`` should be a unique identifier for that DAG run, and
        the payload has to be a picklable object that will be made available
        to your tasks while executing that DAG run. Your function header
        should look like ``def foo(context, dag_run_obj):``
    :type python_callable: python callable
    """
    template_fields = tuple()
    template_ext = tuple()
    ui_color = '#bbefeb'

    @apply_defaults
    def __init__(
            self,
            folder,
            trigger_dag_id,
            python_callable=default_trigger_dagrun,
            is_valid_session_id=default_is_valid_session_id,
            look_for_ready_file_marker=default_look_for_ready_file_marker,
            ready_file_marker='.ready',
            *args, **kwargs):
        super(ScanFolderOperator, self).__init__(*args, **kwargs)
        self.folder = folder
        self.python_callable = python_callable
        self.trigger_dag_id = trigger_dag_id
        self.is_valid_session_id = is_valid_session_id
        self.look_for_ready_file_marker = look_for_ready_file_marker
        self.ready_file_marker = ready_file_marker

    def execute(self, context):
        self.scan_dirs(self.folder, context)

    def scan_dirs(self, folder, context):
        daily_folder_date = context['execution_date']

        if not os.path.exists(folder):
            raise AirflowSkipException

        daily_folder = os.path.join(folder, daily_folder_date.strftime(
            '%Y'), daily_folder_date.strftime('%Y%m%d'))

        if not os.path.isdir(daily_folder):
            daily_folder = os.path.join(
                folder, '2014', daily_folder_date.strftime('%Y%m%d'))

        if not os.path.isdir(daily_folder):
            raise AirflowSkipException

        for fname in os.listdir(daily_folder):
            path = os.path.join(daily_folder, fname)
            if os.path.isdir(path):

                ready_file_marker = os.path.join(path, self.ready_file_marker)
                if self.is_valid_session_id(fname) and not self.look_for_ready_file_marker(daily_folder_date) or os.access(ready_file_marker, os.R_OK):

                    logging.info(
                        'Prepare trigger for preprocessing : %s', str(fname))

                    self.trigger_dag_run(context, path, fname)

                    # Avoid creating Dags at the same time, overwise may
                    # get 'Duplicate entry pre_process_dicom-2016-06-06
                    # 00:01:00 for key dag_id'
                    sleep(60)

    def trigger_dag_run(self, context, path, session_dir_name):
        context = copy.copy(context)
        context_params = context['params']
        # Folder containing the DICOM files to process
        context_params['folder'] = path
        # Session ID identifies the session for a scan. The
        # last part of the folder path should match session_id
        context_params['session_id'] = session_dir_name

        dr_time = roundTime(datetime.now() + timedelta(minutes=1))
        context['start_date'] = dr_time

        dro = DagRunOrder(run_id='trig__' + dr_time.isoformat())
        dro = self.python_callable(context, dro)
        if dro:
            session = settings.Session()
            dr = DagRun(
                dag_id=self.trigger_dag_id,
                run_id=dro.run_id,
                conf=dro.payload,
                start_date=dr_time,
                external_trigger=True)
            logging.info("Creating DagRun {}".format(dr))
            session.add(dr)
            session.commit()
            session.close()
        else:
            logging.info("Criteria not met, moving on")
<|endoftext|>


<commit_before># Django settings for editorsnotes project.

ADMINS = (
    ('Ryan Shaw', 'ryanshaw@unc.edu'),
    ('Patrick Golden', 'ptgolden@berkeley.edu'),
)

MANAGERS = ADMINS

TIME_ZONE = 'America/Los_Angeles'
LANGUAGE_CODE = 'en-us'
DATETIME_FORMAT = 'N j, Y, P'

SITE_ID = 1

USE_L10N = False
USE_I18N = False

MEDIA_URL = '/media/'
ADMIN_MEDIA_PREFIX = '/django_admin_media/'

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.load_template_source',
    'django.template.loaders.app_directories.load_template_source',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.request',
    'django.contrib.messages.context_processors.messages',
    'editorsnotes.main.context_processors.list_projects',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
)

ROOT_URLCONF = 'editorsnotes.urls'

import os.path

TEMPLATE_DIRS = (
    os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates'))
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.admin',
    'django.contrib.admindocs',
    'reversion',
    'south',
    'haystack',
    'editorsnotes.main',
    'editorsnotes.djotero',
    'editorsnotes.refine',
)

FIXTURE_DIRS = ( 'fixtures', )

HAYSTACK_SITECONF = 'editorsnotes.search_sites'
HAYSTACK_SEARCH_ENGINE = 'xapian'

AUTH_PROFILE_MODULE = 'main.UserProfile'

from settings_local import *

try:
    INSTALLED_APPS = INSTALLED_APPS + LOCAL_APPS
except NameError:
    pass
<commit_msg>Fix deprecation warning.<commit_after># Django settings for editorsnotes project.

ADMINS = (
    ('Ryan Shaw', 'ryanshaw@unc.edu'),
    ('Patrick Golden', 'ptgolden@berkeley.edu'),
)

MANAGERS = ADMINS

TIME_ZONE = 'America/Los_Angeles'
LANGUAGE_CODE = 'en-us'
DATETIME_FORMAT = 'N j, Y, P'

SITE_ID = 1

USE_L10N = False
USE_I18N = False

MEDIA_URL = '/media/'
ADMIN_MEDIA_PREFIX = '/django_admin_media/'

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.request',
    'django.contrib.messages.context_processors.messages',
    'editorsnotes.main.context_processors.list_projects',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
)

ROOT_URLCONF = 'editorsnotes.urls'

import os.path

TEMPLATE_DIRS = (
    os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates'))
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.admin',
    'django.contrib.admindocs',
    'reversion',
    'south',
    'haystack',
    'editorsnotes.main',
    'editorsnotes.djotero',
    'editorsnotes.refine',
)

FIXTURE_DIRS = ( 'fixtures', )

HAYSTACK_SITECONF = 'editorsnotes.search_sites'
HAYSTACK_SEARCH_ENGINE = 'xapian'

AUTH_PROFILE_MODULE = 'main.UserProfile'

from settings_local import *

try:
    INSTALLED_APPS = INSTALLED_APPS + LOCAL_APPS
except NameError:
    pass
<|endoftext|>


<commit_before>"""Tests for eliot.dask."""

from unittest import TestCase, skipUnless

from ..testing import capture_logging, LoggedAction
from .. import start_action, Message
try:
    import dask
    from dask.bag import from_sequence
except ImportError:
    dask = None
else:
    from ..dask import compute_with_trace, _RunWithEliotContext, _add_logging

@skipUnless(dask, "Dask not available.")
class DaskTests(TestCase):
    """Tests for end-to-end functionality."""

    def setUp(self):
        dask.config.set(scheduler="threading")

    def test_compute(self):
        """compute_with_trace() runs the same logic as compute()."""
        bag = from_sequence([1, 2, 3])
        bag = bag.map(lambda x: x * 7).map(lambda x: x * 4)
        bag = bag.fold(lambda x, y: x + y)
        self.assertEqual(dask.compute(bag), compute_with_trace(bag))

    @capture_logging(None)
    def test_logging(self, logger):
        """compute_with_trace() preserves Eliot context."""
        def mult(x):
            Message.log(message_type="mult")
            return x * 4

        def summer(x, y):
            Message.log(message_type="sum")
            return x + y

        bag = from_sequence([1, 2], partition_size=1)
        bag = bag.map(mult).fold(summer)
        with start_action(action_type="act1"):
            compute_with_trace(bag)
        for message in logger.messages:
            print(message)
        [logged_action] = LoggedAction.ofType(logger.messages, "act1")
        self.assertEqual(
            logged_action.type_tree(),
            {"act1": {"dask:compute": ""}}
        )

class AddLoggingTests(TestCase):
    """Tests for _add_logging()."""

    def test_add_logging_to_full_graph(self):
        """_add_logging() recreates Dask graph with wrappers."""
        bag = from_sequence([1, 2, 3])
        bag = bag.map(lambda x: x * 7).map(lambda x: x * 4)
        bag = bag.fold(lambda x, y: x + y)
        graph = bag.__dask_graph__()

        # Add logging:
        with start_action(action_type="bleh"):
            logging_added = _add_logging(graph, [])

        # Ensure resulting graph hasn't changed substantively:
        logging_removed = {}
        for key, value in logging_added.items():
            if callable(value[0]):
                func, args = value[0], value[1:]
                self.assertIsInstance(func, _RunWithEliotContext)
                value = (func.func,) + args
            logging_removed[key] = value

        self.assertEqual(logging_removed, graph)
<commit_msg>Basic end-to-end logging test passes.<commit_after>"""Tests for eliot.dask."""

from unittest import TestCase, skipUnless

from ..testing import capture_logging, LoggedAction
from .. import start_action, Message
try:
    import dask
    from dask.bag import from_sequence
except ImportError:
    dask = None
else:
    from ..dask import compute_with_trace, _RunWithEliotContext, _add_logging

@skipUnless(dask, "Dask not available.")
class DaskTests(TestCase):
    """Tests for end-to-end functionality."""

    def setUp(self):
        dask.config.set(scheduler="threading")

    def test_compute(self):
        """compute_with_trace() runs the same logic as compute()."""
        bag = from_sequence([1, 2, 3])
        bag = bag.map(lambda x: x * 7).map(lambda x: x * 4)
        bag = bag.fold(lambda x, y: x + y)
        self.assertEqual(dask.compute(bag), compute_with_trace(bag))

    @capture_logging(None)
    def test_logging(self, logger):
        """compute_with_trace() preserves Eliot context."""
        def mult(x):
            Message.log(message_type="mult")
            return x * 4

        def summer(x, y):
            Message.log(message_type="sum")
            return x + y

        bag = from_sequence([1, 2])
        bag = bag.map(mult).fold(summer)
        with start_action(action_type="act1"):
            compute_with_trace(bag)

        [logged_action] = LoggedAction.ofType(logger.messages, "act1")
        self.assertEqual(
            logged_action.type_tree(),
            {'act1': [{'dask:compute':
                       [{'eliot:remote_task': ['dask:task', 'mult']},
                        {'eliot:remote_task': ['dask:task', 'mult']},
                        {'eliot:remote_task': ['dask:task', 'sum']}]}]}
        )

class AddLoggingTests(TestCase):
    """Tests for _add_logging()."""

    def test_add_logging_to_full_graph(self):
        """_add_logging() recreates Dask graph with wrappers."""
        bag = from_sequence([1, 2, 3])
        bag = bag.map(lambda x: x * 7).map(lambda x: x * 4)
        bag = bag.fold(lambda x, y: x + y)
        graph = bag.__dask_graph__()

        # Add logging:
        with start_action(action_type="bleh"):
            logging_added = _add_logging(graph)

        # Ensure resulting graph hasn't changed substantively:
        logging_removed = {}
        for key, value in logging_added.items():
            if callable(value[0]):
                func, args = value[0], value[1:]
                self.assertIsInstance(func, _RunWithEliotContext)
                value = (func.func,) + args
            logging_removed[key] = value

        self.assertEqual(logging_removed, graph)
<|endoftext|>


<commit_before>from datetime import datetime

from django.utils.translation import ugettext_lazy as _, ugettext
from django.db import models
from django.db.models import Q
from django.contrib.contenttypes.models import ContentType
from django.template import Template

from ella.core.models import Category
from ella.core.box import Box
from ella.core.cache import CACHE_DELETER, cache_this, CachedGenericForeignKey

def get_position_key(func, self, category, name, nofallback=False):
    return 'ella.positions.models.PositionManager.get_active_position:%d:%s:%s' % (
            category.pk, name, nofallback and '1' or '0'
    )
def invalidate_cache(key,  self, category, name, nofallback=False):
    CACHE_DELETER.register_test(Position, "category_id:%s;name:%s" % (category.pk, name) , key)

class PositionManager(models.Manager):
    @cache_this(get_position_key, invalidate_cache)
    def get_active_position(self, category, name, nofallback=False):
        """
        Get active position for given position name.

        params:
            category - Category model to look for
            name - name of the position
            nofallback - if True than do not fall back to parent
                        category if active position is not found for category
        """
        now = datetime.now()
        lookup = (Q(active_from__isnull=True) | Q(active_from__lte=now)) & (Q(active_till__isnull=True) | Q(active_till__gt=now))
        while True:
            try:
                return self.get(lookup, category=category, name=name, disabled=False)
            except Position.DoesNotExist:
                # if nofallback was specified, do not look into parent categories
                if nofallback:
                    raise

                # traverse the category tree to the top otherwise
                category = category.get_tree_parent()

                # we reached the top and still haven't found the position - raise
                if category is None:
                    raise

def PositionBox(position, *args, **kwargs):
    " Delegate the boxing. "
    obj = position.target
    return getattr(position.target, 'box_class', Box)(obj, *args, **kwargs)

class Position(models.Model):
    " Represents a position on a page belonging to a certain category. "
    box_class = staticmethod(PositionBox)

    category = models.ForeignKey(Category, verbose_name=_('Category'))
    name = models.CharField(_('Name'), max_length=200)

    target_ct = models.ForeignKey(ContentType, verbose_name=_('Target content type'), null=True, blank=True)
    target_id = models.PositiveIntegerField(_('Target id'), null=True, blank=True)

    target = CachedGenericForeignKey('target_ct', 'target_id')

    active_from = models.DateTimeField(_('Position active from'), null=True, blank=True)
    active_till = models.DateTimeField(_('Position active till'), null=True, blank=True)

    box_type = models.CharField(_('Box type'), max_length=200, blank=True)
    text = models.TextField(_('Definition'), blank=True)
    disabled = models.BooleanField(_('Disabled'), default=False)

    objects = PositionManager()

    def render(self, context, nodelist, box_type):
        " Render the position. "
        if not self.target:
            if self.target_ct:
                # broken Generic FK:
                return ''
            return Template(self.text, name="position-%s" % self.name).render(context)

        if self.box_type:
            box_type = self.box_type
        if self.text:
            nodelist = Template('%s\n%s' % (nodelist.render({}), self.text), name="position-%s" % self.name).nodelist

        b = self.box_class(self, box_type, nodelist)
        b.prepare(context)
        return b.render()

    def __unicode__(self):
        return u'%s:%s' % (self.category, self.name)

    class Meta:
        verbose_name = _('Position')
        verbose_name_plural = _('Positions')
<commit_msg>Log invalid positions<commit_after>import logging
from datetime import datetime

from django.utils.translation import ugettext_lazy as _, ugettext
from django.db import models
from django.db.models import Q
from django.contrib.contenttypes.models import ContentType
from django.template import Template

from ella.core.models import Category
from ella.core.box import Box
from ella.core.cache import CACHE_DELETER, cache_this, CachedGenericForeignKey

log = logging.getLogger('ella.positions.models')

def get_position_key(func, self, category, name, nofallback=False):
    return 'ella.positions.models.PositionManager.get_active_position:%d:%s:%s' % (
            category.pk, name, nofallback and '1' or '0'
    )
def invalidate_cache(key,  self, category, name, nofallback=False):
    CACHE_DELETER.register_test(Position, "category_id:%s;name:%s" % (category.pk, name) , key)

class PositionManager(models.Manager):
    @cache_this(get_position_key, invalidate_cache)
    def get_active_position(self, category, name, nofallback=False):
        """
        Get active position for given position name.

        params:
            category - Category model to look for
            name - name of the position
            nofallback - if True than do not fall back to parent
                        category if active position is not found for category
        """
        now = datetime.now()
        lookup = (Q(active_from__isnull=True) | Q(active_from__lte=now)) & (Q(active_till__isnull=True) | Q(active_till__gt=now))
        while True:
            try:
                return self.get(lookup, category=category, name=name, disabled=False)
            except Position.DoesNotExist:
                # if nofallback was specified, do not look into parent categories
                if nofallback:
                    raise

                # traverse the category tree to the top otherwise
                category = category.get_tree_parent()

                # we reached the top and still haven't found the position - raise
                if category is None:
                    raise

def PositionBox(position, *args, **kwargs):
    " Delegate the boxing. "
    obj = position.target
    return getattr(position.target, 'box_class', Box)(obj, *args, **kwargs)

class Position(models.Model):
    " Represents a position on a page belonging to a certain category. "
    box_class = staticmethod(PositionBox)

    category = models.ForeignKey(Category, verbose_name=_('Category'))
    name = models.CharField(_('Name'), max_length=200)

    target_ct = models.ForeignKey(ContentType, verbose_name=_('Target content type'), null=True, blank=True)
    target_id = models.PositiveIntegerField(_('Target id'), null=True, blank=True)

    target = CachedGenericForeignKey('target_ct', 'target_id')

    active_from = models.DateTimeField(_('Position active from'), null=True, blank=True)
    active_till = models.DateTimeField(_('Position active till'), null=True, blank=True)

    box_type = models.CharField(_('Box type'), max_length=200, blank=True)
    text = models.TextField(_('Definition'), blank=True)
    disabled = models.BooleanField(_('Disabled'), default=False)

    objects = PositionManager()

    def render(self, context, nodelist, box_type):
        " Render the position. "
        if not self.target:
            if self.target_ct:
                # broken Generic FK:
                log.warning('Broken target for position with pk %r', self.pk)
                return ''
            return Template(self.text, name="position-%s" % self.name).render(context)

        if self.box_type:
            box_type = self.box_type
        if self.text:
            nodelist = Template('%s\n%s' % (nodelist.render({}), self.text), name="position-%s" % self.name).nodelist

        b = self.box_class(self, box_type, nodelist)
        b.prepare(context)
        return b.render()

    def __unicode__(self):
        return u'%s:%s' % (self.category, self.name)

    class Meta:
        verbose_name = _('Position')
        verbose_name_plural = _('Positions')
<|endoftext|>


<commit_before>import re
import sys
import time
import codecs
import scrapy
import locale
from jobcrawl.items import JobItem

class AllJobsSpider(scrapy.Spider):
    """ Spider to scrape job information from site http://www.alljobs.co.il """

    name = "alljobs"
    allowed_domains = ["http://www.alljobs.co.il"]
    start_urls = [
        "http://www.alljobs.co.il/SearchResultsGuest.aspx?" +
        "page=1&position=&type=&freetxt=&city=&region=",
    ]

    def __init__(self):

        sys.stdout = codecs.getwriter(
            locale.getpreferredencoding())(sys.stdout)
        reload(sys)
        sys.setdefaultencoding('utf-8')

    def parse(self, response):
        if response.status != 200:
            self.logger.error("{}\n ERROR Code {}: {} \n {}".format(
                "*" * 30, response.status, response.url, "*" * 30)
            )
        else:
            self.logger.info(
                "{}\n  Status Code {} OK: {} \n {}".format(
                    "*" * 30, response.status, response.url, "*" * 30)
            )

        job_container_div_list = response.xpath("//div[@class='open-board']")

        for job_item_sel in job_container_div_list:
            job_id_container = job_item_sel.xpath(".//@id").extract_first()
            job_id_group = re.findall(r'[\d]+', job_id_container)
            if job_id_group:
                job_id = job_id_group[0]
                job_link = "http://www.alljobs.co.il/Search/" \
                           "UploadSingle.aspx?JobID={}".format(job_id)
            else:
                job_id = ''
                job_link = ''

            try:
                job_date = job_item_sel.xpath(
                    './/div[@class="job-content-top-date"]/text()'
                ).extract_first()
                date = job_date.split(' ')[-1]
            except:
                date = ""

            try:
                job_class = job_item_sel.xpath(
                    './/div[@class="job-content-top-status-text"]/text()'
                ).extract_first()
            except:
                job_class = ""

            try:

                job_title = job_item_sel.xpath(
                    './/div[contains(@class, "job-content-top-title")]'
                    '//div/a/h2/text()').extract_first()
            except:
                job_title = ""

            try:
                company = job_item_sel.xpath(
                    './/div[@class="T14"]/text()').extract_first()
            except:
                company = ""

            try:
                company_jobs = job_item_sel.xpath(
                    './/div[@class="job-company-details"]'
                    '//a[@class="L_Blue gad"]/@href').extract_first()
                company_jobs = response.urljoin(company_jobs)
            except:
                company_jobs = ""

            if 'SearchResultsGuest.aspx' in company_jobs:
                # if no company jobs link, then set job link
                company_jobs = job_link
            try:
                location_list_sel = response.xpath(
                    "//div[@class='job-regions-box']")
                if location_list_sel:
                    location_list = location_list_sel.xpath(
                        ".//a/text()").extract()
                    country_areas = ", ".join(location_list)
                else:
                    country_areas = job_item_sel.xpath(
                        './/div[@class="job-content-top-location"]/a/text()'
                    ).extract_first()

            except:
                country_areas = ""

            job_description = ""
            try:
                description_div_id = "job-body-content" + \
                                     str(job_id)
                description_div = job_item_sel.xpath(
                    './/div[@id="' + description_div_id + '"]/*')
                for dv in description_div:
                    description_text = dv.xpath(
                        "normalize-space(string())").extract_first()
                    if description_text:
                        job_description += description_text
                        job_description += "\n"

            except:
                job_description = ""

            item = JobItem()

            item['Job'] = {
                'Site': 'AllJobs',
                'Company': company,
                'Company_jobs': company_jobs,
                'Job_id': job_id,
                'Job_title': job_title,
                'Job_Description': job_description,
                'Job_Post_Date': date,
                'Job_URL': job_link,
                'Country_Areas': country_areas,
                'Job_categories': '',
                'AllJobs_Job_class': job_class,
                'unique_id': 'alljobs_{}'.format(job_id)
            }

            yield item

        next_page = response.xpath('//div[@class="jobs-paging-next"]'
            '/a/@href').extract_first()
        if next_page:
            time.sleep(1)
            yield scrapy.Request(
                response.urljoin(next_page), self.parse, dont_filter=True)
<commit_msg>Fix alljobs company name parsing<commit_after>import re
import sys
import time
import codecs
import scrapy
import locale
from jobcrawl.items import JobItem

class AllJobsSpider(scrapy.Spider):
    """ Spider to scrape job information from site http://www.alljobs.co.il """

    name = "alljobs"
    allowed_domains = ["http://www.alljobs.co.il"]
    start_urls = [
        "https://www.alljobs.co.il/SearchResultsGuest.aspx?" +
        "page=1&position=&type=&freetxt=&city=&region=",
    ]

    def __init__(self):

        sys.stdout = codecs.getwriter(
            locale.getpreferredencoding())(sys.stdout)
        reload(sys)
        sys.setdefaultencoding('utf-8')

    def parse(self, response):
        if response.status != 200:
            self.logger.error("{}\n ERROR Code {}: {} \n {}".format(
                "*" * 30, response.status, response.url, "*" * 30)
            )
        else:
            self.logger.info(
                "{}\n  Status Code {} OK: {} \n {}".format(
                    "*" * 30, response.status, response.url, "*" * 30)
            )

        job_container_div_list = response.xpath("//div[@class='open-board']")

        for job_item_sel in job_container_div_list:
            job_id_container = job_item_sel.xpath(".//@id").extract_first()
            job_id_group = re.findall(r'[\d]+', job_id_container)
            if job_id_group:
                job_id = job_id_group[0]
                job_link = "http://www.alljobs.co.il/Search/" \
                           "UploadSingle.aspx?JobID={}".format(job_id)
            else:
                job_id = ''
                job_link = ''

            try:
                job_date = job_item_sel.xpath(
                    './/div[@class="job-content-top-date"]/text()'
                ).extract_first()
                date = job_date.split(' ')[-1]
            except:
                date = ""

            try:
                job_class = job_item_sel.xpath(
                    './/div[@class="job-content-top-status-text"]/text()'
                ).extract_first()
            except:
                job_class = ""

            try:

                job_title = job_item_sel.xpath(
                    './/div[contains(@class, "job-content-top-title")]'
                    '//div/a/h2/text()').extract_first()
            except:
                job_title = ""

            try:
                company = job_item_sel.xpath(
                    './/div[@class="T14"]/a/text()').extract_first()
                if company:
                    company = company.strip()
            except:
                company = ""

            try:
                company_jobs = job_item_sel.xpath(
                    './/div[@class="job-company-details"]'
                    '//a[@class="L_Blue gad"]/@href').extract_first()
                company_jobs = response.urljoin(company_jobs)
            except:
                company_jobs = ""

            if 'SearchResultsGuest.aspx' in company_jobs:
                # if no company jobs link, then set job link
                company_jobs = job_link
            try:
                location_list_sel = response.xpath(
                    "//div[@class='job-regions-box']")
                if location_list_sel:
                    location_list = location_list_sel.xpath(
                        ".//a/text()").extract()
                    country_areas = ", ".join(location_list)
                else:
                    country_areas = job_item_sel.xpath(
                        './/div[@class="job-content-top-location"]/a/text()'
                    ).extract_first()

            except:
                country_areas = ""

            job_description = ""
            try:
                description_div_id = "job-body-content" + \
                                     str(job_id)
                description_div = job_item_sel.xpath(
                    './/div[@id="' + description_div_id + '"]/*')
                for dv in description_div:
                    description_text = dv.xpath(
                        "normalize-space(string())").extract_first()
                    if description_text:
                        job_description += description_text
                        job_description += "\n"

            except:
                job_description = ""

            item = JobItem()

            item['Job'] = {
                'Site': 'AllJobs',
                'Company': company,
                'Company_jobs': company_jobs,
                'Job_id': job_id,
                'Job_title': job_title,
                'Job_Description': job_description,
                'Job_Post_Date': date,
                'Job_URL': job_link,
                'Country_Areas': country_areas,
                'Job_categories': '',
                'AllJobs_Job_class': job_class,
                'unique_id': 'alljobs_{}'.format(job_id)
            }

            yield item

        next_page = response.xpath('//div[@class="jobs-paging-next"]'
            '/a/@href').extract_first()
        if next_page:
            time.sleep(1)
            yield scrapy.Request(
                response.urljoin(next_page), self.parse, dont_filter=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import random

def job(bot):
    print("Sending message...")
    msg = "Fui!"
    bot.sendMessage("-58208727", msg)
    bot.sendChatAction("-58208727", 'upload_document')
    gifs = [
        'BQADBAADeQMAAqwbZAeNjNm2fzVR0wI',
        'BQADBAADEgMAAnwYZAdLuk2uOkqL2gI',
        'BQADBAADSwMAAoQYZAevJ86wPaeRxwI',
        'BQADBAADuwMAAj8bZAfJzPzXdPh7jwI',
    ]
    bot.sendDocument("-58208727", random.choice(gifs))
<commit_msg>shuffle night gifs<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import random
from utils.utils import my_shuffle

def job(bot):
    print("Sending message...")
    msg = "Fui!"
    bot.sendMessage("-58208727", msg)
    bot.sendChatAction("-58208727", 'upload_document')
    gifs = [
        'BQADBAADeQMAAqwbZAeNjNm2fzVR0wI',
        'BQADBAADEgMAAnwYZAdLuk2uOkqL2gI',
        'BQADBAADSwMAAoQYZAevJ86wPaeRxwI',
        'BQADBAADuwMAAj8bZAfJzPzXdPh7jwI',
    ]
    bot.sendDocument("-58208727", random.choice(my_shuffle(gifs)))
    print 'Sent'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals

import calendar
from datetime import timedelta

import frappe
from frappe import _
from frappe.model.document import Document
from frappe.utils import (format_time, get_link_to_form, get_url_to_report,
	global_date_format, now, now_datetime, validate_email_address, today, add_to_date)
from frappe.model.naming import append_number_if_name_exists
from frappe.utils.csvutils import to_csv
from frappe.utils.xlsxutils import make_xlsx

max_reports_per_user = frappe.local.conf.max_reports_per_user or 3

class AutoEmailReport(Document):
	def autoname(self):
		self.name = _(self.report)
		if frappe.db.exists('Auto Email Report', self.name):
			self.name = append_number_if_name_exists('Auto Email Report', self.name)

	def validate(self):
		self.validate_report_count()
		self.validate_emails()
		self.validate_report_format()
		self.validate_mandatory_fields()

	def validate_emails(self):
		'''Cleanup list of emails'''
		if ',' in self.email_to:
			self.email_to.replace(',', '\n')

		valid = []
		for email in self.email_to.split():
			if email:
				validate_email_address(email, True)
				valid.append(email)

		self.email_to = '\n'.join(valid)

	def validate_report_count(self):
		'''check that there are only 3 enabled reports per user'''
		count = frappe.db.sql('select count(*) from `tabAuto Email Report` where user=%s and enabled=1', self.user)[0][0]
		if count > max_reports_per_user + (-1 if self.flags.in_insert else 0):
			frappe.throw(_('Only {0} emailed reports are allowed per user').format(max_reports_per_user))

	def validate_report_format(self):
		""" check if user has select correct report format """
		valid_report_formats = ["HTML", "XLSX", "CSV"]
		if self.format not in valid_report_formats:
			frappe.throw(_("{0} is not a valid report format. Report format should one of the following {1}")
				.format(frappe.bold(self.format), frappe.bold(", ".join(valid_report_formats))))

	def validate_mandatory_fields(self):
		# Check if all Mandatory Report Filters are filled by the User
		filters = frappe.parse_json(self.filters) if self.filters else {}
		filter_meta = frappe.parse_json(self.filter_meta) if self.filter_meta else {}
		throw_list = []
		for meta in filter_meta:
			if meta.get("reqd") and not filters.get(meta["fieldname"]):
				throw_list.append(meta['label'])
		if throw_list:
			frappe.throw(
				title= _('Missing Filters Required'),
				msg= _('Following Report Filters have missing values:') +
					'

' + '
'.join(throw_list) + '
',
			)

	def get_report_content(self):
		'''Returns file in for the report in given format'''
		report = frappe.get_doc('Report', self.report)

		self.filters = frappe.parse_json(self.filters) if self.filters else {}

		if self.report_type=='Report Builder' and self.data_modified_till:
			self.filters['modified'] = ('>', now_datetime() - timedelta(hours=self.data_modified_till))

		if self.report_type != 'Report Builder' and self.dynamic_date_filters_set():
			self.prepare_dynamic_filters()

		columns, data = report.get_data(limit=self.no_of_rows or 100, user = self.user,
			filters = self.filters, as_dict=True, ignore_prepared_report=True)

		# add serial numbers
		columns.insert(0, frappe._dict(fieldname='idx', label='', width='30px'))
		for i in range(len(data)):
			data[i]['idx'] = i+1

		if len(data)==0 and self.send_if_data:
			return None

		if self.format == 'HTML':
			columns, data = make_links(columns, data)
			columns = update_field_types(columns)
			return self.get_html_table(columns, data)

		elif self.format == 'XLSX':
			spreadsheet_data = self.get_spreadsheet_data(columns, data)
			xlsx_file = make_xlsx(spreadsheet_data, "Auto Email Report")
			return xlsx_file.getvalue()

		elif self.format == 'CSV':
			spreadsheet_data = self.get_spreadsheet_data(columns, data)
			return to_csv(spreadsheet_data)

		else:
			frappe.throw(_('Invalid Output Format'))

	def get_html_table(self, columns=None, data=None):

		date_time = global_date_format(now()) + ' ' + format_time(now())
		report_doctype = frappe.db.get_value('Report', self.report, 'ref_doctype')

		return frappe.render_template('frappe/templates/emails/auto_email_report.html', {
			'title': self.name,
			'description': self.description,
			'date_time': date_time,
			'columns': columns,
			'data': data,
			'report_url': get_url_to_report(self.report, self.report_type, report_doctype),
			'report_name': self.report,
			'edit_report_settings': get_link_to_form('Auto Email Report', self.name)
		})

	@staticmethod
	def get_spreadsheet_data(columns, data):
		out = [[_(df.label) for df in columns], ]
		for row in data:
			new_row = []
			out.append(new_row)
			for df in columns:
				if df.fieldname not in row: continue
				new_row.append(frappe.format(row[df.fieldname], df, row))

		return out

	def get_file_name(self):
		return "{0}.{1}".format(self.report.replace(" ", "-").replace("/", "-"), self.format.lower())

	def prepare_dynamic_filters(self):
		self.filters = frappe.parse_json(self.filters)

		to_date = today()
		from_date_value = {
			'Daily': ('days', -1),
			'Weekly': ('weeks', -1),
			'Monthly': ('months', -1),
			'Quarterly': ('months', -3),
			'Half Yearly': ('months', -6),
			'Yearly': ('years', -1)
		}[self.dynamic_date_period]

		from_date = add_to_date(to_date, **{from_date_value[0]: from_date_value[1]})

		self.filters[self.from_date_field] = from_date
		self.filters[self.to_date_field] = to_date

	def send(self):
		if self.filter_meta and not self.filters:
			frappe.throw(_("Please set filters value in Report Filter table."))

		data = self.get_report_content()
		if not data:
			return

		attachments = None
		if self.format == "HTML":
			message = data
		else:
			message = self.get_html_table()

		if not self.format=='HTML':
			attachments = [{
				'fname': self.get_file_name(),
				'fcontent': data
			}]

		frappe.sendmail(
			recipients = self.email_to.split(),
			subject = self.name,
			message = message,
			attachments = attachments,
			reference_doctype = self.doctype,
			reference_name = self.name
		)

	def dynamic_date_filters_set(self):
		return self.dynamic_date_period and self.from_date_field and self.to_date_field

@frappe.whitelist()
def download(name):
	'''Download report locally'''
	auto_email_report = frappe.get_doc('Auto Email Report', name)
	auto_email_report.check_permission()
	data = auto_email_report.get_report_content()

	if not data:
		frappe.msgprint(_('No Data'))
		return

	frappe.local.response.filecontent = data
	frappe.local.response.type = "download"
	frappe.local.response.filename = auto_email_report.get_file_name()

@frappe.whitelist()
def send_now(name):
	'''Send Auto Email report now'''
	auto_email_report = frappe.get_doc('Auto Email Report', name)
	auto_email_report.check_permission()
	auto_email_report.send()

def send_daily():
	'''Check reports to be sent daily'''

	current_day = calendar.day_name[now_datetime().weekday()]
	enabled_reports = frappe.get_all('Auto Email Report',
		filters={'enabled': 1, 'frequency': ('in', ('Daily', 'Weekdays', 'Weekly'))})

	for report in enabled_reports:
		auto_email_report = frappe.get_doc('Auto Email Report', report.name)

		# if not correct weekday, skip
		if auto_email_report.frequency == "Weekdays":
			if current_day in ("Saturday", "Sunday"):
				continue
		elif auto_email_report.frequency == 'Weekly':
			if auto_email_report.day_of_week != current_day:
				continue
		try:
			auto_email_report.send()
		except Exception as e:
			frappe.log_error(e, _('Failed to send {0} Auto Email Report').format(auto_email_report.name))

def send_monthly():
	'''Check reports to be sent monthly'''
	for report in frappe.get_all('Auto Email Report', {'enabled': 1, 'frequency': 'Monthly'}):
		frappe.get_doc('Auto Email Report', report.name).send()

def make_links(columns, data):
	for row in data:
		for col in columns:
			if col.fieldtype == "Link" and col.options != "Currency":
				if col.options and row.get(col.fieldname):
					row[col.fieldname] = get_link_to_form(col.options, row[col.fieldname])
			elif col.fieldtype == "Dynamic Link":
				if col.options and row.get(col.fieldname) and row.get(col.options):
					row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname])
			elif col.fieldtype == "Currency":
				row[col.fieldname] = frappe.format_value(row[col.fieldname], col)

	return columns, data

def update_field_types(columns):
	for col in columns:
		if col.fieldtype in  ("Link", "Dynamic Link", "Currency")  and col.options != "Currency":
			col.fieldtype = "Data"
			col.options = ""
	return columns<commit_msg>fix: none type error for empty lines<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals

import calendar
from datetime import timedelta

import frappe
from frappe import _
from frappe.model.document import Document
from frappe.utils import (format_time, get_link_to_form, get_url_to_report,
	global_date_format, now, now_datetime, validate_email_address, today, add_to_date)
from frappe.model.naming import append_number_if_name_exists
from frappe.utils.csvutils import to_csv
from frappe.utils.xlsxutils import make_xlsx

max_reports_per_user = frappe.local.conf.max_reports_per_user or 3

class AutoEmailReport(Document):
	def autoname(self):
		self.name = _(self.report)
		if frappe.db.exists('Auto Email Report', self.name):
			self.name = append_number_if_name_exists('Auto Email Report', self.name)

	def validate(self):
		self.validate_report_count()
		self.validate_emails()
		self.validate_report_format()
		self.validate_mandatory_fields()

	def validate_emails(self):
		'''Cleanup list of emails'''
		if ',' in self.email_to:
			self.email_to.replace(',', '\n')

		valid = []
		for email in self.email_to.split():
			if email:
				validate_email_address(email, True)
				valid.append(email)

		self.email_to = '\n'.join(valid)

	def validate_report_count(self):
		'''check that there are only 3 enabled reports per user'''
		count = frappe.db.sql('select count(*) from `tabAuto Email Report` where user=%s and enabled=1', self.user)[0][0]
		if count > max_reports_per_user + (-1 if self.flags.in_insert else 0):
			frappe.throw(_('Only {0} emailed reports are allowed per user').format(max_reports_per_user))

	def validate_report_format(self):
		""" check if user has select correct report format """
		valid_report_formats = ["HTML", "XLSX", "CSV"]
		if self.format not in valid_report_formats:
			frappe.throw(_("{0} is not a valid report format. Report format should one of the following {1}")
				.format(frappe.bold(self.format), frappe.bold(", ".join(valid_report_formats))))

	def validate_mandatory_fields(self):
		# Check if all Mandatory Report Filters are filled by the User
		filters = frappe.parse_json(self.filters) if self.filters else {}
		filter_meta = frappe.parse_json(self.filter_meta) if self.filter_meta else {}
		throw_list = []
		for meta in filter_meta:
			if meta.get("reqd") and not filters.get(meta["fieldname"]):
				throw_list.append(meta['label'])
		if throw_list:
			frappe.throw(
				title= _('Missing Filters Required'),
				msg= _('Following Report Filters have missing values:') +
					'

' + '
'.join(throw_list) + '
',
			)

	def get_report_content(self):
		'''Returns file in for the report in given format'''
		report = frappe.get_doc('Report', self.report)

		self.filters = frappe.parse_json(self.filters) if self.filters else {}

		if self.report_type=='Report Builder' and self.data_modified_till:
			self.filters['modified'] = ('>', now_datetime() - timedelta(hours=self.data_modified_till))

		if self.report_type != 'Report Builder' and self.dynamic_date_filters_set():
			self.prepare_dynamic_filters()

		columns, data = report.get_data(limit=self.no_of_rows or 100, user = self.user,
			filters = self.filters, as_dict=True, ignore_prepared_report=True)

		# add serial numbers
		columns.insert(0, frappe._dict(fieldname='idx', label='', width='30px'))
		for i in range(len(data)):
			data[i]['idx'] = i+1

		if len(data)==0 and self.send_if_data:
			return None

		if self.format == 'HTML':
			columns, data = make_links(columns, data)
			columns = update_field_types(columns)
			return self.get_html_table(columns, data)

		elif self.format == 'XLSX':
			spreadsheet_data = self.get_spreadsheet_data(columns, data)
			xlsx_file = make_xlsx(spreadsheet_data, "Auto Email Report")
			return xlsx_file.getvalue()

		elif self.format == 'CSV':
			spreadsheet_data = self.get_spreadsheet_data(columns, data)
			return to_csv(spreadsheet_data)

		else:
			frappe.throw(_('Invalid Output Format'))

	def get_html_table(self, columns=None, data=None):

		date_time = global_date_format(now()) + ' ' + format_time(now())
		report_doctype = frappe.db.get_value('Report', self.report, 'ref_doctype')

		return frappe.render_template('frappe/templates/emails/auto_email_report.html', {
			'title': self.name,
			'description': self.description,
			'date_time': date_time,
			'columns': columns,
			'data': data,
			'report_url': get_url_to_report(self.report, self.report_type, report_doctype),
			'report_name': self.report,
			'edit_report_settings': get_link_to_form('Auto Email Report', self.name)
		})

	@staticmethod
	def get_spreadsheet_data(columns, data):
		out = [[_(df.label) for df in columns], ]
		for row in data:
			new_row = []
			out.append(new_row)
			for df in columns:
				if df.fieldname not in row: continue
				new_row.append(frappe.format(row[df.fieldname], df, row))

		return out

	def get_file_name(self):
		return "{0}.{1}".format(self.report.replace(" ", "-").replace("/", "-"), self.format.lower())

	def prepare_dynamic_filters(self):
		self.filters = frappe.parse_json(self.filters)

		to_date = today()
		from_date_value = {
			'Daily': ('days', -1),
			'Weekly': ('weeks', -1),
			'Monthly': ('months', -1),
			'Quarterly': ('months', -3),
			'Half Yearly': ('months', -6),
			'Yearly': ('years', -1)
		}[self.dynamic_date_period]

		from_date = add_to_date(to_date, **{from_date_value[0]: from_date_value[1]})

		self.filters[self.from_date_field] = from_date
		self.filters[self.to_date_field] = to_date

	def send(self):
		if self.filter_meta and not self.filters:
			frappe.throw(_("Please set filters value in Report Filter table."))

		data = self.get_report_content()
		if not data:
			return

		attachments = None
		if self.format == "HTML":
			message = data
		else:
			message = self.get_html_table()

		if not self.format=='HTML':
			attachments = [{
				'fname': self.get_file_name(),
				'fcontent': data
			}]

		frappe.sendmail(
			recipients = self.email_to.split(),
			subject = self.name,
			message = message,
			attachments = attachments,
			reference_doctype = self.doctype,
			reference_name = self.name
		)

	def dynamic_date_filters_set(self):
		return self.dynamic_date_period and self.from_date_field and self.to_date_field

@frappe.whitelist()
def download(name):
	'''Download report locally'''
	auto_email_report = frappe.get_doc('Auto Email Report', name)
	auto_email_report.check_permission()
	data = auto_email_report.get_report_content()

	if not data:
		frappe.msgprint(_('No Data'))
		return

	frappe.local.response.filecontent = data
	frappe.local.response.type = "download"
	frappe.local.response.filename = auto_email_report.get_file_name()

@frappe.whitelist()
def send_now(name):
	'''Send Auto Email report now'''
	auto_email_report = frappe.get_doc('Auto Email Report', name)
	auto_email_report.check_permission()
	auto_email_report.send()

def send_daily():
	'''Check reports to be sent daily'''

	current_day = calendar.day_name[now_datetime().weekday()]
	enabled_reports = frappe.get_all('Auto Email Report',
		filters={'enabled': 1, 'frequency': ('in', ('Daily', 'Weekdays', 'Weekly'))})

	for report in enabled_reports:
		auto_email_report = frappe.get_doc('Auto Email Report', report.name)

		# if not correct weekday, skip
		if auto_email_report.frequency == "Weekdays":
			if current_day in ("Saturday", "Sunday"):
				continue
		elif auto_email_report.frequency == 'Weekly':
			if auto_email_report.day_of_week != current_day:
				continue
		try:
			auto_email_report.send()
		except Exception as e:
			frappe.log_error(e, _('Failed to send {0} Auto Email Report').format(auto_email_report.name))

def send_monthly():
	'''Check reports to be sent monthly'''
	for report in frappe.get_all('Auto Email Report', {'enabled': 1, 'frequency': 'Monthly'}):
		frappe.get_doc('Auto Email Report', report.name).send()

def make_links(columns, data):
	for row in data:
		for col in columns:
			if col.fieldtype == "Link" and col.options != "Currency":
				if col.options and row.get(col.fieldname):
					row[col.fieldname] = get_link_to_form(col.options, row[col.fieldname])
			elif col.fieldtype == "Dynamic Link":
				if col.options and row.get(col.fieldname) and row.get(col.options):
					row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname])
			elif col.fieldtype == "Currency" and row.get(col.fieldname):
				row[col.fieldname] = frappe.format_value(row[col.fieldname], col)

	return columns, data

def update_field_types(columns):
	for col in columns:
		if col.fieldtype in  ("Link", "Dynamic Link", "Currency")  and col.options != "Currency":
			col.fieldtype = "Data"
			col.options = ""
	return columns<|endoftext|>


<commit_before>''' -- imports from installed packages -- '''
from django.http import StreamingHttpResponse, HttpResponseRedirect
from django.shortcuts import render_to_response, render
from django.template import RequestContext
from django.contrib.auth.models import User
from django.contrib.auth.decorators import user_passes_test

from gnowsys_ndf.settings import LANGUAGES
from gnowsys_ndf.ndf.models import node_collection, triple_collection
from gnowsys_ndf.ndf.views.methods import *

import json
import datetime

@user_passes_test(lambda u: u.is_superuser)
def adminDesignerDashboardClass(request, class_name='GSystemType'):
    '''
    fetching class's objects
    '''
    if request.method == "POST":
        search = request.POST.get("search","")
        classtype = request.POST.get("class","")
        nodes = node_collection.find({'name':{'$regex':search,'$options': 'i' },'_type':classtype}).sort('last_update', -1)
    else :
        nodes = node_collection.find({'_type':class_name}).sort('last_update', -1)

    objects_details = []
    for each in nodes:
        member = []
        member_of_list = []
        collection_list = []
        attribute_type_set = []
        relation_type_set = []
        for e in each.member_of:
            member_of_list.append(node_collection.one({'_id':e}).name+" ")
            # member_of_list.append(node_collection.one({'_id':e}).name+" - "+str(e))

        for members in each.member_of:
            member.append(node_collection.one({ '_id': members}).name+" ")
            # member.append(node_collection.one({ '_id': members}).name+" - "+str(members))

        # for coll in each.collection_set:
        #     collection_list.append(node_collection.one({ '_id': coll}).name+" - "+str(coll))

        if class_name in ("GSystemType"):
            for at_set in each.attribute_type_set:
                attribute_type_set.append(at_set.name+" ")
                # attribute_type_set.append(at_set.name+" - "+str(at_set._id))
            for rt_set in each.relation_type_set:
                relation_type_set.append(rt_set.name+" ")
                # relation_type_set.append(rt_set.name+" - "+str(rt_set._id))
            objects_details.append({"Id":each._id,"Title":each.name,"Type":",".join(member),"Author":User.objects.get(id=each.created_by).username,"Creation":each.created_at,'member_of':",".join(member_of_list), "collection_list":",".join(collection_list), "attribute_type_set":",".join(attribute_type_set), "relation_type_set":",".join(relation_type_set)})
        else :
		objects_details.append({"Id":each._id,"Title":each.name,"Type":",".join(member),"Author":User.objects.get(id=each.created_by).username,"Creation":each.created_at,'member_of':",".join(member_of_list), "collection_list":",".join(collection_list)})
    groups = []
    group = node_collection.find({'_type':"Group"})
    for each in group:
        groups.append({'id':each._id,"title":each.name})

    systemtypes = []
    systemtype = node_collection.find({'_type':"GSystemType"})
    for each in systemtype:
        systemtypes.append({'id':each._id,"title":each.name})

    meta_types = []
    meta_type = node_collection.find({'_type':"MetaType"})
    for each in meta_type:
        meta_types.append({'id':each._id,"title":each.name})

    groupid = ""
    group_obj= node_collection.find({'$and':[{"_type":u'Group'},{"name":u'home'}]})
    if group_obj:
	groupid = str(group_obj[0]._id)

    template = "ndf/adminDashboard.html"
    variable = RequestContext(request, {'class_name':class_name,"nodes":objects_details,"Groups":groups,"systemtypes":systemtypes,"url":"designer","groupid":groupid,'meta_types':meta_types,'group_id':groupid})
    return render_to_response(template, variable)

@user_passes_test(lambda u: u.is_superuser)
def adminDesignerDashboardClassCreate(request, class_name='GSystemType', node_id=None):
    '''
    delete class's objects
    '''
    global LANGUAGES
    new_instance_type = None
    LANGUAGES = '' if not LANGUAGES else LANGUAGES
    definitionlist = []
    contentlist = []
    dependencylist = []
    options = []
    translate=request.GET.get('translate','')
    if class_name == "AttributeType":
        definitionlist = ['name','altnames','language','subject_type','data_type','applicable_node_type','member_of','verbose_name','null','blank','help_text','max_digits','decimal_places','auto_now','auto_now_add','path','verify_exist','status']
        # definitionlist = ['name','altnames','language','subject_type','data_type','member_of','verbose_name','null','blank','help_text','max_digits','decimal_places','auto_now','auto_now_add','path','verify_exist','status']
        contentlist = ['content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_at','start_publication','tags','url','last_update','login_required']
    elif class_name == "GSystemType":
        definitionlist = ['name','altnames','language','status','member_of','meta_type_set','attribute_type_set','relation_type_set','type_of']
        contentlist = ['content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_at','start_publication','tags','url','last_update','login_required']
    elif class_name == "RelationType":
        definitionlist = ['name','inverse_name','altnames','language','subject_type','object_type','subject_cardinality','object_cardinality','subject_applicable_nodetype','object_applicable_nodetype','is_symmetric','is_reflexive','is_transitive','status','member_of']
        # definitionlist = ['name','inverse_name','altnames','language','subject_type','object_type','subject_cardinality','object_cardinality','is_symmetric','is_reflexive','is_transitive','status','member_of']
        contentlist = ['content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_at','start_publication','tags','url','last_update','login_required']
    else :
        definitionlist = []
        contentlist = []
        dependencylist = []
        options = []

    class_structure = eval(class_name).structure

    required_fields = eval(class_name).required_fields

    newdict = {}
    if node_id:
        new_instance_type = node_collection.one({'_type': unicode(class_name), '_id': ObjectId(node_id)})

    else:
        new_instance_type = eval("node_collection.collection"+"."+class_name)()

    if request.method=="POST":
        if translate:
            new_instance_type = eval("node_collection.collection"+"."+class_name)()
        # print new_instance_type
        for key,value in class_structure.items():
            if value == bool:
                if request.POST.get(key,""):
                    if request.POST.get(key,"") in ('1','2'):
                        if request.POST.get(key,"") == '1':
                            new_instance_type[key] = True
                        else :
                            new_instance_type[key] = False

            elif value == unicode:
                if request.POST.get(key,""):

                    if key == "content_org":
                        new_instance_type[key] = unicode(request.POST.get(key,""))
                        # Required to link temporary files with the current user who is modifying this document
                        usrname = request.user.username
                        filename = slugify(new_instance_type['name']) + "-" + usrname + "-"
                        new_instance_type['content'] = org2html(new_instance_type[key], file_prefix=filename)
                    else :
                        if translate:
                            if key in ("name","inverse_name"):
                                new_instance_type[key] = unicode(request.POST.get(key+"_trans",""))
                                language= request.POST.get('lan')
                                new_instance_type.language = get_language_tuple(language)

                            else:
                                new_instance_type[key] = unicode(request.POST.get(key,""))
                        else:
                            new_instance_type[key] = unicode(request.POST.get(key,""))

            elif value == list:
                if request.POST.get(key,""):
                    new_instance_type[key] = request.POST.get(key,"").split(",")

            elif type(value) == list:
                if request.POST.get(key,""):
                    if key in ("tags","applicable_node_type"):
                        new_instance_type[key] = request.POST.get(key,"").split(",")
                    elif key in ["meta_type_set","attribute_type_set","relation_type_set"]:
                        listoflist = []
                        for each in request.POST.get(key,"").split(","):
                            listoflist.append(node_collection.one({"_id":ObjectId(each)}))
                        new_instance_type[key] = listoflist
                    else :
                        listoflist = []

                        for each in request.POST.get(key,"").split(","):

                            listoflist.append(ObjectId(each))
                        new_instance_type[key] = listoflist

            elif type(value) == tuple:
                new_instance_type[key] = tuple(eval(request.POST.get(key,"")))

            elif value == datetime.datetime:
                if key == "last_update":
                    new_instance_type[key] = datetime.datetime.now()

            elif key == "status":
                if request.POST.get(key,""):
                    new_instance_type[key] = unicode(request.POST.get(key,""))

            # elif key == "created_by":
            #     new_instance_type[key] = request.user.id

            elif value == int:
                if request.POST.get(key,""):
                    new_instance_type[key] = int(request.POST.get(key,""))

            else:

                if request.POST.get(key,""):
                    new_instance_type[key] = request.POST.get(key,"")

        user_id = request.user.id
        if not new_instance_type.has_key('_id'):
            new_instance_type.created_by = user_id

        new_instance_type.modified_by = user_id

        if user_id not in new_instance_type.contributors:
            new_instance_type.contributors.append(user_id)
        parent_node=node_collection.one({'_id':ObjectId(node_id)})
        if translate and class_name == "RelationType":
            new_instance_type.subject_type = parent_node.subject_type
            new_instance_type.object_type = parent_node.object_type
        if translate and class_name == "AttributeType":
            new_instance_type.data_type = parent_node.data_type
            new_instance_type.subject_type = parent_node.subject_type

        new_instance_type.save()
        if translate:
            relation_type=node_collection.one({'$and':[{'name':'translation_of'},{'_type':'RelationType'}]})
            grelation=node_collection.collection.GRelation()
            grelation.relation_type=relation_type
            grelation.subject=new_instance_type['_id']
            grelation.right_subject=ObjectId(node_id)
            grelation.name=u""
            grelation.save()

        return HttpResponseRedirect("/admin/designer/"+class_name)

    # If GET request ---------------------------------------------------------------------------------------

    for key,value in class_structure.items():
        if value == bool:
            # newdict[key] = "bool"
            newdict[key] = ["bool", new_instance_type[key]]
        elif value == unicode:
            if key == "language":
                newdict[key] = ["list", new_instance_type[key]]
            else:
                # newdict[key] = "unicode"
                newdict[key] = ["unicode", new_instance_type[key]]

        elif value == list:
            # newdict[key] = "list"

            newdict[key] = ["list", new_instance_type[key]]
        elif type(value) == list:
            # newdict[key] = "list"
            newdict[key] = ["list", new_instance_type[key]]
        elif value == datetime.datetime:
            # newdict[key] = "datetime"
            newdict[key] = ["datetime", new_instance_type[key]]
        elif value == int:
            # newdict[key] = "int"
            newdict[key] = ["int", new_instance_type[key]]
        elif key == "status":
            # newdict[key] = "status"
            newdict[key] = ["status", new_instance_type[key]]
        else:
            # newdict[key] = value
            newdict[key] = [value, new_instance_type[key]]

    class_structure = newdict

    groupid = ""
    group_obj= node_collection.find({'$and':[{"_type":u'Group'},{"name":u'home'}]})
    if group_obj:
	groupid = str(group_obj[0]._id)

    template = "ndf/adminDashboardCreate.html"

    variable =  None
    class_structure_with_values = {}
    if node_id:

        for key, value in class_structure.items():
            class_structure_with_values[key] = [class_structure[key][0], new_instance_type[key]]

        variable = RequestContext(request, {'node': new_instance_type,
                                            'class_name': class_name, 'class_structure': class_structure_with_values, 'url': "designer",
                                            'definitionlist': definitionlist, 'contentlist': contentlist, 'dependencylist': dependencylist,
                                            'options': options, 'required_fields': required_fields,"translate":translate,"lan":LANGUAGES,
                                            'groupid': groupid,'group_id':groupid
                                        })

    else:

        variable = RequestContext(request, {'class_name':class_name, "url":"designer", "class_structure":class_structure, 'definitionlist':definitionlist, 'contentlist':contentlist, 'dependencylist':dependencylist, 'options':options, "required_fields":required_fields,"groupid":groupid,"translate":translate,"lan":LANGUAGES,'group_id':groupid })

    return render_to_response(template, variable)<commit_msg>code to save elements from admin designer dashboard<commit_after>''' -- imports from installed packages -- '''
from django.http import StreamingHttpResponse, HttpResponseRedirect
from django.shortcuts import render_to_response, render
from django.template import RequestContext
from django.contrib.auth.models import User
from django.contrib.auth.decorators import user_passes_test

from gnowsys_ndf.settings import LANGUAGES
from gnowsys_ndf.ndf.models import node_collection, triple_collection
from gnowsys_ndf.ndf.views.methods import *

import json
import datetime

@user_passes_test(lambda u: u.is_superuser)
def adminDesignerDashboardClass(request, class_name='GSystemType'):
    '''
    fetching class's objects
    '''
    if request.method == "POST":
        search = request.POST.get("search","")
        classtype = request.POST.get("class","")
        nodes = node_collection.find({'name':{'$regex':search,'$options': 'i' },'_type':classtype}).sort('last_update', -1)
    else :
        nodes = node_collection.find({'_type':class_name}).sort('last_update', -1)

    objects_details = []
    for each in nodes:
        member = []
        member_of_list = []
        collection_list = []
        attribute_type_set = []
        relation_type_set = []
        for e in each.member_of:
            member_of_list.append(node_collection.one({'_id':e}).name+" ")
            # member_of_list.append(node_collection.one({'_id':e}).name+" - "+str(e))

        for members in each.member_of:
            member.append(node_collection.one({ '_id': members}).name+" ")
            # member.append(node_collection.one({ '_id': members}).name+" - "+str(members))

        # for coll in each.collection_set:
        #     collection_list.append(node_collection.one({ '_id': coll}).name+" - "+str(coll))

        if class_name in ("GSystemType"):
            for at_set in each.attribute_type_set:
                attribute_type_set.append(at_set.name+" ")
                # attribute_type_set.append(at_set.name+" - "+str(at_set._id))
            for rt_set in each.relation_type_set:
                relation_type_set.append(rt_set.name+" ")
                # relation_type_set.append(rt_set.name+" - "+str(rt_set._id))
            objects_details.append({"Id":each._id,"Title":each.name,"Type":",".join(member),"Author":User.objects.get(id=each.created_by).username,"Creation":each.created_at,'member_of':",".join(member_of_list), "collection_list":",".join(collection_list), "attribute_type_set":",".join(attribute_type_set), "relation_type_set":",".join(relation_type_set)})
        else :
		objects_details.append({"Id":each._id,"Title":each.name,"Type":",".join(member),"Author":User.objects.get(id=each.created_by).username,"Creation":each.created_at,'member_of':",".join(member_of_list), "collection_list":",".join(collection_list)})
    groups = []
    group = node_collection.find({'_type':"Group"})
    for each in group:
        groups.append({'id':each._id,"title":each.name})

    systemtypes = []
    systemtype = node_collection.find({'_type':"GSystemType"})
    for each in systemtype:
        systemtypes.append({'id':each._id,"title":each.name})

    meta_types = []
    meta_type = node_collection.find({'_type':"MetaType"})
    for each in meta_type:
        meta_types.append({'id':each._id,"title":each.name})

    groupid = ""
    group_obj= node_collection.find({'$and':[{"_type":u'Group'},{"name":u'home'}]})
    if group_obj:
	groupid = str(group_obj[0]._id)

    template = "ndf/adminDashboard.html"
    variable = RequestContext(request, {'class_name':class_name,"nodes":objects_details,"Groups":groups,"systemtypes":systemtypes,"url":"designer","groupid":groupid,'meta_types':meta_types,'group_id':groupid})
    return render_to_response(template, variable)

@user_passes_test(lambda u: u.is_superuser)
def adminDesignerDashboardClassCreate(request, class_name='GSystemType', node_id=None):
    '''
    delete class's objects
    '''
    global LANGUAGES
    new_instance_type = None
    LANGUAGES = '' if not LANGUAGES else LANGUAGES
    definitionlist = []
    contentlist = []
    dependencylist = []
    options = []
    translate=request.GET.get('translate','')
    if class_name == "AttributeType":
        definitionlist = ['name','altnames','language','subject_type','data_type','applicable_node_type','member_of','verbose_name','null','blank','help_text','max_digits','decimal_places','auto_now','auto_now_add','path','verify_exist','status']
        # definitionlist = ['name','altnames','language','subject_type','data_type','member_of','verbose_name','null','blank','help_text','max_digits','decimal_places','auto_now','auto_now_add','path','verify_exist','status']
        contentlist = ['content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_at','start_publication','tags','url','last_update','login_required']
    elif class_name == "GSystemType":
        definitionlist = ['name','altnames','language','status','member_of','meta_type_set','attribute_type_set','relation_type_set','type_of']
        contentlist = ['content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_at','start_publication','tags','url','last_update','login_required']
    elif class_name == "RelationType":
        definitionlist = ['name','inverse_name','altnames','language','subject_type','object_type','subject_cardinality','object_cardinality','subject_applicable_nodetype','object_applicable_nodetype','is_symmetric','is_reflexive','is_transitive','status','member_of']
        # definitionlist = ['name','inverse_name','altnames','language','subject_type','object_type','subject_cardinality','object_cardinality','is_symmetric','is_reflexive','is_transitive','status','member_of']
        contentlist = ['content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_at','start_publication','tags','url','last_update','login_required']
    else :
        definitionlist = []
        contentlist = []
        dependencylist = []
        options = []

    class_structure = eval(class_name).structure

    required_fields = eval(class_name).required_fields

    newdict = {}
    if node_id:
        new_instance_type = node_collection.one({'_type': unicode(class_name), '_id': ObjectId(node_id)})
    else:
        new_instance_type = eval("node_collection.collection"+"."+class_name)()

    if request.method=="POST":
        if translate:
            new_instance_type = eval("node_collection.collection"+"."+class_name)()
        for key,value in class_structure.items():

            if value == bool:
                if request.POST.get(key,""):
                    if request.POST.get(key,"") in ('1','2'):
                        if request.POST.get(key,"") == '1':
                            new_instance_type[key] = True
                        else :
                            new_instance_type[key] = False

            elif value == unicode:
                if request.POST.get(key,""):

                    if key == "content_org":
                        new_instance_type[key] = unicode(request.POST.get(key,""))
                        # Required to link temporary files with the current user who is modifying this document
                        usrname = request.user.username
                        filename = slugify(new_instance_type['name']) + "-" + usrname + "-"
                        new_instance_type['content'] = org2html(new_instance_type[key], file_prefix=filename)
                    else :
                        if translate:
                            if key in ("name","inverse_name"):
                                new_instance_type[key] = unicode(request.POST.get(key+"_trans",""))
                                language= request.POST.get('lan')
                                new_instance_type.language = get_language_tuple(language)

                            else:
                                new_instance_type[key] = unicode(request.POST.get(key,""))
                        else:
                            new_instance_type[key] = unicode(request.POST.get(key,""))

            elif value == list:
                if request.POST.get(key,""):
                    new_instance_type[key] = request.POST.get(key,"").split(",")

            elif type(value) == list:
                if request.POST.get(key,""):
                    if key in ("tags","applicable_node_type"):
                        new_instance_type[key] = request.POST.get(key,"").split(",")
                    elif key in ["meta_type_set","attribute_type_set","relation_type_set"]:
                        listoflist = []
                        for each in request.POST.get(key,"").split(","):
                            listoflist.append(node_collection.one({"_id":ObjectId(each)}))
                        new_instance_type[key] = listoflist

                    elif key in ["member_of","prior_node","type_of"]:
                        new_mem = request.POST.get(key,"").split(",")
                        new_mem_list = [ObjectId(each) for each in new_mem]
                        new_instance_type[key] = new_mem_list

                    else :
                        listoflist = []
                        for each in request.POST.get(key,"").split(","):
                            listoflist.append(ObjectId(each))
                        new_instance_type[key] = listoflist

            elif type(value) == tuple:
                new_instance_type[key] = tuple(eval(request.POST.get(key,"")))

            elif value == datetime.datetime:
                if key == "last_update":
                    new_instance_type[key] = datetime.datetime.now()

            elif key == "status":
                if request.POST.get(key,""):
                    new_instance_type[key] = unicode(request.POST.get(key,""))

            # elif key == "created_by":
            #     new_instance_type[key] = request.user.id

            elif value == int:
                if request.POST.get(key,""):
                    new_instance_type[key] = int(request.POST.get(key,""))

            else:

                if request.POST.get(key,""):
                    new_instance_type[key] = request.POST.get(key,"")

        user_id = request.user.id
        if not new_instance_type.has_key('_id'):
            new_instance_type.created_by = user_id

        new_instance_type.modified_by = user_id

        if user_id not in new_instance_type.contributors:
            new_instance_type.contributors.append(user_id)
        parent_node=node_collection.one({'_id':ObjectId(node_id)})
        if translate and class_name == "RelationType":
            new_instance_type.subject_type = parent_node.subject_type
            new_instance_type.object_type = parent_node.object_type
        if translate and class_name == "AttributeType":
            new_instance_type.data_type = parent_node.data_type
            new_instance_type.subject_type = parent_node.subject_type

        new_instance_type.save()
        if translate:
            relation_type=node_collection.one({'$and':[{'name':'translation_of'},{'_type':'RelationType'}]})
            grelation=node_collection.collection.GRelation()
            grelation.relation_type=relation_type
            grelation.subject=new_instance_type['_id']
            grelation.right_subject=ObjectId(node_id)
            grelation.name=u""
            grelation.save()

        return HttpResponseRedirect("/admin/designer/"+class_name)

    # If GET request ---------------------------------------------------------------------------------------
    for key,value in class_structure.items():

        if value == bool:
            # newdict[key] = "bool"
            newdict[key] = ["bool", new_instance_type[key]]
        elif value == unicode:
            if key == "language":
                newdict[key] = ["list", new_instance_type[key]]
            else:
                # newdict[key] = "unicode"
                newdict[key] = ["unicode", new_instance_type[key]]

        elif value == list:
            # newdict[key] = "list"
            newdict[key] = ["list", new_instance_type[key]]

        elif type(value) == list:
            # newdict[key] = "list"
            newdict[key] = ["list", new_instance_type[key]]

        elif value == datetime.datetime:
            # newdict[key] = "datetime"
            newdict[key] = ["datetime", new_instance_type[key]]
        elif value == int:
            # newdict[key] = "int"
            newdict[key] = ["int", new_instance_type[key]]
        elif key == "status":
            # newdict[key] = "status"
            newdict[key] = ["status", new_instance_type[key]]
        else:
            # newdict[key] = value
            newdict[key] = [value, new_instance_type[key]]

    class_structure = newdict

    groupid = ""
    group_obj= node_collection.find({'$and':[{"_type":u'Group'},{"name":u'home'}]})
    if group_obj:
	groupid = str(group_obj[0]._id)

    template = "ndf/adminDashboardCreate.html"

    variable =  None
    class_structure_with_values = {}
    if node_id:

        for key, value in class_structure.items():
            class_structure_with_values[key] = [class_structure[key][0], new_instance_type[key]]

        variable = RequestContext(request, {'node': new_instance_type,
                                            'class_name': class_name, 'class_structure': class_structure_with_values, 'url': "designer",
                                            'definitionlist': definitionlist, 'contentlist': contentlist, 'dependencylist': dependencylist,
                                            'options': options, 'required_fields': required_fields,"translate":translate,"lan":LANGUAGES,
                                            'groupid': groupid,'group_id':groupid
                                        })

    else:

        variable = RequestContext(request, {'class_name':class_name, "url":"designer", "class_structure":class_structure, 'definitionlist':definitionlist, 'contentlist':contentlist, 'dependencylist':dependencylist, 'options':options, "required_fields":required_fields,"groupid":groupid,"translate":translate,"lan":LANGUAGES,'group_id':groupid })

    return render_to_response(template, variable)<|endoftext|>