<commit_before>import re
from collections import defaultdict

from openerp.osv import expression
from openerp.tools.safe_eval import safe_eval

MODE_VARIATION = 'p'
MODE_INITIAL = 'i'
MODE_END = 'e'

class AccountingExpressionProcessor(object):
    """ Processor for accounting expressions.

    Expressions of the form <field><mode>[accounts][optional move line domain]
    are supported, where:
        * field is bal, crd, deb
        * mode is i (initial balance), e (ending balance),
          p (moves over period)
        * accounts is a list of accounts, possibly containing % wildcards
        * an optional domain on move lines allowing filters on eg analytic
          accounts or journal

    Examples:
        * bal[70]: variation of the balance of moves on account 70
          over the period (it is the same as balp[70]);
        * bali[70,60]: initial balance of accounts 70 and 60;
        * bale[1%]: balance of accounts starting with 1 at end of period.

    How to use:
        * repeatedly invoke parse_expr() for each expression containing
          accounting variables as described above; this lets the processor
          group domains and modes and accounts;
        * when all expressions have been parsed, invoke done_parsing()
          to notify the processor that it can prepare to query (mainly
          search all accounts - children, consolidation - that will need to
          be queried;
        * for each period, call do_queries(), then call replace_expr() for each
          expression to replace accounting variables with their resulting value
          for the given period.

    How it works:
        * by accumulating the expressions before hand, it ensures to do the
          strict minimum number of queries to the database (for each period,
          one query per domain and mode);
        * it queries using the orm read_group which reduces to a query with
          sum on debit and credit and group by on account_id (note: it seems
          the orm then does one query per account to fetch the account
          name...);
        * additionally, one query per view/consolidation account is done to
          discover the children accounts.
    """

    ACC_RE = re.compile(r"(?P<field>\bbal|\bcrd|\bdeb)"
                        r"(?P<mode>[pise])?"
                        r"(?P<accounts>_[0-9]+|\[.*?\])"
                        r"(?P<domain>\[.*?\])?")

    def __init__(self, env):
        self.env = env
        # before done_parsing: {(domain, mode): set(account_codes)}
        # after done_parsing: {(domain, mode): set(account_ids)}
        self._map_account_ids = defaultdict(set)
        self._account_ids_by_code = defaultdict(set)

    def _load_account_codes(self, account_codes, root_account):
        account_model = self.env['account.account']
        # TODO: account_obj is necessary because _get_children_and_consol
        #       does not work in new API?
        account_obj = self.env.registry('account.account')
        exact_codes = set()
        like_codes = set()
        for account_code in account_codes:
            if account_code in self._account_ids_by_code:
                continue
            if account_code is None:
                # by convention the root account is keyed as
                # None in _account_ids_by_code, so it is consistent
                # with what _parse_match_object returns for an
                # empty list of account codes, ie [None]
                exact_codes.add(root_account.code)
            elif '%' in account_code:
                like_codes.add(account_code)
            else:
                exact_codes.add(account_code)
        for account in account_model.\
                search([('code', 'in', list(exact_codes)),
                        ('parent_id', 'child_of', root_account.id)]):
            if account.code == root_account.code:
                code = None
            else:
                code = account.code
            if account.type in ('view', 'consolidation'):
                self._account_ids_by_code[code].update(
                    account_obj._get_children_and_consol(
                        self.env.cr, self.env.uid,
                        [account.id],
                        self.env.context))
            else:
                self._account_ids_by_code[code].add(account.id)
        for like_code in like_codes:
            for account in account_model.\
                    search([('code', 'like', like_code),
                            ('parent_id', 'child_of', root_account.id)]):
                if account.type in ('view', 'consolidation'):
                    self._account_ids_by_code[like_code].update(
                        account_obj._get_children_and_consol(
                            self.env.cr, self.env.uid,
                            [account.id],
                            self.env.context))
                else:
                    self._account_ids_by_code[like_code].add(account.id)

    def _parse_match_object(self, mo):
        """Split a match object corresponding to an accounting variable

        Returns field, mode, [account codes], [domain expression].
        """
        field, mode, account_codes, domain = mo.groups()
        if not mode:
            mode = MODE_VARIATION
        elif mode == 's':
            mode = MODE_END
        if account_codes.startswith('_'):
            account_codes = account_codes[1:]
        else:
            account_codes = account_codes[1:-1]
        if account_codes.strip():
            account_codes = [a.strip() for a in account_codes.split(',')]
        else:
            account_codes = [None]
        domain = domain or '[]'
        domain = tuple(safe_eval(domain))
        return field, mode, account_codes, domain

    def parse_expr(self, expr):
        """Parse an expression, extracting accounting variables.

        Domains and accounts are extracted and stored in the map
        so when all expressions have been parsed, we know what to query.
        """
        for mo in self.ACC_RE.finditer(expr):
            _, mode, account_codes, domain = self._parse_match_object(mo)
            key = (domain, mode)
            self._map_account_ids[key].update(account_codes)

    def done_parsing(self, root_account):
        # load account codes and replace account codes by account ids in _map
        for key, account_codes in self._map_account_ids.items():
            self._load_account_codes(account_codes, root_account)
            account_ids = set()
            for account_code in account_codes:
                account_ids.update(self._account_ids_by_code[account_code])
            self._map_account_ids[key] = list(account_ids)

    def has_account_var(self, expr):
        return bool(self.ACC_RE.search(expr))

    def get_aml_domain_for_expr(self, expr,
                                date_from, date_to,
                                period_from, period_to,
                                target_move):
        """ Get a domain on account.move.line for an expression.

        Prerequisite: done_parsing() must have been invoked.

        Returns a domain that can be used to search on account.move.line.
        To be meaningful, this domain must be extended with a domain
        obtained from get_aml_domain_for_dates()
        """
        aml_domains = []
        domain_by_mode = {}
        for mo in self.ACC_RE.finditer(expr):
            field, mode, account_codes, domain = self._parse_match_object(mo)
            aml_domain = list(domain)
            account_ids = set()
            for account_code in account_codes:
                account_ids.update(self._account_ids_by_code[account_code])
            aml_domain.append(('account_id', 'in', tuple(account_ids)))
            if field == 'crd':
                aml_domain.append(('credit', '>', 0))
            elif field == 'deb':
                aml_domain.append(('debit', '>', 0))
            aml_domains.append(expression.normalize_domain(aml_domain))
            if mode not in domain_by_mode:
                domain_by_mode[mode] = \
                    self.get_aml_domain_for_dates(date_from, date_to,
                                                  period_from, period_to,
                                                  mode, target_move)
        return expression.OR(aml_domains) + \
            expression.OR(domain_by_mode.values())

    def _period_has_moves(self, period):
        move_model = self.env['account.move']
        return bool(move_model.search([('period_id', '=', period.id)],
                                      limit=1))

    def _get_previous_opening_period(self, period, company_id):
        period_model = self.env['account.period']
        periods = period_model.search(
            [('date_start', '<=', period.date_start),
             ('special', '=', True),
             ('company_id', '=', company_id)],
            order="date_start desc",
            limit=1)
        return periods and periods[0]

    def _get_previous_normal_period(self, period, company_id):
        period_model = self.env['account.period']
        periods = period_model.search(
            [('date_start', '<', period.date_start),
             ('special', '=', False),
             ('company_id', '=', company_id)],
            order="date_start desc",
            limit=1)
        return periods and periods[0]

    def _get_first_normal_period(self, company_id):
        period_model = self.env['account.period']
        periods = period_model.search(
            [('special', '=', False),
             ('company_id', '=', company_id)],
            order="date_start asc",
            limit=1)
        return periods and periods[0]

    def _get_period_ids_between(self, period_from, period_to, company_id):
        period_model = self.env['account.period']
        periods = period_model.search(
            [('date_start', '>=', period_from.date_start),
             ('date_stop', '<=', period_to.date_stop),
             ('special', '=', False),
             ('company_id', '=', company_id)])
        period_ids = [p.id for p in periods]
        if period_from.special:
            period_ids.append(period_from.id)
        return period_ids

    def _get_period_company_ids(self, period_from, period_to):
        period_model = self.env['account.period']
        periods = period_model.search(
            [('date_start', '>=', period_from.date_start),
             ('date_stop', '<=', period_to.date_stop),
             ('special', '=', False)])
        return set([p.company_id.id for p in periods])

    def _get_period_ids_for_mode(self, period_from, period_to, mode):
        assert not period_from.special
        assert not period_to.special
        assert period_from.company_id == period_to.company_id
        assert period_from.date_start <= period_to.date_start
        period_ids = []
        for company_id in self._get_period_company_ids(period_from, period_to):
            if mode == MODE_VARIATION:
                period_ids.extend(self._get_period_ids_between(
                    period_from, period_to, company_id))
            else:
                if mode == MODE_INITIAL:
                    period_to = self._get_previous_normal_period(
                        period_from, company_id)
                # look for opening period with moves
                opening_period = self._get_previous_opening_period(
                    period_from, company_id)
                if opening_period and \
                        self._period_has_moves(opening_period[0]):
                    # found opening period with moves
                    if opening_period.date_start == period_from.date_start and \
                            mode == MODE_INITIAL:
                        # if the opening period has the same start date as
                        # period_from, then we'll find the initial balance
                        # in the initial period and that's it
                        period_ids.append(opening_period[0].id)
                        continue
                    period_from = opening_period[0]
                else:
                    # no opening period with moves,
                    # use very first normal period
                    period_from = self._get_first_normal_period(company_id)
                if period_to:
                    period_ids.extend(self._get_period_ids_between(
                        period_from, period_to, company_id))
        return period_ids

    def get_aml_domain_for_dates(self, date_from, date_to,
                                 period_from, period_to,
                                 mode,
                                 target_move):
        if period_from and period_to:
            period_ids = self._get_period_ids_for_mode(
                period_from, period_to, mode)
            domain = [('period_id', 'in', period_ids)]
        else:
            domain = [('date', '>=', date_from), ('date', '<=', date_to)]
        if target_move == 'posted':
            domain.append(('move_id.state', '=', 'posted'))
        return expression.normalize_domain(domain)

    def do_queries(self, date_from, date_to, period_from, period_to,
                   target_move):
        aml_model = self.env['account.move.line']
        # {(domain, mode): {account_id: (debit, credit)}}
        self._data = defaultdict(dict)
        domain_by_mode = {}
        for key in self._map_account_ids:
            domain, mode = key
            if mode not in domain_by_mode:
                domain_by_mode[mode] = \
                    self.get_aml_domain_for_dates(date_from, date_to,
                                                  period_from, period_to,
                                                  mode, target_move)
            domain = list(domain) + domain_by_mode[mode]
            domain.append(('account_id', 'in', self._map_account_ids[key]))
            # fetch sum of debit/credit, grouped by account_id
            accs = aml_model.read_group(domain,
                                        ['debit', 'credit', 'account_id'],
                                        ['account_id'])
            for acc in accs:
                self._data[key][acc['account_id'][0]] = \
                    (acc['debit'] or 0.0, acc['credit'] or 0.0)

    def replace_expr(self, expr):
        """Replace accounting variables in an expression by their amount.

        Returns a new expression.

        This method must be executed after do_queries().
        """
        def f(mo):
            field, mode, account_codes, domain = self._parse_match_object(mo)
            key = (domain, mode)
            account_ids_data = self._data[key]
            v = 0.0
            for account_code in account_codes:
                account_ids = self._account_ids_by_code[account_code]
                for account_id in account_ids:
                    debit, credit = \
                        account_ids_data.get(account_id, (0.0, 0.0))
                    if field == 'bal':
                        v += debit - credit
                    elif field == 'deb':
                        v += debit
                    elif field == 'crd':
                        v += credit
            return '(' + repr(v) + ')'
        return self.ACC_RE.sub(f, expr)
<commit_msg>[IMP] mis_builder: date and modes i/e are not compatibles<commit_after>import re
from collections import defaultdict

from openerp.exceptions import Warning
from openerp.osv import expression
from openerp.tools.safe_eval import safe_eval
from openerp.tools.translate import _

MODE_VARIATION = 'p'
MODE_INITIAL = 'i'
MODE_END = 'e'

class AccountingExpressionProcessor(object):
    """ Processor for accounting expressions.

    Expressions of the form <field><mode>[accounts][optional move line domain]
    are supported, where:
        * field is bal, crd, deb
        * mode is i (initial balance), e (ending balance),
          p (moves over period)
        * accounts is a list of accounts, possibly containing % wildcards
        * an optional domain on move lines allowing filters on eg analytic
          accounts or journal

    Examples:
        * bal[70]: variation of the balance of moves on account 70
          over the period (it is the same as balp[70]);
        * bali[70,60]: initial balance of accounts 70 and 60;
        * bale[1%]: balance of accounts starting with 1 at end of period.

    How to use:
        * repeatedly invoke parse_expr() for each expression containing
          accounting variables as described above; this lets the processor
          group domains and modes and accounts;
        * when all expressions have been parsed, invoke done_parsing()
          to notify the processor that it can prepare to query (mainly
          search all accounts - children, consolidation - that will need to
          be queried;
        * for each period, call do_queries(), then call replace_expr() for each
          expression to replace accounting variables with their resulting value
          for the given period.

    How it works:
        * by accumulating the expressions before hand, it ensures to do the
          strict minimum number of queries to the database (for each period,
          one query per domain and mode);
        * it queries using the orm read_group which reduces to a query with
          sum on debit and credit and group by on account_id (note: it seems
          the orm then does one query per account to fetch the account
          name...);
        * additionally, one query per view/consolidation account is done to
          discover the children accounts.
    """

    ACC_RE = re.compile(r"(?P<field>\bbal|\bcrd|\bdeb)"
                        r"(?P<mode>[pise])?"
                        r"(?P<accounts>_[0-9]+|\[.*?\])"
                        r"(?P<domain>\[.*?\])?")

    def __init__(self, env):
        self.env = env
        # before done_parsing: {(domain, mode): set(account_codes)}
        # after done_parsing: {(domain, mode): set(account_ids)}
        self._map_account_ids = defaultdict(set)
        self._account_ids_by_code = defaultdict(set)

    def _load_account_codes(self, account_codes, root_account):
        account_model = self.env['account.account']
        # TODO: account_obj is necessary because _get_children_and_consol
        #       does not work in new API?
        account_obj = self.env.registry('account.account')
        exact_codes = set()
        like_codes = set()
        for account_code in account_codes:
            if account_code in self._account_ids_by_code:
                continue
            if account_code is None:
                # by convention the root account is keyed as
                # None in _account_ids_by_code, so it is consistent
                # with what _parse_match_object returns for an
                # empty list of account codes, ie [None]
                exact_codes.add(root_account.code)
            elif '%' in account_code:
                like_codes.add(account_code)
            else:
                exact_codes.add(account_code)
        for account in account_model.\
                search([('code', 'in', list(exact_codes)),
                        ('parent_id', 'child_of', root_account.id)]):
            if account.code == root_account.code:
                code = None
            else:
                code = account.code
            if account.type in ('view', 'consolidation'):
                self._account_ids_by_code[code].update(
                    account_obj._get_children_and_consol(
                        self.env.cr, self.env.uid,
                        [account.id],
                        self.env.context))
            else:
                self._account_ids_by_code[code].add(account.id)
        for like_code in like_codes:
            for account in account_model.\
                    search([('code', 'like', like_code),
                            ('parent_id', 'child_of', root_account.id)]):
                if account.type in ('view', 'consolidation'):
                    self._account_ids_by_code[like_code].update(
                        account_obj._get_children_and_consol(
                            self.env.cr, self.env.uid,
                            [account.id],
                            self.env.context))
                else:
                    self._account_ids_by_code[like_code].add(account.id)

    def _parse_match_object(self, mo):
        """Split a match object corresponding to an accounting variable

        Returns field, mode, [account codes], [domain expression].
        """
        field, mode, account_codes, domain = mo.groups()
        if not mode:
            mode = MODE_VARIATION
        elif mode == 's':
            mode = MODE_END
        if account_codes.startswith('_'):
            account_codes = account_codes[1:]
        else:
            account_codes = account_codes[1:-1]
        if account_codes.strip():
            account_codes = [a.strip() for a in account_codes.split(',')]
        else:
            account_codes = [None]
        domain = domain or '[]'
        domain = tuple(safe_eval(domain))
        return field, mode, account_codes, domain

    def parse_expr(self, expr):
        """Parse an expression, extracting accounting variables.

        Domains and accounts are extracted and stored in the map
        so when all expressions have been parsed, we know what to query.
        """
        for mo in self.ACC_RE.finditer(expr):
            _, mode, account_codes, domain = self._parse_match_object(mo)
            key = (domain, mode)
            self._map_account_ids[key].update(account_codes)

    def done_parsing(self, root_account):
        # load account codes and replace account codes by account ids in _map
        for key, account_codes in self._map_account_ids.items():
            self._load_account_codes(account_codes, root_account)
            account_ids = set()
            for account_code in account_codes:
                account_ids.update(self._account_ids_by_code[account_code])
            self._map_account_ids[key] = list(account_ids)

    def has_account_var(self, expr):
        return bool(self.ACC_RE.search(expr))

    def get_aml_domain_for_expr(self, expr,
                                date_from, date_to,
                                period_from, period_to,
                                target_move):
        """ Get a domain on account.move.line for an expression.

        Prerequisite: done_parsing() must have been invoked.

        Returns a domain that can be used to search on account.move.line.
        To be meaningful, this domain must be extended with a domain
        obtained from get_aml_domain_for_dates()
        """
        aml_domains = []
        domain_by_mode = {}
        for mo in self.ACC_RE.finditer(expr):
            field, mode, account_codes, domain = self._parse_match_object(mo)
            aml_domain = list(domain)
            account_ids = set()
            for account_code in account_codes:
                account_ids.update(self._account_ids_by_code[account_code])
            aml_domain.append(('account_id', 'in', tuple(account_ids)))
            if field == 'crd':
                aml_domain.append(('credit', '>', 0))
            elif field == 'deb':
                aml_domain.append(('debit', '>', 0))
            aml_domains.append(expression.normalize_domain(aml_domain))
            if mode not in domain_by_mode:
                domain_by_mode[mode] = \
                    self.get_aml_domain_for_dates(date_from, date_to,
                                                  period_from, period_to,
                                                  mode, target_move)
        return expression.OR(aml_domains) + \
            expression.OR(domain_by_mode.values())

    def _period_has_moves(self, period):
        move_model = self.env['account.move']
        return bool(move_model.search([('period_id', '=', period.id)],
                                      limit=1))

    def _get_previous_opening_period(self, period, company_id):
        period_model = self.env['account.period']
        periods = period_model.search(
            [('date_start', '<=', period.date_start),
             ('special', '=', True),
             ('company_id', '=', company_id)],
            order="date_start desc",
            limit=1)
        return periods and periods[0]

    def _get_previous_normal_period(self, period, company_id):
        period_model = self.env['account.period']
        periods = period_model.search(
            [('date_start', '<', period.date_start),
             ('special', '=', False),
             ('company_id', '=', company_id)],
            order="date_start desc",
            limit=1)
        return periods and periods[0]

    def _get_first_normal_period(self, company_id):
        period_model = self.env['account.period']
        periods = period_model.search(
            [('special', '=', False),
             ('company_id', '=', company_id)],
            order="date_start asc",
            limit=1)
        return periods and periods[0]

    def _get_period_ids_between(self, period_from, period_to, company_id):
        period_model = self.env['account.period']
        periods = period_model.search(
            [('date_start', '>=', period_from.date_start),
             ('date_stop', '<=', period_to.date_stop),
             ('special', '=', False),
             ('company_id', '=', company_id)])
        period_ids = [p.id for p in periods]
        if period_from.special:
            period_ids.append(period_from.id)
        return period_ids

    def _get_period_company_ids(self, period_from, period_to):
        period_model = self.env['account.period']
        periods = period_model.search(
            [('date_start', '>=', period_from.date_start),
             ('date_stop', '<=', period_to.date_stop),
             ('special', '=', False)])
        return set([p.company_id.id for p in periods])

    def _get_period_ids_for_mode(self, period_from, period_to, mode):
        assert not period_from.special
        assert not period_to.special
        assert period_from.company_id == period_to.company_id
        assert period_from.date_start <= period_to.date_start
        period_ids = []
        for company_id in self._get_period_company_ids(period_from, period_to):
            if mode == MODE_VARIATION:
                period_ids.extend(self._get_period_ids_between(
                    period_from, period_to, company_id))
            else:
                if mode == MODE_INITIAL:
                    period_to = self._get_previous_normal_period(
                        period_from, company_id)
                # look for opening period with moves
                opening_period = self._get_previous_opening_period(
                    period_from, company_id)
                if opening_period and \
                        self._period_has_moves(opening_period[0]):
                    # found opening period with moves
                    if opening_period.date_start == period_from.date_start and \
                            mode == MODE_INITIAL:
                        # if the opening period has the same start date as
                        # period_from, then we'll find the initial balance
                        # in the initial period and that's it
                        period_ids.append(opening_period[0].id)
                        continue
                    period_from = opening_period[0]
                else:
                    # no opening period with moves,
                    # use very first normal period
                    period_from = self._get_first_normal_period(company_id)
                if period_to:
                    period_ids.extend(self._get_period_ids_between(
                        period_from, period_to, company_id))
        return period_ids

    def get_aml_domain_for_dates(self, date_from, date_to,
                                 period_from, period_to,
                                 mode,
                                 target_move):
        if period_from and period_to:
            period_ids = self._get_period_ids_for_mode(
                period_from, period_to, mode)
            domain = [('period_id', 'in', period_ids)]
        else:
            if mode == MODE_VARIATION:
                domain = [('date', '>=', date_from), ('date', '<=', date_to)]
            else:
                raise Warning(_("Modes i and e are only applicable for "
                                "fiscal periods"))
        if target_move == 'posted':
            domain.append(('move_id.state', '=', 'posted'))
        return expression.normalize_domain(domain)

    def do_queries(self, date_from, date_to, period_from, period_to,
                   target_move):
        aml_model = self.env['account.move.line']
        # {(domain, mode): {account_id: (debit, credit)}}
        self._data = defaultdict(dict)
        domain_by_mode = {}
        for key in self._map_account_ids:
            domain, mode = key
            if mode not in domain_by_mode:
                domain_by_mode[mode] = \
                    self.get_aml_domain_for_dates(date_from, date_to,
                                                  period_from, period_to,
                                                  mode, target_move)
            domain = list(domain) + domain_by_mode[mode]
            domain.append(('account_id', 'in', self._map_account_ids[key]))
            # fetch sum of debit/credit, grouped by account_id
            accs = aml_model.read_group(domain,
                                        ['debit', 'credit', 'account_id'],
                                        ['account_id'])
            for acc in accs:
                self._data[key][acc['account_id'][0]] = \
                    (acc['debit'] or 0.0, acc['credit'] or 0.0)

    def replace_expr(self, expr):
        """Replace accounting variables in an expression by their amount.

        Returns a new expression.

        This method must be executed after do_queries().
        """
        def f(mo):
            field, mode, account_codes, domain = self._parse_match_object(mo)
            key = (domain, mode)
            account_ids_data = self._data[key]
            v = 0.0
            for account_code in account_codes:
                account_ids = self._account_ids_by_code[account_code]
                for account_id in account_ids:
                    debit, credit = \
                        account_ids_data.get(account_id, (0.0, 0.0))
                    if field == 'bal':
                        v += debit - credit
                    elif field == 'deb':
                        v += debit
                    elif field == 'crd':
                        v += credit
            return '(' + repr(v) + ')'
        return self.ACC_RE.sub(f, expr)
<|endoftext|>


<commit_before>__author__ = 'sergey'

import subprocess

def get_table_engines():

    try:
        output = subprocess.check_output(["mysqld", "--version"])
        table_engines = ('MyISAM', 'InnoDB',)
        if output.find(b'MariaDB'):
            table_engines += ('Aria', 'TokuDB',)
    except:
        # No MySQL?
        table_engines = ()

    return table_engines
<commit_msg>Avoid error-log output from mysqld (MariaDB)<commit_after>__author__ = 'sergey'

import subprocess

def get_table_engines():

    try:
        output = subprocess.check_output(["mysqld", "--version"], stderr=subprocess.DEVNULL)
        table_engines = ('MyISAM', 'InnoDB',)
        if output.find(b'MariaDB'):
            table_engines += ('Aria', 'TokuDB',)
    except:
        # No MySQL?
        table_engines = ()

    return table_engines
<|endoftext|>


<commit_before>import os, time, sys
import logging

from .get_subreddit_submissions import GetSubredditSubmissions
from .general_utility import slugify

# Exceptions
from downloaders.imgur_downloader.imgurdownloader import FileExistsException
from downloaders.imgur_downloader.imgurdownloader import ImgurException

# downloaders
from downloaders.direct_link_download import direct_link_download
from downloaders.imgur_downloader.imgurdownloader import ImgurDownloader
from downloaders.gfycat.gfycat.gfycat import Gfycat
from downloaders.deviantart import download_deviantart_url

class DownloadSubredditSubmissions(GetSubredditSubmissions):
    """Downloads subreddit submissions"""

    def __init__(self, *args, **kwargs):
        # call constructor of GetSubredditSubmissions class passing args
        super().__init__(*args, **kwargs)

        # setup logging
        # logging.basicConfig(filename='download_history.log',
        #     format='%(levelname)s|%(name)s|%(asctime)s|%(message)s',
        #     datefmt='%m/%d/%y %H:%M:%S',
        #     level=logging.DEBUG)
        self.log = logging.getLogger('DownloadSubredditSubmissions')

    def download(self):
        """Download media from submissions"""
        media_extensions = ('.png', '.jpg', '.jpeg', '.webm', '.gif', '.mp4')
        submissions = self.get_submissions_info()

        for submission in submissions:
            url = submission['url']
            title = submission['title']
            filename = slugify(title)
            file_path = os.path.join(self.path, filename)

            self.log.info('Attempting to save %s as %s' % (url, file_path))

            try:
                if url.endswith(media_extensions):
                    direct_link_download(url, file_path)

                elif 'imgur.com' in url:
                    imgur = ImgurDownloader(imgur_url=url, dir_download=self.path,
                        file_name=filename, delete_dne=True, debug=False)
                    imgur.save_images()

                elif 'gfycat.com' in url:
                    gfycat_id = url.split('/')[-1]
                    gfycat = Gfycat().more(gfycat_id).download(save_dir)

                elif 'deviantart.com' in url:
                    download_deviantart_url(url, file_path)

            except (FileExistsException, FileExistsError) as e:
                msg = '%s already exists (url = %s)' % (file_path, url)
                self.log.warning(msg)
                print(msg)

            except ImugrException as e:
                msg = 'ImgurException: %s' % e.msg
                self.log.warning(msg)
                print(msg)

            except HTTPError as e:
                msg = 'HTTPError: %s' % e.msg
                self.log.warning(msg)
                print(msg)

            except Exception as e:
                msg = 'Exception: %s' % e.msg
                self.log.warning(msg)
                print(msg)
<commit_msg>condense exceptions into single tuple for catching<commit_after>import os, time, sys
import logging

from .get_subreddit_submissions import GetSubredditSubmissions
from .general_utility import slugify

# Exceptions
from downloaders.imgur_downloader.imgurdownloader import FileExistsException
from downloaders.imgur_downloader.imgurdownloader import ImgurException

# downloaders
from downloaders.direct_link_download import direct_link_download
from downloaders.imgur_downloader.imgurdownloader import ImgurDownloader
from downloaders.gfycat.gfycat.gfycat import Gfycat
from downloaders.deviantart import download_deviantart_url

class DownloadSubredditSubmissions(GetSubredditSubmissions):
    """Downloads subreddit submissions"""

    def __init__(self, *args, **kwargs):
        # call constructor of GetSubredditSubmissions class passing args
        super().__init__(*args, **kwargs)

        # setup logging
        # logging.basicConfig(filename='download_history.log',
        #     format='%(levelname)s|%(name)s|%(asctime)s|%(message)s',
        #     datefmt='%m/%d/%y %H:%M:%S',
        #     level=logging.DEBUG)
        self.log = logging.getLogger('DownloadSubredditSubmissions')
        self.Exceptions = (FileExistsException, FileExistsError,
            ImgurException, HTTPError)

    def download(self):
        """Download media from submissions"""
        submissions = self.get_submissions_info()
        media_extensions = ('.png', '.jpg', '.jpeg', '.webm', '.gif', '.mp4')

        # counters to keep track of how many submissions we downloaded & more
        download_count, error_count, skip_count = 0

        for submission in submissions:
            url = submission['url']
            title = submission['title']
            filename = slugify(title)
            file_path = os.path.join(self.path, filename)

            self.log.info('Attempting to save %s as %s' % (url, file_path))

            # check domain and call corresponding downloader download functions
            # or methods
            try:
                if url.endswith(media_extensions):
                    direct_link_download(url, file_path)

                elif 'imgur.com' in url:
                    imgur = ImgurDownloader(imgur_url=url,
                        dir_download=self.path, file_name=filename,
                        delete_dne=True, debug=False)
                    imgur.save_images()

                elif 'gfycat.com' in url:
                    gfycat_id = url.split('/')[-1]
                    gfycat = Gfycat().more(gfycat_id).download(save_dir)

                elif 'deviantart.com' in url:
                    download_deviantart_url(url, file_path)

            # except (FileExistsException, FileExistsError) as e:
            #     msg = '%s already exists (url = %s)' % (file_path, url)
            #     self.log.warning(msg)
            #     print(msg)

            except self.Exceptions as e:
                msg = '%s: %s' % (type(e).__name__, e.msg)
                self.log.warning(msg)
                print(msg)
<|endoftext|>


<commit_before># Authors: Denis Engemann <denis.engemann@gmail.com>
#          Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#
# License: Simplified BSD

import os.path as op
import warnings

from numpy.testing import assert_raises

from mne import io, read_events, Epochs, read_cov
from mne import pick_types
from mne.utils import run_tests_if_main, requires_sklearn
from mne.preprocessing import ICA, create_ecg_epochs, create_eog_epochs

# Set our plotters to test mode
import matplotlib
matplotlib.use('Agg')  # for testing don't use X server

warnings.simplefilter('always')  # enable b/c these tests throw warnings

base_dir = op.join(op.dirname(__file__), '..', '..', 'io', 'tests', 'data')
evoked_fname = op.join(base_dir, 'test-ave.fif')
raw_fname = op.join(base_dir, 'test_raw.fif')
cov_fname = op.join(base_dir, 'test-cov.fif')
event_name = op.join(base_dir, 'test-eve.fif')
event_id, tmin, tmax = 1, -0.1, 0.2

def _get_raw(preload=False):
    return io.Raw(raw_fname, preload=preload)

def _get_events():
    return read_events(event_name)

def _get_picks(raw):
    return [0, 1, 2, 6, 7, 8, 12, 13, 14]  # take a only few channels

def _get_epochs():
    raw = _get_raw()
    events = _get_events()
    picks = _get_picks(raw)
    epochs = Epochs(raw, events[:10], event_id, tmin, tmax, picks=picks,
                    baseline=(None, 0))
    return epochs

@requires_sklearn
def test_plot_ica_components():
    """Test plotting of ICA solutions
    """
    import matplotlib.pyplot as plt
    raw = _get_raw()
    ica = ICA(noise_cov=read_cov(cov_fname), n_components=2,
              max_pca_components=3, n_pca_components=3)
    ica_picks = _get_picks(raw)
    ica.fit(raw, picks=ica_picks)
    warnings.simplefilter('always', UserWarning)
    with warnings.catch_warnings(record=True):
        for components in [0, [0], [0, 1], [0, 1] * 2, None]:
            ica.plot_components(components, image_interp='bilinear', res=16)
    ica.info = None
    assert_raises(RuntimeError, ica.plot_components, 1)
    plt.close('all')

@requires_sklearn
def test_plot_ica_sources():
    """Test plotting of ICA panel
    """
    import matplotlib.pyplot as plt
    raw = io.Raw(raw_fname, preload=True)
    raw.crop(0, 1, copy=False)
    picks = _get_picks(raw)
    epochs = _get_epochs()
    raw.pick_channels([raw.ch_names[k] for k in picks])
    ica_picks = pick_types(raw.info, meg=True, eeg=False, stim=False,
                           ecg=False, eog=False, exclude='bads')
    ica = ICA(n_components=2, max_pca_components=3, n_pca_components=3)
    ica.fit(raw, picks=ica_picks)
    raw.info['bads'] = ['MEG 0113']
    assert_raises(RuntimeError, ica.plot_sources, inst=raw)
    ica.plot_sources(epochs)
    epochs.info['bads'] = ['MEG 0113']
    assert_raises(RuntimeError, ica.plot_sources, inst=epochs)
    epochs.info['bads'] = []
    with warnings.catch_warnings(record=True):  # no labeled objects mpl
        ica.plot_sources(epochs.average())
    assert_raises(ValueError, ica.plot_sources, 'meeow')
    plt.close('all')

@requires_sklearn
def test_plot_ica_overlay():
    """Test plotting of ICA cleaning
    """
    import matplotlib.pyplot as plt
    raw = _get_raw(preload=True)
    picks = _get_picks(raw)
    ica = ICA(noise_cov=read_cov(cov_fname), n_components=2,
              max_pca_components=3, n_pca_components=3)
    ica.fit(raw, picks=picks)
    # don't test raw, needs preload ...
    ecg_epochs = create_ecg_epochs(raw, picks=picks)
    ica.plot_overlay(ecg_epochs.average())
    eog_epochs = create_eog_epochs(raw, picks=picks)
    ica.plot_overlay(eog_epochs.average())
    assert_raises(ValueError, ica.plot_overlay, raw[:2, :3][0])
    ica.plot_overlay(raw)
    plt.close('all')

@requires_sklearn
def test_plot_ica_scores():
    """Test plotting of ICA scores
    """
    import matplotlib.pyplot as plt
    raw = _get_raw()
    picks = _get_picks(raw)
    ica = ICA(noise_cov=read_cov(cov_fname), n_components=2,
              max_pca_components=3, n_pca_components=3)
    ica.fit(raw, picks=picks)
    ica.plot_scores([0.3, 0.2], axhline=[0.1, -0.1])
    assert_raises(ValueError, ica.plot_scores, [0.2])
    plt.close('all')

run_tests_if_main()
<commit_msg>ENH  test_ica<commit_after># Authors: Denis Engemann <denis.engemann@gmail.com>
#          Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#
# License: Simplified BSD

import os.path as op
import warnings

from numpy.testing import assert_raises

from mne import io, read_events, Epochs, read_cov
from mne import pick_types
from mne.utils import run_tests_if_main, requires_sklearn
from mne.preprocessing import ICA, create_ecg_epochs, create_eog_epochs

# Set our plotters to test mode
import matplotlib
matplotlib.use('Agg')  # for testing don't use X server

warnings.simplefilter('always')  # enable b/c these tests throw warnings

base_dir = op.join(op.dirname(__file__), '..', '..', 'io', 'tests', 'data')
evoked_fname = op.join(base_dir, 'test-ave.fif')
raw_fname = op.join(base_dir, 'test_raw.fif')
cov_fname = op.join(base_dir, 'test-cov.fif')
event_name = op.join(base_dir, 'test-eve.fif')
event_id, tmin, tmax = 1, -0.1, 0.2

def _get_raw(preload=False):
    return io.Raw(raw_fname, preload=preload)

def _get_events():
    return read_events(event_name)

def _get_picks(raw):
    return [0, 1, 2, 6, 7, 8, 12, 13, 14]  # take a only few channels

def _get_epochs():
    raw = _get_raw()
    events = _get_events()
    picks = _get_picks(raw)
    epochs = Epochs(raw, events[:10], event_id, tmin, tmax, picks=picks,
                    baseline=(None, 0))
    return epochs

@requires_sklearn
def test_plot_ica_components():
    """Test plotting of ICA solutions
    """
    import matplotlib.pyplot as plt
    raw = _get_raw()
    ica = ICA(noise_cov=read_cov(cov_fname), n_components=2,
              max_pca_components=3, n_pca_components=3)
    ica_picks = _get_picks(raw)
    ica.fit(raw, picks=ica_picks)
    warnings.simplefilter('always', UserWarning)
    with warnings.catch_warnings(record=True):
        for components in [0, [0], [0, 1], [0, 1] * 2, None]:
            ica.plot_components(components, image_interp='bilinear', res=16)
    ica.info = None
    assert_raises(RuntimeError, ica.plot_components, 1)
    plt.close('all')

@requires_sklearn
def test_plot_ica_sources():
    """Test plotting of ICA panel
    """
    import matplotlib.pyplot as plt
    raw = io.Raw(raw_fname, preload=False)
    raw.crop(0, 1, copy=False)
    raw.preload_data()
    picks = _get_picks(raw)
    epochs = _get_epochs()
    raw.pick_channels([raw.ch_names[k] for k in picks])
    ica_picks = pick_types(raw.info, meg=True, eeg=False, stim=False,
                           ecg=False, eog=False, exclude='bads')
    ica = ICA(n_components=2, max_pca_components=3, n_pca_components=3)
    ica.fit(raw, picks=ica_picks)
    raw.info['bads'] = ['MEG 0113']
    assert_raises(RuntimeError, ica.plot_sources, inst=raw)
    ica.plot_sources(epochs)
    epochs.info['bads'] = ['MEG 0113']
    assert_raises(RuntimeError, ica.plot_sources, inst=epochs)
    epochs.info['bads'] = []
    with warnings.catch_warnings(record=True):  # no labeled objects mpl
        ica.plot_sources(epochs.average())
    assert_raises(ValueError, ica.plot_sources, 'meeow')
    plt.close('all')

@requires_sklearn
def test_plot_ica_overlay():
    """Test plotting of ICA cleaning
    """
    import matplotlib.pyplot as plt
    raw = _get_raw(preload=True)
    picks = _get_picks(raw)
    ica = ICA(noise_cov=read_cov(cov_fname), n_components=2,
              max_pca_components=3, n_pca_components=3)
    ica.fit(raw, picks=picks)
    # don't test raw, needs preload ...
    ecg_epochs = create_ecg_epochs(raw, picks=picks)
    ica.plot_overlay(ecg_epochs.average())
    eog_epochs = create_eog_epochs(raw, picks=picks)
    ica.plot_overlay(eog_epochs.average())
    assert_raises(ValueError, ica.plot_overlay, raw[:2, :3][0])
    ica.plot_overlay(raw)
    plt.close('all')

@requires_sklearn
def test_plot_ica_scores():
    """Test plotting of ICA scores
    """
    import matplotlib.pyplot as plt
    raw = _get_raw()
    picks = _get_picks(raw)
    ica = ICA(noise_cov=read_cov(cov_fname), n_components=2,
              max_pca_components=3, n_pca_components=3)
    ica.fit(raw, picks=picks)
    ica.plot_scores([0.3, 0.2], axhline=[0.1, -0.1])
    assert_raises(ValueError, ica.plot_scores, [0.2])
    plt.close('all')

run_tests_if_main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import copy
import re
from lxml import etree

from mobify.source import MobifySource

class HistmagSource(MobifySource):

    HEADER = u"""

{title}

{author}

"""

    FOOTER = u"""

Wolna licencja – ten materiał został opublikowany na licencji Creative Commons Uznanie autorstwa
Na tych samych warunkach 3.0 Polska.

Redakcja i autor zezwalają na jego dowolny przedruk i wykorzystanie (również w celach komercyjnych) pod
następującymi warunkami: należy wyraźnie wskazać autora materiału oraz miejsce pierwotnej publikacji –
Portal historyczny Histmag.org, a także nazwę licencji (CC BY-SA 3.0) wraz z odnośnikiem do jej postanowień.
W przypadku przedruku w internecie konieczne jest także zamieszczenie dokładnego aktywnego
odnośnika do materiału objętego licencją.

Źródło: {url}

    """

    def set_up(self):
        self._url = self.extend_url(self._url)

    @staticmethod
    def extend_url(url):
        # extend the histmag.org URL to make it a single page article
        # http://histmag.org/Margaret-Thatcher-tajfun-reform-7896;0
        if not url.endswith(';0'):
            url += ';0'

        return url

    @staticmethod
    def _cleanup(tree):
        tree = copy.copy(tree)

        # clean up the HTML
        xpaths = [
            'table',
            'p[1]',  # first paragraph
            'h4',  # "Zobacz też"
            'hr',
            'p[small]',
            'div[contains(@class, "social")]',
            'script',
            'p[@class="article-info"]',
            'p[@class="article-tags"]',
            'ul[li[a]]',
            'p[span[@class="center"]]',  # big pictures
            'p/span/a/img',  # inline pictures
            'p[iframe]',  # video
        ]

        for xpath in xpaths:
            nodes = tree.xpath(xpath)
            [node.getparent().remove(node) for node in nodes]

        return tree

    @staticmethod
    def is_my_url(url):
        return url.startswith('http://histmag.org/')

    def get_html(self):
        article = self.xpath('//*[@id="article"]')

        article = self._cleanup(article)
        html = etree.tostring(article, pretty_print=True, method="html", encoding='utf8').decode('utf8')

        # tags cleanup
        html = re.sub(r'

', '', html)
        html = re.sub(r'
\s*
', '', html)
        html = re.sub(r'</?(span|a|img|em|div)[^>]*>', '', html)

        # add a title and a footer
        return '\n'.join([
            self.HEADER.format(title=self.get_title(), author=self.get_author()).strip(),
            html,
            self.FOOTER.format(url=self._url).strip()
        ])

    def get_title(self):
        return self.get_node('//title').strip()

    def get_author(self):
        return self.get_node('//p[@class="article-info"]//a').strip()

    def get_language(self):
        return 'pl'
<commit_msg>Histmag: use get_node_html() helper<commit_after># -*- coding: utf-8 -*-
import copy
import re
from lxml import etree

from mobify.source import MobifySource

class HistmagSource(MobifySource):

    HEADER = u"""

{title}

{author}

"""

    FOOTER = u"""

Wolna licencja – ten materiał został opublikowany na licencji Creative Commons Uznanie autorstwa
Na tych samych warunkach 3.0 Polska.

Redakcja i autor zezwalają na jego dowolny przedruk i wykorzystanie (również w celach komercyjnych) pod
następującymi warunkami: należy wyraźnie wskazać autora materiału oraz miejsce pierwotnej publikacji –
Portal historyczny Histmag.org, a także nazwę licencji (CC BY-SA 3.0) wraz z odnośnikiem do jej postanowień.
W przypadku przedruku w internecie konieczne jest także zamieszczenie dokładnego aktywnego
odnośnika do materiału objętego licencją.

Źródło: {url}

    """

    def set_up(self):
        self._url = self.extend_url(self._url)

    @staticmethod
    def extend_url(url):
        # extend the histmag.org URL to make it a single page article
        # http://histmag.org/Margaret-Thatcher-tajfun-reform-7896;0
        if not url.endswith(';0'):
            url += ';0'

        return url

    @staticmethod
    def _cleanup(tree):
        tree = copy.copy(tree)

        # clean up the HTML
        xpaths = [
            'table',
            'p[1]',  # first paragraph
            'h4',  # "Zobacz też"
            'hr',
            'p[small]',
            'div[contains(@class, "social")]',
            'script',
            'p[@class="article-info"]',
            'p[@class="article-tags"]',
            'ul[li[a]]',
            'p[span[@class="center"]]',  # big pictures
            'p/span/a/img',  # inline pictures
            'p[iframe]',  # video
        ]

        for xpath in xpaths:
            nodes = tree.xpath(xpath)
            [node.getparent().remove(node) for node in nodes]

        return tree

    @staticmethod
    def is_my_url(url):
        return url.startswith('http://histmag.org/')

    def get_html(self):
        article = self.xpath('//*[@id="article"]')

        article = self._cleanup(article)
        html = self.get_node_html(article)

        # tags cleanup
        html = re.sub(r'

', '', html)
        html = re.sub(r'
\s*
', '', html)
        html = re.sub(r'</?(span|a|img|em|div)[^>]*>', '', html)

        # add a title and a footer
        return '\n'.join([
            self.HEADER.format(title=self.get_title(), author=self.get_author()).strip(),
            html,
            self.FOOTER.format(url=self._url).strip()
        ])

    def get_title(self):
        return self.get_node('//title').strip()

    def get_author(self):
        return self.get_node('//p[@class="article-info"]//a').strip()

    def get_language(self):
        return 'pl'
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-

import sys
import os
import time
import yaml
import random

from twisted.internet import reactor
from twisted.python import usage
from twisted.python.util import spewer

from ooni import errors

from ooni.settings import config
from ooni.director import Director
from ooni.deck import Deck, nettest_to_path
from ooni.reporter import YAMLReporter, OONIBReporter
from ooni.nettest import NetTestLoader

from ooni.utils import log, checkForRoot

class Options(usage.Options):
    synopsis = """%s [options] [path to test].py
    """ % (os.path.basename(sys.argv[0]),)

    longdesc = ("ooniprobe loads and executes a suite or a set of suites of"
                " network tests. These are loaded from modules, packages and"
                " files listed on the command line")

    optFlags = [["help", "h"],
                ["resume", "r"],
                ["no-collector", "n"],
                ["list", "s"],
                ["printdeck", "p"]
                ]

    optParameters = [["reportfile", "o", None, "report file name"],
                     ["testdeck", "i", None,
                         "Specify as input a test deck: a yaml file containig the tests to run an their arguments"],
                     ["collector", "c", None,
                         "Address of the collector of test results. This option should not be used, but you should always use a bouncer."],
                     ["bouncer", "b", 'httpo://nkvphnp3p6agi5qq.onion',
                         "Address of the bouncer for test helpers. default: httpo://nkvphnp3p6agi5qq.onion"],
                     ["logfile", "l", None, "log file name"],
                     ["pcapfile", "O", None, "pcap file name"],
                     ["configfile", "f", None,
                         "Specify a path to the ooniprobe configuration file"],
                     ["datadir", "d", None,
                         "Specify a path to the ooniprobe data directory"]
                     ]

    compData = usage.Completions(
        extraActions=[usage.CompleteFiles(
                "*.py", descr="file | module | package | TestCase | testMethod",
                repeat=True)],)

    tracer = None

    def __init__(self):
        self['test'] = None
        usage.Options.__init__(self)

    def opt_asciilulz(self):
        from ooni.utils import logo
        print logo.getlogo()

    def opt_spew(self):
        """
        Print an insanely verbose log of everything that happens.  Useful
        when debugging freezes or locks in complex code.
        """
        sys.settrace(spewer)

    def parseArgs(self, *args):
        if self['testdeck'] or self['list']:
            return
        try:
            self['test_file'] = args[0]
            self['subargs'] = args[1:]
        except:
            raise usage.UsageError("No test filename specified!")

def parseOptions():
    cmd_line_options = Options()
    if len(sys.argv) == 1:
        cmd_line_options.getUsage()
    try:
        cmd_line_options.parseOptions()
    except usage.UsageError, ue:
        print cmd_line_options.getUsage()
        raise SystemExit, "%s: %s" % (sys.argv[0], ue)

    return dict(cmd_line_options)

def shutdown(result):
    """
    This will get called once all the operations that need to be done in the
    current oonicli session have been completed.
    """
    log.debug("Halting reactor")
    try: reactor.stop()
    except: pass

def runWithDirector():
    """
    Instance the director, parse command line options and start an ooniprobe
    test!
    """
    global_options = parseOptions()
    config.global_options = global_options
    config.set_paths()
    config.read_config_file()

    log.start(global_options['logfile'])

    if config.privacy.includepcap:
        try:
            checkForRoot()
        except errors.InsufficientPrivileges:
             log.err("Insufficient Privileges to capture packets."
                     " See ooniprobe.conf privacy.includepcap")
             sys.exit(2)

    director = Director()
    if global_options['list']:
        print "# Installed nettests"
        for net_test_id, net_test in director.getNetTests().items():
            print "* %s (%s/%s)" % (net_test['name'],
                                    net_test['category'],
                                    net_test['id'])
            print "  %s" % net_test['description']

        sys.exit(0)

    elif global_options['printdeck']:
        del global_options['printdeck']
        print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
        print yaml.safe_dump([{'options': global_options}]).strip()

        sys.exit(0)

    d = director.start()

    #XXX: This should mean no bouncer either!
    if global_options['no-collector']:
        log.msg("Not reporting using a collector")
        collector = global_options['collector'] = None
        global_options['bouncer'] = None

    deck = Deck()
    deck.bouncer = global_options['bouncer']

    try:
        if global_options['testdeck']:
            deck.loadDeck(global_options['testdeck'])
        else:
            log.debug("No test deck detected")
            test_file = nettest_to_path(global_options['test_file'])
            net_test_loader = NetTestLoader(global_options['subargs'],
                    test_file=test_file)
            deck.insert(net_test_loader)
    except errors.MissingRequiredOption, option_name:
        log.err('Missing required option: "%s"' % option_name)
        print net_test_loader.usageOptions().getUsage()
        sys.exit(2)
    except errors.NetTestNotFound, path:
        log.err('Requested NetTest file not found (%s)' % path)
        sys.exit(3)
    except usage.UsageError, e:
        log.err(e)
        print net_test_loader.usageOptions().getUsage()
        sys.exit(2)

    def setup_nettest(_):
        try:
            return deck.setup()
        except errors.UnableToLoadDeckInput as error:
            return defer.failure.Failure(error)

    def director_startup_failed(failure):
        log.err("Failed to start the director")
        r = failure.trap(errors.TorNotRunning,
                errors.InvalidOONIBCollectorAddress,
                errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper,
                errors.CouldNotFindTestCollector, errors.ProbeIPUnknown)

        if isinstance(failure.value, errors.TorNotRunning):
            log.err("Tor does not appear to be running")
            log.err("Reporting with the collector %s is not possible" %
                    global_options['collector'])
            log.msg("Try with a different collector or disable collector reporting with -n")

        elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
            log.err("Invalid format for oonib collector address.")
            log.msg("Should be in the format http://<collector_address>:<port>")
            log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")

        elif isinstance(failure.value, errors.UnableToLoadDeckInput):
            log.err("Unable to fetch the required inputs for the test deck.")
            log.msg("Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues")

        elif isinstance(failure.value, errors.CouldNotFindTestHelper):
            log.err("Unable to obtain the required test helpers.")
            log.msg("Try with a different bouncer or check that Tor is running properly.")

        elif isinstance(failure.value, errors.CouldNotFindTestCollector):
            log.err("Could not find a valid collector.")
            log.msg("Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.")

        elif isinstance(failure.value, errors.ProbeIPUnknown):
            log.err("Failed to lookup probe IP address.")
            log.msg("Check your internet connection.")

        if config.advanced.debug:
            log.exception(failure)

        reactor.stop()

    # Wait until director has started up (including bootstrapping Tor)
    # before adding tests
    def post_director_start(_):
        for net_test_loader in deck.netTestLoaders:
            # Decks can specify different collectors
            # for each net test, so that each NetTest
            # may be paired with a test_helper and its collector
            # However, a user can override this behavior by
            # specifying a collector from the command-line (-c).
            # If a collector is not specified in the deck, or the
            # deck is a singleton, the default collector set in
            # ooniprobe.conf will be used

            collector = None
            if not global_options['no-collector']:
                if global_options['collector']:
                    collector = global_options['collector']
                elif net_test_loader.collector:
                    collector = net_test_loader.collector

            if collector and collector.startswith('httpo:') \
                    and (not (config.tor_state or config.tor.socks_port)):
                raise errors.TorNotRunning

            test_details = net_test_loader.testDetails
            yaml_reporter = YAMLReporter(test_details)
            reporters = [yaml_reporter]

            if collector:
                log.msg("Reporting using collector: %s" % collector)
                try:
                    oonib_reporter = OONIBReporter(test_details, collector)
                    reporters.append(oonib_reporter)
                except errors.InvalidOONIBCollectorAddress, e:
                    raise e

            log.debug("adding callback for startNetTest")
            director.startNetTest(net_test_loader, reporters)

        director.allTestsDone.addBoth(shutdown)

    def start():
        d.addCallback(setup_nettest)
        d.addCallback(post_director_start)
        d.addErrback(director_startup_failed)

    reactor.callWhenRunning(start)
    reactor.run()
<commit_msg>Fix typo in help.<commit_after>#-*- coding: utf-8 -*-

import sys
import os
import time
import yaml
import random

from twisted.internet import reactor
from twisted.python import usage
from twisted.python.util import spewer

from ooni import errors

from ooni.settings import config
from ooni.director import Director
from ooni.deck import Deck, nettest_to_path
from ooni.reporter import YAMLReporter, OONIBReporter
from ooni.nettest import NetTestLoader

from ooni.utils import log, checkForRoot

class Options(usage.Options):
    synopsis = """%s [options] [path to test].py
    """ % (os.path.basename(sys.argv[0]),)

    longdesc = ("ooniprobe loads and executes a suite or a set of suites of"
                " network tests. These are loaded from modules, packages and"
                " files listed on the command line")

    optFlags = [["help", "h"],
                ["resume", "r"],
                ["no-collector", "n"],
                ["list", "s"],
                ["printdeck", "p"]
                ]

    optParameters = [["reportfile", "o", None, "report file name"],
                     ["testdeck", "i", None,
                         "Specify as input a test deck: a yaml file containing the tests to run and their arguments"],
                     ["collector", "c", None,
                         "Address of the collector of test results. This option should not be used, but you should always use a bouncer."],
                     ["bouncer", "b", 'httpo://nkvphnp3p6agi5qq.onion',
                         "Address of the bouncer for test helpers. default: httpo://nkvphnp3p6agi5qq.onion"],
                     ["logfile", "l", None, "log file name"],
                     ["pcapfile", "O", None, "pcap file name"],
                     ["configfile", "f", None,
                         "Specify a path to the ooniprobe configuration file"],
                     ["datadir", "d", None,
                         "Specify a path to the ooniprobe data directory"]
                     ]

    compData = usage.Completions(
        extraActions=[usage.CompleteFiles(
                "*.py", descr="file | module | package | TestCase | testMethod",
                repeat=True)],)

    tracer = None

    def __init__(self):
        self['test'] = None
        usage.Options.__init__(self)

    def opt_asciilulz(self):
        from ooni.utils import logo
        print logo.getlogo()

    def opt_spew(self):
        """
        Print an insanely verbose log of everything that happens.  Useful
        when debugging freezes or locks in complex code.
        """
        sys.settrace(spewer)

    def parseArgs(self, *args):
        if self['testdeck'] or self['list']:
            return
        try:
            self['test_file'] = args[0]
            self['subargs'] = args[1:]
        except:
            raise usage.UsageError("No test filename specified!")

def parseOptions():
    cmd_line_options = Options()
    if len(sys.argv) == 1:
        cmd_line_options.getUsage()
    try:
        cmd_line_options.parseOptions()
    except usage.UsageError, ue:
        print cmd_line_options.getUsage()
        raise SystemExit, "%s: %s" % (sys.argv[0], ue)

    return dict(cmd_line_options)

def shutdown(result):
    """
    This will get called once all the operations that need to be done in the
    current oonicli session have been completed.
    """
    log.debug("Halting reactor")
    try: reactor.stop()
    except: pass

def runWithDirector():
    """
    Instance the director, parse command line options and start an ooniprobe
    test!
    """
    global_options = parseOptions()
    config.global_options = global_options
    config.set_paths()
    config.read_config_file()

    log.start(global_options['logfile'])

    if config.privacy.includepcap:
        try:
            checkForRoot()
        except errors.InsufficientPrivileges:
             log.err("Insufficient Privileges to capture packets."
                     " See ooniprobe.conf privacy.includepcap")
             sys.exit(2)

    director = Director()
    if global_options['list']:
        print "# Installed nettests"
        for net_test_id, net_test in director.getNetTests().items():
            print "* %s (%s/%s)" % (net_test['name'],
                                    net_test['category'],
                                    net_test['id'])
            print "  %s" % net_test['description']

        sys.exit(0)

    elif global_options['printdeck']:
        del global_options['printdeck']
        print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
        print yaml.safe_dump([{'options': global_options}]).strip()

        sys.exit(0)

    d = director.start()

    #XXX: This should mean no bouncer either!
    if global_options['no-collector']:
        log.msg("Not reporting using a collector")
        collector = global_options['collector'] = None
        global_options['bouncer'] = None

    deck = Deck()
    deck.bouncer = global_options['bouncer']

    try:
        if global_options['testdeck']:
            deck.loadDeck(global_options['testdeck'])
        else:
            log.debug("No test deck detected")
            test_file = nettest_to_path(global_options['test_file'])
            net_test_loader = NetTestLoader(global_options['subargs'],
                    test_file=test_file)
            deck.insert(net_test_loader)
    except errors.MissingRequiredOption, option_name:
        log.err('Missing required option: "%s"' % option_name)
        print net_test_loader.usageOptions().getUsage()
        sys.exit(2)
    except errors.NetTestNotFound, path:
        log.err('Requested NetTest file not found (%s)' % path)
        sys.exit(3)
    except usage.UsageError, e:
        log.err(e)
        print net_test_loader.usageOptions().getUsage()
        sys.exit(2)

    def setup_nettest(_):
        try:
            return deck.setup()
        except errors.UnableToLoadDeckInput as error:
            return defer.failure.Failure(error)

    def director_startup_failed(failure):
        log.err("Failed to start the director")
        r = failure.trap(errors.TorNotRunning,
                errors.InvalidOONIBCollectorAddress,
                errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper,
                errors.CouldNotFindTestCollector, errors.ProbeIPUnknown)

        if isinstance(failure.value, errors.TorNotRunning):
            log.err("Tor does not appear to be running")
            log.err("Reporting with the collector %s is not possible" %
                    global_options['collector'])
            log.msg("Try with a different collector or disable collector reporting with -n")

        elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
            log.err("Invalid format for oonib collector address.")
            log.msg("Should be in the format http://<collector_address>:<port>")
            log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")

        elif isinstance(failure.value, errors.UnableToLoadDeckInput):
            log.err("Unable to fetch the required inputs for the test deck.")
            log.msg("Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues")

        elif isinstance(failure.value, errors.CouldNotFindTestHelper):
            log.err("Unable to obtain the required test helpers.")
            log.msg("Try with a different bouncer or check that Tor is running properly.")

        elif isinstance(failure.value, errors.CouldNotFindTestCollector):
            log.err("Could not find a valid collector.")
            log.msg("Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.")

        elif isinstance(failure.value, errors.ProbeIPUnknown):
            log.err("Failed to lookup probe IP address.")
            log.msg("Check your internet connection.")

        if config.advanced.debug:
            log.exception(failure)

        reactor.stop()

    # Wait until director has started up (including bootstrapping Tor)
    # before adding tests
    def post_director_start(_):
        for net_test_loader in deck.netTestLoaders:
            # Decks can specify different collectors
            # for each net test, so that each NetTest
            # may be paired with a test_helper and its collector
            # However, a user can override this behavior by
            # specifying a collector from the command-line (-c).
            # If a collector is not specified in the deck, or the
            # deck is a singleton, the default collector set in
            # ooniprobe.conf will be used

            collector = None
            if not global_options['no-collector']:
                if global_options['collector']:
                    collector = global_options['collector']
                elif net_test_loader.collector:
                    collector = net_test_loader.collector

            if collector and collector.startswith('httpo:') \
                    and (not (config.tor_state or config.tor.socks_port)):
                raise errors.TorNotRunning

            test_details = net_test_loader.testDetails
            yaml_reporter = YAMLReporter(test_details)
            reporters = [yaml_reporter]

            if collector:
                log.msg("Reporting using collector: %s" % collector)
                try:
                    oonib_reporter = OONIBReporter(test_details, collector)
                    reporters.append(oonib_reporter)
                except errors.InvalidOONIBCollectorAddress, e:
                    raise e

            log.debug("adding callback for startNetTest")
            director.startNetTest(net_test_loader, reporters)

        director.allTestsDone.addBoth(shutdown)

    def start():
        d.addCallback(setup_nettest)
        d.addCallback(post_director_start)
        d.addErrback(director_startup_failed)

    reactor.callWhenRunning(start)
    reactor.run()
<|endoftext|>


<commit_before>'''A CKAN plugin that enables logging into CKAN using BC IDIR Credentials.

'''
import httplib
import json
import uuid
import urllib2
from urllib2 import HTTPError
import requests
import socket
# Unfortunately we need to import pylons directly here, because we need to
# put stuff into the Beaker session and CKAN's plugins toolkit doesn't let
# us do that yet.
import pylons

import pylons.config as config
import ckan.lib.base as base
import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
import ckan.lib.helpers as helpers
from ckan.common import _, c, g, request, response

render = base.render

class IdirVerificationError(Exception):
    '''The exception class that is raised if trying to verify a Idir
    assertion fails.

    '''
    pass

def user_list(context, data_dict=None):
     # Get the user name of the logged-in user.
    user_name = context['user']
    # We have the logged-in user's user name, get their user id.
    convert_user_name_or_id_to_id = toolkit.get_converter(
        'convert_user_name_or_id_to_id')
    try:
        user_id = convert_user_name_or_id_to_id(user_name, context)
    except toolkit.Invalid:
        # The user doesn't exist (e.g. they're not logged-in).
        return {'success': False,
                'msg': 'You must be logged-in to list users.'}

    return {'success': True, 'msg': 'You must be authorized to list users'}

def user_show(context, data_dict=None):

     # Get the user name of the logged-in user.
    user_name = context['user']
    try:
        print user_name
        socket.inet_aton(user_name)
        # legal
    except socket.error:
        # Not legal
        return {'success': True,
                'msg': 'You must be logged-in to list users.'}
    return {'success': True, 'msg': 'You must be authorized to list users'}

class IdirPlugin(plugins.SingletonPlugin):
    '''A CKAN plugin that enables logging into CKAN using IDIR.

    '''
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.IAuthenticator)
    plugins.implements(plugins.IAuthFunctions)

    def get_auth_functions(self):
        return {'user_list': user_list}

    def update_config(self, config):
        '''Update CKAN's config with settings needed by this plugin.

        '''
        toolkit.add_template_directory(config, 'templates')
        toolkit.add_public_directory(config, 'public')
        toolkit.add_resource('fanstatic', 'idir')

    def login(self, error=None):
        '''Handle an attempt to login using IDIR.

        '''
        print '**** Login invoked'
        # Get the params that were posted to /user/login.
        params = toolkit.request.params

        if 'login' in params:
             try:

                #check for edc cookie

                edc_cookie = toolkit.request.cookies.get( 'edc_auth' )

                if edc_cookie:
                    print edc_cookie

                    #username in cookie must match login name
                    universalId,guid = edc_cookie.split('_')

                    print universalId
                else:
                    helpers.redirect_to(controller='user', action='login', error='You are not authorized to login')

                if universalId == params['login']:
					user = toolkit.get_action('user_show')(data_dict={'id': params['login']})
					#get username

					# Store the name of the verified logged-in user in the Beaker
					# sesssion store.
					pylons.session['ckanext-idir-user'] = user['name']
					print user['name']
					#pylons.session['ckanext-persona-email'] = user['email']
					pylons.session.save()
					# Try to get the item that login() placed in the session.
					user = pylons.session.get('ckanext-idir-user')
					if user:
						# We've found a logged-in user. Set c.user to let CKAN know.
						toolkit.c.user = user
						helpers.redirect_to(controller='user', action='dashboard')
					else:

						helpers.redirect_to(controller='user', action='login', error='You are not authorized to login')

             except urllib2.HTTPError as e:
                 print e.code
                 print str(e)
                 if e.code == 302:
                     pass
                 else:
                     helpers.redirect_to(controller='user', action='login', error='You are not authorized to login')

             #except Exception as e:
                  #print str(e)
                  #print str(e.code)
                  #helpers.redirect_to(controller='user', action='login', error='An error occured. You are not authorized to login')

    def identify(self):
        '''Identify which user (if any) is logged-in via idir.

        If a logged-in user is found, set toolkit.c.user to be their user name.

        '''
        # Try to get the item that login() placed in the session.
        user = pylons.session.get('ckanext-idir-user')
        if user:
            # We've found a logged-in user. Set c.user to let CKAN know.
            toolkit.c.user = user

    def _delete_session_items(self):
        import pylons
        if 'ckanext-idir-user' in pylons.session:
            del pylons.session['ckanext-idir-user']
        if 'ckanext-idir-email' in pylons.session:
            del pylons.session['ckanext-idir-email']
        pylons.session.save()

    def logout(self):
        '''Handle a logout.'''

        toolkit.request.cookies.pop( 'edc_auth', None )
        response.delete_cookie('edc_auth', path='/', domain=None)
        response.delete_cookie('edc_auth2', path='/', domain=None)
        print 'deleted edc_auth cookie'
        # Delete the session item, so that identify() will no longer find it.
        self._delete_session_items()

    def abort(self, status_code, detail, headers, comment):
        '''Handle an abort.'''

        # Delete the session item, so that identify() will no longer find it.
        self._delete_session_items()

    def no_registering(context, data_dict):
        return {'success': False, 'msg': toolkit._('''You cannot register for this
            site.''')}

    class NoSelfRegistration(plugins.SingletonPlugin):
        plugins.implements(plugins.IAuthFunctions, inherit=True)

        def get_auth_functions(self):
            return {
                'user_create': no_registering
            }

<commit_msg>Added debug info to plugin.py<commit_after>'''A CKAN plugin that enables logging into CKAN using BC IDIR Credentials.

'''

import pprint
import httplib
import json
import uuid
import urllib2
from urllib2 import HTTPError
import requests
import socket
# Unfortunately we need to import pylons directly here, because we need to
# put stuff into the Beaker session and CKAN's plugins toolkit doesn't let
# us do that yet.
import pylons

import pylons.config as config
import ckan.lib.base as base
import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
import ckan.lib.helpers as helpers
from ckan.common import _, c, g, request, response

render = base.render

class IdirVerificationError(Exception):
    '''The exception class that is raised if trying to verify a Idir
    assertion fails.

    '''
    pass

def user_list(context, data_dict=None):
     # Get the user name of the logged-in user.
    user_name = context['user']
    # We have the logged-in user's user name, get their user id.
    convert_user_name_or_id_to_id = toolkit.get_converter(
        'convert_user_name_or_id_to_id')
    try:
        user_id = convert_user_name_or_id_to_id(user_name, context)
    except toolkit.Invalid:
        # The user doesn't exist (e.g. they're not logged-in).
        return {'success': False,
                'msg': 'You must be logged-in to list users.'}

    return {'success': True, 'msg': 'You must be authorized to list users'}

def user_show(context, data_dict=None):

     # Get the user name of the logged-in user.
    user_name = context['user']
    try:
        print user_name
        socket.inet_aton(user_name)
        # legal
    except socket.error:
        # Not legal
        return {'success': True,
                'msg': 'You must be logged-in to list users.'}
    return {'success': True, 'msg': 'You must be authorized to list users'}

class IdirPlugin(plugins.SingletonPlugin):
    '''A CKAN plugin that enables logging into CKAN using IDIR.

    '''
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.IAuthenticator)
    plugins.implements(plugins.IAuthFunctions)

    def get_auth_functions(self):
        return {'user_list': user_list}

    def update_config(self, config):
        '''Update CKAN's config with settings needed by this plugin.

        '''
        toolkit.add_template_directory(config, 'templates')
        toolkit.add_public_directory(config, 'public')
        toolkit.add_resource('fanstatic', 'idir')

    def login(self, error=None):
        '''Handle an attempt to login using IDIR.

        '''
        print '**** Login invoked'
        # Get the params that were posted to /user/login.
        params = toolkit.request.params

        print '******************************  Request params  ******************************'
        pprint.pprint(params)

        if 'login' in params:
            try:
                #url = config.get('edc.eas.idir.login.url', False)  # Commented By Khalegh Mamakani the value of url never been used.
                #check for edc cookie

                edc_cookie = toolkit.request.cookies.get( 'edc_auth' )

                print '*********** edc_auth cookie : ************'
                print edc_cookie

                if edc_cookie:
                    print edc_cookie

                    #username in cookie must match login name
                    universalId,guid = edc_cookie.split('_')

                    print '******************************  universalId,guid  ******************************'
                    print universalId, guid
                else:
                    helpers.redirect_to(controller='user', action='login', error='You are not authorized to login')

                if universalId == params['login']:
                    user = toolkit.get_action('user_show')(data_dict={'id': params['login']})
                    #get username
                    print '******************************  User info  ******************************'
                    pprint.pprint(user)
                    # Store the name of the verified logged-in user in the Beaker
                    # sesssion store.
                    pylons.session['ckanext-idir-user'] = user['name']
                    print user['name']
                    #pylons.session['ckanext-persona-email'] = user['email']
                    pylons.session.save()
                    # Try to get the item that login() placed in the session.
                    user = pylons.session.get('ckanext-idir-user')

                    print '******************************  ckanext-idir-user  ******************************'
                    pprint.pprint(user)
                    if user:
                        # We've found a logged-in user. Set c.user to let CKAN know.
                        toolkit.c.user = user
                        helpers.redirect_to(controller='user', action='dashboard')
                    else:
                        helpers.redirect_to(controller='user', action='login', error='You are not authorized to login')
            except urllib2.HTTPError as e:
                print 'Http Error ------------------------------>', e.code
                print str(e)
                if e.code == 302:
                    pass
                else:
                    helpers.redirect_to(controller='user', action='login', error='You are not authorized to login')

             #except Exception as e:
                  #print str(e)
                  #print str(e.code)
                  #helpers.redirect_to(controller='user', action='login', error='An error occured. You are not authorized to login')

    def identify(self):
        '''Identify which user (if any) is logged-in via idir.

        If a logged-in user is found, set toolkit.c.user to be their user name.

        '''
        # Try to get the item that login() placed in the session.
        user = pylons.session.get('ckanext-idir-user')
        if user:
            # We've found a logged-in user. Set c.user to let CKAN know.
            toolkit.c.user = user

    def _delete_session_items(self):
        import pylons
        if 'ckanext-idir-user' in pylons.session:
            del pylons.session['ckanext-idir-user']
        if 'ckanext-idir-email' in pylons.session:
            del pylons.session['ckanext-idir-email']
        pylons.session.save()

    def logout(self):
        '''Handle a logout.'''

        toolkit.request.cookies.pop( 'edc_auth', None )
        response.delete_cookie('edc_auth', path='/', domain=None)
        response.delete_cookie('edc_auth2', path='/', domain=None)
        print 'deleted edc_auth cookie'
        # Delete the session item, so that identify() will no longer find it.
        self._delete_session_items()

    def abort(self, status_code, detail, headers, comment):
        '''Handle an abort.'''

        # Delete the session item, so that identify() will no longer find it.
        self._delete_session_items()

    def no_registering(context, data_dict):
        return {'success': False, 'msg': toolkit._('''You cannot register for this
            site.''')}

    class NoSelfRegistration(plugins.SingletonPlugin):
        plugins.implements(plugins.IAuthFunctions, inherit=True)

        def get_auth_functions(self):
            return {
                'user_create': no_registering
            }

<|endoftext|>


<commit_before>from django.conf.urls.defaults import *

urlpatterns = patterns('devilry.addons.examiner',
    url(r'^$',
        'views.list_assignments',
        name='devilry-examiner-list_assignments'),

    url(r'^list-groups/(?P<assignment_id>\d+)$',
        'views.list_assignmentgroups',
        name='devilry-examiner-list_assignmentgroups'),
    url(r'^list-groups/json/(?P<assignment_id>\d+)$',
        'views.list_assignmentgroups_json',
        name='devilry-examiner-list_assignmentgroups_json'),
    url(r'^assignments/(?P<assignmentgroup_id>\d+)$',
        'views.show_assignmentgroup',
        name='devilry-examiner-show_assignmentgroup'),
    url(r'^assignments-as-examiner/(?P<assignmentgroup_id>\d+)$',
        'views.show_assignmentgroup',
        kwargs={'is_admin':False},
        name='devilry-examiner-show_assignmentgroup-as-examiner'),
    url(r'^assignments-as-admin/(?P<assignmentgroup_id>\d+)$',
        'views.show_assignmentgroup',
        kwargs={'is_admin':True},
        name='devilry-examiner-show_assignmentgroup-as-admin'),

    url(r'^assignments/(?P<assignmentgroup_id>\d+)/edit-deadline/(?P<deadline_id>\d+)$',
        'views.edit_deadline',
        name='devilry-examiner-edit_deadline'),
    url(r'^assignments/(?P<assignmentgroup_id>\d+)/create-deadline$',
        'views.edit_deadline',
        name='devilry-examiner-create_deadline'),

    url(r'^open-group/(?P<assignmentgroup_id>\d+)$',
        'views.open_assignmentgroup',
        name='devilry-examiner-open_assignmentgroup'),
    url(r'^close-group/(?P<assignmentgroup_id>\d+)$',
        'views.close_assignmentgroup',
        name='devilry-examiner-close_assignmentgroup'),
    url(r'^delete-deadline/(?P<assignmentgroup_id>\d+)/(?P<deadline_id>\d+)$',
        'views.delete_deadline',
        name='devilry-examiner-delete_deadline'),
    url(r'^assignments/(?P<assignment_id>\d+)/download_file_collection_as_zip$',
        'views.download_file_collection_as_zip',
        name='devilry-examiner-download_file_collection_as_zip'),
    url(r'^assignments/(?P<assignment_id>\d+)/download_file_collection_as_tar$',
        'views.download_file_collection_as_tar',
        name='devilry-examiner-download_file_collection_as_tar'),
    url(r'^edit-feedback/(?P<delivery_id>\d+)$',
        'views.edit_feedback',
        name='devilry-examiner-edit-feedback'),
    url(r'^edit-feedback-as-examiner/(?P<delivery_id>\d+)$',
        'views.edit_feedback',
        kwargs={"is_admin":False},
        name='devilry-examiner-edit-feedback-as-examiner'),
    url(r'^edit-feedback-as-admin/(?P<delivery_id>\d+)$',
        'views.edit_feedback',
        kwargs={"is_admin":True},
        name='devilry-examiner-edit-feedback-as-admin'),
)
<commit_msg>Added url for downloading one deliver<commit_after>from django.conf.urls.defaults import *

urlpatterns = patterns('devilry.addons.examiner',
    url(r'^$',
        'views.list_assignments',
        name='devilry-examiner-list_assignments'),

    url(r'^list-groups/(?P<assignment_id>\d+)$',
        'views.list_assignmentgroups',
        name='devilry-examiner-list_assignmentgroups'),
    url(r'^list-groups/json/(?P<assignment_id>\d+)$',
        'views.list_assignmentgroups_json',
        name='devilry-examiner-list_assignmentgroups_json'),
    url(r'^assignments/(?P<assignmentgroup_id>\d+)$',
        'views.show_assignmentgroup',
        name='devilry-examiner-show_assignmentgroup'),
    url(r'^assignments-as-examiner/(?P<assignmentgroup_id>\d+)$',
        'views.show_assignmentgroup',
        kwargs={'is_admin':False},
        name='devilry-examiner-show_assignmentgroup-as-examiner'),
    url(r'^assignments-as-admin/(?P<assignmentgroup_id>\d+)$',
        'views.show_assignmentgroup',
        kwargs={'is_admin':True},
        name='devilry-examiner-show_assignmentgroup-as-admin'),

    url(r'^assignments/(?P<assignmentgroup_id>\d+)/edit-deadline/(?P<deadline_id>\d+)$',
        'views.edit_deadline',
        name='devilry-examiner-edit_deadline'),
    url(r'^assignments/(?P<assignmentgroup_id>\d+)/create-deadline$',
        'views.edit_deadline',
        name='devilry-examiner-create_deadline'),

    url(r'^open-group/(?P<assignmentgroup_id>\d+)$',
        'views.open_assignmentgroup',
        name='devilry-examiner-open_assignmentgroup'),
    url(r'^close-group/(?P<assignmentgroup_id>\d+)$',
        'views.close_assignmentgroup',
        name='devilry-examiner-close_assignmentgroup'),
    url(r'^delete-deadline/(?P<assignmentgroup_id>\d+)/(?P<deadline_id>\d+)$',
        'views.delete_deadline',
        name='devilry-examiner-delete_deadline'),
    url(r'^assignments/(?P<assignment_id>\d+)/download_file_collection_as_zip$',
        'views.download_file_collection_as_zip',
        name='devilry-examiner-download_file_collection_as_zip'),
    url(r'^assignments/(?P<assignment_id>\d+)/download_file_collection_as_tar$',
        'views.download_file_collection_as_tar',
        name='devilry-examiner-download_file_collection_as_tar'),

    url(r'^edit-feedback/(?P<delivery_id>\d+)/download_delivery_as_tar$',
        'views.download_delivery_as_tar',
        name='devilry-examiner-download_delivery_as_tar'),

    url(r'^edit-feedback/(?P<delivery_id>\d+)$',
        'views.edit_feedback',
        name='devilry-examiner-edit-feedback'),
    url(r'^edit-feedback-as-examiner/(?P<delivery_id>\d+)$',
        'views.edit_feedback',
        kwargs={"is_admin":False},
        name='devilry-examiner-edit-feedback-as-examiner'),
    url(r'^edit-feedback-as-admin/(?P<delivery_id>\d+)$',
        'views.edit_feedback',
        kwargs={"is_admin":True},
        name='devilry-examiner-edit-feedback-as-admin'),
)
<|endoftext|>