<commit_before># -*- coding: utf-8 -*-
'''
osquery wrapper for HubbleStack Nebula

:maintainer: basepi
:maturity: 2016.10.2
:platform: All
:requires: SaltStack, osquery

Designed to run sets of osquery queries defined in pillar. These sets will have
a unique identifier, and be targeted by identifier. Usually, this identifier
will be a frequency. ('15 minutes', '1 day', etc). Identifiers are
case-insensitive.

You can then use the scheduler of your choice to run sets os queries at
whatever frequency you choose.

Sample pillar data:

nebula_osquery:
  hour:
    - crontab: query: select c.*,t.iso_8601 as _time from crontab as c join time as t;
    - query_name: suid_binaries
      query: select sb.*, t.iso_8601 as _time from suid_bin as sb join time as t;
  day:
    - query_name: rpm_packages
      query: select rpm.*, t.iso_8601 from rpm_packages as rpm join time as t;
'''
from __future__ import absolute_import

import copy
import json
import logging
import os
import sys
import yaml

import salt.utils
from salt.exceptions import CommandExecutionError
from hubblestack import __version__

log = logging.getLogger(__name__)

__virtualname__ = 'nebula'

def __virtual__():
    return __virtualname__

def queries(query_group,
            query_file=None,
            verbose=False,
            report_version_with_day=True):
    '''
    Run the set of queries represented by ``query_group`` from the
    configuration in the file query_file

    query_group
        Group of queries to run

    query_file
        salt:// file which will be parsed for osquery queries

    verbose
        Defaults to False. If set to True, more information (such as the query
        which was run) will be included in the result.

    CLI Examples:

    .. code_block:: bash

        salt '*' nebula.queries day
        salt '*' nebula.queries hour verbose=True
        salt '*' nebula.queries hour pillar_key=sec_osqueries
    '''
    MAX_FILE_SIZE = 104857600
    if query_file is None:
        if salt.utils.is_windows():
            query_file = 'salt://hubblestack_nebula/hubblestack_nebula_win_queries.yaml'
        else:
            query_file = 'salt://hubblestack_nebula/hubblestack_nebula_queries.yaml'
    if not salt.utils.which('osqueryi'):
        if query_group == 'day':
            log.warning('osquery not installed on this host. Returning baseline data')
            # Match the formatting of normal osquery results. Not super
            #   readable, but just add new dictionaries to the list as we need
            #   more data
            ret = []
            ret.append(
                    {'fallback_osfinger': {
                         'data': [{'osfinger': __grains__.get('osfinger', __grains__.get('osfullname')),
                                   'osrelease': __grains__.get('osrelease', __grains__.get('lsb_distrib_release'))}],
                         'result': True
                    }}
            )
            if 'pkg.list_pkgs' in __salt__:
                ret.append(
                        {'fallback_pkgs': {
                             'data': [{'name': k, 'version': v} for k, v in __salt__['pkg.list_pkgs']().iteritems()],
                             'result': True
                        }}
                )
            uptime = __salt__['status.uptime']()
            if isinstance(uptime, dict):
                uptime = uptime.get('seconds', __salt__['cmd.run']('uptime'))
            ret.append(
                    {'fallback_uptime': {
                         'data': [{'uptime': uptime}],
                         'result': True
                    }}
            )
            if report_version_with_day:
                ret.append(hubble_versions())
            return ret
        else:
            log.debug('osquery not installed on this host. Skipping.')
            return None

    if salt.utils.is_windows():
        win_version = __grains__['osfullname']
        if '2008' not in win_version and '2012' not in win_version and '2016' not in win_version:
            log.error('osquery does not run on windows versions earlier than Server 2008 and Windows 7')
            if query_group == 'day':
                ret = []
                ret.append(
                        {'fallback_osfinger': {
                             'data': [{'osfinger': __grains__.get('osfinger', __grains__.get('osfullname')),
                                       'osrelease': __grains__.get('osrelease', __grains__.get('lsb_distrib_release'))}],
                             'result': True
                        }}
                )
                ret.append(
                        {'fallback_error': {
                             'data': 'osqueryi is installed but not compatible with this version of windows',
                             'result': True
                        }}
                )
                return ret
            else:
               return None

    orig_filename = query_file
    query_file = __salt__['cp.cache_file'](query_file)
    if query_file is None:
        log.error('Could not find file {0}.'.format(orig_filename))
        return None
    with open(query_file, 'r') as fh:
        query_data = yaml.safe_load(fh)

    if not isinstance(query_data, dict):
        raise CommandExecutionError('Query data is not formed as a dict {0}'
                                    .format(query_data))

    query_data = query_data.get(query_group, [])

    if not query_data:
        return None

    ret = []
    for query in query_data:
        name = query.get('query_name')
        query_sql = query.get('query')
        if not query_sql:
            continue

        # Run the osqueryi query
        query_ret = {
            'result': True,
        }

        cmd = ['osqueryi', '--read_max', MAX_FILE_SIZE, '--json', query_sql]
        res = __salt__['cmd.run_all'](cmd)
        if res['retcode'] == 0:
            query_ret['data'] = json.loads(res['stdout'])
        else:
            query_ret['result'] = False
            query_ret['error'] = res['stderr']

        if verbose:
            tmp = copy.deepcopy(query)
            tmp['query_result'] = query_ret
            ret.append(tmp)
        else:
            ret.append({name: query_ret})

    if query_group == 'day' and report_version_with_day:
        ret.append(hubble_versions())

    for r in ret:
        for query_name, query_ret in r.iteritems():
            for result in query_ret['data']:
                for key, value in result.iteritems():
                    if value.startswith('__JSONIFY__'):
                        result[key] = json.loads(value[len('__JSONIFY__'):])

    return ret

def fields(*args):
    '''
    Use config.get to retrieve custom data based on the keys in the `*args`
    list.

    Arguments:

    *args
        List of keys to retrieve
    '''
    ret = {}
    for field in args:
        ret[field] = __salt__['config.get'](field)
    # Return it as nebula data
    if ret:
        return [{'custom_fields': {
                     'data': [ret],
                     'result': True
                }}]
    return []

def version():
    '''
    Report version of this module
    '''
    return __version__

def hubble_versions():
    '''
    Report version of all hubble modules as query
    '''
    versions = {'nova': __version__,
                'nebula': __version__,
                'pulsar': __version__,
                'quasar': __version__}

    return {'hubble_versions': {'data': [versions],
                                'result': True}}
<commit_msg>Add `custom_` prefix to nebula.fields<commit_after># -*- coding: utf-8 -*-
'''
osquery wrapper for HubbleStack Nebula

:maintainer: basepi
:maturity: 2016.10.2
:platform: All
:requires: SaltStack, osquery

Designed to run sets of osquery queries defined in pillar. These sets will have
a unique identifier, and be targeted by identifier. Usually, this identifier
will be a frequency. ('15 minutes', '1 day', etc). Identifiers are
case-insensitive.

You can then use the scheduler of your choice to run sets os queries at
whatever frequency you choose.

Sample pillar data:

nebula_osquery:
  hour:
    - crontab: query: select c.*,t.iso_8601 as _time from crontab as c join time as t;
    - query_name: suid_binaries
      query: select sb.*, t.iso_8601 as _time from suid_bin as sb join time as t;
  day:
    - query_name: rpm_packages
      query: select rpm.*, t.iso_8601 from rpm_packages as rpm join time as t;
'''
from __future__ import absolute_import

import copy
import json
import logging
import os
import sys
import yaml

import salt.utils
from salt.exceptions import CommandExecutionError
from hubblestack import __version__

log = logging.getLogger(__name__)

__virtualname__ = 'nebula'

def __virtual__():
    return __virtualname__

def queries(query_group,
            query_file=None,
            verbose=False,
            report_version_with_day=True):
    '''
    Run the set of queries represented by ``query_group`` from the
    configuration in the file query_file

    query_group
        Group of queries to run

    query_file
        salt:// file which will be parsed for osquery queries

    verbose
        Defaults to False. If set to True, more information (such as the query
        which was run) will be included in the result.

    CLI Examples:

    .. code_block:: bash

        salt '*' nebula.queries day
        salt '*' nebula.queries hour verbose=True
        salt '*' nebula.queries hour pillar_key=sec_osqueries
    '''
    MAX_FILE_SIZE = 104857600
    if query_file is None:
        if salt.utils.is_windows():
            query_file = 'salt://hubblestack_nebula/hubblestack_nebula_win_queries.yaml'
        else:
            query_file = 'salt://hubblestack_nebula/hubblestack_nebula_queries.yaml'
    if not salt.utils.which('osqueryi'):
        if query_group == 'day':
            log.warning('osquery not installed on this host. Returning baseline data')
            # Match the formatting of normal osquery results. Not super
            #   readable, but just add new dictionaries to the list as we need
            #   more data
            ret = []
            ret.append(
                    {'fallback_osfinger': {
                         'data': [{'osfinger': __grains__.get('osfinger', __grains__.get('osfullname')),
                                   'osrelease': __grains__.get('osrelease', __grains__.get('lsb_distrib_release'))}],
                         'result': True
                    }}
            )
            if 'pkg.list_pkgs' in __salt__:
                ret.append(
                        {'fallback_pkgs': {
                             'data': [{'name': k, 'version': v} for k, v in __salt__['pkg.list_pkgs']().iteritems()],
                             'result': True
                        }}
                )
            uptime = __salt__['status.uptime']()
            if isinstance(uptime, dict):
                uptime = uptime.get('seconds', __salt__['cmd.run']('uptime'))
            ret.append(
                    {'fallback_uptime': {
                         'data': [{'uptime': uptime}],
                         'result': True
                    }}
            )
            if report_version_with_day:
                ret.append(hubble_versions())
            return ret
        else:
            log.debug('osquery not installed on this host. Skipping.')
            return None

    if salt.utils.is_windows():
        win_version = __grains__['osfullname']
        if '2008' not in win_version and '2012' not in win_version and '2016' not in win_version:
            log.error('osquery does not run on windows versions earlier than Server 2008 and Windows 7')
            if query_group == 'day':
                ret = []
                ret.append(
                        {'fallback_osfinger': {
                             'data': [{'osfinger': __grains__.get('osfinger', __grains__.get('osfullname')),
                                       'osrelease': __grains__.get('osrelease', __grains__.get('lsb_distrib_release'))}],
                             'result': True
                        }}
                )
                ret.append(
                        {'fallback_error': {
                             'data': 'osqueryi is installed but not compatible with this version of windows',
                             'result': True
                        }}
                )
                return ret
            else:
               return None

    orig_filename = query_file
    query_file = __salt__['cp.cache_file'](query_file)
    if query_file is None:
        log.error('Could not find file {0}.'.format(orig_filename))
        return None
    with open(query_file, 'r') as fh:
        query_data = yaml.safe_load(fh)

    if not isinstance(query_data, dict):
        raise CommandExecutionError('Query data is not formed as a dict {0}'
                                    .format(query_data))

    query_data = query_data.get(query_group, [])

    if not query_data:
        return None

    ret = []
    for query in query_data:
        name = query.get('query_name')
        query_sql = query.get('query')
        if not query_sql:
            continue

        # Run the osqueryi query
        query_ret = {
            'result': True,
        }

        cmd = ['osqueryi', '--read_max', MAX_FILE_SIZE, '--json', query_sql]
        res = __salt__['cmd.run_all'](cmd)
        if res['retcode'] == 0:
            query_ret['data'] = json.loads(res['stdout'])
        else:
            query_ret['result'] = False
            query_ret['error'] = res['stderr']

        if verbose:
            tmp = copy.deepcopy(query)
            tmp['query_result'] = query_ret
            ret.append(tmp)
        else:
            ret.append({name: query_ret})

    if query_group == 'day' and report_version_with_day:
        ret.append(hubble_versions())

    for r in ret:
        for query_name, query_ret in r.iteritems():
            for result in query_ret['data']:
                for key, value in result.iteritems():
                    if value.startswith('__JSONIFY__'):
                        result[key] = json.loads(value[len('__JSONIFY__'):])

    return ret

def fields(*args):
    '''
    Use config.get to retrieve custom data based on the keys in the `*args`
    list.

    Arguments:

    *args
        List of keys to retrieve
    '''
    ret = {}
    for field in args:
        ret['custom_{0}'.format(field)] = __salt__['config.get'](field)
    # Return it as nebula data
    if ret:
        return [{'custom_fields': {
                     'data': [ret],
                     'result': True
                }}]
    return []

def version():
    '''
    Report version of this module
    '''
    return __version__

def hubble_versions():
    '''
    Report version of all hubble modules as query
    '''
    versions = {'nova': __version__,
                'nebula': __version__,
                'pulsar': __version__,
                'quasar': __version__}

    return {'hubble_versions': {'data': [versions],
                                'result': True}}
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: Cyril Sester <csester@compassion.ch>
#
#    The licence is in the file __openerp__.py
#
##############################################################################

from openerp import api, exceptions, fields, models, _
from openerp.tools import mod10r

from .product import GIFT_CATEGORY, SPONSORSHIP_CATEGORY

class invoice_line(models.Model):
    _inherit = 'account.invoice.line'

    last_payment = fields.Date(
        compute='_set_last_payment', store=True)

    @api.multi
    @api.depends('invoice_id.payment_ids', 'state')
    def _set_last_payment(self):
        for line in self:
            last_date = None
            for payment in line.invoice_id.payment_ids:
                if payment.credit > 0 and payment.date > last_date:
                    last_date = payment.date
            line.last_payment = last_date

class account_invoice(models.Model):
    """Generate automatically a BVR Reference for LSV Invoices"""
    _inherit = 'account.invoice'

    children = fields.Char(
        'Children', compute='_set_children')

    @api.multi
    def _set_children(self):
        """ View children contained in invoice. """
        for invoice in self:
            children = invoice.mapped('invoice_line.contract_id.child_id')
            if len(children) > 1:
                invoice.children = _("{0} children".format(str(len(
                    children))))
            elif children:
                invoice.children = children.local_id
            else:
                invoice.children = False

    @api.multi
    def action_date_assign(self):
        """Method called when invoice is validated.
            - Add BVR Reference if payment term is LSV and no reference is
              set.
            - Prevent validating invoices missing related contract.
        """
        for invoice in self:
            if invoice.payment_term and 'LSV' in invoice.payment_term.name \
                    and not invoice.bvr_reference:
                seq = self.env['ir.sequence']
                ref = mod10r(seq.next_by_code('contract.bvr.ref'))
                invoice.write({'bvr_reference': ref})
            for invl in invoice.invoice_line:
                if not invl.contract_id and invl.product_id.categ_name in (
                        SPONSORSHIP_CATEGORY, GIFT_CATEGORY):
                    raise exceptions.Warning(
                        _('Sponsorship missing in invoice'),
                        _("Invoice %s for '%s' is missing a sponsorship.") %
                        (str(invoice.id), invoice.partner_id.name))

        return super(account_invoice, self).action_date_assign()
<commit_msg>FIX Last payment computation<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: Cyril Sester <csester@compassion.ch>
#
#    The licence is in the file __openerp__.py
#
##############################################################################

from openerp import api, exceptions, fields, models, _
from openerp.tools import mod10r

from .product import GIFT_CATEGORY, SPONSORSHIP_CATEGORY

class invoice_line(models.Model):
    _inherit = 'account.invoice.line'

    last_payment = fields.Date(
        compute='_set_last_payment', store=True)

    @api.multi
    @api.depends('invoice_id.payment_ids', 'state')
    def _set_last_payment(self):
        for line in self:
            last_date = None
            for payment in line.invoice_id.payment_ids.filtered('credit'):
                if payment.date > last_date:
                    last_date = payment.date
            line.last_payment = last_date

class account_invoice(models.Model):
    """Generate automatically a BVR Reference for LSV Invoices"""
    _inherit = 'account.invoice'

    children = fields.Char(
        'Children', compute='_set_children')

    @api.multi
    def _set_children(self):
        """ View children contained in invoice. """
        for invoice in self:
            children = invoice.mapped('invoice_line.contract_id.child_id')
            if len(children) > 1:
                invoice.children = _("{0} children".format(str(len(
                    children))))
            elif children:
                invoice.children = children.local_id
            else:
                invoice.children = False

    @api.multi
    def action_date_assign(self):
        """Method called when invoice is validated.
            - Add BVR Reference if payment term is LSV and no reference is
              set.
            - Prevent validating invoices missing related contract.
        """
        for invoice in self:
            if invoice.payment_term and 'LSV' in invoice.payment_term.name \
                    and not invoice.bvr_reference:
                seq = self.env['ir.sequence']
                ref = mod10r(seq.next_by_code('contract.bvr.ref'))
                invoice.write({'bvr_reference': ref})
            for invl in invoice.invoice_line:
                if not invl.contract_id and invl.product_id.categ_name in (
                        SPONSORSHIP_CATEGORY, GIFT_CATEGORY):
                    raise exceptions.Warning(
                        _('Sponsorship missing in invoice'),
                        _("Invoice %s for '%s' is missing a sponsorship.") %
                        (str(invoice.id), invoice.partner_id.name))

        return super(account_invoice, self).action_date_assign()
<|endoftext|>


<commit_before>import pandas as pd
import pandas.testing as tm
import pytest

import ibis
import ibis.config as config
import ibis.expr.datatypes as dt
import ibis.expr.types as ir
from ibis import util

pytest.importorskip("clickhouse_driver")

def test_run_sql(con):
    query = 'SELECT * FROM ibis_testing.functional_alltypes'
    table = con.sql(query)

    fa = con.table('functional_alltypes')
    assert isinstance(table, ir.TableExpr)
    assert table.schema() == fa.schema()

    expr = table.limit(10)
    result = expr.execute()
    assert len(result) == 10

def test_get_schema(con):
    t = con.table('functional_alltypes')
    schema = con.get_schema('functional_alltypes')
    assert t.schema() == schema

def test_result_as_dataframe(con, alltypes):
    expr = alltypes.limit(10)

    ex_names = list(expr.schema().names)
    result = con.execute(expr)

    assert isinstance(result, pd.DataFrame)
    assert result.columns.tolist() == ex_names
    assert len(result) == 10

def test_array_default_limit(con, alltypes):
    result = con.execute(alltypes.float_col, limit=100)
    assert len(result) == 100

def test_limit_overrides_expr(con, alltypes):
    result = con.execute(alltypes.limit(10), limit=5)
    assert len(result) == 5

def test_limit_equals_none_no_limit(alltypes):
    with config.option_context('sql.default_limit', 10):
        result = alltypes.execute(limit=None)
        assert len(result) > 10

def test_verbose_log_queries(con):
    queries = []

    def logger(x):
        queries.append(x)

    with config.option_context('verbose', True):
        with config.option_context('verbose_log', logger):
            con.table('functional_alltypes')

    expected = 'DESCRIBE ibis_testing.`functional_alltypes`'

    assert len(queries) == 1
    assert queries[0] == expected

def test_sql_query_limits(alltypes):
    table = alltypes
    with config.option_context('sql.default_limit', 100000):
        # table has 25 rows
        assert len(table.execute()) == 7300
        # comply with limit arg for TableExpr
        assert len(table.execute(limit=10)) == 10
        # state hasn't changed
        assert len(table.execute()) == 7300
        # non-TableExpr ignores default_limit
        assert table.count().execute() == 7300
        # non-TableExpr doesn't observe limit arg
        assert table.count().execute(limit=10) == 7300
    with config.option_context('sql.default_limit', 20):
        # TableExpr observes default limit setting
        assert len(table.execute()) == 20
        # explicit limit= overrides default
        assert len(table.execute(limit=15)) == 15
        assert len(table.execute(limit=23)) == 23
        # non-TableExpr ignores default_limit
        assert table.count().execute() == 7300
        # non-TableExpr doesn't observe limit arg
        assert table.count().execute(limit=10) == 7300
    # eliminating default_limit doesn't break anything
    with config.option_context('sql.default_limit', None):
        assert len(table.execute()) == 7300
        assert len(table.execute(limit=15)) == 15
        assert len(table.execute(limit=10000)) == 7300
        assert table.count().execute() == 7300
        assert table.count().execute(limit=10) == 7300

def test_embedded_identifier_quoting(alltypes):
    t = alltypes

    expr = t[[(t.double_col * 2).name('double(fun)')]]['double(fun)'].sum()
    expr.execute()

@pytest.fixture
def temporary_alltypes(con):
    id = util.guid()
    con.raw_sql(f"CREATE TABLE temporary_alltypes_{id} AS functional_alltypes")
    try:
        yield con.table(f"temporary_alltypes_{id}")
    finally:
        con.raw_sql(f"DROP TABLE temporary_alltypes_{id}")

def test_insert(temporary_alltypes, df):
    temporary = temporary_alltypes
    records = df[:10]

    assert len(temporary.execute()) == 0
    temporary.insert(records)

    tm.assert_frame_equal(temporary.execute(), records)

def test_insert_with_less_columns(temporary_alltypes, df):
    temporary = temporary_alltypes
    records = df.loc[:10, ['string_col']].copy()
    records['date_col'] = None

    with pytest.raises(AssertionError):
        temporary.insert(records)

def test_insert_with_more_columns(temporary_alltypes, df):
    temporary = temporary_alltypes
    records = df[:10].copy()
    records['non_existing_column'] = 'raise on me'

    with pytest.raises(AssertionError):
        temporary.insert(records)

@pytest.mark.parametrize(
    ("query", "expected_schema"),
    [
        (
            "SELECT 1 as a, 2 + dummy as b",
            ibis.schema(
                dict(a=dt.UInt8(nullable=False), b=dt.UInt16(nullable=False))
            ),
        ),
        (
            "SELECT string_col, sum(double_col) as b FROM functional_alltypes GROUP BY string_col",  # noqa: E501
            ibis.schema(
                dict(
                    string_col=dt.String(nullable=False),
                    b=dt.Float64(nullable=False),
                )
            ),
        ),
    ],
)
def test_get_schema_using_query(con, query, expected_schema):
    result = con._get_schema_using_query(query)
    assert result == expected_schema
<commit_msg>test(clickhouse): fix field nullability for new test cases<commit_after>import pandas as pd
import pandas.testing as tm
import pytest

import ibis
import ibis.config as config
import ibis.expr.datatypes as dt
import ibis.expr.types as ir
from ibis import util

pytest.importorskip("clickhouse_driver")

def test_run_sql(con):
    query = 'SELECT * FROM ibis_testing.functional_alltypes'
    table = con.sql(query)

    fa = con.table('functional_alltypes')
    assert isinstance(table, ir.TableExpr)
    assert table.schema() == fa.schema()

    expr = table.limit(10)
    result = expr.execute()
    assert len(result) == 10

def test_get_schema(con):
    t = con.table('functional_alltypes')
    schema = con.get_schema('functional_alltypes')
    assert t.schema() == schema

def test_result_as_dataframe(con, alltypes):
    expr = alltypes.limit(10)

    ex_names = list(expr.schema().names)
    result = con.execute(expr)

    assert isinstance(result, pd.DataFrame)
    assert result.columns.tolist() == ex_names
    assert len(result) == 10

def test_array_default_limit(con, alltypes):
    result = con.execute(alltypes.float_col, limit=100)
    assert len(result) == 100

def test_limit_overrides_expr(con, alltypes):
    result = con.execute(alltypes.limit(10), limit=5)
    assert len(result) == 5

def test_limit_equals_none_no_limit(alltypes):
    with config.option_context('sql.default_limit', 10):
        result = alltypes.execute(limit=None)
        assert len(result) > 10

def test_verbose_log_queries(con):
    queries = []

    def logger(x):
        queries.append(x)

    with config.option_context('verbose', True):
        with config.option_context('verbose_log', logger):
            con.table('functional_alltypes')

    expected = 'DESCRIBE ibis_testing.`functional_alltypes`'

    assert len(queries) == 1
    assert queries[0] == expected

def test_sql_query_limits(alltypes):
    table = alltypes
    with config.option_context('sql.default_limit', 100000):
        # table has 25 rows
        assert len(table.execute()) == 7300
        # comply with limit arg for TableExpr
        assert len(table.execute(limit=10)) == 10
        # state hasn't changed
        assert len(table.execute()) == 7300
        # non-TableExpr ignores default_limit
        assert table.count().execute() == 7300
        # non-TableExpr doesn't observe limit arg
        assert table.count().execute(limit=10) == 7300
    with config.option_context('sql.default_limit', 20):
        # TableExpr observes default limit setting
        assert len(table.execute()) == 20
        # explicit limit= overrides default
        assert len(table.execute(limit=15)) == 15
        assert len(table.execute(limit=23)) == 23
        # non-TableExpr ignores default_limit
        assert table.count().execute() == 7300
        # non-TableExpr doesn't observe limit arg
        assert table.count().execute(limit=10) == 7300
    # eliminating default_limit doesn't break anything
    with config.option_context('sql.default_limit', None):
        assert len(table.execute()) == 7300
        assert len(table.execute(limit=15)) == 15
        assert len(table.execute(limit=10000)) == 7300
        assert table.count().execute() == 7300
        assert table.count().execute(limit=10) == 7300

def test_embedded_identifier_quoting(alltypes):
    t = alltypes

    expr = t[[(t.double_col * 2).name('double(fun)')]]['double(fun)'].sum()
    expr.execute()

@pytest.fixture
def temporary_alltypes(con):
    id = util.guid()
    con.raw_sql(f"CREATE TABLE temporary_alltypes_{id} AS functional_alltypes")
    try:
        yield con.table(f"temporary_alltypes_{id}")
    finally:
        con.raw_sql(f"DROP TABLE temporary_alltypes_{id}")

def test_insert(temporary_alltypes, df):
    temporary = temporary_alltypes
    records = df[:10]

    assert len(temporary.execute()) == 0
    temporary.insert(records)

    tm.assert_frame_equal(temporary.execute(), records)

def test_insert_with_less_columns(temporary_alltypes, df):
    temporary = temporary_alltypes
    records = df.loc[:10, ['string_col']].copy()
    records['date_col'] = None

    with pytest.raises(AssertionError):
        temporary.insert(records)

def test_insert_with_more_columns(temporary_alltypes, df):
    temporary = temporary_alltypes
    records = df[:10].copy()
    records['non_existing_column'] = 'raise on me'

    with pytest.raises(AssertionError):
        temporary.insert(records)

@pytest.mark.parametrize(
    ("query", "expected_schema"),
    [
        (
            "SELECT 1 as a, 2 + dummy as b",
            ibis.schema(
                dict(a=dt.UInt8(nullable=False), b=dt.UInt16(nullable=False))
            ),
        ),
        (
            "SELECT string_col, sum(double_col) as b FROM functional_alltypes GROUP BY string_col",  # noqa: E501
            ibis.schema(
                dict(
                    string_col=dt.String(nullable=True),
                    b=dt.Float64(nullable=True),
                )
            ),
        ),
    ],
)
def test_get_schema_using_query(con, query, expected_schema):
    result = con._get_schema_using_query(query)
    assert result == expected_schema
<|endoftext|>


<commit_before>import logging
import ibmsecurity.utilities.tools
import os.path
from ibmsecurity.utilities.tools import files_same, get_random_temp_dir
import shutil

logger = logging.getLogger(__name__)

def get(isamAppliance, path, check_mode=False, force=False):
    """
    Retrieving the current runtime template files directory contents
    """
    return isamAppliance.invoke_get("Retrieving the current runtime template files directory contents",
                                    "/mga/template_files/{0}".format(path))

def _check(isamAppliance, path, name):
    ret_obj = get(isamAppliance, path)

    for obj in ret_obj['data']['contents']:
        if obj['name'] == name and obj['type'] == 'File':
            logger.info("File .{0}".format(obj['name']))
            return True

    return None

def create(isamAppliance, path, name, contents=None, check_mode=False, force=False):
    """
    Creating a file in the runtime template files directory

    :param isamAppliance:
    :param path:
    :param name:
    :param contents:
    :param check_mode:
    :param force:
    :return:
    """
    warnings = []
    check_file = _check(isamAppliance, path, name)
    if check_file != None:
        warnings.append("File {0} exists in path {1}. Ignoring create.".format(name,path))

    if force is True or check_file == None:
        if check_mode is True:
            return isamAppliance.create_return_object(changed=True, warnings=warnings)
        else:
            return isamAppliance.invoke_post(
                "Creating a file in the runtime template files directory",
                "/mga/template_files/{0}".format(path),
                {
                    'file_name': name,
                    'type': 'file',
                    'contents': contents
                })
    return isamAppliance.create_return_object(warnings=warnings)

def update(isamAppliance, path, name, contents=None, check_mode=False, force=False):
    """
    Update a file in the runtime template files directory
    :param isamAppliance:
    :param path:
    :param name:
    :param contents:
    :param check_mode:
    :param force:
    :return:
    """
    warnings = []
    check_file = _check(isamAppliance, path, name)
    if check_file == None:
        warnings.append("File {0} does not exists in path {1}. Ignoring update.".format(name,path))
        return isamAppliance.create_return_object(warnings=warnings)

    if force is True or check_file is True:
        if check_mode is True:
            return isamAppliance.create_return_object(changed=True)
        else:
            if contents is not None:
                return isamAppliance.invoke_put_files(
                    "Update a file in the runtime template files directory",
                    "/mga/template_files/{0}/{1}".format(path, name),
                    {
                        'contents': contents,
                        'type': 'file'
                    })
            else:
                warnings.append("Either contents or filename parameter need to be provided. Skipping update request.")
                return isamAppliance.create_return_object(warnings=warnings)

def delete(isamAppliance, path, name, check_mode=False, force=False):
    """
    Deleting a file in the runtime template files directory

    :param isamAppliance:
    :param path:
    :param name:
    :param check_mode:
    :param force:
    :return:
    """
    warnings = []
    check_file = _check(isamAppliance, path, name)
    if check_file == None:
        warnings.append("File {0} does not exists in path {1}. Ignoring delete.".format(name,path))
        return isamAppliance.create_return_object(warnings=warnings)

    if force is True or check_file != None:
        if check_mode is True:
            return isamAppliance.create_return_object(changed=True)
        else:
            return isamAppliance.invoke_delete(
                "Deleting a file in the runtime template files directory",
                "/mga/template_files/{0}/{1}".format(path, name))

    return isamAppliance.create_return_object()

def rename(isamAppliance, path, name, new_name, check_mode=False, force=False):
    """
    Deleting a file in the runtime template files directory

    :param isamAppliance:
    :param path:
    :param name:
    :param new_name:
    :param check_mode:
    :param force:
    :return:
    """
    warnings = []
    check_file = _check(isamAppliance, path, name)
    if check_file == None:
        warnings.append("File {0} does not exists in path {1}. Ignoring rename.".format(name,path))
        return isamAppliance.create_return_object(warnings=warnings)

    if force is True or check_file != None:
        if check_mode is True:
            return isamAppliance.create_return_object(changed=True)
        else:
            return isamAppliance.invoke_put(
                "Renaming a file in the runtime template files directory",
                "/mga/template_files/{0}/{1}".format(path, name),
                {
                    'new_name': new_name,
                    'type': 'file'
                })

    return isamAppliance.create_return_object()

def export_file(isamAppliance, path, name, filename, check_mode=False, force=False):
    """
    Exporting a file from the runtime template files directory

    :param isamAppliance:
    :param path:
    :param name:
    :param check_mode:
    :param force:
    :return:
    """
    warnings = []
    check_file = _check(isamAppliance, path, name)
    if check_file == None:
        warnings.append("File {0} does not exists in path {1}. Ignoring export.".format(name,path))
        return isamAppliance.create_return_object(warnings=warnings)

    if force is True or check_file != None:
        if check_mode is False:
            return isamAppliance.invoke_get_file(
                "Exporting a file from the runtime template files directory",
                "/mga/template_files/{0}/{1}?type=File&export=true".format(path, name), filename)

    return isamAppliance.create_return_object()

def _check_import(isamAppliance, path, name, filename, check_mode=False):
    """
    Checks if file on the Appliance (name) exists and if so, whether it is different from filename
    :param isamAppliance:
    :param path:
    :param name:
    :param filename:
    :return:
    """
    tmpdir = get_random_temp_dir()
    tmp_original_file = os.path.join(tmpdir, os.path.basename(name))
    if _check(isamAppliance, path, name):
        export_file(isamAppliance, path, name, tmp_original_file, check_mode=False, force=True)
        logger.debug("file already exists on appliance")
        if files_same(tmp_original_file, filename):
            logger.debug("files are the same, so we don't want to do anything")
            shutil.rmtree(tmpdir)
            return False
        else:
            logger.debug("files are different, so we delete existing file in preparation for import")
            delete(isamAppliance, path, name, check_mode=check_mode, force=True)
            shutil.rmtree(tmpdir)
            return True
    else:
        logger.debug("file does not exist on appliance, so we'll want to import")
        shutil.rmtree(tmpdir)
        return True

def import_file(isamAppliance, path, name, filename, check_mode=False, force=False):
    """
    Importing a file in the runtime template files directory.
    """
    warnings = []
    check_file = _check(isamAppliance, path, name)
    if check_file != None and force == False:
        warnings.append("File {0} exists in path {1}. Ignoring import.".format(name,path))
        return isamAppliance.create_return_object(warnings=warnings)

    if force is True or _check_import(isamAppliance, path, name, filename, check_mode=check_mode):
        if check_mode is True:
            return isamAppliance.create_return_object(changed=True)
        else:
            return isamAppliance.invoke_post_files(
                "Importing a file in the runtime template files directory",
                "/mga/template_files/{0}/{1}".format(path, name),
                [
                    {
                        'file_formfield': 'file',
                        'filename': filename,
                        'mimetype': 'application/octet-stream'
                    }
                ],
                {
                    'type': 'file',
                    'force': force
                })

    return isamAppliance.create_return_object()

def compare(isamAppliance1, isamAppliance2, instance_id):
    ret_obj1 = get_all(isamAppliance1, instance_id)
    ret_obj2 = get_all(isamAppliance2, instance_id)

    return ibmsecurity.utilities.tools.json_compare(ret_obj1, ret_obj2, deleted_keys=[])
<commit_msg>fixed a bug where an already exisiting file would not be replaced, when the new file is different.<commit_after>import logging
import ibmsecurity.utilities.tools
import os.path
from ibmsecurity.utilities.tools import files_same, get_random_temp_dir
import shutil

logger = logging.getLogger(__name__)

def get(isamAppliance, path, check_mode=False, force=False):
    """
    Retrieving the current runtime template files directory contents
    """
    return isamAppliance.invoke_get("Retrieving the current runtime template files directory contents",
                                    "/mga/template_files/{0}".format(path))

def _check(isamAppliance, path, name):
    ret_obj = get(isamAppliance, path)

    for obj in ret_obj['data']['contents']:
        if obj['name'] == name and obj['type'] == 'File':
            logger.info("File .{0}".format(obj['name']))
            return True

    return None

def create(isamAppliance, path, name, contents=None, check_mode=False, force=False):
    """
    Creating a file in the runtime template files directory

    :param isamAppliance:
    :param path:
    :param name:
    :param contents:
    :param check_mode:
    :param force:
    :return:
    """
    warnings = []
    check_file = _check(isamAppliance, path, name)
    if check_file != None:
        warnings.append("File {0} exists in path {1}. Ignoring create.".format(name,path))

    if force is True or check_file == None:
        if check_mode is True:
            return isamAppliance.create_return_object(changed=True, warnings=warnings)
        else:
            return isamAppliance.invoke_post(
                "Creating a file in the runtime template files directory",
                "/mga/template_files/{0}".format(path),
                {
                    'file_name': name,
                    'type': 'file',
                    'contents': contents
                })
    return isamAppliance.create_return_object(warnings=warnings)

def update(isamAppliance, path, name, contents=None, check_mode=False, force=False):
    """
    Update a file in the runtime template files directory
    :param isamAppliance:
    :param path:
    :param name:
    :param contents:
    :param check_mode:
    :param force:
    :return:
    """
    warnings = []
    check_file = _check(isamAppliance, path, name)
    if check_file == None:
        warnings.append("File {0} does not exists in path {1}. Ignoring update.".format(name,path))
        return isamAppliance.create_return_object(warnings=warnings)

    if force is True or check_file is True:
        if check_mode is True:
            return isamAppliance.create_return_object(changed=True)
        else:
            if contents is not None:
                return isamAppliance.invoke_put_files(
                    "Update a file in the runtime template files directory",
                    "/mga/template_files/{0}/{1}".format(path, name),
                    {
                        'contents': contents,
                        'type': 'file'
                    })
            else:
                warnings.append("Either contents or filename parameter need to be provided. Skipping update request.")
                return isamAppliance.create_return_object(warnings=warnings)

def delete(isamAppliance, path, name, check_mode=False, force=False):
    """
    Deleting a file in the runtime template files directory

    :param isamAppliance:
    :param path:
    :param name:
    :param check_mode:
    :param force:
    :return:
    """
    warnings = []
    check_file = _check(isamAppliance, path, name)
    if check_file == None:
        warnings.append("File {0} does not exists in path {1}. Ignoring delete.".format(name,path))
        return isamAppliance.create_return_object(warnings=warnings)

    if force is True or check_file != None:
        if check_mode is True:
            return isamAppliance.create_return_object(changed=True)
        else:
            return isamAppliance.invoke_delete(
                "Deleting a file in the runtime template files directory",
                "/mga/template_files/{0}/{1}".format(path, name))

    return isamAppliance.create_return_object()

def rename(isamAppliance, path, name, new_name, check_mode=False, force=False):
    """
    Deleting a file in the runtime template files directory

    :param isamAppliance:
    :param path:
    :param name:
    :param new_name:
    :param check_mode:
    :param force:
    :return:
    """
    warnings = []
    check_file = _check(isamAppliance, path, name)
    if check_file == None:
        warnings.append("File {0} does not exists in path {1}. Ignoring rename.".format(name,path))
        return isamAppliance.create_return_object(warnings=warnings)

    if force is True or check_file != None:
        if check_mode is True:
            return isamAppliance.create_return_object(changed=True)
        else:
            return isamAppliance.invoke_put(
                "Renaming a file in the runtime template files directory",
                "/mga/template_files/{0}/{1}".format(path, name),
                {
                    'new_name': new_name,
                    'type': 'file'
                })

    return isamAppliance.create_return_object()

def export_file(isamAppliance, path, name, filename, check_mode=False, force=False):
    """
    Exporting a file from the runtime template files directory

    :param isamAppliance:
    :param path:
    :param name:
    :param check_mode:
    :param force:
    :return:
    """
    warnings = []
    check_file = _check(isamAppliance, path, name)
    if check_file == None:
        warnings.append("File {0} does not exists in path {1}. Ignoring export.".format(name,path))
        return isamAppliance.create_return_object(warnings=warnings)

    if force is True or check_file != None:
        if check_mode is False:
            return isamAppliance.invoke_get_file(
                "Exporting a file from the runtime template files directory",
                "/mga/template_files/{0}/{1}?type=File&export=true".format(path, name), filename)

    return isamAppliance.create_return_object()

def _check_import(isamAppliance, path, name, filename, check_mode=False):
    """
    Checks if file on the Appliance (name) exists and if so, whether it is different from filename
    :param isamAppliance:
    :param path:
    :param name:
    :param filename:
    :return:
    """
    tmpdir = get_random_temp_dir()
    tmp_original_file = os.path.join(tmpdir, os.path.basename(name))
    if _check(isamAppliance, path, name):
        export_file(isamAppliance, path, name, tmp_original_file, check_mode=False, force=True)
        logger.debug("file already exists on appliance")
        if files_same(tmp_original_file, filename):
            logger.debug("files are the same, so we don't want to do anything")
            shutil.rmtree(tmpdir)
            return False
        else:
            logger.debug("files are different, so we delete existing file in preparation for import")
            delete(isamAppliance, path, name, check_mode=check_mode, force=True)
            shutil.rmtree(tmpdir)
            return True
    else:
        logger.debug("file does not exist on appliance, so we'll want to import")
        shutil.rmtree(tmpdir)
        return True

def import_file(isamAppliance, path, name, filename, check_mode=False, force=False):
    """
    Importing a file in the runtime template files directory.
    """
    warnings = []
    check_file = _check(isamAppliance, path, name)
    if check_file != None and force == False:
        warnings.append("File {0} exists in path {1}.".format(name,path))

    if force is True or _check_import(isamAppliance, path, name, filename, check_mode=check_mode):
        if check_mode is True:
            return isamAppliance.create_return_object(changed=True)
        else:
            return isamAppliance.invoke_post_files(
                "Importing a file in the runtime template files directory",
                "/mga/template_files/{0}/{1}".format(path, name),
                [
                    {
                        'file_formfield': 'file',
                        'filename': filename,
                        'mimetype': 'application/octet-stream'
                    }
                ],
                {
                    'type': 'file',
                    'force': force
                })

    return isamAppliance.create_return_object(warnings=warnings)

def compare(isamAppliance1, isamAppliance2, instance_id):
    ret_obj1 = get_all(isamAppliance1, instance_id)
    ret_obj2 = get_all(isamAppliance2, instance_id)

    return ibmsecurity.utilities.tools.json_compare(ret_obj1, ret_obj2, deleted_keys=[])
<|endoftext|>


<commit_before>from django.test import TestCase
from django.contrib.auth.models import User
from django.db import IntegrityError
from nose.tools import raises, assert_equals
from api.models import Procedure
import uuid

class ProcedureTest(TestCase):
    def setUp(self):
        self.test_user = User.objects.create_user(
            'TestUser',
            'test@sanaprotocolbuilder.me',
            'testpassword'
        )
        self.test_user.save()

    def test_procedures_required(self):
        Procedure.objects.create(
            author='tester',
            title='test procedure',
            owner=self.test_user
        )

        proc = Procedure.objects.get(author='tester')
        assert_equals(proc.author, 'tester')
        assert_equals(proc.title, 'test procedure')
        assert_equals(proc.version, None)
        assert_equals(proc.uuid, None)

    def test_procedures_all_properties(self):
        test_uuid = str(uuid.uuid1())

        Procedure.objects.create(
            author='tester2',
            title='full test procedure',
            version='1.0',
            uuid=test_uuid,
            owner=self.test_user
        )

        proc = Procedure.objects.get(author='tester2')
        assert_equals(proc.author, 'tester2')
        assert_equals(proc.title, 'full test procedure')
        assert_equals(proc.version, '1.0')
        assert_equals(proc.uuid, test_uuid)

    @raises(IntegrityError)
    def test_author_none(self):
        Procedure.objects.create(
            author=None,
            title='full test procedure',
            owner=self.test_user
        )

    @raises(IntegrityError)
    def test_title_none(self):
        Procedure.objects.create(
            author='author',
            title=None,
            owner=self.test_user
        )

    @raises(IntegrityError)
    def test_owner_none(self):
        Procedure.objects.create(
            author='author',
            title=None
        )
<commit_msg>Test the procedure's owner<commit_after>from django.test import TestCase
from django.contrib.auth.models import User
from django.db import IntegrityError
from nose.tools import raises, assert_equals
from api.models import Procedure
import uuid

class ProcedureTest(TestCase):
    def setUp(self):
        self.test_user = User.objects.create_user(
            'TestUser',
            'test@sanaprotocolbuilder.me',
            'testpassword'
        )
        self.test_user.save()

    def test_procedures_required(self):
        Procedure.objects.create(
            author='tester',
            title='test procedure',
            owner=self.test_user
        )

        proc = Procedure.objects.get(author='tester')
        assert_equals(proc.author, 'tester')
        assert_equals(proc.title, 'test procedure')
        assert_equals(proc.version, None)
        assert_equals(proc.uuid, None)
        assert_equals(proc.owner, self.test_user)

    def test_procedures_all_properties(self):
        test_uuid = str(uuid.uuid1())

        Procedure.objects.create(
            author='tester2',
            title='full test procedure',
            version='1.0',
            uuid=test_uuid,
            owner=self.test_user
        )

        proc = Procedure.objects.get(author='tester2')
        assert_equals(proc.author, 'tester2')
        assert_equals(proc.title, 'full test procedure')
        assert_equals(proc.version, '1.0')
        assert_equals(proc.uuid, test_uuid)
        assert_equals(proc.owner, self.test_user)

    @raises(IntegrityError)
    def test_author_none(self):
        Procedure.objects.create(
            author=None,
            title='full test procedure',
            owner=self.test_user
        )

    @raises(IntegrityError)
    def test_title_none(self):
        Procedure.objects.create(
            author='author',
            title=None,
            owner=self.test_user
        )

    @raises(IntegrityError)
    def test_owner_none(self):
        Procedure.objects.create(
            author='author',
            title=None
        )
<|endoftext|>


<commit_before># Copyright 2010-2012 RethinkDB, all rights reserved.
from distutils.core import setup

setup(name="rethinkdb"
     ,version="1.2.0-0"
     ,description="This package provides the Python driver library for the RethinkDB database server."
     ,url="http://rethinkdb.com"
     ,maintainer="RethinkDB Inc."
     ,maintainer_email="bugs@rethinkdb.com"
     ,packages=['rethinkdb']
     )
<commit_msg>adds protobuf dependency to python setup.py<commit_after># Copyright 2010-2012 RethinkDB, all rights reserved.
from distutils.core import setup

setup(name="rethinkdb"
     ,version="1.2.0-0"
     ,description="This package provides the Python driver library for the RethinkDB database server."
     ,url="http://rethinkdb.com"
     ,maintainer="RethinkDB Inc."
     ,maintainer_email="bugs@rethinkdb.com"
     ,packages=['rethinkdb']
     ,requires=['protobuf']
     )
<|endoftext|>


<commit_before>import json
import os
import re
import urllib.request

target_dir = "~/epnamer/test"

show_search_url = "http://api.tvmaze.com/singlesearch/shows?q={}"

epcode_re = re.compile(r"S(\d\d)E(\d\d)", re.IGNORECASE)

class ShowInfo:
    id = None
    name = None
    summary = None

def generate_name(season, episode):
    return '{}.s{}e{}'.format(show_name, season, episode)

def get_name_map(old_filenames):
    name_map = {}
    for filename in old_filenames:
        match = epcode_re.search(filename)
        if match:
            name_map[filename] = generate_name(filename, *match.groups())

def get_show(show_name):
    query_url = show_search_url.format(urllib.parse.quote(show_name))
    with urllib.request.urlopen(query_url) as response:
        json_data = json.loads(response.readall().decode('utf-8'))
    show_info = ShowInfo()
    show_info.id = json_data['id']
    show_info.name = json_data['name']
    show_info.summary = json_data['summary']
    return show_info

def confirm_show(show_info):
    print("Data obtained via TVmaze <tvmaze.com>")
    print("Selected '{}'".format(show_info.name))
    print("'''{}'''".format(show_info.summary))
    print()
    return input("Correct? [Y/n] ").lower() in ('y', 'yes', '')

def main():
    show_name = "Penn & Teller Bullshit!"
    show_info = get_show(show_name)
    print(confirm_show(show_info))

if __name__ == "__main__":
    main()
<commit_msg>Get episode data<commit_after>import json
import os
import re
import urllib.request

target_dir = "~/epnamer/test"

show_search_url = "http://api.tvmaze.com/singlesearch/shows?q={}"
episode_search_url = "http://api.tvmaze.com/shows/{}/episodes"

epcode_re = re.compile(r"S(\d\d)E(\d\d)", re.IGNORECASE)

class ShowInfo:
    id = None
    name = None
    summary = None

class EpisodeInfo:
    season = None
    num = None
    name = None

def json_query(url):
    with urllib.request.urlopen(url) as response:
        return json.loads(response.readall().decode('utf-8'))

def get_show_info(show_name):
    query_url = show_search_url.format(urllib.parse.quote(show_name))
    json_data = json_query(query_url)
    show_info = ShowInfo()
    show_info.id = json_data['id']
    show_info.name = json_data['name']
    show_info.summary = json_data['summary']
    return show_info

def confirm_show(show_info):
    print("Data obtained via TVmaze <tvmaze.com>")
    print("Selected '{}'".format(show_info.name))
    print("'''{}'''".format(show_info.summary))
    print()
    return input("Correct? [Y/n] ").lower() in ('y', 'yes', '')

def get_episodes_info(show_id):
    query_url = episode_search_url.format(show_id)
    json_data = json_query(query_url)
    episodes_info = []
    for episode_data in json_data:
        episode_info = EpisodeInfo()
        episode_info.season = episode_data['season']
        episode_info.num = episode_data['number']
        episode_info.name = episode_data['name']
        episodes_info.append(episode_info)
    return episodes_info

def main():
    show_name = "Penn & Teller Bullshit!"
    show_info = get_show_info(show_name)
    print(confirm_show(show_info))

    for ep in get_episodes_info(show_info.id):
        print(ep.name)

def generate_name(season, episode):
    return '{}.s{}e{}'.format(show_name, season, episode)

def get_name_map(old_filenames):
    name_map = {}
    for filename in old_filenames:
        match = epcode_re.search(filename)
        if match:
            name_map[filename] = generate_name(filename, *match.groups())

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import sys
import tensorflow as tf
import numpy as np
from src.Utilities import Utilities
from src.Autoencoder import InputLayer
from src.Autoencoder import HiddenLayer
from src.Autoencoder import OutputLayer

# Number of epochs to run
numEpochs = 250
# Size of each batch
batchSize = 10
# Scaling factor for sparsity cost function
alpha = 0.01
# Sparsity parameter
rho = 0.05
# Scaling factor for l2 regularization cost function
beta = 0.01

# Read in data from csv file
Utilities.progress(1, 7, status='Reading in data        ')
inputArray = Utilities.readData()
dictFeeder = Utilities.batchBuilder(inputArray, batchSize)

# Build input layer
Utilities.progress(2, 7, status='Building input layer   ')
with tf.variable_scope("input"):
    iLayer = InputLayer(len(inputArray[1]))

# Build hidden layer 1
Utilities.progress(3, 7, status='Building hidden layer 1')
with tf.variable_scope("hidden1"):
    hidden1 = HiddenLayer(100, iLayer.inputLayer)
    hidden1.buildTrainer(alpha, beta, rho)

# Build hidden layer 2
Utilities.progress(4, 7, status='Building hidden layer 2')
with tf.variable_scope("hidden2"):
    hidden2 = HiddenLayer(50, hidden1.y1)
    hidden2.buildTrainer(alpha, beta, rho)

# Build hidden layer 3
Utilities.progress(5, 7, status='Building hidden layer 3')
with tf.variable_scope("hidden3"):
    hidden3 = HiddenLayer(16, hidden2.y1)
    hidden3.buildTrainer(alpha, beta, rho)

# Build output layer
Utilities.progress(6, 7, status='Building output layer  ')
with tf.variable_scope("output"):
    oLayer = OutputLayer(2, hidden3.y1)
    oLayer.buildTrainer()

Utilities.progress(7, 7, status='Starting session       ')
sess = Utilities.startSession()

# Print the shape of each layer
iLayer.printLayerShape()
hidden1.printLayerShape()
hidden2.printLayerShape()
hidden3.printLayerShape()
oLayer.printLayerShape()

# Training the hidden layers
testCase = Utilities.numpyReshape(inputArray[0])

for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Layer 1 ')
    np.random.shuffle(dictFeeder)
    for j in range(len(dictFeeder)):
        sess.run(hidden1.trainStep,
                 feed_dict = {iLayer.inputLayer: dictFeeder[j]})
    print("Squared Difference: ", sess.run(hidden1.squareDifference,
                                           feed_dict = {iLayer.inputLayer: testCase}))

for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Layer 2 ')
    np.random.shuffle(dictFeeder)
    for j in range(len(dictFeeder)):
        sess.run(hidden2.trainStep,
                 feed_dict = {iLayer.inputLayer: dictFeeder[j]})
    print("Squared Difference: ", sess.run(hidden2.squareDifference,
                                           feed_dict = {iLayer.inputLayer: testCase}))

for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Layer 3 ')
    np.random.shuffle(dictFeeder)
    for j in range(len(dictFeeder)):
        sess.run(hidden3.trainStep,
                 feed_dict = {iLayer.inputLayer: dictFeeder[j]})
    print("Squared Difference: ", sess.run(hidden3.squareDifference,
                                           feed_dict = {iLayer.inputLayer: testCase}))

outputList = []
for i in range(len(inputArray)):
    Utilities.progress(i + 1, len(inputArray), status='Gathering Output')
    outputList.append(sess.run(hidden3.y1, feed_dict = {iLayer.inputLayer: Utilities.numpyReshape(inputArray[i])}))

writeStream = open('results.txt', 'w')
for i in outputList:
    outputString = str(i[0][0])
    count = 0
    for j in range(len(i[0])):
        if count != 0:
            outputString += ", " + str(i[0][j])
        else:
            count = 1
    outputString += "\n"
    writeStream(outputString)
writeStream.close()

# Training the output layer
# for i in range(numEpochs):
    # Utilities.progress(i + 1, numEpochs, status='Training Ouput Layer')
    # logits = Utilities.batchBuilder(inputArray, batchSize)
    # labels = []
    # for i in logits:
        # if i[0] > 0.5:
            # labels.append([1.0, 0.0])
        # else:
            # labels.append([0.0, 1.0])
    # sess.run(oLayer.trainStep, feed_dict = {iLayer.inputLayer: logits, oLayer.labelTensor: labels})

# Gathers the results for analysis
# for i in range(len(inputArray)):
    # Utilities.progress(i + 1, len(inputArray), status='Gathering Output')
    # outputList.append(sess.run(oLayer.yo,
                      # feed_dict = {iLayer.inputLayer: Utilities.numpyReshape(inputArray[i])}))

num1 = 0
num2 = 0

for i in outputList:
    if i[0][0] > i[0][1]:
        num1 += 1
    else:
        num2 += 1

# Output results
print("\nNumber of 1: ", num1)
print("Number of 2: ", num2)

print("\nHidden Layer 1:")
print("Squared Difference: ", sess.run(hidden1.squareDifference,
                                       feed_dict = {iLayer.inputLayer: testCase}))
print("\nHidden Layer 2:")
print("Squared Difference: ", sess.run(hidden2.squareDifference,
                                       feed_dict = {iLayer.inputLayer: testCase}))
print("\nHidden Layer 3:")
print("Squared Difference: ", sess.run(hidden3.squareDifference,
                                       feed_dict = {iLayer.inputLayer: testCase}))
<commit_msg>added write function<commit_after>import sys
import tensorflow as tf
import numpy as np
from src.Utilities import Utilities
from src.Autoencoder import InputLayer
from src.Autoencoder import HiddenLayer
from src.Autoencoder import OutputLayer

# Number of epochs to run
numEpochs = 250
# Size of each batch
batchSize = 10
# Scaling factor for sparsity cost function
alpha = 0.01
# Sparsity parameter
rho = 0.05
# Scaling factor for l2 regularization cost function
beta = 0.01

# Read in data from csv file
Utilities.progress(1, 7, status='Reading in data        ')
inputArray = Utilities.readData()
dictFeeder = Utilities.batchBuilder(inputArray, batchSize)

# Build input layer
Utilities.progress(2, 7, status='Building input layer   ')
with tf.variable_scope("input"):
    iLayer = InputLayer(len(inputArray[1]))

# Build hidden layer 1
Utilities.progress(3, 7, status='Building hidden layer 1')
with tf.variable_scope("hidden1"):
    hidden1 = HiddenLayer(100, iLayer.inputLayer)
    hidden1.buildTrainer(alpha, beta, rho)

# Build hidden layer 2
Utilities.progress(4, 7, status='Building hidden layer 2')
with tf.variable_scope("hidden2"):
    hidden2 = HiddenLayer(50, hidden1.y1)
    hidden2.buildTrainer(alpha, beta, rho)

# Build hidden layer 3
Utilities.progress(5, 7, status='Building hidden layer 3')
with tf.variable_scope("hidden3"):
    hidden3 = HiddenLayer(16, hidden2.y1)
    hidden3.buildTrainer(alpha, beta, rho)

# Build output layer
Utilities.progress(6, 7, status='Building output layer  ')
with tf.variable_scope("output"):
    oLayer = OutputLayer(2, hidden3.y1)
    oLayer.buildTrainer()

Utilities.progress(7, 7, status='Starting session       ')
sess = Utilities.startSession()

# Print the shape of each layer
iLayer.printLayerShape()
hidden1.printLayerShape()
hidden2.printLayerShape()
hidden3.printLayerShape()
oLayer.printLayerShape()

# Training the hidden layers
testCase = Utilities.numpyReshape(inputArray[0])

for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Layer 1 ')
    np.random.shuffle(dictFeeder)
    for j in range(len(dictFeeder)):
        sess.run(hidden1.trainStep,
                 feed_dict = {iLayer.inputLayer: dictFeeder[j]})
    print("Squared Difference: ", sess.run(hidden1.squareDifference,
                                           feed_dict = {iLayer.inputLayer: testCase}))

for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Layer 2 ')
    np.random.shuffle(dictFeeder)
    for j in range(len(dictFeeder)):
        sess.run(hidden2.trainStep,
                 feed_dict = {iLayer.inputLayer: dictFeeder[j]})
    print("Squared Difference: ", sess.run(hidden2.squareDifference,
                                           feed_dict = {iLayer.inputLayer: testCase}))

for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Layer 3 ')
    np.random.shuffle(dictFeeder)
    for j in range(len(dictFeeder)):
        sess.run(hidden3.trainStep,
                 feed_dict = {iLayer.inputLayer: dictFeeder[j]})
    print("Squared Difference: ", sess.run(hidden3.squareDifference,
                                           feed_dict = {iLayer.inputLayer: testCase}))

outputList = []
for i in range(len(inputArray)):
    Utilities.progress(i + 1, len(inputArray), status='Gathering Output')
    outputList.append(sess.run(hidden3.y1, feed_dict = {iLayer.inputLayer: Utilities.numpyReshape(inputArray[i])}))

writeStream = open('results.txt', 'w')
for i in outputList:
    outputString = str(i[0][0])
    count = 0
    for j in range(len(i[0])):
        if count != 0:
            outputString += ", " + str(i[0][j])
        else:
            count = 1
    outputString += "\n"
    writeStream.write(outputString)
writeStream.close()

# Training the output layer
# for i in range(numEpochs):
    # Utilities.progress(i + 1, numEpochs, status='Training Ouput Layer')
    # logits = Utilities.batchBuilder(inputArray, batchSize)
    # labels = []
    # for i in logits:
        # if i[0] > 0.5:
            # labels.append([1.0, 0.0])
        # else:
            # labels.append([0.0, 1.0])
    # sess.run(oLayer.trainStep, feed_dict = {iLayer.inputLayer: logits, oLayer.labelTensor: labels})

# Gathers the results for analysis
# for i in range(len(inputArray)):
    # Utilities.progress(i + 1, len(inputArray), status='Gathering Output')
    # outputList.append(sess.run(oLayer.yo,
                      # feed_dict = {iLayer.inputLayer: Utilities.numpyReshape(inputArray[i])}))

num1 = 0
num2 = 0

for i in outputList:
    if i[0][0] > i[0][1]:
        num1 += 1
    else:
        num2 += 1

# Output results
print("\nNumber of 1: ", num1)
print("Number of 2: ", num2)

print("\nHidden Layer 1:")
print("Squared Difference: ", sess.run(hidden1.squareDifference,
                                       feed_dict = {iLayer.inputLayer: testCase}))
print("\nHidden Layer 2:")
print("Squared Difference: ", sess.run(hidden2.squareDifference,
                                       feed_dict = {iLayer.inputLayer: testCase}))
print("\nHidden Layer 3:")
print("Squared Difference: ", sess.run(hidden3.squareDifference,
                                       feed_dict = {iLayer.inputLayer: testCase}))
<|endoftext|>


<commit_before>import tensorflow as tf
import numpy as np
from src.Utilities import Utilities
from src.Autoencoder import InputLayer
from src.Autoencoder import HiddenLayer
from src.Autoencoder import OutputLayer

# Number of epochs to run
numEpochs = 10000
# Size of each batch
batchSize = 10
# Scaling factor for sparsity cost function
beta = 0.01

# Read in data from csv file
Utilities.progress(1, 7, status='Reading in data        ')
inputArray = Utilities.readData()

# Build input layer
Utilities.progress(2, 7, status='Building input layer   ')
with tf.variable_scope("input"):
    iLayer = InputLayer(len(inputArray[1]))

# Build hidden layer 1
Utilities.progress(3, 7, status='Building hidden layer 1')
with tf.variable_scope("hidden1"):
    hidden1 = HiddenLayer(100, iLayer.inputLayer)
    hidden1.buildTrainer(beta)

# Build hidden layer 2
Utilities.progress(4, 7, status='Building hidden layer 2')
with tf.variable_scope("hidden2"):
    hidden2 = HiddenLayer(50, hidden1.y1)
    hidden2.buildTrainer(beta)

# Build hidden layer 3
Utilities.progress(5, 7, status='Building hidden layer 3')
with tf.variable_scope("hidden3"):
    hidden3 = HiddenLayer(16, hidden2.y1)
    hidden3.buildTrainer(beta)

# Build output layer
Utilities.progress(6, 7, status='Building output layer  ')
with tf.variable_scope("output"):
    oLayer = OutputLayer(2, hidden3.y1)
    oLayer.buildTrainer()

Utilities.progress(7, 7, status='Starting session       ')
sess = Utilities.startSession()

# Print the shape of each layer
iLayer.printLayerShape()
hidden1.printLayerShape()
hidden2.printLayerShape()
hidden3.printLayerShape()
oLayer.printLayerShape()

# Training the hidden layers
for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Layer 1 ')
    sess.run(hidden1.trainStep,
             feed_dict = {iLayer.inputLayer: Utilities.batchBuilder(inputArray, batchSize)})

for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Layer 2 ')
    sess.run(hidden2.trainStep,
             feed_dict = {iLayer.inputLayer: Utilities.batchBuilder(inputArray, batchSize)})

for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Layer 3 ')
    sess.run(hidden3.trainStep,
             feed_dict = {iLayer.inputLayer: Utilities.batchBuilder(inputArray, batchSize)})

# Training the output layer
for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Ouput Layer')
    logits = Utilities.batchBuilder(inputArray, batchSize)
    labels = []
    for i in logits:
        if i[0] > 0.5:
            labels.append([1.0, 0.0])
        else:
            labels.append([0.0, 1.0])
    sess.run(oLayer.trainStep, feed_dict = {iLayer.inputLayer: logits, oLayer.labelTensor: labels})

# Gathers the results for analysis
outputList = []
for i in range(len(inputArray)):
    Utilities.progress(i + 1, len(inputArray), status='Gathering Output')
    outputList.append(sess.run(oLayer.yo,
                      feed_dict = {iLayer.inputLayer: Utilities.numpyReshape(inputArray[i])}))

num1 = 0
num2 = 0

for i in outputList:
    if i[0][0] > i[0][1]:
        num1 += 1
    else:
        num2 += 1

# Output results
print("\nNumber of 1: ", num1)
print("Number of 2: ", num2)

testCase = Utilities.numpyReshape(inputArray[0])

print("\nHidden Layer 1:")
print("Squared Difference: ", sess.run(hidden1.squareDifference,
                                       feed_dict = {iLayer.inputLayer: testCase}))
print("\nHidden Layer 2:")
print("Squared Difference: ", sess.run(hidden2.squareDifference,
                                       feed_dict = {iLayer.inputLayer: testCase}))
print("\nHidden Layer 3:")
print("Squared Difference: ", sess.run(hidden3.squareDifference,
                                       feed_dict = {iLayer.inputLayer: testCase}))
<commit_msg>Outputting results from hidden3.y1<commit_after>import tensorflow as tf
import numpy as np
from src.Utilities import Utilities
from src.Autoencoder import InputLayer
from src.Autoencoder import HiddenLayer
from src.Autoencoder import OutputLayer

# Number of epochs to run
numEpochs = 10000
# Size of each batch
batchSize = 10
# Scaling factor for sparsity cost function
beta = 0.01

# Read in data from csv file
Utilities.progress(1, 7, status='Reading in data        ')
inputArray = Utilities.readData()

# Build input layer
Utilities.progress(2, 7, status='Building input layer   ')
with tf.variable_scope("input"):
    iLayer = InputLayer(len(inputArray[1]))

# Build hidden layer 1
Utilities.progress(3, 7, status='Building hidden layer 1')
with tf.variable_scope("hidden1"):
    hidden1 = HiddenLayer(100, iLayer.inputLayer)
    hidden1.buildTrainer(beta)

# Build hidden layer 2
Utilities.progress(4, 7, status='Building hidden layer 2')
with tf.variable_scope("hidden2"):
    hidden2 = HiddenLayer(50, hidden1.y1)
    hidden2.buildTrainer(beta)

# Build hidden layer 3
Utilities.progress(5, 7, status='Building hidden layer 3')
with tf.variable_scope("hidden3"):
    hidden3 = HiddenLayer(16, hidden2.y1)
    hidden3.buildTrainer(beta)

# Build output layer
Utilities.progress(6, 7, status='Building output layer  ')
with tf.variable_scope("output"):
    oLayer = OutputLayer(2, hidden3.y1)
    oLayer.buildTrainer()

Utilities.progress(7, 7, status='Starting session       ')
sess = Utilities.startSession()

# Print the shape of each layer
iLayer.printLayerShape()
hidden1.printLayerShape()
hidden2.printLayerShape()
hidden3.printLayerShape()
oLayer.printLayerShape()

# Training the hidden layers
for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Layer 1 ')
    sess.run(hidden1.trainStep,
             feed_dict = {iLayer.inputLayer: Utilities.batchBuilder(inputArray, batchSize)})

for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Layer 2 ')
    sess.run(hidden2.trainStep,
             feed_dict = {iLayer.inputLayer: Utilities.batchBuilder(inputArray, batchSize)})

for i in range(numEpochs):
    Utilities.progress(i + 1, numEpochs, status='Training Layer 3 ')
    sess.run(hidden3.trainStep,
             feed_dict = {iLayer.inputLayer: Utilities.batchBuilder(inputArray, batchSize)})

outputList = []
for i in range(len(inputArray)):
    Utilities.progress(i + 1, len(inputArray), status='Gathering Output')
    outputList.append(sess.run(hidden3.y1, feed_dict = {iLayer.inputLayer: Utilities.numpyReshape(inputArray[i])}))

for i in outputList:
    print(i)
    print()

# Training the output layer
# for i in range(numEpochs):
    # Utilities.progress(i + 1, numEpochs, status='Training Ouput Layer')
    # logits = Utilities.batchBuilder(inputArray, batchSize)
    # labels = []
    # for i in logits:
        # if i[0] > 0.5:
            # labels.append([1.0, 0.0])
        # else:
            # labels.append([0.0, 1.0])
    # sess.run(oLayer.trainStep, feed_dict = {iLayer.inputLayer: logits, oLayer.labelTensor: labels})

# Gathers the results for analysis
# for i in range(len(inputArray)):
    # Utilities.progress(i + 1, len(inputArray), status='Gathering Output')
    # outputList.append(sess.run(oLayer.yo,
                      # feed_dict = {iLayer.inputLayer: Utilities.numpyReshape(inputArray[i])}))

num1 = 0
num2 = 0

for i in outputList:
    if i[0][0] > i[0][1]:
        num1 += 1
    else:
        num2 += 1

# Output results
print("\nNumber of 1: ", num1)
print("Number of 2: ", num2)

testCase = Utilities.numpyReshape(inputArray[0])

print("\nHidden Layer 1:")
print("Squared Difference: ", sess.run(hidden1.squareDifference,
                                       feed_dict = {iLayer.inputLayer: testCase}))
print("\nHidden Layer 2:")
print("Squared Difference: ", sess.run(hidden2.squareDifference,
                                       feed_dict = {iLayer.inputLayer: testCase}))
print("\nHidden Layer 3:")
print("Squared Difference: ", sess.run(hidden3.squareDifference,
                                       feed_dict = {iLayer.inputLayer: testCase}))
<|endoftext|>


<commit_before>
#Christina Hammer
#new client Gui

from tkinter import *
from tkinter import messagebox

'''
from sqlalchemy import Column, DateTime, String, Integer, ForeignKey, func
from sqlalchemy.orm import relationship, backref
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from datetime import datetime
from cdbtabledef import Household, Person, Volunteer, Visit'''

def clientinfoconfirm():
    confirm=messagebox.OKCANCEL(title="Confirm Client Entry", message="Confirm the following client information:")
    if confirm==True:
        #newclientfunction
        pass
        return

def cancelentry():
    ncGui.destroy()
    return

#new client GUI
ncGui=Tk()

ncGui.geometry('600x200+400+200')

ncGui.title('Add New Client')

#Label(ncGui, text="Please enter the following information: \n").grid(row=0, column=0)

#entry variables
fname = StringVar()
lname = StringVar()
dob = IntVar()
age= IntVar()
phone=IntVar()
address=StringVar()
apt = StringVar()
cityv = StringVar()
statev = StringVar()
zipcv = StringVar()
doe = IntVar()

#Labels and entries
Label(ncGui, text="First Name").grid(row=1, column=0)
fname = Entry(ncGui).grid(row=2, column=0)

Label(ncGui, text="Last Name \t\t").grid(row=1, column=1)
lname = Entry(ncGui).grid(row=2, column=1)

Label(ncGui, text="Date of Birth").grid(row=1, column=2)
dob=Entry(ncGui).grid(row=2, column=2)

Label(ncGui, text="Age").grid(row=1, column=3)
age = Entry(ncGui).grid(row=2, column=3)

Label(ncGui, text="Phone").grid(row=3, column=0)
phone = Entry(ncGui).grid(row=4, column=0)

Label(ncGui, text="Address").grid(row=3, column=1)
address=Entry(ncGui).grid(row=4, column=1)

Label(ncGui, text="Apt.").grid(row=3, column=2)
apt=Entry(ncGui).grid(row=4, column=2)

Label(ncGui, text="City").grid(row=3, column=3)
city=Entry(ncGui,textvariable=cityv).grid(row=4, column=3)
cityv.set('Troy')

Label(ncGui, text="State").grid(row=5, column=0)
state=Entry(ncGui, textvariable=statev).grid(row=6, column=0)
statev.set('NY')

Label(ncGui, text="Zip").grid(row=5, column=1)
zipc=Entry(ncGui,textvariable=zipcv).grid(row=6, column=1)
zipcv.set('12180')

Label(ncGui, text="Date of Entry").grid(row=5, column=2)
doe=Entry(ncGui).grid(row=6, column=2)

Cancel=Button(ncGui,text='Cancel', command=cancelentry).grid(row=7,column=1)
Confirm=Button(ncGui,text='Add Client', command=clientinfoconfirm).grid(row=7,column=2)

ncGui.mainloop()

<commit_msg>Delete NewclientInter2.py<commit_after><|endoftext|>


<commit_before>import logging

from django.utils.translation import ugettext_lazy as _

import plata
from plata.shop import signals

logger = logging.getLogger('plata.payment')

class ProcessorBase(object):
    """Payment processor base class"""

    #: Safe key for this payment module (shouldn't contain special chars, spaces etc.)
    key = 'unnamed'

    #: Human-readable name for this payment module. You may even use i18n here.
    default_name = 'unnamed'

    def __init__(self, shop):
        self.shop = shop

    @property
    def name(self):
        """
        Return name of this payment module suitable for human consumption

        Defaults to ``default_name`` but can be overridden by placing an entry
        in ``PLATA_PAYMENT_MODULE_NAMES``. Example::

            PLATA_PAYMENT_MODULE_NAMES = {
                'paypal': _('Paypal and credit cards'),
                }
        """
        return plata.settings.PLATA_PAYMENT_MODULE_NAMES.get(
            self.key,
            self.default_name)

    @property
    def urls(self):
        """
        Return URLconf definitions used by this payment processor

        This is especially useful for processors offering server-to-server
        communication such as Paypal's IPN (Instant Payment Notification) where
        Paypal communicates payment success immediately and directly, without
        involving the client.

        Define your own URLs in ``get_urls``.
        """
        return self.get_urls()

    def get_urls(self):
        """
        Define URLs for this payment processor

        Note that these URLs are added directly to the shop views URLconf
        without prefixes. It is your responsability to namespace these URLs
        so they don't clash with shop views and other payment processors.
        """
        from django.conf.urls import patterns
        return patterns('')

    def enabled_for_request(self, request):
        """
        Decides whether or not this payment modules is available for a given request.

        Defaults to ``True``. If you need to disable payment modules for certain
        visitors or group of visitors, that is the method you are searching for.
        """
        return True

    def process_order_confirmed(self, request, order):
        """
        This is the initial entry point of payment modules and is called when
        the user has selected a payment module and accepted the terms and
        conditions of the shop.

        Must return a response which is presented to the user, i.e. a form with
        hidden values forwarding the user to the PSP or a redirect to the success
        page if no further processing is needed.
        """
        raise NotImplementedError

    def clear_pending_payments(self, order):
        """
        Clear pending payments
        """
        logger.info('Clearing pending payments on %s' % order)
        if plata.settings.PLATA_STOCK_TRACKING:
            StockTransaction = plata.stock_model()
            for transaction in order.stock_transactions.filter(
                    type=StockTransaction.PAYMENT_PROCESS_RESERVATION):
                transaction.delete()

        order.payments.pending().delete()

    def create_pending_payment(self, order):
        """
        Create a pending payment
        """
        self.clear_pending_payments(order)
        logger.info('Creating pending payment on %s' % order)
        return order.payments.create(
            currency=order.currency,
            amount=order.balance_remaining,
            payment_module_key=self.key,
            payment_module=u'%s' % self.name,
            )

    def create_transactions(self, order, stage, **kwargs):
        """
        Create transactions for all order items. The real work is offloaded
        to ``StockTransaction.objects.bulk_create``.
        """

        # TODO remove this check now that create_transactions is only called
        # when PLATA_STOCK_TRACKING = True ?
        if not plata.settings.PLATA_STOCK_TRACKING:
            return
        StockTransaction = plata.stock_model()
        StockTransaction.objects.bulk_create(order,
            notes=_('%(stage)s: %(order)s processed by %(payment_module)s') % {
                'stage': stage,
                'order': order,
                'payment_module': self.name,
                },
            **kwargs)

    def order_paid(self, order, payment=None):
        """
        Call this when the order has been fully paid for

        This method does the following:

        - Sets order status to ``PAID``.
        - Calculates the remaining discount amount (if any) and calls the
          ``order_paid`` signal.
        - Clears pending payments which aren't interesting anymore anyway.
        """

        if order.status < order.PAID:
            logger.info('Order %s has been completely paid for using %s' % (
                order, self.name))
            order.update_status(order.PAID, 'Order has been paid')

            signal_kwargs = dict(sender=self, order=order, payment=payment)

            if order.discount_remaining:
                logger.info('Creating discount for remaining amount %s on order %s' % (
                    order.discount_remaining, order))
                discount_model = self.shop.discount_model
                try:
                    discount = order.applied_discounts.filter(
                        type__in=(discount_model.AMOUNT_VOUCHER_EXCL_TAX, discount_model.AMOUNT_VOUCHER_INCL_TAX),
                        ).order_by('type')[0]
                except IndexError:
                    # XXX: Remaining discount will be applicable to ALL products,
                    # not sure if this behavior is correct...
                    discount = None

                signal_kwargs['remaining_discount'] = discount_model.objects.create(
                    name='Remaining discount amount for order #%s' % order.pk,
                    type=self.shop.discount_model.AMOUNT_VOUCHER_EXCL_TAX,
                    value=order.discount_remaining,
                    currency=order.currency,
                    config=getattr(discount, 'config', '{"all": {}}'),
                    allowed_uses=1,
                    )

            signals.order_paid.send(**signal_kwargs)
        self.clear_pending_payments(order)

    def already_paid(self, order):
        """
        Handles the case where a payment module is selected but the order
        is already completely paid for (f.e. because an amount discount has been
        used which covers the order).

        Does nothing if the order **status** is ``PAID`` already.
        """
        if order.status < order.PAID:
            logger.info('Order %s is already completely paid' % order)

            if plata.settings.PLATA_STOCK_TRACKING:
                StockTransaction = plata.stock_model()
                self.create_transactions(order, _('sale'),
                    type=StockTransaction.SALE, negative=True)

            self.order_paid(order)

        return self.shop.redirect('plata_order_success')
<commit_msg>Emit a deprecation warning when create_transactions is called when tracking is inactive<commit_after>import logging
import warnings

from django.utils.translation import ugettext_lazy as _

import plata
from plata.shop import signals

logger = logging.getLogger('plata.payment')

class ProcessorBase(object):
    """Payment processor base class"""

    #: Safe key for this payment module (shouldn't contain special chars, spaces etc.)
    key = 'unnamed'

    #: Human-readable name for this payment module. You may even use i18n here.
    default_name = 'unnamed'

    def __init__(self, shop):
        self.shop = shop

    @property
    def name(self):
        """
        Return name of this payment module suitable for human consumption

        Defaults to ``default_name`` but can be overridden by placing an entry
        in ``PLATA_PAYMENT_MODULE_NAMES``. Example::

            PLATA_PAYMENT_MODULE_NAMES = {
                'paypal': _('Paypal and credit cards'),
                }
        """
        return plata.settings.PLATA_PAYMENT_MODULE_NAMES.get(
            self.key,
            self.default_name)

    @property
    def urls(self):
        """
        Return URLconf definitions used by this payment processor

        This is especially useful for processors offering server-to-server
        communication such as Paypal's IPN (Instant Payment Notification) where
        Paypal communicates payment success immediately and directly, without
        involving the client.

        Define your own URLs in ``get_urls``.
        """
        return self.get_urls()

    def get_urls(self):
        """
        Define URLs for this payment processor

        Note that these URLs are added directly to the shop views URLconf
        without prefixes. It is your responsability to namespace these URLs
        so they don't clash with shop views and other payment processors.
        """
        from django.conf.urls import patterns
        return patterns('')

    def enabled_for_request(self, request):
        """
        Decides whether or not this payment modules is available for a given request.

        Defaults to ``True``. If you need to disable payment modules for certain
        visitors or group of visitors, that is the method you are searching for.
        """
        return True

    def process_order_confirmed(self, request, order):
        """
        This is the initial entry point of payment modules and is called when
        the user has selected a payment module and accepted the terms and
        conditions of the shop.

        Must return a response which is presented to the user, i.e. a form with
        hidden values forwarding the user to the PSP or a redirect to the success
        page if no further processing is needed.
        """
        raise NotImplementedError

    def clear_pending_payments(self, order):
        """
        Clear pending payments
        """
        logger.info('Clearing pending payments on %s' % order)
        if plata.settings.PLATA_STOCK_TRACKING:
            StockTransaction = plata.stock_model()
            for transaction in order.stock_transactions.filter(
                    type=StockTransaction.PAYMENT_PROCESS_RESERVATION):
                transaction.delete()

        order.payments.pending().delete()

    def create_pending_payment(self, order):
        """
        Create a pending payment
        """
        self.clear_pending_payments(order)
        logger.info('Creating pending payment on %s' % order)
        return order.payments.create(
            currency=order.currency,
            amount=order.balance_remaining,
            payment_module_key=self.key,
            payment_module=u'%s' % self.name,
            )

    def create_transactions(self, order, stage, **kwargs):
        """
        Create transactions for all order items. The real work is offloaded
        to ``StockTransaction.objects.bulk_create``.
        """

        if not plata.settings.PLATA_STOCK_TRACKING:
            warnings.warn('StockTransaction.objects.create_transactions'
                ' currently has no effect when PLATA_STOCK_TRACKING = False.'
                ' This will change in the future. Change your code to only'
                ' call create_transactions when'
                ' plata.settings.PLATA_STOCK_TRACKING = True',
                DeprecationWarning, stacklevel=2)
            return
        StockTransaction = plata.stock_model()
        StockTransaction.objects.bulk_create(order,
            notes=_('%(stage)s: %(order)s processed by %(payment_module)s') % {
                'stage': stage,
                'order': order,
                'payment_module': self.name,
                },
            **kwargs)

    def order_paid(self, order, payment=None):
        """
        Call this when the order has been fully paid for

        This method does the following:

        - Sets order status to ``PAID``.
        - Calculates the remaining discount amount (if any) and calls the
          ``order_paid`` signal.
        - Clears pending payments which aren't interesting anymore anyway.
        """

        if order.status < order.PAID:
            logger.info('Order %s has been completely paid for using %s' % (
                order, self.name))
            order.update_status(order.PAID, 'Order has been paid')

            signal_kwargs = dict(sender=self, order=order, payment=payment)

            if order.discount_remaining:
                logger.info('Creating discount for remaining amount %s on order %s' % (
                    order.discount_remaining, order))
                discount_model = self.shop.discount_model
                try:
                    discount = order.applied_discounts.filter(
                        type__in=(discount_model.AMOUNT_VOUCHER_EXCL_TAX, discount_model.AMOUNT_VOUCHER_INCL_TAX),
                        ).order_by('type')[0]
                except IndexError:
                    # XXX: Remaining discount will be applicable to ALL products,
                    # not sure if this behavior is correct...
                    discount = None

                signal_kwargs['remaining_discount'] = discount_model.objects.create(
                    name='Remaining discount amount for order #%s' % order.pk,
                    type=self.shop.discount_model.AMOUNT_VOUCHER_EXCL_TAX,
                    value=order.discount_remaining,
                    currency=order.currency,
                    config=getattr(discount, 'config', '{"all": {}}'),
                    allowed_uses=1,
                    )

            signals.order_paid.send(**signal_kwargs)
        self.clear_pending_payments(order)

    def already_paid(self, order):
        """
        Handles the case where a payment module is selected but the order
        is already completely paid for (f.e. because an amount discount has been
        used which covers the order).

        Does nothing if the order **status** is ``PAID`` already.
        """
        if order.status < order.PAID:
            logger.info('Order %s is already completely paid' % order)

            if plata.settings.PLATA_STOCK_TRACKING:
                StockTransaction = plata.stock_model()
                self.create_transactions(order, _('sale'),
                    type=StockTransaction.SALE, negative=True)

            self.order_paid(order)

        return self.shop.redirect('plata_order_success')
<|endoftext|>