<commit_before>"""
Linux on Hyper-V and Azure Test Code, ver. 1.0.0
Copyright (c) Microsoft Corporation

All rights reserved
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

See the Apache Version 2.0 License for specific language governing
permissions and limitations under the License.
"""
import os
import logging
import ConfigParser

from sqlalchemy import Table, Column, Date, DECIMAL, INT, BIGINT, NVARCHAR, MetaData, create_engine
from sqlalchemy.pool import NullPool
from sqlalchemy.orm import create_session, mapper

logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
                    datefmt='%y/%m/%d %H:%M:%S', level=logging.INFO)
log = logging.getLogger(__name__)

COLUMNS = [{'name': 'TestCaseName', 'type': NVARCHAR(50)},
           {'name': 'DataPath', 'type': NVARCHAR(10)},
           {'name': 'TestDate', 'type': Date},
           {'name': 'HostBy', 'type': NVARCHAR(50)},
           {'name': 'HostOS', 'type': NVARCHAR(100)},
           {'name': 'HostType', 'type': NVARCHAR(50)},
           {'name': 'InstanceSize', 'type': NVARCHAR(20)},
           {'name': 'GuestOS', 'type': NVARCHAR(50)},
           {'name': 'GuestSize', 'type': NVARCHAR(50)},
           {'name': 'GuestOSType', 'type': NVARCHAR(50)},
           {'name': 'GuestDistro', 'type': NVARCHAR(50)},
           {'name': 'KernelVersion', 'type': NVARCHAR(50)},
           {'name': 'DiskSetup', 'type': NVARCHAR(20)},
           {'name': 'TestMode', 'type': NVARCHAR(30)},
           {'name': 'FileTestMode', 'type': NVARCHAR(20)},
           {'name': 'WebServerVersion', 'type': NVARCHAR(20)},
           {'name': 'Driver', 'type': NVARCHAR(10)},
           {'name': 'IPVersion', 'type': NVARCHAR(4)},
           {'name': 'ProtocolType', 'type': NVARCHAR(3)},
           {'name': 'ClusterSetup', 'type': NVARCHAR(25)},
           {'name': 'HadoopVersion', 'type': NVARCHAR(12)},
           {'name': 'Threads', 'type': DECIMAL(4, 0)},
           {'name': 'BufferSize_Bytes', 'type': DECIMAL(5, 0)},
           {'name': 'TestConnections', 'type': DECIMAL(4, 0)},
           {'name': 'NumberOfConnections', 'type': INT},
           {'name': 'TestPipelines', 'type': DECIMAL(4, 0)},
           {'name': 'TestConcurrency', 'type': DECIMAL(4, 0)},
           {'name': 'NodeSize_bytes', 'type': DECIMAL(4, 0)},
           {'name': 'TeragenRecords', 'type': DECIMAL(12, 0)},
           {'name': 'SortDuration_sec', 'type': DECIMAL(10, 4)},
           {'name': 'TotalCreatedCallsPerSec', 'type': DECIMAL(10, 4)},
           {'name': 'TotalSetCallsPerSec', 'type': DECIMAL(10, 4)},
           {'name': 'TotalGetCallsPerSec', 'type': DECIMAL(10, 4)},
           {'name': 'TotalDeletedCallsPerSec', 'type': DECIMAL(10, 4)},
           {'name': 'TotalWatchedCallsPerSec', 'type': DECIMAL(10, 4)},
           {'name': 'TotalQueries', 'type': DECIMAL(9, 0)},
           {'name': 'TransactionsPerSec', 'type': DECIMAL(8, 3)},
           {'name': 'DeadlocksPerSec', 'type': DECIMAL(8, 3)},
           {'name': 'RWRequestsPerSec', 'type': DECIMAL(10, 3)},
           {'name': 'NumberOfAbInstances', 'type': DECIMAL(3, 0)},
           {'name': 'ConcurrencyPerAbInstance', 'type': DECIMAL(3, 0)},
           {'name': 'Document_bytes', 'type': DECIMAL(7, 0)},
           {'name': 'CompleteRequests', 'type': DECIMAL(7, 0)},
           {'name': 'RequestsPerSec', 'type': DECIMAL(8, 3)},
           {'name': 'TransferRate_KBps', 'type': DECIMAL(10, 3)},
           {'name': 'MeanConnectionTimes_ms', 'type': DECIMAL(8, 4)},
           {'name': 'TotalRequests', 'type': DECIMAL(9, 0)},
           {'name': 'ParallelClients', 'type': DECIMAL(5, 0)},
           {'name': 'Payload_bytes', 'type': DECIMAL(5, 0)},
           {'name': 'ConnectionsPerThread', 'type': DECIMAL(8, 0)},
           {'name': 'RequestsPerThread', 'type': DECIMAL(8, 0)},
           {'name': 'BestLatency_ms', 'type': DECIMAL(7, 4)},
           {'name': 'WorstLatency_ms', 'type': DECIMAL(7, 4)},
           {'name': 'AverageLatency_ms', 'type': DECIMAL(7, 4)},
           {'name': 'BestOpsPerSec', 'type': DECIMAL(10, 3)},
           {'name': 'WorstOpsPerSec', 'type': DECIMAL(10, 3)},
           {'name': 'AverageOpsPerSec', 'type': DECIMAL(10, 3)},
           {'name': 'SETRequestsPerSec', 'type': DECIMAL(10, 3)},
           {'name': 'GETRequestsPerSec', 'type': DECIMAL(10, 3)},
           {'name': 'NumOutstandingSmall_IO', 'type': DECIMAL(3, 0)},
           {'name': 'NumOutstandingLarge_IO', 'type': DECIMAL(3, 0)},
           {'name': 'BlockSize_Kb', 'type': DECIMAL(3, 0)},
           {'name': 'BlockSize_KB', 'type': INT},
           {'name': 'QDepth', 'type': INT},
           {'name': 'Throughput_MBps', 'type': DECIMAL(10, 2)},
           {'name': 'Latency95Percentile_ms', 'type': DECIMAL(10, 2)},
           {'name': 'RequestsExecutedPerSec', 'type': DECIMAL(10, 2)},
           {'name': 'Latency_ms', 'type': DECIMAL(7, 3)},
           {'name': 'IOPS', 'type': DECIMAL(10, 2)},
           {'name': 'TotalOpsPerSec', 'type': DECIMAL(10, 4)},
           {'name': 'ReadOps', 'type': DECIMAL(9, 1)},
           {'name': 'ReadLatency95Percentile_us', 'type': DECIMAL(6, 0)},
           {'name': 'CleanupOps', 'type': DECIMAL(9, 1)},
           {'name': 'CleanupLatency95Percentile_us', 'type': DECIMAL(6, 0)},
           {'name': 'UpdateOps', 'type': DECIMAL(9, 1)},
           {'name': 'UpdateLatency95Percentile_us', 'type': DECIMAL(6, 0)},
           {'name': 'ReadFailedOps', 'type': DECIMAL(9, 1)},
           {'name': 'ReadFailedLatency95Percentile_us', 'type': DECIMAL(6, 0)},
           {'name': 'Throughput_Gbps', 'type': DECIMAL(5, 3)},
           {'name': 'Latency_ms', 'type': DECIMAL(9, 3)},
           {'name': 'PacketSize_KBytes', 'type': DECIMAL(5, 3)},
           {'name': 'MaxLatency_us', 'type': DECIMAL(9, 3)},
           {'name': 'AverageLatency_us', 'type': DECIMAL(9, 3)},
           {'name': 'MinLatency_us', 'type': DECIMAL(9, 3)},
           {'name': 'Latency95Percentile_us', 'type': DECIMAL(9, 3)},
           {'name': 'Latency99Percentile_us', 'type': DECIMAL(9, 3)},
           {'name': 'RxThroughput_Gbps', 'type': DECIMAL(5, 3)},
           {'name': 'TxThroughput_Gbps', 'type': DECIMAL(5, 3)},
           {'name': 'RetransmittedSegments', 'type': DECIMAL(6, 0)},
           {'name': 'CongestionWindowSize_KB', 'type': DECIMAL(6, 0)},
           {'name': 'SendBufSize_KBytes', 'type': DECIMAL(9, 0)},
           {'name': 'DatagramLoss', 'type': DECIMAL(5, 3)},
           {'name': 'seq_read_iops', 'type': DECIMAL(8, 1)},
           {'name': 'seq_read_lat_usec', 'type': DECIMAL(10, 2)},
           {'name': 'rand_read_iops', 'type': DECIMAL(8, 1)},
           {'name': 'rand_read_lat_usec', 'type': DECIMAL(10, 2)},
           {'name': 'seq_write_iops', 'type': DECIMAL(8, 1)},
           {'name': 'seq_write_lat_usec', 'type': DECIMAL(10, 2)},
           {'name': 'rand_write_iops', 'type': DECIMAL(8, 1)},
           {'name': 'rand_write_lat_usec', 'type': DECIMAL(10, 2)},
           {'name': 'ScaleFactor', 'type': DECIMAL(4, 0)},
           {'name': 'SQLServerVersion', 'type': NVARCHAR(20)},
           {'name': 'ResponseTime95thPercentile', 'type': DECIMAL(8, 3)},
           {'name': 'TxnPerSec', 'type': DECIMAL(10, 2)},
           {'name': 'PostgreSQLVersion', 'type': NVARCHAR(20)},
           {'name': 'TransactionType', 'type': NVARCHAR(50)},
           {'name': 'TestClients', 'type': DECIMAL(3, 0)},
           {'name': 'ScalingFactor', 'type': DECIMAL(5, 0)},
           {'name': 'TestDuration_s', 'type': DECIMAL(5, 0)},
           {'name': 'TransactionsPerSecIncEstablishing', 'type': DECIMAL(8, 3)},
           {'name': 'TransactionsPerSecExcEstablishing', 'type': DECIMAL(8, 3)},
           ]

class TestResults(object):
    def __getitem__(self, item):
        return getattr(self, item)

    def __setitem__(self, item, value):
        return setattr(self, item, value)

# def upload_results(localpath=None, table_name=None, test_results=None):
def upload_results(localpath=None, table_name=None, results_path=None, parser=None, **kwargs):
    """
    Connect to DB and upload results
    """
    if not test_results:
        log.error('Results missing. Skipping DB upload.')
        return None
    if localpath:
        log.info('Looking up DB credentials for results upload in {}.' .format(localpath))
        db_creds_file = [os.path.join(localpath, c) for c in os.listdir(localpath)
                         if c.endswith('.config')][0]
        # read credentials from file - should be present in the localpath provided to runner
        config = ConfigParser.ConfigParser()
        config.read(db_creds_file)
    else:
        log.error('No credentials file path provided. Skipping results upload.')
        return None

    test_results = parser(log_path=results_path, **kwargs).process_logs()

    import pprint
    pprint.pprint(test_results)

    e = create_engine('mssql+pyodbc://{}:{}@{}/{}?driver={}'.format(
            config.get('Credentials', 'User'), config.get('Credentials', 'Password'),
            config.get('Credentials', 'Server'), config.get('Credentials', 'Database'),
            config.get('Credentials', 'Driver'), poolclass=NullPool, echo=True))
    metadata = MetaData(bind=e)

    table_columns = [column for column in COLUMNS if column['name'] in test_results[0]]
    t = Table(table_name, metadata,
              Column('TestId', BIGINT, primary_key=True, nullable=False, index=True),
              *(Column(column['name'], column['type']) for column in table_columns))

    # When creating db is also necessary
    # metadata.create_all(checkfirst=True)

    mapper(TestResults, t)
    session = create_session(bind=e, autocommit=False, autoflush=True)

    for row in test_results:
        test_data = TestResults()
        for key, value in row.items():
            test_data[key] = value
        session.add(test_data)
        try:
            session.commit()
        except:
            print("Failed to commit {} data. Rolling back.".format(row))
            session.rollback()
            raise
<commit_msg>Fix db parsing changes<commit_after>"""
Linux on Hyper-V and Azure Test Code, ver. 1.0.0
Copyright (c) Microsoft Corporation

All rights reserved
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

See the Apache Version 2.0 License for specific language governing
permissions and limitations under the License.
"""
import os
import logging
import ConfigParser

from sqlalchemy import Table, Column, Date, DECIMAL, INT, BIGINT, NVARCHAR, MetaData, create_engine
from sqlalchemy.pool import NullPool
from sqlalchemy.orm import create_session, mapper

logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
                    datefmt='%y/%m/%d %H:%M:%S', level=logging.INFO)
log = logging.getLogger(__name__)

COLUMNS = [{'name': 'TestCaseName', 'type': NVARCHAR(50)},
           {'name': 'DataPath', 'type': NVARCHAR(10)},
           {'name': 'TestDate', 'type': Date},
           {'name': 'HostBy', 'type': NVARCHAR(50)},
           {'name': 'HostOS', 'type': NVARCHAR(100)},
           {'name': 'HostType', 'type': NVARCHAR(50)},
           {'name': 'InstanceSize', 'type': NVARCHAR(20)},
           {'name': 'GuestOS', 'type': NVARCHAR(50)},
           {'name': 'GuestSize', 'type': NVARCHAR(50)},
           {'name': 'GuestOSType', 'type': NVARCHAR(50)},
           {'name': 'GuestDistro', 'type': NVARCHAR(50)},
           {'name': 'KernelVersion', 'type': NVARCHAR(50)},
           {'name': 'DiskSetup', 'type': NVARCHAR(20)},
           {'name': 'TestMode', 'type': NVARCHAR(30)},
           {'name': 'FileTestMode', 'type': NVARCHAR(20)},
           {'name': 'WebServerVersion', 'type': NVARCHAR(20)},
           {'name': 'Driver', 'type': NVARCHAR(10)},
           {'name': 'IPVersion', 'type': NVARCHAR(4)},
           {'name': 'ProtocolType', 'type': NVARCHAR(3)},
           {'name': 'ClusterSetup', 'type': NVARCHAR(25)},
           {'name': 'HadoopVersion', 'type': NVARCHAR(12)},
           {'name': 'Threads', 'type': DECIMAL(4, 0)},
           {'name': 'BufferSize_Bytes', 'type': DECIMAL(5, 0)},
           {'name': 'TestConnections', 'type': DECIMAL(4, 0)},
           {'name': 'NumberOfConnections', 'type': INT},
           {'name': 'TestPipelines', 'type': DECIMAL(4, 0)},
           {'name': 'TestConcurrency', 'type': DECIMAL(4, 0)},
           {'name': 'NodeSize_bytes', 'type': DECIMAL(4, 0)},
           {'name': 'TeragenRecords', 'type': DECIMAL(12, 0)},
           {'name': 'SortDuration_sec', 'type': DECIMAL(10, 4)},
           {'name': 'TotalCreatedCallsPerSec', 'type': DECIMAL(10, 4)},
           {'name': 'TotalSetCallsPerSec', 'type': DECIMAL(10, 4)},
           {'name': 'TotalGetCallsPerSec', 'type': DECIMAL(10, 4)},
           {'name': 'TotalDeletedCallsPerSec', 'type': DECIMAL(10, 4)},
           {'name': 'TotalWatchedCallsPerSec', 'type': DECIMAL(10, 4)},
           {'name': 'TotalQueries', 'type': DECIMAL(9, 0)},
           {'name': 'TransactionsPerSec', 'type': DECIMAL(8, 3)},
           {'name': 'DeadlocksPerSec', 'type': DECIMAL(8, 3)},
           {'name': 'RWRequestsPerSec', 'type': DECIMAL(10, 3)},
           {'name': 'NumberOfAbInstances', 'type': DECIMAL(3, 0)},
           {'name': 'ConcurrencyPerAbInstance', 'type': DECIMAL(3, 0)},
           {'name': 'Document_bytes', 'type': DECIMAL(7, 0)},
           {'name': 'CompleteRequests', 'type': DECIMAL(7, 0)},
           {'name': 'RequestsPerSec', 'type': DECIMAL(8, 3)},
           {'name': 'TransferRate_KBps', 'type': DECIMAL(10, 3)},
           {'name': 'MeanConnectionTimes_ms', 'type': DECIMAL(8, 4)},
           {'name': 'TotalRequests', 'type': DECIMAL(9, 0)},
           {'name': 'ParallelClients', 'type': DECIMAL(5, 0)},
           {'name': 'Payload_bytes', 'type': DECIMAL(5, 0)},
           {'name': 'ConnectionsPerThread', 'type': DECIMAL(8, 0)},
           {'name': 'RequestsPerThread', 'type': DECIMAL(8, 0)},
           {'name': 'BestLatency_ms', 'type': DECIMAL(7, 4)},
           {'name': 'WorstLatency_ms', 'type': DECIMAL(7, 4)},
           {'name': 'AverageLatency_ms', 'type': DECIMAL(7, 4)},
           {'name': 'BestOpsPerSec', 'type': DECIMAL(10, 3)},
           {'name': 'WorstOpsPerSec', 'type': DECIMAL(10, 3)},
           {'name': 'AverageOpsPerSec', 'type': DECIMAL(10, 3)},
           {'name': 'SETRequestsPerSec', 'type': DECIMAL(10, 3)},
           {'name': 'GETRequestsPerSec', 'type': DECIMAL(10, 3)},
           {'name': 'NumOutstandingSmall_IO', 'type': DECIMAL(3, 0)},
           {'name': 'NumOutstandingLarge_IO', 'type': DECIMAL(3, 0)},
           {'name': 'BlockSize_Kb', 'type': DECIMAL(3, 0)},
           {'name': 'BlockSize_KB', 'type': INT},
           {'name': 'QDepth', 'type': INT},
           {'name': 'Throughput_MBps', 'type': DECIMAL(10, 2)},
           {'name': 'Latency95Percentile_ms', 'type': DECIMAL(10, 2)},
           {'name': 'RequestsExecutedPerSec', 'type': DECIMAL(10, 2)},
           {'name': 'Latency_ms', 'type': DECIMAL(7, 3)},
           {'name': 'IOPS', 'type': DECIMAL(10, 2)},
           {'name': 'TotalOpsPerSec', 'type': DECIMAL(10, 4)},
           {'name': 'ReadOps', 'type': DECIMAL(9, 1)},
           {'name': 'ReadLatency95Percentile_us', 'type': DECIMAL(6, 0)},
           {'name': 'CleanupOps', 'type': DECIMAL(9, 1)},
           {'name': 'CleanupLatency95Percentile_us', 'type': DECIMAL(6, 0)},
           {'name': 'UpdateOps', 'type': DECIMAL(9, 1)},
           {'name': 'UpdateLatency95Percentile_us', 'type': DECIMAL(6, 0)},
           {'name': 'ReadFailedOps', 'type': DECIMAL(9, 1)},
           {'name': 'ReadFailedLatency95Percentile_us', 'type': DECIMAL(6, 0)},
           {'name': 'Throughput_Gbps', 'type': DECIMAL(5, 3)},
           {'name': 'Latency_ms', 'type': DECIMAL(9, 3)},
           {'name': 'PacketSize_KBytes', 'type': DECIMAL(5, 3)},
           {'name': 'MaxLatency_us', 'type': DECIMAL(9, 3)},
           {'name': 'AverageLatency_us', 'type': DECIMAL(9, 3)},
           {'name': 'MinLatency_us', 'type': DECIMAL(9, 3)},
           {'name': 'Latency95Percentile_us', 'type': DECIMAL(9, 3)},
           {'name': 'Latency99Percentile_us', 'type': DECIMAL(9, 3)},
           {'name': 'RxThroughput_Gbps', 'type': DECIMAL(5, 3)},
           {'name': 'TxThroughput_Gbps', 'type': DECIMAL(5, 3)},
           {'name': 'RetransmittedSegments', 'type': DECIMAL(6, 0)},
           {'name': 'CongestionWindowSize_KB', 'type': DECIMAL(6, 0)},
           {'name': 'SendBufSize_KBytes', 'type': DECIMAL(9, 0)},
           {'name': 'DatagramLoss', 'type': DECIMAL(5, 3)},
           {'name': 'seq_read_iops', 'type': DECIMAL(8, 1)},
           {'name': 'seq_read_lat_usec', 'type': DECIMAL(10, 2)},
           {'name': 'rand_read_iops', 'type': DECIMAL(8, 1)},
           {'name': 'rand_read_lat_usec', 'type': DECIMAL(10, 2)},
           {'name': 'seq_write_iops', 'type': DECIMAL(8, 1)},
           {'name': 'seq_write_lat_usec', 'type': DECIMAL(10, 2)},
           {'name': 'rand_write_iops', 'type': DECIMAL(8, 1)},
           {'name': 'rand_write_lat_usec', 'type': DECIMAL(10, 2)},
           {'name': 'ScaleFactor', 'type': DECIMAL(4, 0)},
           {'name': 'SQLServerVersion', 'type': NVARCHAR(20)},
           {'name': 'ResponseTime95thPercentile', 'type': DECIMAL(8, 3)},
           {'name': 'TxnPerSec', 'type': DECIMAL(10, 2)},
           {'name': 'PostgreSQLVersion', 'type': NVARCHAR(20)},
           {'name': 'TransactionType', 'type': NVARCHAR(50)},
           {'name': 'TestClients', 'type': DECIMAL(3, 0)},
           {'name': 'ScalingFactor', 'type': DECIMAL(5, 0)},
           {'name': 'TestDuration_s', 'type': DECIMAL(5, 0)},
           {'name': 'TransactionsPerSecIncEstablishing', 'type': DECIMAL(8, 3)},
           {'name': 'TransactionsPerSecExcEstablishing', 'type': DECIMAL(8, 3)},
           ]

class TestResults(object):
    def __getitem__(self, item):
        return getattr(self, item)

    def __setitem__(self, item, value):
        return setattr(self, item, value)

# def upload_results(localpath=None, table_name=None, test_results=None):
def upload_results(localpath=None, table_name=None, results_path=None, parser=None, **kwargs):
    """
    Connect to DB and upload results
    """
    # if not test_results:
    #     log.error('Results missing. Skipping DB upload.')
    #     return None
    if localpath:
        log.info('Looking up DB credentials for results upload in {}.' .format(localpath))
        db_creds_file = [os.path.join(localpath, c) for c in os.listdir(localpath)
                         if c.endswith('.config')][0]
        # read credentials from file - should be present in the localpath provided to runner
        config = ConfigParser.ConfigParser()
        config.read(db_creds_file)
    else:
        log.error('No credentials file path provided. Skipping results upload.')
        return None

    test_results = parser(log_path=results_path, **kwargs).process_logs()

    import pprint
    pprint.pprint(test_results)

    e = create_engine('mssql+pyodbc://{}:{}@{}/{}?driver={}'.format(
            config.get('Credentials', 'User'), config.get('Credentials', 'Password'),
            config.get('Credentials', 'Server'), config.get('Credentials', 'Database'),
            config.get('Credentials', 'Driver'), poolclass=NullPool, echo=True))
    metadata = MetaData(bind=e)

    table_columns = [column for column in COLUMNS if column['name'] in test_results[0]]
    t = Table(table_name, metadata,
              Column('TestId', BIGINT, primary_key=True, nullable=False, index=True),
              *(Column(column['name'], column['type']) for column in table_columns))

    # When creating db is also necessary
    # metadata.create_all(checkfirst=True)

    mapper(TestResults, t)
    session = create_session(bind=e, autocommit=False, autoflush=True)

    for row in test_results:
        test_data = TestResults()
        for key, value in row.items():
            test_data[key] = value
        session.add(test_data)
        try:
            session.commit()
        except:
            print("Failed to commit {} data. Rolling back.".format(row))
            session.rollback()
            raise
<|endoftext|>


<commit_before>
from django.core.management.base import BaseCommand

from corehq.apps.es import CaseES, FormES
from corehq.motech.repeaters.dbaccessors import (
    get_domains_that_have_repeat_records,
    get_repeat_records_by_payload_id,
    get_repeaters_by_domain,
)
from corehq.motech.repeaters.models import FormRepeater, ShortFormRepeater, CaseRepeater, CreateCaseRepeater, \
    UpdateCaseRepeater
from corehq.util.argparse_types import date_type

from dimagi.utils.parsing import string_to_utc_datetime

REPEATERS_WITH_FORM_PAYLOADS = (
    FormRepeater,
    ShortFormRepeater,
)

REPEATERS_WITH_CASE_PAYLOADS = (
    CaseRepeater,
)

def obtain_missing_form_repeat_records(startdate,
                                       enddate,
                                       domains,
                                       should_create=False):
    """
    :param startdate: filter out forms with a server_modified_on prior to this date
    :param enddate: filter out forms with a server_modified_on after this date
    :param domains: list of domains to check
    :param should_create: if  True, missing repeat records that are discovered will be registered with the repeater
    :return: a dictionary containing stats about number of missing records, impacted form_ids, percentage missing
    """
    stats_per_domain = {}
    for domain in domains:
        total_missing_count = 0
        total_count = 0
        form_repeaters_in_domain = get_form_repeaters_in_domain(domain)

        for form in get_forms_in_domain_between_dates(domain, startdate, enddate):
            # results returned from scroll() do not include '_id'
            missing_count, successful_count = obtain_missing_form_repeat_records_in_domain(
                domain, form_repeaters_in_domain, form, should_create
            )
            total_missing_count += missing_count
            total_count += missing_count + successful_count

        if total_missing_count > 0:
            stats_per_domain[domain] = {
                'forms': {
                    'missing_count': total_missing_count,
                    'percentage_missing': f'{round((total_missing_count / total_count) * 100, 2)}%',
                }
            }

    return stats_per_domain

def obtain_missing_form_repeat_records_in_domain(domain, repeaters, form, should_create):
    if form.is_duplicate:
        return 0, 0

    missing_count = 0
    successful_count = 0
    repeat_records = get_repeat_records_by_payload_id(domain, form.get_id)
    triggered_repeater_ids = [record.repeater_id for record in repeat_records]
    for repeater in repeaters:
        if not repeater.allowed_to_forward(form):
            continue

        if repeater.get_id in triggered_repeater_ids:
            successful_count += 1
        else:
            missing_count += 1
            if should_create:
                # will attempt to send now
                repeater.register(form)

    return missing_count, successful_count

def get_forms_in_domain_between_dates(domain, startdate, enddate):
    return FormES().domain(domain).date_range('server_modified_on', gte=startdate, lte=enddate).run().hits

def obtain_missing_case_repeat_records(startdate, domains):
    stats_per_domain = {}
    for domain in domains:
        total_missing_count = 0
        total_count = 0
        case_repeaters_in_domain = get_case_repeaters_in_domain(domain)

        for case in get_cases_in_domain_since_date(domain, startdate):
            missing_count, successful_count = obtain_missing_case_repeat_records_in_domain(
                domain, case_repeaters_in_domain, case
            )
            total_missing_count += missing_count
            total_count += missing_count + successful_count

        if total_missing_count > 0:
            stats_per_domain[domain] = {
                'cases': {
                    'missing_count': total_missing_count,
                    'percentage_missing': f'{round((total_missing_count / total_count) * 100, 2)}%',
                }
            }

def obtain_missing_case_repeat_records_in_domain(domain, repeaters, case):
    successful_count = 0
    missing_count = 0

    repeat_records = get_repeat_records_by_payload_id(domain, case.get_id)
    # triggered_repeater_ids = [record.repeater_id for record in repeat_records]
    triggered_repeater_ids_and_counts = {}
    for record in repeat_records:
        current_count = triggered_repeater_ids_and_counts.get(record.repeater_id, 0)
        if current_count > 0:
            triggered_repeater_ids_and_counts[record.repeater_id] += 1
        else:
            triggered_repeater_ids_and_counts[record.repeater_id] = 1

    transactions = case['transactions']

    for repeater in repeaters:
        expected_record_count = number_of_repeat_records_triggered_by_case(transactions, repeater)
        actual_record_count = triggered_repeater_ids_and_counts[repeater.get_id]

        # worry about specifying create vs update vs normal later
        temp_missing_count = expected_record_count - actual_record_count
        missing_count += temp_missing_count if temp_missing_count > 0 else 0
        successful_count += actual_record_count

    return missing_count, successful_count

def number_of_repeat_records_triggered_by_case(transactions, repeater):
    if isinstance(repeater, CreateCaseRepeater):
        filtered_transactions = transactions[0:1]
    elif isinstance(repeater, UpdateCaseRepeater):
        filtered_transactions = transactions[1:]
    else:
        filtered_transactions = transactions

    return len(filtered_transactions)

def get_transaction_date(transaction):
    return string_to_utc_datetime(transaction['server_date']).date()

def get_cases_in_domain_since_date(domain, startdate):
    """
    Can only search for cases modified since a date
    """
    return CaseES().domain(domain).server_modified_range(gte=startdate).run().hits

def get_form_repeaters_in_domain(domain):
    return [repeater for repeater in get_repeaters_by_domain(domain)
            if isinstance(repeater, REPEATERS_WITH_FORM_PAYLOADS)]

def get_case_repeaters_in_domain(domain):
    return [repeater for repeater in get_repeaters_by_domain(domain)
            if isinstance(repeater, REPEATERS_WITH_CASE_PAYLOADS)]

class Command(BaseCommand):
    help = """
    Find case/form submissions/updates that do not have a corresponding repeat record and create a repeat record
    """

    def add_arguments(self, parser):
        parser.add_argument('-s', '--startdate', default="2021-06-19", type=date_type, help='Format YYYY-MM-DD')
        parser.add_argument('-e', '--enddate', default="2021-06-23", type=date_type, help='Format YYYY-MM-DD')
        parser.add_argument('-d', '--domain', default=None, type=str, help='Run on a specific domain')
        parser.add_argument('-c', '--create', action='store_true', help='Create missing repeat records')

    def handle(self, startdate, enddate, domain, create, **options):
        if domain:
            domains_with_repeaters = [domain]
        else:
            domains_with_repeaters = get_domains_that_have_repeat_records()

        _ = obtain_missing_form_repeat_records(startdate, enddate, domains_with_repeaters, create)
<commit_msg>missed a few things<commit_after>
from django.core.management.base import BaseCommand

from corehq.apps.es import CaseES, FormES
from corehq.motech.repeaters.dbaccessors import (
    get_domains_that_have_repeat_records,
    get_repeat_records_by_payload_id,
    get_repeaters_by_domain,
)
from corehq.motech.repeaters.models import FormRepeater, ShortFormRepeater, CaseRepeater, CreateCaseRepeater, \
    UpdateCaseRepeater
from corehq.util.argparse_types import date_type

from dimagi.utils.parsing import string_to_utc_datetime

REPEATERS_WITH_FORM_PAYLOADS = (
    FormRepeater,
    ShortFormRepeater,
)

REPEATERS_WITH_CASE_PAYLOADS = (
    CaseRepeater,
)

def obtain_missing_form_repeat_records(startdate,
                                       enddate,
                                       domains,
                                       should_create=False):
    """
    :param startdate: filter out forms with a server_modified_on prior to this date
    :param enddate: filter out forms with a server_modified_on after this date
    :param domains: list of domains to check
    :param should_create: if  True, missing repeat records that are discovered will be registered with the repeater
    :return: a dictionary containing stats about number of missing records, impacted form_ids, percentage missing
    """
    stats_per_domain = {}
    for domain in domains:
        total_missing_count = 0
        total_count = 0
        form_repeaters_in_domain = get_form_repeaters_in_domain(domain)

        for form in get_forms_in_domain_between_dates(domain, startdate, enddate):
            # results returned from scroll() do not include '_id'
            missing_count, successful_count = obtain_missing_form_repeat_records_in_domain(
                domain, form_repeaters_in_domain, form, should_create
            )
            total_missing_count += missing_count
            total_count += missing_count + successful_count

        if total_missing_count > 0:
            stats_per_domain[domain] = {
                'forms': {
                    'missing_count': total_missing_count,
                    'percentage_missing': f'{round((total_missing_count / total_count) * 100, 2)}%',
                }
            }

    return stats_per_domain

def obtain_missing_form_repeat_records_in_domain(domain, repeaters, form, should_create):
    # hack for now
    if form['state'] == 8:
        return 0, 0

    missing_count = 0
    successful_count = 0
    repeat_records = get_repeat_records_by_payload_id(domain, form.get_id)
    triggered_repeater_ids = [record.repeater_id for record in repeat_records]
    for repeater in repeaters:
        if not repeater.allowed_to_forward(form):
            continue

        if repeater.get_id in triggered_repeater_ids:
            successful_count += 1
        else:
            missing_count += 1
            if should_create:
                # will attempt to send now
                repeater.register(form)

    return missing_count, successful_count

def get_forms_in_domain_between_dates(domain, startdate, enddate):
    return FormES().domain(domain).date_range('server_modified_on', gte=startdate, lte=enddate).run().hits

def obtain_missing_case_repeat_records(startdate, domains):
    stats_per_domain = {}
    for domain in domains:
        total_missing_count = 0
        total_count = 0
        case_repeaters_in_domain = get_case_repeaters_in_domain(domain)

        for case in get_cases_in_domain_since_date(domain, startdate):
            missing_count, successful_count = obtain_missing_case_repeat_records_in_domain(
                domain, case_repeaters_in_domain, case
            )
            total_missing_count += missing_count
            total_count += missing_count + successful_count

        if total_missing_count > 0:
            stats_per_domain[domain] = {
                'cases': {
                    'missing_count': total_missing_count,
                    'percentage_missing': f'{round((total_missing_count / total_count) * 100, 2)}%',
                }
            }

    return stats_per_domain

def obtain_missing_case_repeat_records_in_domain(domain, repeaters, case):
    successful_count = 0
    missing_count = 0

    repeat_records = get_repeat_records_by_payload_id(domain, case.get_id)
    # triggered_repeater_ids = [record.repeater_id for record in repeat_records]
    triggered_repeater_ids_and_counts = {}
    for record in repeat_records:
        current_count = triggered_repeater_ids_and_counts.get(record.repeater_id, 0)
        if current_count > 0:
            triggered_repeater_ids_and_counts[record.repeater_id] += 1
        else:
            triggered_repeater_ids_and_counts[record.repeater_id] = 1

    transactions = case['transactions']

    for repeater in repeaters:
        expected_record_count = number_of_repeat_records_triggered_by_case(transactions, repeater)
        actual_record_count = triggered_repeater_ids_and_counts[repeater.get_id]

        # worry about specifying create vs update vs normal later
        temp_missing_count = expected_record_count - actual_record_count
        missing_count += temp_missing_count if temp_missing_count > 0 else 0
        successful_count += actual_record_count

    return missing_count, successful_count

def number_of_repeat_records_triggered_by_case(transactions, repeater):
    if isinstance(repeater, CreateCaseRepeater):
        filtered_transactions = transactions[0:1]
    elif isinstance(repeater, UpdateCaseRepeater):
        filtered_transactions = transactions[1:]
    else:
        filtered_transactions = transactions

    return len(filtered_transactions)

def get_transaction_date(transaction):
    return string_to_utc_datetime(transaction['server_date']).date()

def get_cases_in_domain_since_date(domain, startdate):
    """
    Can only search for cases modified since a date
    """
    return CaseES().domain(domain).server_modified_range(gte=startdate).run().hits

def get_form_repeaters_in_domain(domain):
    return [repeater for repeater in get_repeaters_by_domain(domain)
            if isinstance(repeater, REPEATERS_WITH_FORM_PAYLOADS)]

def get_case_repeaters_in_domain(domain):
    return [repeater for repeater in get_repeaters_by_domain(domain)
            if isinstance(repeater, REPEATERS_WITH_CASE_PAYLOADS)]

class Command(BaseCommand):
    help = """
    Find case/form submissions/updates that do not have a corresponding repeat record and create a repeat record
    """

    def add_arguments(self, parser):
        parser.add_argument('-s', '--startdate', default="2021-06-19", type=date_type, help='Format YYYY-MM-DD')
        parser.add_argument('-e', '--enddate', default="2021-06-23", type=date_type, help='Format YYYY-MM-DD')
        parser.add_argument('-d', '--domain', default=None, type=str, help='Run on a specific domain')
        parser.add_argument('-c', '--create', action='store_true', help='Create missing repeat records')

    def handle(self, startdate, enddate, domain, create, **options):
        if domain:
            domains_with_repeaters = [domain]
        else:
            domains_with_repeaters = get_domains_that_have_repeat_records()

        _ = obtain_missing_form_repeat_records(startdate, enddate, domains_with_repeaters, create)
<|endoftext|>


<commit_before>"""Database Schema v3

Revision ID: 24a4a626ff7c
Revises: None
Create Date: 2015-07-17

"""

# revision identifiers, used by Alembic.
revision = '24a4a626ff7c'
down_revision = None

import os
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), *['..'] * 5)))

from alembic import op
from king_phisher.server.database import manager as db_manager
import sqlalchemy

def upgrade():
	op.create_table(
		'campaign_types',
		sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
		sqlalchemy.Column('name', sqlalchemy.String, nullable=False),
		sqlalchemy.Column('description', sqlalchemy.String)
	)

	op.create_table(
		'company_departments',
		sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
		sqlalchemy.Column('name', sqlalchemy.String, nullable=False),
		sqlalchemy.Column('description', sqlalchemy.String)
	)

	op.create_table(
		'industries',
		sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
		sqlalchemy.Column('name', sqlalchemy.String, nullable=False),
		sqlalchemy.Column('description', sqlalchemy.String)
	)

	op.create_table(
		'companies',
		sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
		sqlalchemy.Column('name', sqlalchemy.String, unique=True, nullable=False),
		sqlalchemy.Column('description', sqlalchemy.String),
		sqlalchemy.Column('industry_id', sqlalchemy.Integer, sqlalchemy.ForeignKey('industries.id')),
		sqlalchemy.Column('url_main', sqlalchemy.String),
		sqlalchemy.Column('url_email', sqlalchemy.String),
		sqlalchemy.Column('url_remote_access', sqlalchemy.String)
	)

	alert_subscriptions_type = sqlalchemy.Enum('email', 'sms', name='alert_subscription_type')
	alert_subscriptions_type.create(op.get_bind(), checkfirst=True)
	op.add_column('alert_subscriptions', sqlalchemy.Column('type', alert_subscriptions_type, default='sms', server_default='sms', nullable=False))
	op.add_column('alert_subscriptions', sqlalchemy.Column('mute_timestamp', sqlalchemy.DateTime))
	op.add_column('campaigns', sqlalchemy.Column('campaign_type_id', sqlalchemy.Integer, sqlalchemy.ForeignKey('companies.id')))
	op.add_column('campaigns', sqlalchemy.Column('company_id', sqlalchemy.Integer, sqlalchemy.ForeignKey('campaign_types.id')))
	op.add_column('campaigns', sqlalchemy.Column('expiration', sqlalchemy.DateTime))
	op.add_column('messages', sqlalchemy.Column('company_department_id', sqlalchemy.Integer, sqlalchemy.ForeignKey('company_departments.id')))
	op.add_column('users', sqlalchemy.Column('email_address', sqlalchemy.String))
	op.add_column('users', sqlalchemy.Column('otp_secret', sqlalchemy.String(16)))

	db_manager.Session.remove()
	db_manager.Session.configure(bind=op.get_bind())
	session = db_manager.Session()
	db_manager.set_meta_data('schema_version', 3, session=session)
	session.commit()

def downgrade():
	op.drop_column('alert_subscriptions', 'type')
	op.drop_column('alert_subscriptions', 'mute_timestamp')
	op.drop_column('campaigns', 'campaign_type_id')
	op.drop_column('campaigns', 'company_id')
	op.drop_column('campaigns', 'expiration')
	op.drop_column('messages', 'company_department_id')
	op.drop_column('users', 'email_address')
	op.drop_column('users', 'otp_secret')

	op.drop_table('campaign_types')
	op.drop_table('company_departments')
	op.drop_table('companies')
	op.drop_table('industries')

	db_manager.Session.remove()
	db_manager.Session.configure(bind=op.get_bind())
	session = db_manager.Session()
	db_manager.set_meta_data('schema_version', 2, session=session)
	session.commit()
<commit_msg>Fix a foreign key mixup in the v3 migration script<commit_after>"""Database Schema v3

Revision ID: 24a4a626ff7c
Revises: None
Create Date: 2015-07-17

"""

# revision identifiers, used by Alembic.
revision = '24a4a626ff7c'
down_revision = None

import os
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), *['..'] * 5)))

from alembic import op
from king_phisher.server.database import manager as db_manager
import sqlalchemy

def upgrade():
	op.create_table(
		'campaign_types',
		sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
		sqlalchemy.Column('name', sqlalchemy.String, nullable=False),
		sqlalchemy.Column('description', sqlalchemy.String)
	)

	op.create_table(
		'company_departments',
		sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
		sqlalchemy.Column('name', sqlalchemy.String, nullable=False),
		sqlalchemy.Column('description', sqlalchemy.String)
	)

	op.create_table(
		'industries',
		sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
		sqlalchemy.Column('name', sqlalchemy.String, nullable=False),
		sqlalchemy.Column('description', sqlalchemy.String)
	)

	op.create_table(
		'companies',
		sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
		sqlalchemy.Column('name', sqlalchemy.String, unique=True, nullable=False),
		sqlalchemy.Column('description', sqlalchemy.String),
		sqlalchemy.Column('industry_id', sqlalchemy.Integer, sqlalchemy.ForeignKey('industries.id')),
		sqlalchemy.Column('url_main', sqlalchemy.String),
		sqlalchemy.Column('url_email', sqlalchemy.String),
		sqlalchemy.Column('url_remote_access', sqlalchemy.String)
	)

	alert_subscriptions_type = sqlalchemy.Enum('email', 'sms', name='alert_subscription_type')
	alert_subscriptions_type.create(op.get_bind(), checkfirst=True)
	op.add_column('alert_subscriptions', sqlalchemy.Column('type', alert_subscriptions_type, default='sms', server_default='sms', nullable=False))
	op.add_column('alert_subscriptions', sqlalchemy.Column('mute_timestamp', sqlalchemy.DateTime))
	op.add_column('campaigns', sqlalchemy.Column('campaign_type_id', sqlalchemy.Integer, sqlalchemy.ForeignKey('campaign_types.id')))
	op.add_column('campaigns', sqlalchemy.Column('company_id', sqlalchemy.Integer, sqlalchemy.ForeignKey('companies.id')))
	op.add_column('campaigns', sqlalchemy.Column('expiration', sqlalchemy.DateTime))
	op.add_column('messages', sqlalchemy.Column('company_department_id', sqlalchemy.Integer, sqlalchemy.ForeignKey('company_departments.id')))
	op.add_column('users', sqlalchemy.Column('email_address', sqlalchemy.String))
	op.add_column('users', sqlalchemy.Column('otp_secret', sqlalchemy.String(16)))

	db_manager.Session.remove()
	db_manager.Session.configure(bind=op.get_bind())
	session = db_manager.Session()
	db_manager.set_meta_data('schema_version', 3, session=session)
	session.commit()

def downgrade():
	op.drop_column('alert_subscriptions', 'type')
	op.drop_column('alert_subscriptions', 'mute_timestamp')
	op.drop_column('campaigns', 'campaign_type_id')
	op.drop_column('campaigns', 'company_id')
	op.drop_column('campaigns', 'expiration')
	op.drop_column('messages', 'company_department_id')
	op.drop_column('users', 'email_address')
	op.drop_column('users', 'otp_secret')

	op.drop_table('campaign_types')
	op.drop_table('company_departments')
	op.drop_table('companies')
	op.drop_table('industries')

	db_manager.Session.remove()
	db_manager.Session.configure(bind=op.get_bind())
	session = db_manager.Session()
	db_manager.set_meta_data('schema_version', 2, session=session)
	session.commit()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models

class Migration(SchemaMigration):

    depends_on = (
        ("instances", "0001_initial"),
    )

    def forwards(self, orm):
        instance = orm['instances.Instance'](label='default')
        instance.save()

        # Adding field 'Meeting.instance'
        db.add_column('speeches_meeting', 'instance',
                      self.gf('django.db.models.fields.related.ForeignKey')(default=instance.id, to=orm['instances.Instance']),
                      keep_default=False)

        # Adding field 'Debate.instance'
        db.add_column('speeches_debate', 'instance',
                      self.gf('django.db.models.fields.related.ForeignKey')(default=instance.id, to=orm['instances.Instance']),
                      keep_default=False)

        # Adding field 'RecordingTimestamp.instance'
        db.add_column('speeches_recordingtimestamp', 'instance',
                      self.gf('django.db.models.fields.related.ForeignKey')(default=instance.id, to=orm['instances.Instance']),
                      keep_default=False)

        # Adding field 'Speech.instance'
        db.add_column('speeches_speech', 'instance',
                      self.gf('django.db.models.fields.related.ForeignKey')(default=instance.id, to=orm['instances.Instance']),
                      keep_default=False)

        # Adding field 'Speaker.instance'
        db.add_column('speeches_speaker', 'instance',
                      self.gf('django.db.models.fields.related.ForeignKey')(default=instance.id, to=orm['instances.Instance']),
                      keep_default=False)

        # Adding field 'Recording.instance'
        db.add_column('speeches_recording', 'instance',
                      self.gf('django.db.models.fields.related.ForeignKey')(default=instance.id, to=orm['instances.Instance']),
                      keep_default=False)

    def backwards(self, orm):
        # Deleting field 'Meeting.instance'
        db.delete_column('speeches_meeting', 'instance_id')

        # Deleting field 'Debate.instance'
        db.delete_column('speeches_debate', 'instance_id')

        # Deleting field 'RecordingTimestamp.instance'
        db.delete_column('speeches_recordingtimestamp', 'instance_id')

        # Deleting field 'Speech.instance'
        db.delete_column('speeches_speech', 'instance_id')

        # Deleting field 'Speaker.instance'
        db.delete_column('speeches_speaker', 'instance_id')

        # Deleting field 'Recording.instance'
        db.delete_column('speeches_recording', 'instance_id')

    models = {
        'instances.instance': {
            'Meta': {'object_name': 'Instance'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'label': ('instances.fields.DNSLabelField', [], {'unique': 'True', 'max_length': '63', 'db_index': 'True'})
        },
        'speeches.debate': {
            'Meta': {'object_name': 'Debate'},
            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['instances.Instance']"}),
            'meeting': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['speeches.Meeting']", 'null': 'True', 'blank': 'True'}),
            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'title': ('django.db.models.fields.CharField', [], {'max_length': '255'})
        },
        'speeches.meeting': {
            'Meta': {'object_name': 'Meeting'},
            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['instances.Instance']"}),
            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'title': ('django.db.models.fields.CharField', [], {'max_length': '255'})
        },
        'speeches.recording': {
            'Meta': {'object_name': 'Recording'},
            'audio': ('django.db.models.fields.files.FileField', [], {'max_length': '255'}),
            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['instances.Instance']"}),
            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'timestamps': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['speeches.RecordingTimestamp']", 'null': 'True', 'blank': 'True'})
        },
        'speeches.recordingtimestamp': {
            'Meta': {'object_name': 'RecordingTimestamp'},
            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['instances.Instance']"}),
            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'speaker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['speeches.Speaker']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}),
            'timestamp': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'})
        },
        'speeches.speaker': {
            'Meta': {'object_name': 'Speaker'},
            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['instances.Instance']"}),
            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'name': ('django.db.models.fields.TextField', [], {'db_index': 'True'}),
            'popit_url': ('django.db.models.fields.TextField', [], {'unique': 'True'})
        },
        'speeches.speech': {
            'Meta': {'object_name': 'Speech'},
            'audio': ('django.db.models.fields.files.FileField', [], {'max_length': '255', 'blank': 'True'}),
            'celery_task_id': ('django.db.models.fields.CharField', [], {'max_length': '256', 'null': 'True', 'blank': 'True'}),
            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'debate': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['speeches.Debate']", 'null': 'True', 'blank': 'True'}),
            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'end_time': ('django.db.models.fields.TimeField', [], {'null': 'True', 'blank': 'True'}),
            'event': ('django.db.models.fields.TextField', [], {'db_index': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['instances.Instance']"}),
            'location': ('django.db.models.fields.TextField', [], {'db_index': 'True', 'blank': 'True'}),
            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'source_url': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
            'speaker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['speeches.Speaker']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}),
            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'start_time': ('django.db.models.fields.TimeField', [], {'null': 'True', 'blank': 'True'}),
            'text': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
            'title': ('django.db.models.fields.TextField', [], {'blank': 'True'})
        }
    }

    complete_apps = ['speeches']
<commit_msg>Specify migration must take place before title is added to instances.<commit_after># -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models

class Migration(SchemaMigration):

    depends_on = (
        ("instances", "0001_initial"),
    )
    needed_by = (
        ("instances", "0002_auto__add_field_instance_description"),
    )

    def forwards(self, orm):
        instance = orm['instances.Instance'](label='default')
        instance.save()

        # Adding field 'Meeting.instance'
        db.add_column('speeches_meeting', 'instance',
                      self.gf('django.db.models.fields.related.ForeignKey')(default=instance.id, to=orm['instances.Instance']),
                      keep_default=False)

        # Adding field 'Debate.instance'
        db.add_column('speeches_debate', 'instance',
                      self.gf('django.db.models.fields.related.ForeignKey')(default=instance.id, to=orm['instances.Instance']),
                      keep_default=False)

        # Adding field 'RecordingTimestamp.instance'
        db.add_column('speeches_recordingtimestamp', 'instance',
                      self.gf('django.db.models.fields.related.ForeignKey')(default=instance.id, to=orm['instances.Instance']),
                      keep_default=False)

        # Adding field 'Speech.instance'
        db.add_column('speeches_speech', 'instance',
                      self.gf('django.db.models.fields.related.ForeignKey')(default=instance.id, to=orm['instances.Instance']),
                      keep_default=False)

        # Adding field 'Speaker.instance'
        db.add_column('speeches_speaker', 'instance',
                      self.gf('django.db.models.fields.related.ForeignKey')(default=instance.id, to=orm['instances.Instance']),
                      keep_default=False)

        # Adding field 'Recording.instance'
        db.add_column('speeches_recording', 'instance',
                      self.gf('django.db.models.fields.related.ForeignKey')(default=instance.id, to=orm['instances.Instance']),
                      keep_default=False)

    def backwards(self, orm):
        # Deleting field 'Meeting.instance'
        db.delete_column('speeches_meeting', 'instance_id')

        # Deleting field 'Debate.instance'
        db.delete_column('speeches_debate', 'instance_id')

        # Deleting field 'RecordingTimestamp.instance'
        db.delete_column('speeches_recordingtimestamp', 'instance_id')

        # Deleting field 'Speech.instance'
        db.delete_column('speeches_speech', 'instance_id')

        # Deleting field 'Speaker.instance'
        db.delete_column('speeches_speaker', 'instance_id')

        # Deleting field 'Recording.instance'
        db.delete_column('speeches_recording', 'instance_id')

    models = {
        'instances.instance': {
            'Meta': {'object_name': 'Instance'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'label': ('instances.fields.DNSLabelField', [], {'unique': 'True', 'max_length': '63', 'db_index': 'True'})
        },
        'speeches.debate': {
            'Meta': {'object_name': 'Debate'},
            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['instances.Instance']"}),
            'meeting': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['speeches.Meeting']", 'null': 'True', 'blank': 'True'}),
            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'title': ('django.db.models.fields.CharField', [], {'max_length': '255'})
        },
        'speeches.meeting': {
            'Meta': {'object_name': 'Meeting'},
            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['instances.Instance']"}),
            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'title': ('django.db.models.fields.CharField', [], {'max_length': '255'})
        },
        'speeches.recording': {
            'Meta': {'object_name': 'Recording'},
            'audio': ('django.db.models.fields.files.FileField', [], {'max_length': '255'}),
            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['instances.Instance']"}),
            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'timestamps': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['speeches.RecordingTimestamp']", 'null': 'True', 'blank': 'True'})
        },
        'speeches.recordingtimestamp': {
            'Meta': {'object_name': 'RecordingTimestamp'},
            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['instances.Instance']"}),
            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'speaker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['speeches.Speaker']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}),
            'timestamp': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'})
        },
        'speeches.speaker': {
            'Meta': {'object_name': 'Speaker'},
            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['instances.Instance']"}),
            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'name': ('django.db.models.fields.TextField', [], {'db_index': 'True'}),
            'popit_url': ('django.db.models.fields.TextField', [], {'unique': 'True'})
        },
        'speeches.speech': {
            'Meta': {'object_name': 'Speech'},
            'audio': ('django.db.models.fields.files.FileField', [], {'max_length': '255', 'blank': 'True'}),
            'celery_task_id': ('django.db.models.fields.CharField', [], {'max_length': '256', 'null': 'True', 'blank': 'True'}),
            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'debate': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['speeches.Debate']", 'null': 'True', 'blank': 'True'}),
            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'end_time': ('django.db.models.fields.TimeField', [], {'null': 'True', 'blank': 'True'}),
            'event': ('django.db.models.fields.TextField', [], {'db_index': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['instances.Instance']"}),
            'location': ('django.db.models.fields.TextField', [], {'db_index': 'True', 'blank': 'True'}),
            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'source_url': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
            'speaker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['speeches.Speaker']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}),
            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
            'start_time': ('django.db.models.fields.TimeField', [], {'null': 'True', 'blank': 'True'}),
            'text': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
            'title': ('django.db.models.fields.TextField', [], {'blank': 'True'})
        }
    }

    complete_apps = ['speeches']
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from datetime import datetime
import dateutil.parser
from django.core.management.base import BaseCommand
from django.contrib.humanize.templatetags.humanize import naturaltime
from clint.textui import columns

from remote_fixtures.utils import S3Mixin, humanize_filesize

class Command(BaseCommand, S3Mixin):
    def handle(self, *args, **options):
        bucket = self.get_bucket()

        fixtures = list(bucket.list('fixture_'))
        for fixture in fixtures:
            fixture.last_modified_dt = dateutil.parser.parse(fixture.last_modified)

        fixtures.sort(key=lambda x: x.last_modified_dt)

        for fixture in fixtures:
            print columns(
                [fixture.name, 40],
                [humanize_filesize(fixture.size), 10],
                [fixture.last_modified_dt.strftime('%d %b %G'), 15],
                [naturaltime(fixture.last_modified_dt), 30],
            )
<commit_msg>tweaking column widths down to only what's needed<commit_after>from __future__ import unicode_literals
from datetime import datetime
import dateutil.parser
from django.core.management.base import BaseCommand
from django.contrib.humanize.templatetags.humanize import naturaltime
from clint.textui import columns

from remote_fixtures.utils import S3Mixin, humanize_filesize

class Command(BaseCommand, S3Mixin):
    def handle(self, *args, **options):
        bucket = self.get_bucket()

        fixtures = list(bucket.list('fixture_'))
        for fixture in fixtures:
            fixture.last_modified_dt = dateutil.parser.parse(fixture.last_modified)

        fixtures.sort(key=lambda x: x.last_modified_dt)

        for fixture in fixtures:
            print columns(
                [fixture.name, 38],
                [humanize_filesize(fixture.size), 9],
                [fixture.last_modified_dt.strftime('%d %b %G'), 12],
                [naturaltime(fixture.last_modified_dt), 25],
            )
<|endoftext|>


<commit_before>#########
# Copyright (c) 2016 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

from manager_rest.security.authorization import authorize
from manager_rest.dsl_functions import evaluate_intrinsic_functions

from .. import rest_decorators
from ..resources_v1.nodes import NodeInstancesId as v1_NodeInstancesId
from ..resources_v2 import Nodes as v2_Nodes

class Nodes(v2_Nodes):
    @authorize('node_list')
    @rest_decorators.evaluate_functions
    def get(self, evaluate_functions=False, *args, **kwargs):
        # We don't skip marshalling, because we want an already marshalled
        # object, to avoid setting evaluated secrets in the node's properties
        nodes = super(Nodes, self).get(*args, **kwargs)
        if evaluate_functions:
            for node in nodes['items']:
                evaluate_intrinsic_functions(node['properties'],
                                             node['deployment_id'])
        return nodes

class NodeInstancesId(v1_NodeInstancesId):
    @authorize('node_instance_list')
    @rest_decorators.evaluate_functions
    def get(self, evaluate_functions=False, *args, **kwargs):
        # We don't skip marshalling, because we want an already marshalled
        # object, to avoid setting evaluated secrets in the node instances's
        # runtime properties
        node_instance = super(NodeInstancesId, self).get(*args, **kwargs)
        if evaluate_functions:
            evaluate_intrinsic_functions(node_instance['runtime_properties'],
                                         node_instance['deployment_id'])
        return node_instance
<commit_msg>NodeInstances get: use the node_instance_get permission (#1452)<commit_after>#########
# Copyright (c) 2016 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

from manager_rest.security.authorization import authorize
from manager_rest.dsl_functions import evaluate_intrinsic_functions

from .. import rest_decorators
from ..resources_v1.nodes import NodeInstancesId as v1_NodeInstancesId
from ..resources_v2 import Nodes as v2_Nodes

class Nodes(v2_Nodes):
    @authorize('node_list')
    @rest_decorators.evaluate_functions
    def get(self, evaluate_functions=False, *args, **kwargs):
        # We don't skip marshalling, because we want an already marshalled
        # object, to avoid setting evaluated secrets in the node's properties
        nodes = super(Nodes, self).get(*args, **kwargs)
        if evaluate_functions:
            for node in nodes['items']:
                evaluate_intrinsic_functions(node['properties'],
                                             node['deployment_id'])
        return nodes

class NodeInstancesId(v1_NodeInstancesId):
    @authorize('node_instance_get')
    @rest_decorators.evaluate_functions
    def get(self, evaluate_functions=False, *args, **kwargs):
        # We don't skip marshalling, because we want an already marshalled
        # object, to avoid setting evaluated secrets in the node instances's
        # runtime properties
        node_instance = super(NodeInstancesId, self).get(*args, **kwargs)
        if evaluate_functions:
            evaluate_intrinsic_functions(node_instance['runtime_properties'],
                                         node_instance['deployment_id'])
        return node_instance
<|endoftext|>


<commit_before>##############################################################################
# For copyright and license notices, see __manifest__.py file in module root
# directory
##############################################################################
from odoo import models, fields, api, _
from odoo.exceptions import ValidationError

class SaleOrderType(models.Model):
    _inherit = 'sale.order.type'

    # TODO this should go in a pr to OCA
    sequence_id = fields.Many2one(
        domain="['|', ('company_id', '=', company_id), "
        "('company_id', '=', False)]"
    )
    # agregamos help
    journal_id = fields.Many2one(
        help='Billing journal to be used by default. No matter invoice being '
        'created automatically or manually. If no journal is set here, '
        'default journal will be used'
    )
    invoice_company_id = fields.Many2one(
        related='journal_id.company_id',
        readonly=True
    )
    invoicing_atomation = fields.Selection([
        ('none', 'None'),
        ('create_invoice', 'Create Invoice'),
        ('validate_invoice', 'Validate Invoice'),
        ('try_validate_invoice', 'Try to Validate Invoice (Not recommended)'),
    ],
        default='none',
        required=True,
        help="On sale order confirmation and on picking confirmation, if:\n"
        "*None: no invoice is created\n"
        "*Create Invoice: create invoice for 'Invoiceable lines' (regarding "
        "product configuration and delivery status)\n"
        "*Validate Invoice: create invoice and validate it\n"
        "*Try to Validate Invoice: create invoice and try to validate it, if "
        "there is a validation error, do not raise it, just log it on SO and "
        "Invoice chatter (invoice will be in draft state till someone validate"
        " it. This option is not recommended because it is less intuiteve for "
        "user. Should only be use if it is common that you are facing errors "
        "on invoice validation."
    )
    payment_atomation = fields.Selection([
        ('none', 'None'),
        # TODO ver si implementamos, por ahora solo con validacion
        # porque asi lo implementamos en payment_group
        # ('create_payment', 'Create Payment'),
        ('validate_payment', 'Validate Payment'),
    ],
        default='none',
        required=True,
        help="On invoice validation, if:\n"
        "*None: no payment is created\n"
        # "*Create Payment: create payment with journal configured\n"
        "*Validate Payment: create payment and validate it\n"
    )
    picking_atomation = fields.Selection([
        ('none', 'None'),
        ('validate', 'Validate'),
        ('validate_no_force', 'Validate No Force')],
        default='none',
        required=True,
        help='Pickings are generated automatically upon sale confirmation.\n'
        'If you set "Validate", '
        'then they will be also confirmed automatically.\n'
        'If you set "Validate No Force", then'
        ' Validate without forcing availabilty'
    )
    payment_journal_id = fields.Many2one(
        'account.journal',
        'Payment Journal',
        domain="[('type','in', ['cash', 'bank']), "
        "('company_id', '=', invoice_company_id), "
        # "('company_id', '=', company_id), "
        "('inbound_payment_method_ids.code', '=', 'manual')]",
        help='Jouranl used only with payment_automation. As manual payment '
        'method is used, only journals with manual method are shown.'
    )
    book_id = fields.Many2one(
        'stock.book',
        'Voucher Book',
    )
    set_done_on_confirmation = fields.Boolean(
        help="Upon confirmation set"
        " sale order done instead of leaving it on"
        " 'sale order' state that allows modifications"
    )
    auto_done_setting = fields.Selection([
        (0, "Allow to edit sales order from the"
         " 'Sales Order' menu (not from the Quotation menu)"),
        (1, "Never allow to modify a confirmed sale order")],
        compute='_compute_auto_done_setting',
    )

    @api.depends()
    def _compute_auto_done_setting(self):
        default = self.env['ir.config_parameter'].sudo().get_param(
            'sale.auto_done_setting')
        self.update({'auto_done_setting': default})

    @api.constrains(
        'payment_atomation',
        'payment_journal_id')
    def validate_invoicing_atomation(self):
        payment_journal_required = self.filtered(
            lambda x: x.payment_atomation != 'none' and
            not x.payment_journal_id)
        if payment_journal_required:
            raise ValidationError(_(
                'If you choose a Payment automation, Payment Journal '
                'is required'))

    @api.constrains(
        'journal_id',
        'payment_journal_id',
        'sequence_id')
    def validate_company_id(self):
        different_company = self.filtered(
            lambda x: x.invoice_company_id and x.payment_journal_id and
            x.invoice_company_id != x.payment_journal_id.company_id
        )
        if different_company:
            raise ValidationError(_(
                'Invoice Journal and Payment Journal must be of the same '
                'company'))

        # la cia es opcional en la secuencia, solo chequeamos si esta
        # seteada
        # TODO this should go in a pr to OCA sot module
        sequence_diff_company = self.filtered(
            lambda x: x.sequence_id.company_id and
            x.sequence_id.company_id != x.company_id
        )
        if sequence_diff_company:
            raise ValidationError(
                '\n'.join([_('The Sequence "%s" company must be the same than'
                             ' sale order type') % sot.sequence_id.name
                           for sot in sequence_diff_company])
            )
<commit_msg>[FIX] company constraint of sale_order_type_automation<commit_after>##############################################################################
# For copyright and license notices, see __manifest__.py file in module root
# directory
##############################################################################
from odoo import models, fields, api, _
from odoo.exceptions import ValidationError

class SaleOrderType(models.Model):
    _inherit = 'sale.order.type'

    # TODO this should go in a pr to OCA
    sequence_id = fields.Many2one(
        domain="['|', ('company_id', '=', company_id), "
        "('company_id', '=', False)]"
    )
    # agregamos help
    journal_id = fields.Many2one(
        help='Billing journal to be used by default. No matter invoice being '
        'created automatically or manually. If no journal is set here, '
        'default journal will be used'
    )
    invoice_company_id = fields.Many2one(
        related='journal_id.company_id',
        readonly=True
    )
    invoicing_atomation = fields.Selection([
        ('none', 'None'),
        ('create_invoice', 'Create Invoice'),
        ('validate_invoice', 'Validate Invoice'),
        ('try_validate_invoice', 'Try to Validate Invoice (Not recommended)'),
    ],
        default='none',
        required=True,
        help="On sale order confirmation and on picking confirmation, if:\n"
        "*None: no invoice is created\n"
        "*Create Invoice: create invoice for 'Invoiceable lines' (regarding "
        "product configuration and delivery status)\n"
        "*Validate Invoice: create invoice and validate it\n"
        "*Try to Validate Invoice: create invoice and try to validate it, if "
        "there is a validation error, do not raise it, just log it on SO and "
        "Invoice chatter (invoice will be in draft state till someone validate"
        " it. This option is not recommended because it is less intuiteve for "
        "user. Should only be use if it is common that you are facing errors "
        "on invoice validation."
    )
    payment_atomation = fields.Selection([
        ('none', 'None'),
        # TODO ver si implementamos, por ahora solo con validacion
        # porque asi lo implementamos en payment_group
        # ('create_payment', 'Create Payment'),
        ('validate_payment', 'Validate Payment'),
    ],
        default='none',
        required=True,
        help="On invoice validation, if:\n"
        "*None: no payment is created\n"
        # "*Create Payment: create payment with journal configured\n"
        "*Validate Payment: create payment and validate it\n"
    )
    picking_atomation = fields.Selection([
        ('none', 'None'),
        ('validate', 'Validate'),
        ('validate_no_force', 'Validate No Force')],
        default='none',
        required=True,
        help='Pickings are generated automatically upon sale confirmation.\n'
        'If you set "Validate", '
        'then they will be also confirmed automatically.\n'
        'If you set "Validate No Force", then'
        ' Validate without forcing availabilty'
    )
    payment_journal_id = fields.Many2one(
        'account.journal',
        'Payment Journal',
        domain="[('type','in', ['cash', 'bank']), "
        "('company_id', '=', invoice_company_id), "
        # "('company_id', '=', company_id), "
        "('inbound_payment_method_ids.code', '=', 'manual')]",
        help='Jouranl used only with payment_automation. As manual payment '
        'method is used, only journals with manual method are shown.'
    )
    book_id = fields.Many2one(
        'stock.book',
        'Voucher Book',
    )
    set_done_on_confirmation = fields.Boolean(
        help="Upon confirmation set"
        " sale order done instead of leaving it on"
        " 'sale order' state that allows modifications"
    )
    auto_done_setting = fields.Selection([
        (0, "Allow to edit sales order from the"
         " 'Sales Order' menu (not from the Quotation menu)"),
        (1, "Never allow to modify a confirmed sale order")],
        compute='_compute_auto_done_setting',
    )

    @api.depends()
    def _compute_auto_done_setting(self):
        default = self.env['ir.config_parameter'].sudo().get_param(
            'sale.auto_done_setting')
        self.update({'auto_done_setting': default})

    @api.constrains(
        'payment_atomation',
        'payment_journal_id')
    def validate_invoicing_atomation(self):
        payment_journal_required = self.filtered(
            lambda x: x.payment_atomation != 'none' and
            not x.payment_journal_id)
        if payment_journal_required:
            raise ValidationError(_(
                'If you choose a Payment automation, Payment Journal '
                'is required'))

    @api.constrains(
        'journal_id',
        'payment_journal_id',
        'sequence_id')
    def validate_company_id(self):
        different_company = self.filtered(
            lambda x: x.invoice_company_id and x.payment_journal_id and
            x.invoice_company_id != x.payment_journal_id.company_id
        )
        if different_company:
            raise ValidationError(_(
                'Invoice Journal and Payment Journal must be of the same '
                'company'))

        # la cia es opcional en la secuencia, solo chequeamos si esta
        # seteada
        # TODO this should go in a pr to OCA sot module
        sequence_diff_company = self.filtered(
            lambda x: x.sequence_id.company_id and
            x.warehouse_id.company_id and
            x.sequence_id.company_id != x.company_id
        )
        if sequence_diff_company:
            raise ValidationError(_(
                    'The company of the sequence and the warehouse must be '
                    'the same'))
<|endoftext|>


<commit_before># Generated by Django 3.1 on 2020-09-04 12:51

from django.db import migrations

class Migration(migrations.Migration):

    dependencies = [
        ("product", "0122_auto_20200828_1135"),
    ]

    operations = [
        migrations.AlterUniqueTogether(
            name="variantimage", unique_together={("variant", "image")},
        ),
    ]
<commit_msg>Drop duplicated VariantImages before migration to unique together<commit_after># Generated by Django 3.1 on 2020-09-04 12:51

from django.db import migrations
from django.db.models import Count

def remove_variant_image_duplicates(apps, schema_editor):
    ProductImage = apps.get_model("product", "ProductImage")
    VariantImage = apps.get_model("product", "VariantImage")

    duplicated_images = (
        ProductImage.objects.values("pk", "variant_images__variant")
        .annotate(variant_count=Count("variant_images__variant"))
        .filter(variant_count__gte=2)
    )

    variant_image_ids_to_remove = []
    for image_data in duplicated_images:
        ids = VariantImage.objects.filter(
            variant=image_data["variant_images__variant"], image__pk=image_data["pk"],
        )[1:].values_list("pk", flat=True)
        variant_image_ids_to_remove += ids

    VariantImage.objects.filter(pk__in=variant_image_ids_to_remove).delete()

class Migration(migrations.Migration):

    dependencies = [
        ("product", "0122_auto_20200828_1135"),
    ]

    operations = [
        migrations.RunPython(
            remove_variant_image_duplicates, migrations.RunPython.noop
        ),
        migrations.AlterUniqueTogether(
            name="variantimage", unique_together={("variant", "image")},
        ),
    ]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from odoo import fields, models

class EventRegistration(models.Model):
    """ Store answers on attendees. """
    _inherit = 'event.registration'

    registration_answer_ids = fields.One2many('event.registration.answer', 'registration_id', string='Attendee Answers')

class EventRegistrationAnswer(models.Model):
    """ Represents the user input answer for a single event.question """
    _name = 'event.registration.answer'
    _description = 'Event Registration Answer'

    question_id = fields.Many2one('event.question', required=True, ondelete='cascade')
    registration_id = fields.Many2one('event.registration', required=True, ondelete='cascade')
    partner_id = fields.Many2one('res.partner', related='registration_id.partner_id')
    event_id = fields.Many2one('event.event', related='registration_id.event_id')
    question_type = fields.Selection(related='question_id.question_type')
    value_answer_id = fields.Many2one('event.question.answer', string="Suggested answer")
    value_text_box = fields.Text('Text answer')

    _sql_constraints = [
        ('value_check', "CHECK(value_answer_id IS NOT NULL OR COALESCE(value_text_box, '') <> '')", "There must be a suggested value or a text value.")
    ]
<commit_msg>[FIX] website_event_questions: add domain on answer question_id<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from odoo import fields, models

class EventRegistration(models.Model):
    """ Store answers on attendees. """
    _inherit = 'event.registration'

    registration_answer_ids = fields.One2many('event.registration.answer', 'registration_id', string='Attendee Answers')

class EventRegistrationAnswer(models.Model):
    """ Represents the user input answer for a single event.question """
    _name = 'event.registration.answer'
    _description = 'Event Registration Answer'

    question_id = fields.Many2one(
        'event.question', ondelete='restrict', required=True,
        domain="[('event_id', '=', event_id)]")
    registration_id = fields.Many2one('event.registration', required=True, ondelete='cascade')
    partner_id = fields.Many2one('res.partner', related='registration_id.partner_id')
    event_id = fields.Many2one('event.event', related='registration_id.event_id')
    question_type = fields.Selection(related='question_id.question_type')
    value_answer_id = fields.Many2one('event.question.answer', string="Suggested answer")
    value_text_box = fields.Text('Text answer')

    _sql_constraints = [
        ('value_check', "CHECK(value_answer_id IS NOT NULL OR COALESCE(value_text_box, '') <> '')", "There must be a suggested value or a text value.")
    ]
<|endoftext|>


<commit_before>from .paged_memory_mixin import PagedMemoryMixin

class PrivilegedPagingMixin(PagedMemoryMixin):
    """
    A mixin for paged memory models which will raise SimSegfaultExceptions if STRICT_PAGE_ACCESS is enabled and
    a segfault condition is detected.

    Segfault conditions include:
    - getting a page for reading which is non-readable
    - getting a page for writing which is non-writable
    - creating a page

    The latter condition means that this should be inserted under any mixins which provide other implementations of
    ``_initialize_page``.
    """
    def _get_page(self, pageno: int, writing: bool, priv: bool=False, **kwargs):
        page = super()._get_page(pageno, writing)
        if self.category == 'mem' and not priv and o.STRICT_PAGE_ACCESS in self.state.options:
            if writing and not self.state.solver.is_true(page.perm_write):
                raise SimSegfaultException(pageno * self.page_size, 'non-writable')
            if not writing and not self.state.solver.is_true(page.perm_read):
                raise SimSegfaultException(pageno * self.page_size, 'non-readable')

        return page

    def _initialize_page(self, pageno: int, priv: bool=False, **kwargs):
        if self.category == 'mem' and not priv and o.STRICT_PAGE_ACCESS in self.state.options:
            raise SimSegfaultException(pageno * self.page_size, 'unmapped')

        return super()._initialize_page(pageno, **kwargs)

from angr.errors import SimSegfaultException
from angr import sim_options as o
<commit_msg>bugfix: pass on kwargs<commit_after>from .paged_memory_mixin import PagedMemoryMixin

class PrivilegedPagingMixin(PagedMemoryMixin):
    """
    A mixin for paged memory models which will raise SimSegfaultExceptions if STRICT_PAGE_ACCESS is enabled and
    a segfault condition is detected.

    Segfault conditions include:
    - getting a page for reading which is non-readable
    - getting a page for writing which is non-writable
    - creating a page

    The latter condition means that this should be inserted under any mixins which provide other implementations of
    ``_initialize_page``.
    """
    def _get_page(self, pageno: int, writing: bool, priv: bool=False, **kwargs):
        page = super()._get_page(pageno, writing, **kwargs)
        if self.category == 'mem' and not priv and o.STRICT_PAGE_ACCESS in self.state.options:
            if writing and not self.state.solver.is_true(page.perm_write):
                raise SimSegfaultException(pageno * self.page_size, 'non-writable')
            if not writing and not self.state.solver.is_true(page.perm_read):
                raise SimSegfaultException(pageno * self.page_size, 'non-readable')

        return page

    def _initialize_page(self, pageno: int, priv: bool=False, **kwargs):
        if self.category == 'mem' and not priv and o.STRICT_PAGE_ACCESS in self.state.options:
            raise SimSegfaultException(pageno * self.page_size, 'unmapped')

        return super()._initialize_page(pageno, **kwargs)

from angr.errors import SimSegfaultException
from angr import sim_options as o
<|endoftext|>


<commit_before>import theano
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams

import DeepLearningStack

#create a deep network defined in AlexNet.xml
if __name__=="__main__":

    #create deep net
    print "Creating deep net..."
    config            = "AlexNet.xml"
    #random number generator used to initialize the weights
    rng               = RandomStreams(seed=int(time.time()))
    #create the inputs to the network
    images            = T.tensor4("images")

    #create the graph structure
    net1              = FeedForwardNet.FeedForwardNet(rng,{"image1":images}, config)

<commit_msg>deepnet renamed to feedforwardnet<commit_after>import theano
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams

import ..DeepLearningStack

#create a deep network defined in AlexNet.xml
if __name__=="__main__":

    #create deep net
    print "Creating deep net..."
    config            = "AlexNet.xml"
    #random number generator used to initialize the weights
    rng               = RandomStreams(seed=int(time.time()))
    #create the inputs to the network
    images            = T.tensor4("images")

    #create the graph structure
    net1              = FeedForwardNet.FeedForwardNet(rng,{"image1":images}, config)

<|endoftext|>