<commit_before># Copyright 2015 Stanford University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import time
import shutil
import stat
from subprocess import Popen, PIPE
from ipykernel.kernelbase import Kernel
from datetime import datetime
import base64

def parse_attribute(attribute, is_first):
    attribute = ''.join(attribute)
    if is_first:
        key, value = attribute.split(': ', 1)
        return (key.lower().replace(' ', '_'), value)
    key, value = attribute.split(' = ', 1)
    return (key.lower().replace(' ', '_'), value)

def parse_status(logs):
    entries = [entry for entry in logs.split('\n\n') if len(entry.strip()) > 0]
    jobs = []
    for entry in entries:
        lines = entry.split('\n')
        job = []
        attribute = None
        is_first = True
        for line in lines:
            if line.startswith('\t'):
                attribute.append(line.strip())
            else:
                if attribute is not None:
                    job.append(parse_attribute(attribute, is_first))
                    is_first = False
                attribute = [line.strip()]
        job.append(parse_attribute(attribute, is_first))
        jobs.append(collections.OrderedDict(job))
    return jobs

class RegentKernel(Kernel):
    implementation = 'Regent'
    implementation_version = '1.0'
    language = 'regent'
    language_version = '0.1'
    language_info = { 'mimetype': 'text/x-regent', 'name': 'regent',
            'pygments_lexer': 'lua', 'file_extension': 'rg' }
    banner = "IPython kernel for Regent"

    def do_execute(self, code, silent, store_history=True,
            user_expressions=None, allow_stdin=False):
        if not silent:
            root_dir = os.path.dirname(os.path.realpath(__file__))
            launcher_dir = os.path.join(os.path.dirname(os.path.dirname(root_dir)), 'launcher')

            dir = datetime.now().strftime("kernellaunch-%Y-%m-%d-%M-%S.%f")
            tmp_dir = os.path.join("/var/jupyterhub/launches", dir)
            os.mkdir(tmp_dir)
            os.chmod(tmp_dir, stat.S_IRWXO + stat.S_IRWXG + stat.S_IRWXU)

            regent_file = "test.rg"
            torque_file = "run.sh"
            launcher_file = "launcher.py"

            regent_file_path = os.path.join(tmp_dir, regent_file)
            torque_file_path = os.path.join(tmp_dir, torque_file)
            launcher_file_path = os.path.join(tmp_dir, launcher_file)

            with open(regent_file_path, "w") as file:
                file.write(code)

            num_nodes = 2
            prof_file = "legion_prof_%.log"
            prof_file_path = os.path.join(tmp_dir, prof_file)
            regent_interpreter_path = "regent"

            with open(torque_file_path, "w") as file:
                file.write("#!/bin/bash -l\n")
                file.write("#PBS -l nodes=%d\n" % num_nodes)
                file.write("%s %s %s -hl:prof %d -level legion_prof=2 -logfile %s -ll:cpu 6 -ll:gpu 1 -fcuda 1\n" % \
                        (launcher_file_path,
                         regent_interpreter_path,
                         regent_file_path,
                         num_nodes,
                         prof_file_path))

            shutil.copy2(os.path.join(launcher_dir, launcher_file), launcher_file_path)

            stdout_file_path = os.path.join(tmp_dir, "stdout")
            stderr_file_path = os.path.join(tmp_dir, "stderr")
            # submit the job to torque
            job_process = Popen(["qsub", torque_file_path,
                "-d", os.path.join(os.getenv("HOME"), "notebooks"),
                "-o", stdout_file_path,
                "-e", stderr_file_path],
                stdout=PIPE, stderr=PIPE)
            job_id = job_process.stdout.read()

            # wait until the job finishes
            delay = 0.0001
            running = True
            exitcode = 0
            error = False
            while running:
                status = parse_status(subprocess.check_output("qstat -f" + job_id))
                if status["job_state"] == "C":
                    running = False
                    exitcode = status["exit_status"]
                    error = exitcode != 0
                    if error:
                        subprocess.check_output("qdel " + job_id)
                    break
                time.sleep(delay)
                delay = delay * 2

            with open(stdout_file_path, "r") as file:
                stdout = file.read()
            stream_content = {'name': 'stdout', 'text': stdout}
            self.send_response(self.iopub_socket, 'stream', stream_content)
            with open(stderr_file_path, "r") as file:
                stderr = file.read()
            stream_content = {'name': 'stderr', 'text': stderr}
            self.send_response(self.iopub_socket, 'stream', stream_content)

            if error:
                return {'status': 'error', 'execution_count': self.execution_count,
                        'ename': '', 'evalue': str(exitcode), 'traceback': []}

            prof_file_paths = " ".join([os.path.join(tmp_dir, "legion_prof_%d.log" % i) for i in range(0, num_nodes)])
            html_file_path = os.path.join("/var/www/files", dir)
            os.mkdir(html_file_path)
            html_file_prefix = os.path.join(html_file_path, "legion_prof")
            legion_prof_path = os.path.join('/usr/local/legion/tools/legion_prof.py')
            os.system("%s -o %s -T %s" % \
                (legion_prof_path, html_file_prefix, prof_file_paths))
            url = os.path.join("/files", dir, "legion_prof.html")
            html = '''
                Legion Prof timeline

                ''' % (url, url)
            display_content = {'source': 'LegionProf', 'data': { 'text/html': html } }
            self.send_response(self.iopub_socket, 'display_data', display_content)

        return {'status': 'ok',
                # The base class increments the execution count
                'execution_count': self.execution_count,
                'payload': [],
                'user_expressions': {},
               }

if __name__ == '__main__':
    from ipykernel.kernelapp import IPKernelApp
    IPKernelApp.launch_instance(kernel_class=RegentKernel)
<commit_msg>missing package import<commit_after># Copyright 2015 Stanford University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import time
import shutil
import stat
from subprocess import Popen, PIPE, check_output
from ipykernel.kernelbase import Kernel
from datetime import datetime
import base64

def parse_attribute(attribute, is_first):
    attribute = ''.join(attribute)
    if is_first:
        key, value = attribute.split(': ', 1)
        return (key.lower().replace(' ', '_'), value)
    key, value = attribute.split(' = ', 1)
    return (key.lower().replace(' ', '_'), value)

def parse_status(logs):
    entries = [entry for entry in logs.split('\n\n') if len(entry.strip()) > 0]
    jobs = []
    for entry in entries:
        lines = entry.split('\n')
        job = []
        attribute = None
        is_first = True
        for line in lines:
            if line.startswith('\t'):
                attribute.append(line.strip())
            else:
                if attribute is not None:
                    job.append(parse_attribute(attribute, is_first))
                    is_first = False
                attribute = [line.strip()]
        job.append(parse_attribute(attribute, is_first))
        jobs.append(collections.OrderedDict(job))
    return jobs

class RegentKernel(Kernel):
    implementation = 'Regent'
    implementation_version = '1.0'
    language = 'regent'
    language_version = '0.1'
    language_info = { 'mimetype': 'text/x-regent', 'name': 'regent',
            'pygments_lexer': 'lua', 'file_extension': 'rg' }
    banner = "IPython kernel for Regent"

    def do_execute(self, code, silent, store_history=True,
            user_expressions=None, allow_stdin=False):
        if not silent:
            root_dir = os.path.dirname(os.path.realpath(__file__))
            launcher_dir = os.path.join(os.path.dirname(os.path.dirname(root_dir)), 'launcher')

            dir = datetime.now().strftime("kernellaunch-%Y-%m-%d-%M-%S.%f")
            tmp_dir = os.path.join("/var/jupyterhub/launches", dir)
            os.mkdir(tmp_dir)
            os.chmod(tmp_dir, stat.S_IRWXO + stat.S_IRWXG + stat.S_IRWXU)

            regent_file = "test.rg"
            torque_file = "run.sh"
            launcher_file = "launcher.py"

            regent_file_path = os.path.join(tmp_dir, regent_file)
            torque_file_path = os.path.join(tmp_dir, torque_file)
            launcher_file_path = os.path.join(tmp_dir, launcher_file)

            with open(regent_file_path, "w") as file:
                file.write(code)

            num_nodes = 2
            prof_file = "legion_prof_%.log"
            prof_file_path = os.path.join(tmp_dir, prof_file)
            regent_interpreter_path = "regent"

            with open(torque_file_path, "w") as file:
                file.write("#!/bin/bash -l\n")
                file.write("#PBS -l nodes=%d\n" % num_nodes)
                file.write("%s %s %s -hl:prof %d -level legion_prof=2 -logfile %s -ll:cpu 6 -ll:gpu 1 -fcuda 1\n" % \
                        (launcher_file_path,
                         regent_interpreter_path,
                         regent_file_path,
                         num_nodes,
                         prof_file_path))

            shutil.copy2(os.path.join(launcher_dir, launcher_file), launcher_file_path)

            stdout_file_path = os.path.join(tmp_dir, "stdout")
            stderr_file_path = os.path.join(tmp_dir, "stderr")
            # submit the job to torque
            job_process = Popen(["qsub", torque_file_path,
                "-d", os.path.join(os.getenv("HOME"), "notebooks"),
                "-o", stdout_file_path,
                "-e", stderr_file_path],
                stdout=PIPE, stderr=PIPE)
            job_id = job_process.stdout.read()

            # wait until the job finishes
            delay = 0.0001
            running = True
            exitcode = 0
            error = False
            while running:
                status = parse_status(check_output("qstat -f" + job_id))
                if status["job_state"] == "C":
                    running = False
                    exitcode = status["exit_status"]
                    error = exitcode != 0
                    if error:
                        check_output("qdel " + job_id)
                    break
                time.sleep(delay)
                delay = delay * 2

            with open(stdout_file_path, "r") as file:
                stdout = file.read()
            stream_content = {'name': 'stdout', 'text': stdout}
            self.send_response(self.iopub_socket, 'stream', stream_content)
            with open(stderr_file_path, "r") as file:
                stderr = file.read()
            stream_content = {'name': 'stderr', 'text': stderr}
            self.send_response(self.iopub_socket, 'stream', stream_content)

            if error:
                return {'status': 'error', 'execution_count': self.execution_count,
                        'ename': '', 'evalue': str(exitcode), 'traceback': []}

            prof_file_paths = " ".join([os.path.join(tmp_dir, "legion_prof_%d.log" % i) for i in range(0, num_nodes)])
            html_file_path = os.path.join("/var/www/files", dir)
            os.mkdir(html_file_path)
            html_file_prefix = os.path.join(html_file_path, "legion_prof")
            legion_prof_path = os.path.join('/usr/local/legion/tools/legion_prof.py')
            os.system("%s -o %s -T %s" % \
                (legion_prof_path, html_file_prefix, prof_file_paths))
            url = os.path.join("/files", dir, "legion_prof.html")
            html = '''
                Legion Prof timeline

                ''' % (url, url)
            display_content = {'source': 'LegionProf', 'data': { 'text/html': html } }
            self.send_response(self.iopub_socket, 'display_data', display_content)

        return {'status': 'ok',
                # The base class increments the execution count
                'execution_count': self.execution_count,
                'payload': [],
                'user_expressions': {},
               }

if __name__ == '__main__':
    from ipykernel.kernelapp import IPKernelApp
    IPKernelApp.launch_instance(kernel_class=RegentKernel)
<|endoftext|>


<commit_before>"""
Calculate frequencies and sample rates using roach state.

All frequencies and rates in the roach state dictionary are assumed to be in Hz, not MHz.

These functions should eventually be merged into the roach interface.
"""
from __future__ import division
import numpy as np

def baseband_frequency(roach_state, tone_bin):
    temporary_frequency = roach_state['adc_sample_rate'] * tone_bin / roach_state['num_tone_samples']
    if roach_state['heterodyne']:
        return np.where(temporary_frequency >= roach_state['adc_sample_rate'] / 2,
                        temporary_frequency - roach_state['adc_sample_rate'],
                        temporary_frequency)
    else:
        return temporary_frequency

def frequency(roach_state, tone_bin):
    if roach_state['heterodyne']:
        return roach_state['lo_frequency'] + baseband_frequency(roach_state, tone_bin)
    else:
        return baseband_frequency(roach_state, tone_bin)

def stream_sample_rate(roach_state):
    if roach_state['heterodyne']:
        # In the heterodyne case, the number of complex samples per FFT is just num_filterbank_channels.
        return roach_state['adc_sample_rate'] / roach_state['num_filterbank_channels']
    else:
        # In the baseband case, the number of real samples per FFT is 2 * num_filterbank_channels.
        return roach_state['adc_sample_rate'] / (2 * roach_state['num_filterbank_channels'])

def modulation_period_samples(roach_state):
    if roach_state.modulation_output != 2:
        return 0
    else:
        return 2**(roach_state.modulation_rate+1)

def packet_phase(seq_no,offset_frequencies,nchan,nfft,ns):
    packet_bins = 1024    #this is hardcoded from the roach. number of fft bins that fit in 1 udp packet
    packet_counts = nfft * packet_bins
    chan_counts = packet_counts / nchan
    shift = int(np.log2(chan_counts)) - 1
    modn = ns / chan_counts
    if modn == 0:
        modn = 1
    multy = ns / nfft
    seq_no = seq_no >> shift
    seq_no %= modn
    return np.exp(-1j * 2. * np.pi * seq_no * offset_frequencies * multy / modn)

def tone_offset_frequency(tone_bin,tone_num_samples,fft_bin,nfft):
    k = tone_bin
    m = fft_bin
    nfft = nfft
    ns = tone_num_samples
    return nfft * (k / float(ns)) - m

def get_offset_frequencies_period(offset_frequencies):
    period = 1
    while not np.all(np.round(offset_frequencies*period)==offset_frequencies*period):
        period = period * 2
    return period<commit_msg>fix modulation period samples calculation for baseband mode this commit is sloppy because it includes some other work in progress<commit_after>"""
Calculate frequencies and sample rates using roach state.

All frequencies and rates in the roach state dictionary are assumed to be in Hz, not MHz.

These functions should eventually be merged into the roach interface.
"""
from __future__ import division
import numpy as np

def baseband_frequency(roach_state, tone_bin):
    temporary_frequency = roach_state['adc_sample_rate'] * tone_bin / roach_state['num_tone_samples']
    if roach_state['heterodyne']:
        return np.where(temporary_frequency >= roach_state['adc_sample_rate'] / 2,
                        temporary_frequency - roach_state['adc_sample_rate'],
                        temporary_frequency)
    else:
        return temporary_frequency

def frequency(roach_state, tone_bin):
    if roach_state['heterodyne']:
        return roach_state['lo_frequency'] + baseband_frequency(roach_state, tone_bin)
    else:
        return baseband_frequency(roach_state, tone_bin)

def stream_sample_rate(roach_state):
    if roach_state['heterodyne']:
        # In the heterodyne case, the number of complex samples per FFT is just num_filterbank_channels.
        return roach_state['adc_sample_rate'] / roach_state['num_filterbank_channels']
    else:
        # In the baseband case, the number of real samples per FFT is 2 * num_filterbank_channels.
        return roach_state['adc_sample_rate'] / (2 * roach_state['num_filterbank_channels'])

def modulation_period_samples(roach_state):
    if roach_state.modulation_output != 2:
        return 0
    else:
        if roach_state.heterodyne:
            return 2**(roach_state.modulation_rate+1)
        else:
            return 2**(roach_state.modulation_rate)

def packet_phase(packet_sequence_number, offset_frequencies, num_channels, maximum_period, num_filterbank_channels):
    samples_per_packet = 1024  # hardcoded in roach FPGA design. 4096 payload bytes and 4 bytes per sample
    samples_per_channel_per_packet = samples_per_packet // num_channels
    # If each packet holds at least as many samples per channel as the longest possible demodulated period,
    # then no packet phase correction is needed
    if samples_per_channel_per_packet >= maximum_period:
        return np.zeros_like(offset_frequencies)
    fft_frame_number = packet_sequence_number // num_filterbank_channels
    sample_number = fft_frame_number * samples_per_channel_per_packet
    reduced_sample_number = sample_number % maximum_period
    phase = -2*np.pi*reduced_sample_number*offset_frequencies
    return phase

def packet_phase_original(seq_no,offset_frequencies,nchan,nfft,ns):
    packet_bins = 1024    #this is hardcoded from the roach. number of fft bins that fit in 1 udp packet
    packet_counts = nfft * packet_bins
    chan_counts = packet_counts // nchan
    shift = int(np.log2(chan_counts)) - 1
    modn = ns / chan_counts
    if modn == 0:
        modn = 1
    multy = ns / nfft
    seq_no = seq_no >> shift
    seq_no %= modn
    return np.exp(-1j * 2. * np.pi * seq_no * offset_frequencies * multy / modn)

def tone_offset_frequency(tone_bin,tone_num_samples,fft_bin,nfft):
    k = tone_bin
    m = fft_bin
    nfft = nfft
    ns = tone_num_samples
    return nfft * (k / float(ns)) - m

def get_offset_frequencies_period(offset_frequencies):
    period = 1
    while not np.all(np.round(offset_frequencies*period)==offset_frequencies*period):
        period = period * 2
    return period<|endoftext|>


<commit_before>from pylm.parts.core import zmq_context, Router
from pylm.standalone.clients import SubscribedClient
from pylm.parts.utils import CacheService
from pylm.parts.services import PullService, PubService
from pylm.persistence.kv import DictDB
import concurrent.futures
import traceback
import logging
import sys
import zmq

pull_address = 'inproc://pull'
pub_address = 'inproc://pub'
db_address = 'inproc://db'
broker_address = 'inproc://broker'

def test_get_config():
    cache = DictDB()
    cache.set('name', b'master')
    cache.set('pub_address', pub_address.encode('utf-8'))
    cache.set('pull_address', pull_address.encode('utf-8'))

    cache = CacheService('db', db_address,
                         cache=cache,
                         logger=logging,
                         messages=3,
                         )

    def boot_client():
        client = SubscribedClient('master',
                                  db_address,
                                  pipeline=None)
        return client.push_address, client.sub_address

    def broker():
        socket = zmq_context.socket(zmq.ROUTER)
        socket.bind(broker_address)

    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        results = [
            executor.submit(cache.start),
            executor.submit(boot_client),
            executor.submit(broker)
        ]

        # This works because servers do not return values.
        for i, future in enumerate(concurrent.futures.as_completed(results)):
            try:
                result = future.result()
                print(result)

            except Exception as exc:
                print(exc)
                lines = traceback.format_exception(*sys.exc_info())
                print(lines)

        assert i == 2

def test_send_job():
    cache = DictDB()
    cache.set('name', b'master')
    cache.set('pub_address', pub_address.encode('utf-8'))
    cache.set('pull_address', pull_address.encode('utf-8'))

    cache_service = CacheService('db', db_address,
                                 cache=cache,
                                 logger=logging,
                                 messages=3,
                                 )

    puller = PullService('puller',
                         pull_address,
                         palm=True,
                         logger=logging,
                         cache=cache,
                         messages=1)

    publisher = PubService('publisher',
                           pub_address,
                           logger=logging,
                           cache=cache,
                           palm=True,
                           messages=1)

    def client_job():
        client = SubscribedClient('master',
                                  db_address,
                                  pipeline=None)
        return [r for r in client.job('master.something', [b'1'], messages=1)]

    def broker():
        socket = zmq_context.socket(zmq.ROUTER)
        socket.bind(broker_address)
        message = socket.recv_multipart()
        # Unblock. Here you see why the actual router is complicated.
        socket.send_multipart(message)
        socket.send_multipart([b'publisher', b'', message[2]])
        socket.close()

        return b'router'

    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        results = [
            executor.submit(cache_service.start),
            executor.submit(puller.start),
            executor.submit(publisher.start),
            executor.submit(client_job),
            executor.submit(broker)
        ]

        # This works because servers do not return values.
        for i, future in enumerate(concurrent.futures.as_completed(results)):
            try:
                result = future.result()
                print(result)

            except Exception as exc:
                print(exc)
                lines = traceback.format_exception(*sys.exc_info())
                print(*lines)

        assert i == 4

def test_multiple_clients():
    cache = DictDB()
    cache.set('name', b'master')
    cache.set('pub_address', pub_address.encode('utf-8'))
    cache.set('pull_address', pull_address.encode('utf-8'))

    router = Router(logger=logging, cache=cache, messages=4)
    router.register_inbound('puller', route='publisher')
    router.register_outbound('publisher')

    cache_service = CacheService('db', db_address,
                                 cache=cache,
                                 logger=logging,
                                 messages=6,
                                 )

    puller = PullService('puller',
                         pull_address,
                         broker_address=router.inbound_address,
                         palm=True,
                         logger=logging,
                         cache=cache,
                         messages=4)

    publisher = PubService('publisher',
                           pub_address,
                           broker_address=router.outbound_address,
                           logger=logging,
                           cache=cache,
                           palm=True,
                           messages=4)

    def client1_job():
        client = SubscribedClient('master',
                                  db_address,
                                  pipeline=None)
        return [r for r in client.job('master.something', [b'1', b'2'], messages=2)]

    def client2_job():
        client = SubscribedClient('master',
                                  db_address,
                                  pipeline=None)
        return [r for r in client.job('master.something', [b'3', b'4'], messages=2)]

    with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor:
        results = [
            executor.submit(cache_service.start),
            executor.submit(puller.start),
            executor.submit(publisher.start),
            executor.submit(router.start),
            executor.submit(client1_job),
            executor.submit(client2_job),
        ]

        # This works because servers do not return values.
        for i, future in enumerate(concurrent.futures.as_completed(results)):
            try:
                result = future.result()
                print(result)

            except Exception as exc:
                print(exc)
                lines = traceback.format_exception(*sys.exc_info())
                print(*lines)

        assert i == 5

if __name__ == '__main__':
    test_get_config()
    test_send_job()
    test_multiple_clients()<commit_msg>Now the test asserts that the results are correct too.<commit_after>from pylm.parts.core import zmq_context, Router
from pylm.parts.messages_pb2 import PalmMessage
from pylm.standalone.clients import SubscribedClient
from pylm.parts.utils import CacheService
from pylm.parts.services import PullService, PubService
from pylm.persistence.kv import DictDB
import concurrent.futures
import traceback
import logging
import sys
import zmq

pull_address = 'inproc://pull'
pub_address = 'inproc://pub'
db_address = 'inproc://db'
broker_address = 'inproc://broker'

def test_get_config():
    cache = DictDB()
    cache.set('name', b'master')
    cache.set('pub_address', pub_address.encode('utf-8'))
    cache.set('pull_address', pull_address.encode('utf-8'))

    cache = CacheService('db', db_address,
                         cache=cache,
                         logger=logging,
                         messages=3,
                         )

    def boot_client():
        client = SubscribedClient('master',
                                  db_address,
                                  pipeline=None)
        return client.push_address, client.sub_address

    def broker():
        socket = zmq_context.socket(zmq.ROUTER)
        socket.bind(broker_address)

    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        results = [
            executor.submit(cache.start),
            executor.submit(boot_client),
            executor.submit(broker)
        ]

        # This works because servers do not return values.
        for i, future in enumerate(concurrent.futures.as_completed(results)):
            try:
                result = future.result()
                print(result)

            except Exception as exc:
                print(exc)
                lines = traceback.format_exception(*sys.exc_info())
                print(lines)

        assert i == 2

def test_send_job():
    cache = DictDB()
    cache.set('name', b'master')
    cache.set('pub_address', pub_address.encode('utf-8'))
    cache.set('pull_address', pull_address.encode('utf-8'))

    cache_service = CacheService('db', db_address,
                                 cache=cache,
                                 logger=logging,
                                 messages=3,
                                 )

    puller = PullService('puller',
                         pull_address,
                         palm=True,
                         logger=logging,
                         cache=cache,
                         messages=1)

    publisher = PubService('publisher',
                           pub_address,
                           logger=logging,
                           cache=cache,
                           palm=True,
                           messages=1)

    def client_job():
        client = SubscribedClient('master',
                                  db_address,
                                  pipeline=None)
        return [r for r in client.job('master.something', [b'1'], messages=1)]

    def broker():
        socket = zmq_context.socket(zmq.ROUTER)
        socket.bind(broker_address)
        message = socket.recv_multipart()
        # Unblock. Here you see why the actual router is complicated.
        socket.send_multipart(message)
        socket.send_multipart([b'publisher', b'', message[2]])
        socket.close()

        return b'router'

    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        results = [
            executor.submit(cache_service.start),
            executor.submit(puller.start),
            executor.submit(publisher.start),
            executor.submit(client_job),
            executor.submit(broker)
        ]

        # This works because servers do not return values.
        for i, future in enumerate(concurrent.futures.as_completed(results)):
            try:
                result = future.result()
                print(result)

            except Exception as exc:
                print(exc)
                lines = traceback.format_exception(*sys.exc_info())
                print(*lines)

        assert i == 4

def test_multiple_clients():
    cache = DictDB()
    cache.set('name', b'master')
    cache.set('pub_address', pub_address.encode('utf-8'))
    cache.set('pull_address', pull_address.encode('utf-8'))

    router = Router(logger=logging, cache=cache, messages=4)
    router.register_inbound('puller', route='publisher')
    router.register_outbound('publisher')

    cache_service = CacheService('db', db_address,
                                 cache=cache,
                                 logger=logging,
                                 messages=6,
                                 )

    puller = PullService('puller',
                         pull_address,
                         broker_address=router.inbound_address,
                         palm=True,
                         logger=logging,
                         cache=cache,
                         messages=4)

    publisher = PubService('publisher',
                           pub_address,
                           broker_address=router.outbound_address,
                           logger=logging,
                           cache=cache,
                           palm=True,
                           messages=4)

    def client1_job():
        client = SubscribedClient('master',
                                  db_address,
                                  pipeline=None)
        return [r for r in client.job('master.something', [b'1', b'2'], messages=2)]

    def client2_job():
        client = SubscribedClient('master',
                                  db_address,
                                  pipeline=None)
        return [r for r in client.job('master.something', [b'3', b'4'], messages=2)]

    with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor:
        results = [
            executor.submit(cache_service.start),
            executor.submit(puller.start),
            executor.submit(publisher.start),
            executor.submit(router.start),
            executor.submit(client1_job),
            executor.submit(client2_job),
        ]

        # This works because servers do not return values.
        for i, future in enumerate(concurrent.futures.as_completed(results)):
            try:
                result = future.result()
                if type(result) == list:
                    got = []
                    for r in result:
                        message = PalmMessage()
                        message.ParseFromString(r)
                        got.append(message.payload)

                    assert got == [b'1', b'2'] or got == [b'3', b'4']

            except Exception as exc:
                print(exc)
                lines = traceback.format_exception(*sys.exc_info())
                print(*lines)

    assert i == 5

if __name__ == '__main__':
    test_get_config()
    test_send_job()
    test_multiple_clients()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2015  ADHOC SA  (http://www.adhoc.com.ar)
#    All Rights Reserved.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    "name": "Argentina - Facturación y documentos AFIP",
    'version': '3.1',
    'category': 'Localization/Argentina',
    'sequence': 14,
    'author':  'ADHOC SA',
    'website': 'www.adhoc.com.ar',
    'license': 'AGPL-3',
    'summary': '',
    'description': """
Argentina - Facturación y documentos AFIP
=========================================
Para actualizar el plan de cuentas de datos demo:
-------------------------------------------------
* borrar la parte de account.chart
* reemplazar ".template" por nada
* reemplazar <!-- <field name="company_id" ref=""/> --> por <field name="company_id" ref="company_ri"/>
* reemplazar <field name="chart_template_id" ref="ri_l10nAR_chart_template"/> por <!-- <field name="chart_template_id" ref="ri_l10nAR_chart_template"/> -->
""",
    'depends': [
          'account',
          'l10n_ar_base_vat',
        ],
    'external_dependencies': {
    },
    'data': [
          'security/l10n_ar_invoice_security.xml',
          'data/data_account_type.xml',
          'data/account_financial_report_data.xml',
          'data/responsability.xml',
          'data/afip.incoterm.csv',
          'data/afip.document_letter.csv',
          'data/afip.document_class.csv',
          'data/document_type.xml',
          'data/partner.xml',
          'data/country.xml',
          'data/res.currency.csv',
          'data/fiscal_position.xml',
          'data/res.country.csv',
          'data/product.uom.csv',
          'data/res_partner.xml',
          # 'data/decimal_precision_data.xml', probando si no es necesario
          'wizard/account_journal_create_wizard_view.xml',
          'view/partner_view.xml',
          'view/company_view.xml',
          'view/country_view.xml',
          'view/afip_menuitem.xml',
          'view/product_uom_view.xml',
          'view/fiscal_position_view.xml',
          'view/afip_document_letter_view.xml',
          'view/afip_document_type_view.xml',
          'view/afip_responsability_view.xml',
          'view/afip_document_class_view.xml',
          'view/afip_point_of_sale_view.xml',
          'view/account_journal_afip_document_class_view.xml',
          'view/account_tax_code_view.xml',
          'view/tax_code_template_view.xml',
          'view/journal_view.xml',
          'view/invoice_view.xml',
          'view/account_move_view.xml',
          'view/account_move_line_view.xml',
          'view/currency_view.xml',
          'view/afip_incoterm_view.xml',
          'invoice_template.xml',
          'security/ir.model.access.csv',
          'security/l10n_ar_invoice_security.xml',
      ],
    'demo': [
          'demo/res_country_state_demo.xml',
          'demo/partner_demo.xml',
          'demo/company_demo.xml',
          # TODO tal vez sea mejor estos dos ejecutarlos con una accion ya que al hacer un init dan error porque ya existen movimientos
          'demo/fiscal_year_mono_demo.xml',
          'demo/fiscal_year_ri_demo.xml',
          'demo/account_chart_respinsc.xml',
          'demo/account_demo.xml',
          # 'demo/ri_purchase_invoice_demo.xml',
          # 'demo/ri_sale_invoice_demo.xml',
    ],
    'test': [
    ],
    'installable': True,
    'auto_install': True,
    'application': False,
 }
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>IMP add ars as bas currency on installation<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2015  ADHOC SA  (http://www.adhoc.com.ar)
#    All Rights Reserved.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    "name": "Argentina - Facturación y documentos AFIP",
    'version': '3.1',
    'category': 'Localization/Argentina',
    'sequence': 14,
    'author':  'ADHOC SA',
    'website': 'www.adhoc.com.ar',
    'license': 'AGPL-3',
    'summary': '',
    'description': """
Argentina - Facturación y documentos AFIP
=========================================
Para actualizar el plan de cuentas de datos demo:
-------------------------------------------------
* borrar la parte de account.chart
* reemplazar ".template" por nada
* reemplazar <!-- <field name="company_id" ref=""/> --> por <field name="company_id" ref="company_ri"/>
* reemplazar <field name="chart_template_id" ref="ri_l10nAR_chart_template"/> por <!-- <field name="chart_template_id" ref="ri_l10nAR_chart_template"/> -->
""",
    'depends': [
          'account',
          'l10n_ar_base_vat',
        ],
    'external_dependencies': {
    },
    'data': [
          'security/l10n_ar_invoice_security.xml',
          'data/data_account_type.xml',
          'data/account_financial_report_data.xml',
          'data/responsability.xml',
          'data/afip.incoterm.csv',
          'data/afip.document_letter.csv',
          'data/afip.document_class.csv',
          'data/document_type.xml',
          'data/partner.xml',
          'data/country.xml',
          'data/res.currency.csv',
          'data/fiscal_position.xml',
          'data/res.country.csv',
          'data/product.uom.csv',
          'data/res_partner.xml',
          'data/res_currency.xml',
          # 'data/decimal_precision_data.xml', probando si no es necesario
          'wizard/account_journal_create_wizard_view.xml',
          'view/partner_view.xml',
          'view/company_view.xml',
          'view/country_view.xml',
          'view/afip_menuitem.xml',
          'view/product_uom_view.xml',
          'view/fiscal_position_view.xml',
          'view/afip_document_letter_view.xml',
          'view/afip_document_type_view.xml',
          'view/afip_responsability_view.xml',
          'view/afip_document_class_view.xml',
          'view/afip_point_of_sale_view.xml',
          'view/account_journal_afip_document_class_view.xml',
          'view/account_tax_code_view.xml',
          'view/tax_code_template_view.xml',
          'view/journal_view.xml',
          'view/invoice_view.xml',
          'view/account_move_view.xml',
          'view/account_move_line_view.xml',
          'view/currency_view.xml',
          'view/afip_incoterm_view.xml',
          'invoice_template.xml',
          'security/ir.model.access.csv',
          'security/l10n_ar_invoice_security.xml',
      ],
    'demo': [
          'demo/res_country_state_demo.xml',
          'demo/partner_demo.xml',
          'demo/company_demo.xml',
          # TODO tal vez sea mejor estos dos ejecutarlos con una accion ya que al hacer un init dan error porque ya existen movimientos
          'demo/fiscal_year_mono_demo.xml',
          'demo/fiscal_year_ri_demo.xml',
          'demo/account_chart_respinsc.xml',
          'demo/account_demo.xml',
          # 'demo/ri_purchase_invoice_demo.xml',
          # 'demo/ri_sale_invoice_demo.xml',
    ],
    'test': [
    ],
    'installable': True,
    'auto_install': True,
    'application': False,
 }
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before># Copyright (C) 2012 - TODAY  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

import logging

from odoo import _, api, fields, models
from odoo.exceptions import ValidationError

_logger = logging.getLogger(__name__)

try:
    from erpbrasil.base import misc
    from erpbrasil.base.fiscal import cnpj_cpf, ie
except ImportError:
    _logger.error("erpbrasil.base library not installed")

class Lead(models.Model):
    """ CRM Lead Case """

    _name = "crm.lead"
    _inherit = [_name, "l10n_br_base.party.mixin"]

    cnpj = fields.Char(string="CNPJ")

    street_name = fields.Char(string="Street Name")

    street_number = fields.Char(string="Street Number")

    name_surname = fields.Char(
        string="Name and Surname", help="Name used in fiscal documents"
    )

    cpf = fields.Char(string="CPF")

    @api.onchange("contact_name")
    def _onchange_contact_name(self):
        if not self.name_surname:
            self.name_surname = self.contact_name

    @api.constrains("cnpj")
    def _check_cnpj(self):
        for record in self:
            country_code = record.country_id.code or ""
            if record.cnpj and country_code.upper() == "BR":
                cnpj = misc.punctuation_rm(record.cnpj)
                if not cnpj_cpf.validar(cnpj):
                    raise ValidationError(_("Invalid CNPJ!"))
            return True

    @api.constrains("cpf")
    def _check_cpf(self):
        for record in self:
            country_code = record.country_id.code or ""
            if record.cpf and country_code.upper() == "BR":
                cpf = misc.punctuation_rm(record.cpf)
                if not cnpj_cpf.validar(cpf):
                    raise ValidationError(_("Invalid CPF!"))
            return True

    @api.constrains("inscr_est")
    def _check_ie(self):
        """Checks if company register number in field insc_est is valid,
        this method call others methods because this validation is State wise

        :Return: True or False.
        """
        for record in self:
            result = True
            if record.inscr_est and record.cnpj and record.state_id:
                state_code = record.state_id.code or ""
                uf = state_code.lower()
                result = ie.validar(uf, record.inscr_est)
            if not result:
                raise ValidationError(_("Invalid State Tax Number!"))

    @api.onchange("cnpj", "country_id")
    def _onchange_cnpj(self):
        self.cnpj = cnpj_cpf.formata(self.cnpj)

    @api.onchange("cpf", "country_id")
    def _onchange_mask_cpf(self):
        self.cpf = cnpj_cpf.formata(self.cpf)

    @api.onchange("city_id")
    def _onchange_city_id(self):
        """Ao alterar o campo l10n_br_city_id que é um campo relacional
        com o l10n_br_base.city que são os municípios do IBGE, copia o nome
        do município para o campo city que é o campo nativo do módulo base
        para manter a compatibilidade entre os demais módulos que usam o
        campo city.
        param int l10n_br_city_id: id do l10n_br_city_id digitado.
        return: dicionário com o nome e id do município.
        """
        if self.city_id:
            self.city = self.city_id.name

    @api.onchange("zip")
    def _onchange_zip(self):
        self.zip = misc.format_zipcode(self.zip, self.country_id.code)

    @api.onchange("partner_id")
    def _onchange_partner_id(self):
        result = super(Lead, self)._prepare_values_from_partner(self.partner_id)

        if self.partner_id:
            result["street_number"] = self.partner_id.street_number
            result["district"] = self.partner_id.district
            result["city_id"] = self.partner_id.city_id.id
            if self.partner_id.is_company:
                result["legal_name"] = self.partner_id.legal_name
                result["cnpj"] = self.partner_id.cnpj_cpf
                result["inscr_est"] = self.partner_id.inscr_est
                result["suframa"] = self.partner_id.suframa
            else:
                result["cpf"] = self.partner_id.cnpj_cpf
                result["name_surname"] = self.partner_id.legal_name
        self.update(result)

    def _prepare_customer_values(self, name, is_company, parent_id=False):
        """Extract data from lead to create a partner.
        :param name : furtur name of the partner
        :param is_company : True if the partner is a company
        :param parent_id : id of the parent partner (False if no parent)
        :return: dictionary of values to give at res_partner.create()
        """
        values = super()._prepare_customer_values(name, is_company, parent_id)
        values.update(
            {
                "legal_name": self.legal_name if is_company else self.name_surname,
                "street_number": self.street_number,
                "district": self.district,
                "city_id": self.city_id.id,
            }
        )
        if is_company:
            values.update(
                {
                    "cnpj_cpf": self.cnpj,
                    "inscr_est": self.inscr_est,
                    "inscr_mun": self.inscr_mun,
                    "suframa": self.suframa,
                }
            )
        else:
            values.update(
                {
                    "cnpj_cpf": self.cpf,
                    "inscr_est": self.rg,
                }
            )
        return values
<commit_msg>[FIX] _onchange_city_id<commit_after># Copyright (C) 2012 - TODAY  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

import logging

from odoo import _, api, fields, models
from odoo.exceptions import ValidationError

_logger = logging.getLogger(__name__)

try:
    from erpbrasil.base import misc
    from erpbrasil.base.fiscal import cnpj_cpf, ie
except ImportError:
    _logger.error("erpbrasil.base library not installed")

class Lead(models.Model):
    """ CRM Lead Case """

    _name = "crm.lead"
    _inherit = [_name, "l10n_br_base.party.mixin"]

    cnpj = fields.Char(string="CNPJ")

    street_name = fields.Char(string="Street Name")

    street_number = fields.Char(string="Street Number")

    name_surname = fields.Char(
        string="Name and Surname", help="Name used in fiscal documents"
    )

    cpf = fields.Char(string="CPF")

    @api.onchange("contact_name")
    def _onchange_contact_name(self):
        if not self.name_surname:
            self.name_surname = self.contact_name

    @api.constrains("cnpj")
    def _check_cnpj(self):
        for record in self:
            country_code = record.country_id.code or ""
            if record.cnpj and country_code.upper() == "BR":
                cnpj = misc.punctuation_rm(record.cnpj)
                if not cnpj_cpf.validar(cnpj):
                    raise ValidationError(_("Invalid CNPJ!"))
            return True

    @api.constrains("cpf")
    def _check_cpf(self):
        for record in self:
            country_code = record.country_id.code or ""
            if record.cpf and country_code.upper() == "BR":
                cpf = misc.punctuation_rm(record.cpf)
                if not cnpj_cpf.validar(cpf):
                    raise ValidationError(_("Invalid CPF!"))
            return True

    @api.constrains("inscr_est")
    def _check_ie(self):
        """Checks if company register number in field insc_est is valid,
        this method call others methods because this validation is State wise

        :Return: True or False.
        """
        for record in self:
            result = True
            if record.inscr_est and record.cnpj and record.state_id:
                state_code = record.state_id.code or ""
                uf = state_code.lower()
                result = ie.validar(uf, record.inscr_est)
            if not result:
                raise ValidationError(_("Invalid State Tax Number!"))

    @api.onchange("cnpj", "country_id")
    def _onchange_cnpj(self):
        self.cnpj = cnpj_cpf.formata(self.cnpj)

    @api.onchange("cpf", "country_id")
    def _onchange_mask_cpf(self):
        self.cpf = cnpj_cpf.formata(self.cpf)

    @api.onchange("city_id")
    def _onchange_city_id(self):
        """Ao alterar o campo l10n_br_city_id que é um campo relacional
        com o l10n_br_base.city que são os municípios do IBGE, copia o nome
        do município para o campo city que é o campo nativo do módulo base
        para manter a compatibilidade entre os demais módulos que usam o
        campo city.
        param int l10n_br_city_id: id do l10n_br_city_id digitado.
        return: dicionário com o nome e id do município.
        """
        if self.city_id:
            self.city = self.city_id.name
        elif self.partner_id.city_id:
            self.city_id = self.partner_id.city_id
            self.city = self.partner_id.city_id.name

    @api.onchange("zip")
    def _onchange_zip(self):
        self.zip = misc.format_zipcode(self.zip, self.country_id.code)

    @api.onchange("partner_id")
    def _onchange_partner_id(self):
        result = super(Lead, self)._prepare_values_from_partner(self.partner_id)

        if self.partner_id:
            result["street_number"] = self.partner_id.street_number
            result["district"] = self.partner_id.district
            result["city_id"] = self.partner_id.city_id.id
            if self.partner_id.is_company:
                result["legal_name"] = self.partner_id.legal_name
                result["cnpj"] = self.partner_id.cnpj_cpf
                result["inscr_est"] = self.partner_id.inscr_est
                result["suframa"] = self.partner_id.suframa
            else:
                result["cpf"] = self.partner_id.cnpj_cpf
                result["name_surname"] = self.partner_id.legal_name
        self.update(result)

    def _prepare_customer_values(self, name, is_company, parent_id=False):
        """Extract data from lead to create a partner.
        :param name : furtur name of the partner
        :param is_company : True if the partner is a company
        :param parent_id : id of the parent partner (False if no parent)
        :return: dictionary of values to give at res_partner.create()
        """
        values = super()._prepare_customer_values(name, is_company, parent_id)
        values.update(
            {
                "legal_name": self.legal_name if is_company else self.name_surname,
                "street_number": self.street_number,
                "district": self.district,
                "city_id": self.city_id.id,
            }
        )
        if is_company:
            values.update(
                {
                    "cnpj_cpf": self.cnpj,
                    "inscr_est": self.inscr_est,
                    "inscr_mun": self.inscr_mun,
                    "suframa": self.suframa,
                }
            )
        else:
            values.update(
                {
                    "cnpj_cpf": self.cpf,
                    "inscr_est": self.rg,
                }
            )
        return values
<|endoftext|>


<commit_before># Copyright (C) 2013  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

{
    "name": "Módulo fiscal brasileiro",
    "summary": "Brazilian fiscal core module.",
    "category": "Localisation",
    "license": "AGPL-3",
    "author": "Akretion, Odoo Community Association (OCA)",
    "maintainers": ["renatonlima"],
    "website": "https://github.com/OCA/l10n-brazil",
    "development_status": "Production/Stable",
    "version": "14.0.7.4.0",
    "depends": [
        "uom",
        "product",
        "l10n_br_base",
    ],
    "data": [
        # security
        "security/fiscal_security.xml",
        "security/ir.model.access.csv",
        # Data
        # Some data is being loaded via post_init_hook in hook file
        "data/l10n_br_fiscal_email_template.xml",
        "data/l10n_br_fiscal_data.xml",
        "data/uom_data.xml",
        "data/product_data.xml",
        "data/partner_profile_data.xml",
        "data/res_partner_data.xml",
        "data/l10n_br_fiscal.tax.group.csv",
        "data/l10n_br_fiscal.icms.relief.csv",
        "data/l10n_br_fiscal.document.type.csv",
        "data/l10n_br_fiscal.product.genre.csv",
        "data/l10n_br_fiscal.cst.csv",
        "data/l10n_br_fiscal.tax.csv",
        "data/l10n_br_fiscal.tax.pis.cofins.csv",
        "data/l10n_br_fiscal_server_action.xml",
        "data/ir_cron.xml",
        "data/l10n_br_fiscal_comment_data.xml",
        # Views
        "views/cnae_view.xml",
        "views/cfop_view.xml",
        "views/comment_view.xml",
        "views/cst_view.xml",
        "views/tax_group_view.xml",
        "views/tax_view.xml",
        "views/tax_definition_view.xml",
        "views/icms_regulation_view.xml",
        "views/icms_relief_view.xml",
        "views/tax_pis_cofins_view.xml",
        "views/tax_pis_cofins_base_view.xml",
        "views/tax_pis_cofins_credit_view.xml",
        "views/tax_ipi_control_seal_view.xml",
        "views/tax_ipi_guideline_view.xml",
        "views/tax_ipi_guideline_class_view.xml",
        "views/ncm_view.xml",
        "views/nbm_view.xml",
        "views/nbs_view.xml",
        "views/service_type_view.xml",
        "views/cest_view.xml",
        "views/product_genre_view.xml",
        "views/document_type_view.xml",
        "views/document_serie_view.xml",
        "views/document_email_view.xml",
        "views/certificate_view.xml",
        "views/simplified_tax_view.xml",
        "views/simplified_tax_range_view.xml",
        "views/operation_view.xml",
        "views/operation_line_view.xml",
        "views/product_template_view.xml",
        "views/product_product_view.xml",
        "views/tax_estimate_view.xml",
        "views/partner_profile_view.xml",
        "views/res_partner_view.xml",
        "views/res_company_view.xml",
        "views/document_view.xml",
        "views/document_related_view.xml",
        "views/document_line_view.xml",
        "views/document_fiscal_line_mixin_view.xml",
        "views/res_config_settings_view.xml",
        "views/subsequent_operation_view.xml",
        "views/subsequent_document_view.xml",
        "views/uom_uom.xml",
        "views/closing.xml",
        "views/invalidate_number_view.xml",
        "views/city_taxation_code.xml",
        #        "views/mdfe/mdfe_views.xml",
        #        "views/dfe/dfe_views.xml",
        "views/operation_dashboard_view.xml",
        "views/document_event_view.xml",
        # Wizards
        "wizards/document_cancel_wizard.xml",
        "wizards/document_correction_wizard.xml",
        "wizards/document_status_wizard.xml",
        "wizards/invalidate_number_wizard.xml",
        # Actions
        "views/l10n_br_fiscal_action.xml",
        # Menus
        "views/l10n_br_fiscal_menu.xml",
    ],
    "demo": [
        # Some demo data is being loaded via post_init_hook in hook file
    ],
    "post_init_hook": "post_init_hook",
    "installable": True,
    "application": True,
    "auto_install": False,
    "external_dependencies": {
        "python": [
            "erpbrasil.base",
            "erpbrasil.assinatura",
        ]
    },
}
<commit_msg>l10n_br_fiscal 14.0.7.4.1<commit_after># Copyright (C) 2013  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

{
    "name": "Módulo fiscal brasileiro",
    "summary": "Brazilian fiscal core module.",
    "category": "Localisation",
    "license": "AGPL-3",
    "author": "Akretion, Odoo Community Association (OCA)",
    "maintainers": ["renatonlima"],
    "website": "https://github.com/OCA/l10n-brazil",
    "development_status": "Production/Stable",
    "version": "14.0.7.4.1",
    "depends": [
        "uom",
        "product",
        "l10n_br_base",
    ],
    "data": [
        # security
        "security/fiscal_security.xml",
        "security/ir.model.access.csv",
        # Data
        # Some data is being loaded via post_init_hook in hook file
        "data/l10n_br_fiscal_email_template.xml",
        "data/l10n_br_fiscal_data.xml",
        "data/uom_data.xml",
        "data/product_data.xml",
        "data/partner_profile_data.xml",
        "data/res_partner_data.xml",
        "data/l10n_br_fiscal.tax.group.csv",
        "data/l10n_br_fiscal.icms.relief.csv",
        "data/l10n_br_fiscal.document.type.csv",
        "data/l10n_br_fiscal.product.genre.csv",
        "data/l10n_br_fiscal.cst.csv",
        "data/l10n_br_fiscal.tax.csv",
        "data/l10n_br_fiscal.tax.pis.cofins.csv",
        "data/l10n_br_fiscal_server_action.xml",
        "data/ir_cron.xml",
        "data/l10n_br_fiscal_comment_data.xml",
        # Views
        "views/cnae_view.xml",
        "views/cfop_view.xml",
        "views/comment_view.xml",
        "views/cst_view.xml",
        "views/tax_group_view.xml",
        "views/tax_view.xml",
        "views/tax_definition_view.xml",
        "views/icms_regulation_view.xml",
        "views/icms_relief_view.xml",
        "views/tax_pis_cofins_view.xml",
        "views/tax_pis_cofins_base_view.xml",
        "views/tax_pis_cofins_credit_view.xml",
        "views/tax_ipi_control_seal_view.xml",
        "views/tax_ipi_guideline_view.xml",
        "views/tax_ipi_guideline_class_view.xml",
        "views/ncm_view.xml",
        "views/nbm_view.xml",
        "views/nbs_view.xml",
        "views/service_type_view.xml",
        "views/cest_view.xml",
        "views/product_genre_view.xml",
        "views/document_type_view.xml",
        "views/document_serie_view.xml",
        "views/document_email_view.xml",
        "views/certificate_view.xml",
        "views/simplified_tax_view.xml",
        "views/simplified_tax_range_view.xml",
        "views/operation_view.xml",
        "views/operation_line_view.xml",
        "views/product_template_view.xml",
        "views/product_product_view.xml",
        "views/tax_estimate_view.xml",
        "views/partner_profile_view.xml",
        "views/res_partner_view.xml",
        "views/res_company_view.xml",
        "views/document_view.xml",
        "views/document_related_view.xml",
        "views/document_line_view.xml",
        "views/document_fiscal_line_mixin_view.xml",
        "views/res_config_settings_view.xml",
        "views/subsequent_operation_view.xml",
        "views/subsequent_document_view.xml",
        "views/uom_uom.xml",
        "views/closing.xml",
        "views/invalidate_number_view.xml",
        "views/city_taxation_code.xml",
        #        "views/mdfe/mdfe_views.xml",
        #        "views/dfe/dfe_views.xml",
        "views/operation_dashboard_view.xml",
        "views/document_event_view.xml",
        # Wizards
        "wizards/document_cancel_wizard.xml",
        "wizards/document_correction_wizard.xml",
        "wizards/document_status_wizard.xml",
        "wizards/invalidate_number_wizard.xml",
        # Actions
        "views/l10n_br_fiscal_action.xml",
        # Menus
        "views/l10n_br_fiscal_menu.xml",
    ],
    "demo": [
        # Some demo data is being loaded via post_init_hook in hook file
    ],
    "post_init_hook": "post_init_hook",
    "installable": True,
    "application": True,
    "auto_install": False,
    "external_dependencies": {
        "python": [
            "erpbrasil.base",
            "erpbrasil.assinatura",
        ]
    },
}
<|endoftext|>


<commit_before>from __future__ import print_function

import pytest
pd = pytest.importorskip('pandas')
import numpy as np

from eli5 import format_as_dataframes, format_as_dataframe, format_as_text
from eli5.base import (
    Explanation, TargetExplanation, FeatureWeight, FeatureWeights,
    FeatureImportances, TransitionFeatureWeights,
)

def test_targets():
    expl = Explanation(
        estimator='some estimator',
        targets=[
            TargetExplanation(
                'y', feature_weights=FeatureWeights(
                    pos=[FeatureWeight('a', 13),
                         FeatureWeight('b', 5)],
                    neg=[FeatureWeight('neg1', -10),
                         FeatureWeight('neg2', -1)],
                )),
            TargetExplanation(
                'y2', feature_weights=FeatureWeights(
                    pos=[FeatureWeight('f', 1)],
                    neg=[],
                )),
        ],
    )
    df_dict = format_as_dataframes(expl)
    assert isinstance(df_dict, dict)
    assert list(df_dict) == ['targets']
    df = df_dict['targets']
    expected_df = pd.DataFrame(
        {'weight': [13, 5, -1, -10, 1]},
        index=pd.MultiIndex.from_tuples(
            [('y', 'a'), ('y', 'b'), ('y', 'neg2'), ('y', 'neg1'),
             ('y2', 'f')], names=['target', 'feature']))
    print(df, expected_df, sep='\n')
    assert expected_df.equals(df)

    single_df = format_as_dataframe(expl)
    assert expected_df.equals(single_df)

def test_targets_with_value():
    expl = Explanation(
        estimator='some estimator',
        targets=[
            TargetExplanation(
                'y', feature_weights=FeatureWeights(
                    pos=[FeatureWeight('a', 13, value=1),
                         FeatureWeight('b', 5, value=2)],
                    neg=[FeatureWeight('neg1', -10, value=3),
                         FeatureWeight('neg2', -1, value=4)],
                )),
            TargetExplanation(
                'y2', feature_weights=FeatureWeights(
                    pos=[FeatureWeight('f', 1, value=5)],
                    neg=[],
                )),
        ],
    )
    df = format_as_dataframe(expl)
    expected_df = pd.DataFrame(
        {'weight': [13, 5, -1, -10, 1],
         'value': [1, 2, 4, 3, 5]},
        columns=['weight', 'value'],
        index=pd.MultiIndex.from_tuples(
            [('y', 'a'), ('y', 'b'), ('y', 'neg2'), ('y', 'neg1'),
             ('y2', 'f')], names=['target', 'feature']))
    print(df, expected_df, sep='\n')
    assert expected_df.equals(df)

def test_feature_importances():
    expl = Explanation(
        estimator='some estimator',
        feature_importances=FeatureImportances(
            importances=[
                FeatureWeight('a', 1),
                FeatureWeight('b', 2),
            ],
            remaining=10,
        )
    )
    df_dict = format_as_dataframes(expl)
    assert isinstance(df_dict, dict)
    assert list(df_dict) == ['feature_importances']
    df = df_dict['feature_importances']
    expected_df = pd.DataFrame({'weight': [1, 2]}, index=['a', 'b'])
    print(df, expected_df, sep='\n')
    assert expected_df.equals(df)

    single_df = format_as_dataframe(expl)
    assert expected_df.equals(single_df)

def test_feature_importances_std():
    expl = Explanation(
        estimator='some estimator',
        feature_importances=FeatureImportances(
            importances=[
                FeatureWeight('a', 1, std=0.1),
                FeatureWeight('b', 2, std=0.2),
            ],
            remaining=10,
        )
    )
    df = format_as_dataframe(expl)
    expected_df = pd.DataFrame(
        {'weight': [1, 2], 'std': [0.1, 0.2]},
        columns=['weight', 'std'],
        index=['a', 'b'])
    print(df, expected_df, sep='\n')
    assert expected_df.equals(df)

def test_transition_features():
    expl = Explanation(
        estimator='some estimator',
        targets=[
            TargetExplanation(
                'class1', feature_weights=FeatureWeights(
                    pos=[FeatureWeight('pos', 13, value=1)],
                    neg=[],
                )),
            TargetExplanation(
                'class2', feature_weights=FeatureWeights(
                    pos=[FeatureWeight('pos', 13, value=1)],
                    neg=[],
                )),
        ],
        transition_features=TransitionFeatureWeights(
            class_names=['class2', 'class1'],  # reverse on purpose
            coef=np.array([[1.5, 2.5], [3.5, 4.5]]),
        )
    )
    df_dict = format_as_dataframes(expl)
    assert isinstance(df_dict, dict)
    assert set(df_dict) == {'targets', 'transition_features'}
    assert df_dict['targets'].equals(format_as_dataframe(expl.targets))
    df = df_dict['transition_features']
    print(df)
    print(format_as_text(expl))
    assert str(df) == (
        'to      class2  class1\n'
        'from                  \n'
        'class2     1.5     2.5\n'
        'class1     3.5     4.5'
    )

    with pytest.warns(UserWarning):
        single_df = format_as_dataframe(expl)
    assert single_df.equals(df)
<commit_msg>Improve test coverage<commit_after>from __future__ import print_function

import pytest
pd = pytest.importorskip('pandas')
import numpy as np

from eli5 import format_as_dataframes, format_as_dataframe, format_as_text
from eli5.base import (
    Explanation, TargetExplanation, FeatureWeight, FeatureWeights,
    FeatureImportances, TransitionFeatureWeights,
)

@pytest.mark.parametrize(
    ['with_std', 'with_value'],
    [[False, False], [True, False], [False, True]])
def test_targets(with_std, with_value):
    expl = Explanation(
        estimator='some estimator',
        targets=[
            TargetExplanation(
                'y', feature_weights=FeatureWeights(
                    pos=[FeatureWeight('a', 13,
                                       std=0.13 if with_std else None,
                                       value=2 if with_value else None),
                         FeatureWeight('b', 5,
                                       std=0.5 if with_std else None,
                                       value=1 if with_value else None)],
                    neg=[FeatureWeight('neg1', -10,
                                       std=0.2 if with_std else None,
                                       value=5 if with_value else None),
                         FeatureWeight('neg2', -1,
                                       std=0.3 if with_std else None,
                                       value=4 if with_value else None)],
                )),
            TargetExplanation(
                'y2', feature_weights=FeatureWeights(
                    pos=[FeatureWeight('f', 1)],
                    neg=[],
                )),
        ],
    )
    df_dict = format_as_dataframes(expl)
    assert isinstance(df_dict, dict)
    assert list(df_dict) == ['targets']
    df = df_dict['targets']
    expected_df = pd.DataFrame(
        {'weight': [13, 5, -1, -10, 1]},
        index=pd.MultiIndex.from_tuples(
            [('y', 'a'), ('y', 'b'), ('y', 'neg2'), ('y', 'neg1'),
             ('y2', 'f')], names=['target', 'feature']))
    if with_std:
        expected_df['std'] = [0.13, 0.5, 0.3, 0.2, None]
    if with_value:
        expected_df['value'] = [2, 1, 4, 5, None]
    print(df, expected_df, sep='\n')
    assert expected_df.equals(df)

    single_df = format_as_dataframe(expl)
    assert expected_df.equals(single_df)

def test_bad_list():
    with pytest.raises(ValueError):
        format_as_dataframe([1])

def test_targets_with_value():
    expl = Explanation(
        estimator='some estimator',
        targets=[
            TargetExplanation(
                'y', feature_weights=FeatureWeights(
                    pos=[FeatureWeight('a', 13, value=1),
                         FeatureWeight('b', 5, value=2)],
                    neg=[FeatureWeight('neg1', -10, value=3),
                         FeatureWeight('neg2', -1, value=4)],
                )),
            TargetExplanation(
                'y2', feature_weights=FeatureWeights(
                    pos=[FeatureWeight('f', 1, value=5)],
                    neg=[],
                )),
        ],
    )
    df = format_as_dataframe(expl)
    expected_df = pd.DataFrame(
        {'weight': [13, 5, -1, -10, 1],
         'value': [1, 2, 4, 3, 5]},
        columns=['weight', 'value'],
        index=pd.MultiIndex.from_tuples(
            [('y', 'a'), ('y', 'b'), ('y', 'neg2'), ('y', 'neg1'),
             ('y2', 'f')], names=['target', 'feature']))
    print(df, expected_df, sep='\n')
    assert expected_df.equals(df)

@pytest.mark.parametrize(
    ['with_std', 'with_value'],
    [[False, False], [True, False], [False, True]])
def test_feature_importances(with_std, with_value):
    expl = Explanation(
        estimator='some estimator',
        feature_importances=FeatureImportances(
            importances=[
                FeatureWeight('a', 1,
                              std=0.1 if with_std else None,
                              value=1 if with_value else None),
                FeatureWeight('b', 2,
                              std=0.2 if with_std else None,
                              value=3 if with_value else None),
            ],
            remaining=10,
        )
    )
    df_dict = format_as_dataframes(expl)
    assert isinstance(df_dict, dict)
    assert list(df_dict) == ['feature_importances']
    df = df_dict['feature_importances']
    expected_df = pd.DataFrame({'weight': [1, 2]}, index=['a', 'b'])
    if with_std:
        expected_df['std'] = [0.1, 0.2]
    if with_value:
        expected_df['value'] = [1, 3]
    print(df, expected_df, sep='\n')
    assert expected_df.equals(df)

    single_df = format_as_dataframe(expl)
    assert expected_df.equals(single_df)

def test_transition_features():
    expl = Explanation(
        estimator='some estimator',
        targets=[
            TargetExplanation(
                'class1', feature_weights=FeatureWeights(
                    pos=[FeatureWeight('pos', 13, value=1)],
                    neg=[],
                )),
            TargetExplanation(
                'class2', feature_weights=FeatureWeights(
                    pos=[FeatureWeight('pos', 13, value=1)],
                    neg=[],
                )),
        ],
        transition_features=TransitionFeatureWeights(
            class_names=['class2', 'class1'],  # reverse on purpose
            coef=np.array([[1.5, 2.5], [3.5, 4.5]]),
        )
    )
    df_dict = format_as_dataframes(expl)
    assert isinstance(df_dict, dict)
    assert set(df_dict) == {'targets', 'transition_features'}
    assert df_dict['targets'].equals(format_as_dataframe(expl.targets))
    df = df_dict['transition_features']
    print(df)
    print(format_as_text(expl))
    assert str(df) == (
        'to      class2  class1\n'
        'from                  \n'
        'class2     1.5     2.5\n'
        'class1     3.5     4.5'
    )

    with pytest.warns(UserWarning):
        single_df = format_as_dataframe(expl)
    assert single_df.equals(df)
<|endoftext|>