<commit_before>#!/usr/bin/env python
import codecs
from cStringIO import StringIO
import gzip
import json
import os
import requests
import stat
import sys
import time

class LockPolicy(object):
    MAX_NUM_SLEEP_IN_WAITING_FOR_LOCK = os.environ.get('MAX_NUM_SLEEP_IN_WAITING_FOR_LOCK', 100)
    try:
        SLEEP_FOR_LOCK_TIME = float(os.environ.get('SLEEP_FOR_LOCK_TIME', 0.05))
    except:
        SLEEP_FOR_LOCK_TIME = 0.05

    def __init__(self):
        self.early_exit_if_locked = False
        self.wait_do_not_relock_if_locked = False
        self._reset_current()

    def _reset_current(self):
        self.curr_lockfile, self.curr_owns_lock, self.curr_was_locked = False, False, False

    def _wait_for_lock(self, lockfile):
        '''Returns a pair of bools: lockfile previously existed, lockfile now owned by caller
        '''
        n = 0
        pid = os.getpid()
        previously_existed = False
        while os.path.exists(lockfile):
            previously_existed = True
            n += 1
            if self.early_exit_if_locked or n > LockPolicy.MAX_NUM_SLEEP_IN_WAITING_FOR_LOCK:
                return True, False
            if VERBOSE:
                sys.stderr.write('Waiting for "%s" iter %d\n' % (lockfile, n))
            time.sleep(LockPolicy.SLEEP_FOR_LOCK_TIME)
        if previously_existed and self.wait_do_not_relock_if_locked:
            return True, False
        try:
            o = open_for_group_write(lockfile, 'w')
            o.write(str(pid) + '\n')
            o.close()
        except:
            try:
                self.remove_lock(lockfile)
            except:
                pass
            return previously_existed, False
        else:
            return previously_existed, True
    def wait_for_lock(self, lockfile):
        t = self._wait_for_lock(lockfile)
        self.curr_lockfile = lockfile
        self.curr_was_locked, self.curr_owns_lock = t
        if VERBOSE:
            sys.stderr.write('Lockfile = "%s" was_locked=%s owns_lock=%s\n'%
                    (lockfile,
                     "TRUE" if self.curr_was_locked else "FALSE",
                     "TRUE" if self.curr_owns_lock else "FALSE",
                     ))
        return t
    def remove_lock(self):
        try:
            if self.curr_lockfile and self.curr_owns_lock:
                self._remove_lock(self.curr_lockfile)
        finally:
            self._reset_current()
    def _remove_lock(self, lockfile):
        if os.path.exists(lockfile):
            os.remove(lockfile)

def get_processing_paths_from_prefix(pref,
                                     nexson_dir='.',
                                     nexson_state_db=None):
    d = {'nexson': os.path.abspath(os.path.join(nexson_dir, 'study', pref, pref + '.json')),
         'nexson_state_db': nexson_state_db,
         'study': pref,
         }
    if d['nexson_state_db'] is None:
        d['nexson_state_db'] = os.path.abspath(os.path.join(nexson_dir, '.to_download.json')), # stores the state of this repo. *very* hacky primitive db.
    return d

def get_default_dir_dict(top_level=None):
    r = '.' if top_level is None else top_level
    t = os.path.abspath(r)
    d = {'nexson_dir': t,
         'nexson_state_db': os.path.join(t, '.to_download.json'), # stores the state of this repo. *very* hacky primitive db.
        }
    return d

def get_previous_list_of_dirty_nexsons(dir_dict):
    '''Returns the previous list of studies to be fetch and dict that contains that list and timestamps.
    The dict will be populated from the filepath `dir_dict['nexson_state_db']` if that entry is not
    found then a default dict of no studies and old timestamps will be returned.
    '''
    filename = dir_dict['nexson_state_db']
    if os.path.exists(filename):
        old = json.load(codecs.open(filename, 'rU', encoding='utf-8'))
    else:
        old = {'from': '2010-01-01T00:00:00',
               'to': '2010-01-01T00:00:00',
               'studies': []
        }
    return old['studies'], old

def get_list_of_dirty_nexsons(dir_dict):
    '''Returns a pair: the list of studies that need to be fetched from phylografter
    and a dict that can be serialized to disk in .to_download.json to cache the details
    of the last call to phylografter's study/modified_list service.

    If PHYLOGRAFTER_DOMAIN_PREF is in the env, it will provide the domain the default
        is the main phylografter site.
    '''
    filename = dir_dict['nexson_state_db']
    slist, old = get_previous_list_of_dirty_nexsons(dir_dict)
    DOMAIN = os.environ.get('PHYLOGRAFTER_DOMAIN_PREF')
    if DOMAIN is None:
        DOMAIN = 'http://www.reelab.net/phylografter'

    SUBMIT_URI = DOMAIN + '/study/modified_list.json/url'
    args = {'from': old['to']}
    headers = {'content-type': 'application/json'}
    resp = requests.get(SUBMIT_URI, params=args, headers=headers)
    resp.raise_for_status()
    try:
        new_resp = resp.json()
    except:
        new_resp = resp.json
    ss = set(new_resp['studies'] + old['studies'])
    sl = list(ss)
    sl.sort()
    new_resp['studies'] = sl
    new_resp['from'] = old['from']
    store_state_JSON(new_resp, filename)
    to_refresh = list(new_resp['studies'])
    return to_refresh, new_resp

def open_for_group_write(fp, mode):
    '''Open with mode=mode and permissions '-rw-rw-r--' group writable is
    the default on some systems/accounts, but it is important that it be present on our deployment machine
    '''
    o = codecs.open(fp, mode, encoding='utf-8')
    o.flush()
    os.chmod(fp, stat.S_IRGRP | stat.S_IROTH | stat.S_IRUSR | stat.S_IWGRP | stat.S_IWUSR)
    return o

def store_state_JSON(s, fp):
    tmpfilename = fp + '.tmpfile'
    td = open_for_group_write(tmpfilename, 'w')
    try:
        json.dump(s, td, sort_keys=True, indent=0)
    finally:
        td.close()
    os.rename(tmpfilename, fp) #atomic on POSIX

def download_nexson_from_phylografter(paths, download_db, lock_policy):
    DOMAIN = os.environ.get('PHYLOGRAFTER_DOMAIN_PREF')
    if DOMAIN is None:
        DOMAIN = 'http://www.reelab.net/phylografter'

    headers = {
            'accept-encoding' : 'gzip',
            'content-type' : 'application/json',
            'accept' : 'application/json',
        }
    nexson_path = paths['nexson']
    lockfile = nexson_path + '.lock'
    was_locked, owns_lock = lock_policy.wait_for_lock(lockfile)
    try:
        if not owns_lock:
            return False
        study = paths['study']
        if VERBOSE:
            sys.stderr.write('Downloading %s...\n' % study)
        SUBMIT_URI = DOMAIN + '/study/export_gzipNexSON.json/' + study
        resp = requests.get(SUBMIT_URI,
                         headers=headers,
                         allow_redirects=True)
        resp.raise_for_status()
        try:
            uncompressed = gzip.GzipFile(mode='rb', fileobj=StringIO(resp.content)).read()
            results = uncompressed
        except:
            raise
        if isinstance(results, unicode) or isinstance(results, str):
            er = json.loads(results)
        else:
            raise RuntimeError('Non gzipped response, but not a string is:', results)
        should_write = False
        if not os.path.exists(nexson_path):
            should_write = True
        else:
            prev_content = json.load(codecs.open(nexson_path, 'rU', encoding='utf-8'))
            if prev_content != er:
                should_write = True
        if should_write:
            store_state_JSON(er, nexson_path)
        if download_db is not None:
            try:
                download_db['studies'].remove(int(study))
            except:
                warn('%s not in %s' % (study, paths['nexson_state_db']))
                pass
            else:
                store_state_JSON(download_db, paths['nexson_state_db'])
    finally:
        lock_policy.remove_lock()
    return True

if __name__ == '__main__':
    if '-h' in sys.argv:
        sys.stderr.write('''refresh_nexsons_from_phylografter.py is a short-term hack.

It stores info about the last communication with phylografter in .to_download.json
Based on this info, it tries to download as few studies as possible to make
the NexSONs in treenexus/studies/#/... match the export from phylografter.

   -h gives this help message.
   -v runs in verbose mode

If other arguments aree supplied, it should be the study #'s to be downloaded.
''')
        sys.exit(0)
    if '-v' in sys.argv:
        VERBOSE = True
        sys.argv.remove('-v')
    lock_policy = LockPolicy()
    dd = get_default_dir_dict()
    if len(sys.argv) == 1:
        to_download, download_db = get_list_of_dirty_nexsons(dd)
        if not to_download:
            sys.stderr.write('No studies have been modified since the last refresh.\n')
            sys.exit(0)
    else:
        to_download, download_db = sys.argv[1:], None
    try:
        SLEEP_BETWEEN_DOWNLOADS_TIME = float(os.environ.get('SLEEP_BETWEEN_DOWNLOADS_TIME', 0.5))
    except:
        SLEEP_BETWEEN_DOWNLOADS_TIME = 0.05
    while len(to_download) > 0:
        n = to_download.pop(0)
        study = str(n)
        paths = get_processing_paths_from_prefix(study, **dd)
        if not download_nexson_from_phylografter(paths, download_db, lock_policy):
            sys.exit('NexSON "%s" could not be refreshed\n' % paths['nexson'])
        if len(to_download) > 0:
            time.sleep(SLEEP_BETWEEN_DOWNLOADS_TIME)

<commit_msg>bug fix: setting VERBOSE correctly<commit_after>#!/usr/bin/env python
import codecs
from cStringIO import StringIO
import gzip
import json
import os
import requests
import stat
import sys
import time

class LockPolicy(object):
    MAX_NUM_SLEEP_IN_WAITING_FOR_LOCK = os.environ.get('MAX_NUM_SLEEP_IN_WAITING_FOR_LOCK', 100)
    try:
        SLEEP_FOR_LOCK_TIME = float(os.environ.get('SLEEP_FOR_LOCK_TIME', 0.05))
    except:
        SLEEP_FOR_LOCK_TIME = 0.05

    def __init__(self):
        self.early_exit_if_locked = False
        self.wait_do_not_relock_if_locked = False
        self._reset_current()

    def _reset_current(self):
        self.curr_lockfile, self.curr_owns_lock, self.curr_was_locked = False, False, False

    def _wait_for_lock(self, lockfile):
        '''Returns a pair of bools: lockfile previously existed, lockfile now owned by caller
        '''
        n = 0
        pid = os.getpid()
        previously_existed = False
        while os.path.exists(lockfile):
            previously_existed = True
            n += 1
            if self.early_exit_if_locked or n > LockPolicy.MAX_NUM_SLEEP_IN_WAITING_FOR_LOCK:
                return True, False
            if VERBOSE:
                sys.stderr.write('Waiting for "%s" iter %d\n' % (lockfile, n))
            time.sleep(LockPolicy.SLEEP_FOR_LOCK_TIME)
        if previously_existed and self.wait_do_not_relock_if_locked:
            return True, False
        try:
            o = open_for_group_write(lockfile, 'w')
            o.write(str(pid) + '\n')
            o.close()
        except:
            try:
                self.remove_lock(lockfile)
            except:
                pass
            return previously_existed, False
        else:
            return previously_existed, True
    def wait_for_lock(self, lockfile):
        t = self._wait_for_lock(lockfile)
        self.curr_lockfile = lockfile
        self.curr_was_locked, self.curr_owns_lock = t
        if VERBOSE:
            sys.stderr.write('Lockfile = "%s" was_locked=%s owns_lock=%s\n'%
                    (lockfile,
                     "TRUE" if self.curr_was_locked else "FALSE",
                     "TRUE" if self.curr_owns_lock else "FALSE",
                     ))
        return t
    def remove_lock(self):
        try:
            if self.curr_lockfile and self.curr_owns_lock:
                self._remove_lock(self.curr_lockfile)
        finally:
            self._reset_current()
    def _remove_lock(self, lockfile):
        if os.path.exists(lockfile):
            os.remove(lockfile)

def get_processing_paths_from_prefix(pref,
                                     nexson_dir='.',
                                     nexson_state_db=None):
    d = {'nexson': os.path.abspath(os.path.join(nexson_dir, 'study', pref, pref + '.json')),
         'nexson_state_db': nexson_state_db,
         'study': pref,
         }
    if d['nexson_state_db'] is None:
        d['nexson_state_db'] = os.path.abspath(os.path.join(nexson_dir, '.to_download.json')), # stores the state of this repo. *very* hacky primitive db.
    return d

def get_default_dir_dict(top_level=None):
    r = '.' if top_level is None else top_level
    t = os.path.abspath(r)
    d = {'nexson_dir': t,
         'nexson_state_db': os.path.join(t, '.to_download.json'), # stores the state of this repo. *very* hacky primitive db.
        }
    return d

def get_previous_list_of_dirty_nexsons(dir_dict):
    '''Returns the previous list of studies to be fetch and dict that contains that list and timestamps.
    The dict will be populated from the filepath `dir_dict['nexson_state_db']` if that entry is not
    found then a default dict of no studies and old timestamps will be returned.
    '''
    filename = dir_dict['nexson_state_db']
    if os.path.exists(filename):
        old = json.load(codecs.open(filename, 'rU', encoding='utf-8'))
    else:
        old = {'from': '2010-01-01T00:00:00',
               'to': '2010-01-01T00:00:00',
               'studies': []
        }
    return old['studies'], old

def get_list_of_dirty_nexsons(dir_dict):
    '''Returns a pair: the list of studies that need to be fetched from phylografter
    and a dict that can be serialized to disk in .to_download.json to cache the details
    of the last call to phylografter's study/modified_list service.

    If PHYLOGRAFTER_DOMAIN_PREF is in the env, it will provide the domain the default
        is the main phylografter site.
    '''
    filename = dir_dict['nexson_state_db']
    slist, old = get_previous_list_of_dirty_nexsons(dir_dict)
    DOMAIN = os.environ.get('PHYLOGRAFTER_DOMAIN_PREF')
    if DOMAIN is None:
        DOMAIN = 'http://www.reelab.net/phylografter'

    SUBMIT_URI = DOMAIN + '/study/modified_list.json/url'
    args = {'from': old['to']}
    headers = {'content-type': 'application/json'}
    resp = requests.get(SUBMIT_URI, params=args, headers=headers)
    resp.raise_for_status()
    try:
        new_resp = resp.json()
    except:
        new_resp = resp.json
    ss = set(new_resp['studies'] + old['studies'])
    sl = list(ss)
    sl.sort()
    new_resp['studies'] = sl
    new_resp['from'] = old['from']
    store_state_JSON(new_resp, filename)
    to_refresh = list(new_resp['studies'])
    return to_refresh, new_resp

def open_for_group_write(fp, mode):
    '''Open with mode=mode and permissions '-rw-rw-r--' group writable is
    the default on some systems/accounts, but it is important that it be present on our deployment machine
    '''
    o = codecs.open(fp, mode, encoding='utf-8')
    o.flush()
    os.chmod(fp, stat.S_IRGRP | stat.S_IROTH | stat.S_IRUSR | stat.S_IWGRP | stat.S_IWUSR)
    return o

def store_state_JSON(s, fp):
    tmpfilename = fp + '.tmpfile'
    td = open_for_group_write(tmpfilename, 'w')
    try:
        json.dump(s, td, sort_keys=True, indent=0)
    finally:
        td.close()
    os.rename(tmpfilename, fp) #atomic on POSIX

def download_nexson_from_phylografter(paths, download_db, lock_policy):
    DOMAIN = os.environ.get('PHYLOGRAFTER_DOMAIN_PREF')
    if DOMAIN is None:
        DOMAIN = 'http://www.reelab.net/phylografter'

    headers = {
            'accept-encoding' : 'gzip',
            'content-type' : 'application/json',
            'accept' : 'application/json',
        }
    nexson_path = paths['nexson']
    lockfile = nexson_path + '.lock'
    was_locked, owns_lock = lock_policy.wait_for_lock(lockfile)
    try:
        if not owns_lock:
            return False
        study = paths['study']
        if VERBOSE:
            sys.stderr.write('Downloading %s...\n' % study)
        SUBMIT_URI = DOMAIN + '/study/export_gzipNexSON.json/' + study
        resp = requests.get(SUBMIT_URI,
                         headers=headers,
                         allow_redirects=True)
        resp.raise_for_status()
        try:
            uncompressed = gzip.GzipFile(mode='rb', fileobj=StringIO(resp.content)).read()
            results = uncompressed
        except:
            raise
        if isinstance(results, unicode) or isinstance(results, str):
            er = json.loads(results)
        else:
            raise RuntimeError('Non gzipped response, but not a string is:', results)
        should_write = False
        if not os.path.exists(nexson_path):
            should_write = True
        else:
            prev_content = json.load(codecs.open(nexson_path, 'rU', encoding='utf-8'))
            if prev_content != er:
                should_write = True
        if should_write:
            store_state_JSON(er, nexson_path)
        if download_db is not None:
            try:
                download_db['studies'].remove(int(study))
            except:
                warn('%s not in %s' % (study, paths['nexson_state_db']))
                pass
            else:
                store_state_JSON(download_db, paths['nexson_state_db'])
    finally:
        lock_policy.remove_lock()
    return True

if __name__ == '__main__':
    if '-h' in sys.argv:
        sys.stderr.write('''refresh_nexsons_from_phylografter.py is a short-term hack.

It stores info about the last communication with phylografter in .to_download.json
Based on this info, it tries to download as few studies as possible to make
the NexSONs in treenexus/studies/#/... match the export from phylografter.

   -h gives this help message.
   -v runs in verbose mode

If other arguments aree supplied, it should be the study #'s to be downloaded.
''')
        sys.exit(0)
    if '-v' in sys.argv:
        VERBOSE = True
        sys.argv.remove('-v')
    else:
        VERBOSE = False
    lock_policy = LockPolicy()
    dd = get_default_dir_dict()
    if len(sys.argv) == 1:
        to_download, download_db = get_list_of_dirty_nexsons(dd)
        if not to_download:
            sys.stderr.write('No studies have been modified since the last refresh.\n')
            sys.exit(0)
    else:
        to_download, download_db = sys.argv[1:], None
    try:
        SLEEP_BETWEEN_DOWNLOADS_TIME = float(os.environ.get('SLEEP_BETWEEN_DOWNLOADS_TIME', 0.5))
    except:
        SLEEP_BETWEEN_DOWNLOADS_TIME = 0.05
    while len(to_download) > 0:
        n = to_download.pop(0)
        study = str(n)
        paths = get_processing_paths_from_prefix(study, **dd)
        if not download_nexson_from_phylografter(paths, download_db, lock_policy):
            sys.exit('NexSON "%s" could not be refreshed\n' % paths['nexson'])
        if len(to_download) > 0:
            time.sleep(SLEEP_BETWEEN_DOWNLOADS_TIME)

<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
#-----------------------------------------------------------------------------
#  Copyright (c) 2015, ROOT Team.
#  Authors: Omar Zapata <Omar.Zapata@cern.ch> http://oproject.org
#           Danilo Piparo <Danilo.Piparo@cern.ch> CERN
#           Enric Tejedor enric.tejedor.saavedra@cern.ch> CERN
#  website: http://oproject.org/ROOT+Jupyter+Kernel (information only for ROOT kernel)
#  Distributed under the terms of the Modified LGPLv3 License.
#
#  The full license is in the file COPYING.rst, distributed with this software.
#-----------------------------------------------------------------------------
import os
from glob import glob

from JupyROOT.handlers import IOHandler, JupyROOTDeclarer, JupyROOTExecutor

import ROOT

import __builtin__

_ioHandler = None
_Executor  = None
_Declarer  = None

def GetIOHandler():
    global _ioHandler
    if not _ioHandler:
        _ioHandler = IOHandler()
    return _ioHandler

def GetExecutor():
    global _Executor
    if not _Executor:
        _Executor = JupyROOTExecutor()
    return _Executor

def GetDeclarer():
    global _Declarer
    if not _Declarer:
        _Declarer = JupyROOTDeclarer()
    return _Declarer

class MagicLoader(object):
    '''Class to load JupyROOT Magics'''
    def __init__(self,kernel):
         magics_path = os.path.dirname(__file__)+"/magics/*.py"
         for file in glob(magics_path):
              if file != magics_path.replace("*.py","__init__.py"):
                  module_path="JupyROOT.kernel.magics."+file.split("/")[-1].replace(".py","")
                  try:
                      module= __builtin__.__import__(module_path, globals(), locals(), ['register_magics'], -1)
                      module.register_magics(kernel)
                  except ImportError:
                      raise Exception("Error importing Magic: %s"%module_path)

<commit_msg>Python3: __builtin__ does not exist, invoke built-in function __import__ directly<commit_after># -*- coding:utf-8 -*-
#-----------------------------------------------------------------------------
#  Copyright (c) 2015, ROOT Team.
#  Authors: Omar Zapata <Omar.Zapata@cern.ch> http://oproject.org
#           Danilo Piparo <Danilo.Piparo@cern.ch> CERN
#           Enric Tejedor enric.tejedor.saavedra@cern.ch> CERN
#  website: http://oproject.org/ROOT+Jupyter+Kernel (information only for ROOT kernel)
#  Distributed under the terms of the Modified LGPLv3 License.
#
#  The full license is in the file COPYING.rst, distributed with this software.
#-----------------------------------------------------------------------------
import os
from glob import glob

from JupyROOT.handlers import IOHandler, JupyROOTDeclarer, JupyROOTExecutor

import ROOT

_ioHandler = None
_Executor  = None
_Declarer  = None

def GetIOHandler():
    global _ioHandler
    if not _ioHandler:
        _ioHandler = IOHandler()
    return _ioHandler

def GetExecutor():
    global _Executor
    if not _Executor:
        _Executor = JupyROOTExecutor()
    return _Executor

def GetDeclarer():
    global _Declarer
    if not _Declarer:
        _Declarer = JupyROOTDeclarer()
    return _Declarer

class MagicLoader(object):
    '''Class to load JupyROOT Magics'''
    def __init__(self,kernel):
         magics_path = os.path.dirname(__file__)+"/magics/*.py"
         for file in glob(magics_path):
              if file != magics_path.replace("*.py","__init__.py"):
                  module_path="JupyROOT.kernel.magics."+file.split("/")[-1].replace(".py","")
                  try:
                      module= __import__(module_path, globals(), locals(), ['register_magics'], -1)
                      module.register_magics(kernel)
                  except ImportError:
                      raise Exception("Error importing Magic: %s"%module_path)

<|endoftext|>


<commit_before>#!/usr/bin/python

from sys import argv, exit
from getopt import getopt, GetoptError
from subprocess import Popen, PIPE
from storage import Environment, Storage, PROBE_NORMAL, PROBE_NORMAL_WRITE_MOCKUP, TARGET_NORMAL
from storage import RemoteCallbacks, RemoteCommand, RemoteFile, set_remote_callbacks

host = "localhost"
save_mockup = False
save_devicegraph = False

def run_command(name):

    cmd = "ssh -l root %s %s" % (host, name)

    p = Popen(cmd, shell = True, stdout = PIPE, stderr = PIPE, close_fds = True)
    stdout, stderr = p.communicate()

    ret = RemoteCommand()

    for line in stdout.rstrip().split('\n'):
        ret.stdout.push_back(line)

    for line in stderr.rstrip().split('\n'):
        ret.stderr.push_back(line)

    ret.exitcode = p.returncode

    return ret

class MyRemoteCallbacks(RemoteCallbacks):

    def __init__(self):
        super(MyRemoteCallbacks, self).__init__()

    def get_command(self, name):
        print "command '%s'" % name
        command = run_command(name)
        return command

    def get_file(self, name):
        print "file '%s'" % name
        command = run_command("cat '%s'" % name)
        file = RemoteFile(command.stdout)
        return file

def doit():
    my_remote_callbacks = MyRemoteCallbacks()

    set_remote_callbacks(my_remote_callbacks)

    environment = Environment(True, PROBE_NORMAL_WRITE_MOCKUP if save_mockup else PROBE_NORMAL, TARGET_NORMAL)
    if save_mockup:
        environment.set_mockup_filename("remote-probe-mockup.xml");

    storage = Storage(environment)

    print

    probed = storage.get_probed()

    print probed

    if save_devicegraph:
        probed.save("remote-probe-devicegraph.xml");

def usage():
    print "usage: remote-probe.py [--host=name] [--save-mockup] [--save-devicegraph]"
    exit(1)

try:
    opts, args = getopt(argv[1:], "", ["host=", "save-mockup", "save-devicegraph"])

except GetoptError:
    usage()

if len(args) > 0:
    usage()

for o, a in opts:
    if o == "--host":
        host = a
    if o == "--save-mockup":
        save_mockup = True
    if o == "--save-devicegraph":
        save_devicegraph = True

doit()
<commit_msg>- adapted enum names<commit_after>#!/usr/bin/python

from sys import argv, exit
from getopt import getopt, GetoptError
from subprocess import Popen, PIPE
from storage import Environment, Storage, ProbeMode_STANDARD, ProbeMode_STANDARD_WRITE_MOCKUP, TargetMode_DIRECT
from storage import RemoteCallbacks, RemoteCommand, RemoteFile, set_remote_callbacks

host = "localhost"
save_mockup = False
save_devicegraph = False

def run_command(name):

    cmd = "ssh -l root %s %s" % (host, name)

    p = Popen(cmd, shell = True, stdout = PIPE, stderr = PIPE, close_fds = True)
    stdout, stderr = p.communicate()

    ret = RemoteCommand()

    for line in stdout.rstrip().split('\n'):
        ret.stdout.push_back(line)

    for line in stderr.rstrip().split('\n'):
        ret.stderr.push_back(line)

    ret.exitcode = p.returncode

    return ret

class MyRemoteCallbacks(RemoteCallbacks):

    def __init__(self):
        super(MyRemoteCallbacks, self).__init__()

    def get_command(self, name):
        print "command '%s'" % name
        command = run_command(name)
        return command

    def get_file(self, name):
        print "file '%s'" % name
        command = run_command("cat '%s'" % name)
        file = RemoteFile(command.stdout)
        return file

def doit():
    my_remote_callbacks = MyRemoteCallbacks()

    set_remote_callbacks(my_remote_callbacks)

    environment = Environment(True, ProbeMode_STANDARD_WRITE_MOCKUP if save_mockup else ProbeMode_STANDARD, TargetMode_DIRECT)
    if save_mockup:
        environment.set_mockup_filename("remote-probe-mockup.xml");

    storage = Storage(environment)

    print

    probed = storage.get_probed()

    print probed

    if save_devicegraph:
        probed.save("remote-probe-devicegraph.xml");

def usage():
    print "usage: remote-probe.py [--host=name] [--save-mockup] [--save-devicegraph]"
    exit(1)

try:
    opts, args = getopt(argv[1:], "", ["host=", "save-mockup", "save-devicegraph"])

except GetoptError:
    usage()

if len(args) > 0:
    usage()

for o, a in opts:
    if o == "--host":
        host = a
    if o == "--save-mockup":
        save_mockup = True
    if o == "--save-devicegraph":
        save_devicegraph = True

doit()
<|endoftext|>


<commit_before><commit_msg>Change behave test username to fix functional tests. with @deniscostadsc<commit_after><|endoftext|>


<commit_before>"""
Rio PL interpreter
------------------

Author: Eduardo de Oliveira Padoan
Email:  eduardo.padoan@gmail.com
"""

from rio.sourceparser import parse
from rio.ast import (
    Expr, Block, Message, Args, ConstantInt, Identifier
)

def test_parse_basic():
    assert parse('50') == Block([Expr([Message(ConstantInt('50'))])])
    assert parse('1 inc') == Block(
        [Expr([Message(ConstantInt('1')), Message(Identifier('inc'))])]
    )
    assert parse('a') == Block([Expr([Message(Identifier('a'))])])
    assert parse('a b c') == Block(
        [Expr([Message(Identifier('a')),
               Message(Identifier('b')),
               Message(Identifier('c'))])]
    )

def test_parse_no_args():
    # Yes, we don't have Python's distinction between getting an attrib
    # and calling a method. But you should be able to 'quote a message.
    assert parse('a') == Block([Expr([Message(Identifier('a'))])])
    assert parse('a()') == Block([Expr([Message(Identifier('a'))])])
    assert parse('a b()') == Block(
        [Expr([Message(Identifier('a')),
               Message(Identifier('b'))])])
    assert parse('a b() c') == Block(
        [Expr([Message(Identifier('a')),
               Message(Identifier('b')),
               Message(Identifier('c'))])])

def test_parse_args():
    assert parse('a(b)') == Block(
        [Expr([Message(Identifier('a'),
                       Args([Expr([Message(Identifier('b'))])]),
                       )]
        )])
    assert parse('a(50)') == Block(
        [Expr([Message(Identifier('a'),
                       Args([Expr([Message(ConstantInt('50'))])]),
                       )]
        )])
    assert parse('5 pow(c)') == Block(
        [Expr([
            Message(ConstantInt('5')),
            Message(Identifier('pow'),
                    Args([Expr([Message(Identifier('c'))])]))]
        )])
    assert parse('a(b, c)') == Block(
        [Expr([Message(Identifier('a'), Args(
            [Expr([Message(Identifier('b'))]),
             Expr([Message(Identifier('c'))])]
        ))])]
    )

def test_parse_blocks():
    pass

<commit_msg>Basic blocks tests pass.<commit_after>"""
Rio PL interpreter
------------------

Author: Eduardo de Oliveira Padoan
Email:  eduardo.padoan@gmail.com
"""

from rio.sourceparser import parse
from rio.ast import (
    Expr, Block, Message, Args, ConstantInt, Identifier
)

def test_parse_basic():
    assert parse('50') == Block([Expr([Message(ConstantInt('50'))])])
    assert parse('1 inc') == Block(
        [Expr([Message(ConstantInt('1')), Message(Identifier('inc'))])]
    )
    assert parse('a') == Block([Expr([Message(Identifier('a'))])])
    assert parse('a b c') == Block(
        [Expr([Message(Identifier('a')),
               Message(Identifier('b')),
               Message(Identifier('c'))])]
    )

def test_parse_no_args():
    # Yes, we don't have Python's distinction between getting an attrib
    # and calling a method. But you should be able to 'quote a message.
    # Also, we can use get_slot
    assert parse('a') == Block([Expr([Message(Identifier('a'))])])
    assert parse('a()') == Block([Expr([Message(Identifier('a'))])])
    assert parse('a b()') == Block(
        [Expr([Message(Identifier('a')),
               Message(Identifier('b'))])])
    assert parse('a b() c') == Block(
        [Expr([Message(Identifier('a')),
               Message(Identifier('b')),
               Message(Identifier('c'))])])

def test_parse_args():
    assert parse('a(b)') == Block(
        [Expr([Message(Identifier('a'),
                       Args([Expr([Message(Identifier('b'))])]),
                       )]
        )])
    assert parse('a(50)') == Block(
        [Expr([Message(Identifier('a'),
                       Args([Expr([Message(ConstantInt('50'))])]),
                       )]
        )])
    assert parse('5 pow(c)') == Block(
        [Expr([
            Message(ConstantInt('5')),
            Message(Identifier('pow'),
                    Args([Expr([Message(Identifier('c'))])]))]
        )])
    assert parse('a(b, c)') == Block(
        [Expr([Message(Identifier('a'), Args(
            [Expr([Message(Identifier('b'))]),
             Expr([Message(Identifier('c'))])]
        ))])]
    )

def test_parse_blocks():
    assert parse('a b c\nd') == Block(
        [Expr([Message(Identifier('a')),
               Message(Identifier('b')),
               Message(Identifier('c'))]),
        Expr([Message(Identifier('d'))])]
    )
    assert parse('a b c; d') == Block(
        [Expr([Message(Identifier('a')),
               Message(Identifier('b')),
               Message(Identifier('c'))]),
        Expr([Message(Identifier('d'))])]
    )

<|endoftext|>


<commit_before>"""Color operations."""

import numpy as np
from .utils import epsilon
from .colorspace import saturate_rgb

# Color manipulation functions
def sigmoidal(arr, contrast, bias):
    """
    Sigmoidal contrast is type of contrast control that
    adjusts the contrast without saturating highlights or shadows.
    It allows control over two factors:
    the contrast range from light to dark, and where the middle value
    of the mid-tones falls. The result is a non-linear and smooth
    contrast change.

    Parameters
    ----------
    arr : ndarray, float, 0 .. 1
        Array of color values to adjust
    contrast : integer
        Enhances the intensity differences between the lighter and darker
        elements of the image. For example, 0 is none, 3 is typical and
        20 is a lot.
    bias : float, between 0 and 1
        Threshold level for the contrast function to center on
        (typically centered at 0.5)

    Notes
    ----------

    Sigmoidal contrast is based on the sigmoidal transfer function:

    .. math:: g(u) = ( 1/(1 + e^{- \alpha * u + \beta)})

    This sigmoid function is scaled so that the output is bound by
    the interval [0, 1].

    .. math:: ( 1/(1 + e^(\beta * (\alpha - u))) - 1/(1 + e^(\beta * \alpha)))/
        ( 1/(1 + e^(\beta*(\alpha - 1))) - 1/(1 + e^(\beta * \alpha)) )

    Where :math: `\alpha` is the threshold level, and :math: `\beta` the
    contrast factor to be applied.

    References
    ----------
    .. [CT] Hany Farid "Fundamentals of Image Processing"
            http://www.cs.dartmouth.edu/farid/downloads/tutorials/fip.pdf

    """
    if (arr.max() > 1.0 + epsilon) or (arr.min() < 0 - epsilon):
        raise ValueError("Input array must have float values between 0 and 1")

    if (bias > 1.0 + epsilon) or (bias < 0 - epsilon):
        raise ValueError("bias must be a scalar float between 0 and 1")

    alpha, beta = bias, contrast
    # We use the names a and b to match documentation.

    if alpha == 0:
        alpha = epsilon

    if beta == 0:
        return arr

    np.seterr(divide="ignore", invalid="ignore")

    if beta > 0:
        numerator = 1 / (1 + np.exp(beta * (alpha - arr))) - 1 / (
            1 + np.exp(beta * alpha)
        )
        denominator = 1 / (1 + np.exp(beta * (alpha - 1))) - 1 / (
            1 + np.exp(beta * alpha)
        )
        output = numerator / denominator

    else:
        # Inverse sigmoidal function:
        # todo: account for 0s
        # todo: formatting ;)
        output = (
            (beta * alpha)
            - np.log(
                (
                    1
                    / (
                        (arr / (1 + np.exp(beta * alpha - beta)))
                        - (arr / (1 + np.exp(beta * alpha)))
                        + (1 / (1 + np.exp(beta * alpha)))
                    )
                )
                - 1
            )
        ) / beta

    return output

def gamma(arr, g):
    """
    Gamma correction is a nonlinear operation that
    adjusts the image's channel values pixel-by-pixel according
    to a power-law:

    .. math:: pixel_{out} = pixel_{in} ^ {\gamma}

    Setting gamma (:math:`\gamma`) to be less than 1.0 darkens the image and
    setting gamma to be greater than 1.0 lightens it.

    Parameters
    ----------
    gamma (:math:`\gamma`): float
        Reasonable values range from 0.8 to 2.4.

    """
    if (arr.max() > 1.0 + epsilon) or (arr.min() < 0 - epsilon):
        raise ValueError("Input array must have float values between 0 and 1")
    if g <= 0 or np.isnan(g):
        raise ValueError("gamma must be greater than 0")

    return arr ** (1.0 / g)

def saturation(arr, proportion):
    """Apply saturation to an RGB array (in LCH color space)

    Multiply saturation by proportion in LCH color space to adjust the intensity
    of color in the image. As saturation increases, colors appear
    more "pure." As saturation decreases, colors appear more "washed-out."

    Parameters
    ----------
    arr: ndarray with shape (3, ..., ...)
    proportion: number

    """
    if arr.shape[0] != 3:
        raise ValueError("saturation requires a 3-band array")
    return saturate_rgb(arr, proportion)

def simple_atmo_opstring(haze, contrast, bias):
    """Make a simple atmospheric correction formula."""
    gamma_b = 1 - haze
    gamma_g = 1 - (haze / 3.0)
    ops = (
        "gamma g {gamma_g}, " "gamma b {gamma_b}, " "sigmoidal rgb {contrast} {bias}"
    ).format(gamma_g=gamma_g, gamma_b=gamma_b, contrast=contrast, bias=bias)
    return ops

def simple_atmo(rgb, haze, contrast, bias):
    """
    A simple, static (non-adaptive) atmospheric correction function.

    Parameters
    ----------
    haze: float
        Amount of haze to adjust for. For example, 0.03
    contrast : integer
        Enhances the intensity differences between the lighter and darker
        elements of the image. For example, 0 is none, 3 is typical and
        20 is a lot.
    bias : float, between 0 and 1
        Threshold level for the contrast function to center on
        (typically centered at 0.5 or 50%)
    """
    gamma_b = 1 - haze
    gamma_g = 1 - (haze / 3.0)
    arr = np.empty(shape=(3, rgb.shape[1], rgb.shape[2]))

    arr[0] = rgb[0]
    arr[1] = gamma(rgb[1], gamma_g)
    arr[2] = gamma(rgb[2], gamma_b)

    output = rgb.copy()
    output[0:3] = sigmoidal(arr, contrast, bias)

    return output

def _op_factory(func, kwargs, opname, bands, rgb_op=False):
    """create an operation function closure
    don't call directly, use parse_operations
    returns a function which itself takes and returns ndarrays
    """

    def f(arr):
        # Avoid mutation by copying
        newarr = arr.copy()
        if rgb_op:
            # apply func to array's first 3 bands, assumed r,g,b
            # additional band(s) are untouched
            newarr[0:3] = func(newarr[0:3], **kwargs)
        else:
            # apply func to array band at a time
            for b in bands:
                newarr[b - 1] = func(arr[b - 1], **kwargs)
        return newarr

    f.__name__ = str(opname)
    return f

def parse_operations(ops_string):
    """Takes a string of operations written with a handy DSL

    "OPERATION-NAME BANDS ARG1 ARG2 OPERATION-NAME BANDS ARG"

    And returns a list of functions, each of which take and return ndarrays
    """
    band_lookup = {"r": 1, "g": 2, "b": 3}
    count = len(band_lookup)

    opfuncs = {"saturation": saturation, "sigmoidal": sigmoidal, "gamma": gamma}

    opkwargs = {
        "saturation": ("proportion",),
        "sigmoidal": ("contrast", "bias"),
        "gamma": ("g",),
    }

    # Operations that assume RGB colorspace
    rgb_ops = ("saturation",)

    # split into tokens, commas are optional whitespace
    tokens = [x.strip() for x in ops_string.replace(",", "").split(" ")]
    operations = []
    current = []
    for token in tokens:
        if token.lower() in opfuncs.keys():
            if len(current) > 0:
                operations.append(current)
                current = []
        current.append(token.lower())
    if len(current) > 0:
        operations.append(current)

    result = []
    for parts in operations:
        opname = parts[0]
        bandstr = parts[1]
        args = parts[2:]

        try:
            func = opfuncs[opname]
        except KeyError:
            raise ValueError("{} is not a valid operation".format(opname))

        if opname in rgb_ops:
            # ignore bands, assumed to be in rgb
            # push 2nd arg into args
            args = [bandstr] + args
            bands = (1, 2, 3)
        else:
            # 2nd arg is bands
            # parse r,g,b ~= 1,2,3
            bands = set()
            for bs in bandstr:
                try:
                    band = int(bs)
                except ValueError:
                    band = band_lookup[bs.lower()]
                if band < 1 or band > count:
                    raise ValueError(
                        "{} BAND must be between 1 and {}".format(opname, count)
                    )
                bands.add(band)

        # assume all args are float
        args = [float(arg) for arg in args]
        kwargs = dict(zip(opkwargs[opname], args))

        # Create opperation function
        f = _op_factory(
            func=func,
            kwargs=kwargs,
            opname=opname,
            bands=bands,
            rgb_op=(opname in rgb_ops),
        )
        result.append(f)

    return result
<commit_msg>Mark docstrings with math directives as "raw"<commit_after>"""Color operations."""

import numpy as np
from .utils import epsilon
from .colorspace import saturate_rgb

# Color manipulation functions
def sigmoidal(arr, contrast, bias):
    r"""
    Sigmoidal contrast is type of contrast control that
    adjusts the contrast without saturating highlights or shadows.
    It allows control over two factors:
    the contrast range from light to dark, and where the middle value
    of the mid-tones falls. The result is a non-linear and smooth
    contrast change.

    Parameters
    ----------
    arr : ndarray, float, 0 .. 1
        Array of color values to adjust
    contrast : integer
        Enhances the intensity differences between the lighter and darker
        elements of the image. For example, 0 is none, 3 is typical and
        20 is a lot.
    bias : float, between 0 and 1
        Threshold level for the contrast function to center on
        (typically centered at 0.5)

    Notes
    ----------

    Sigmoidal contrast is based on the sigmoidal transfer function:

    .. math:: g(u) = ( 1/(1 + e^{- \alpha * u + \beta)})

    This sigmoid function is scaled so that the output is bound by
    the interval [0, 1].

    .. math:: ( 1/(1 + e^(\beta * (\alpha - u))) - 1/(1 + e^(\beta * \alpha)))/
        ( 1/(1 + e^(\beta*(\alpha - 1))) - 1/(1 + e^(\beta * \alpha)) )

    Where :math: `\alpha` is the threshold level, and :math: `\beta` the
    contrast factor to be applied.

    References
    ----------
    .. [CT] Hany Farid "Fundamentals of Image Processing"
            http://www.cs.dartmouth.edu/farid/downloads/tutorials/fip.pdf

    """
    if (arr.max() > 1.0 + epsilon) or (arr.min() < 0 - epsilon):
        raise ValueError("Input array must have float values between 0 and 1")

    if (bias > 1.0 + epsilon) or (bias < 0 - epsilon):
        raise ValueError("bias must be a scalar float between 0 and 1")

    alpha, beta = bias, contrast
    # We use the names a and b to match documentation.

    if alpha == 0:
        alpha = epsilon

    if beta == 0:
        return arr

    np.seterr(divide="ignore", invalid="ignore")

    if beta > 0:
        numerator = 1 / (1 + np.exp(beta * (alpha - arr))) - 1 / (
            1 + np.exp(beta * alpha)
        )
        denominator = 1 / (1 + np.exp(beta * (alpha - 1))) - 1 / (
            1 + np.exp(beta * alpha)
        )
        output = numerator / denominator

    else:
        # Inverse sigmoidal function:
        # todo: account for 0s
        # todo: formatting ;)
        output = (
            (beta * alpha)
            - np.log(
                (
                    1
                    / (
                        (arr / (1 + np.exp(beta * alpha - beta)))
                        - (arr / (1 + np.exp(beta * alpha)))
                        + (1 / (1 + np.exp(beta * alpha)))
                    )
                )
                - 1
            )
        ) / beta

    return output

def gamma(arr, g):
    r"""
    Gamma correction is a nonlinear operation that
    adjusts the image's channel values pixel-by-pixel according
    to a power-law:

    .. math:: pixel_{out} = pixel_{in} ^ {\gamma}

    Setting gamma (:math:`\gamma`) to be less than 1.0 darkens the image and
    setting gamma to be greater than 1.0 lightens it.

    Parameters
    ----------
    gamma (:math:`\gamma`): float
        Reasonable values range from 0.8 to 2.4.

    """
    if (arr.max() > 1.0 + epsilon) or (arr.min() < 0 - epsilon):
        raise ValueError("Input array must have float values between 0 and 1")
    if g <= 0 or np.isnan(g):
        raise ValueError("gamma must be greater than 0")

    return arr ** (1.0 / g)

def saturation(arr, proportion):
    """Apply saturation to an RGB array (in LCH color space)

    Multiply saturation by proportion in LCH color space to adjust the intensity
    of color in the image. As saturation increases, colors appear
    more "pure." As saturation decreases, colors appear more "washed-out."

    Parameters
    ----------
    arr: ndarray with shape (3, ..., ...)
    proportion: number

    """
    if arr.shape[0] != 3:
        raise ValueError("saturation requires a 3-band array")
    return saturate_rgb(arr, proportion)

def simple_atmo_opstring(haze, contrast, bias):
    """Make a simple atmospheric correction formula."""
    gamma_b = 1 - haze
    gamma_g = 1 - (haze / 3.0)
    ops = (
        "gamma g {gamma_g}, " "gamma b {gamma_b}, " "sigmoidal rgb {contrast} {bias}"
    ).format(gamma_g=gamma_g, gamma_b=gamma_b, contrast=contrast, bias=bias)
    return ops

def simple_atmo(rgb, haze, contrast, bias):
    """
    A simple, static (non-adaptive) atmospheric correction function.

    Parameters
    ----------
    haze: float
        Amount of haze to adjust for. For example, 0.03
    contrast : integer
        Enhances the intensity differences between the lighter and darker
        elements of the image. For example, 0 is none, 3 is typical and
        20 is a lot.
    bias : float, between 0 and 1
        Threshold level for the contrast function to center on
        (typically centered at 0.5 or 50%)
    """
    gamma_b = 1 - haze
    gamma_g = 1 - (haze / 3.0)
    arr = np.empty(shape=(3, rgb.shape[1], rgb.shape[2]))

    arr[0] = rgb[0]
    arr[1] = gamma(rgb[1], gamma_g)
    arr[2] = gamma(rgb[2], gamma_b)

    output = rgb.copy()
    output[0:3] = sigmoidal(arr, contrast, bias)

    return output

def _op_factory(func, kwargs, opname, bands, rgb_op=False):
    """create an operation function closure
    don't call directly, use parse_operations
    returns a function which itself takes and returns ndarrays
    """

    def f(arr):
        # Avoid mutation by copying
        newarr = arr.copy()
        if rgb_op:
            # apply func to array's first 3 bands, assumed r,g,b
            # additional band(s) are untouched
            newarr[0:3] = func(newarr[0:3], **kwargs)
        else:
            # apply func to array band at a time
            for b in bands:
                newarr[b - 1] = func(arr[b - 1], **kwargs)
        return newarr

    f.__name__ = str(opname)
    return f

def parse_operations(ops_string):
    """Takes a string of operations written with a handy DSL

    "OPERATION-NAME BANDS ARG1 ARG2 OPERATION-NAME BANDS ARG"

    And returns a list of functions, each of which take and return ndarrays
    """
    band_lookup = {"r": 1, "g": 2, "b": 3}
    count = len(band_lookup)

    opfuncs = {"saturation": saturation, "sigmoidal": sigmoidal, "gamma": gamma}

    opkwargs = {
        "saturation": ("proportion",),
        "sigmoidal": ("contrast", "bias"),
        "gamma": ("g",),
    }

    # Operations that assume RGB colorspace
    rgb_ops = ("saturation",)

    # split into tokens, commas are optional whitespace
    tokens = [x.strip() for x in ops_string.replace(",", "").split(" ")]
    operations = []
    current = []
    for token in tokens:
        if token.lower() in opfuncs.keys():
            if len(current) > 0:
                operations.append(current)
                current = []
        current.append(token.lower())
    if len(current) > 0:
        operations.append(current)

    result = []
    for parts in operations:
        opname = parts[0]
        bandstr = parts[1]
        args = parts[2:]

        try:
            func = opfuncs[opname]
        except KeyError:
            raise ValueError("{} is not a valid operation".format(opname))

        if opname in rgb_ops:
            # ignore bands, assumed to be in rgb
            # push 2nd arg into args
            args = [bandstr] + args
            bands = (1, 2, 3)
        else:
            # 2nd arg is bands
            # parse r,g,b ~= 1,2,3
            bands = set()
            for bs in bandstr:
                try:
                    band = int(bs)
                except ValueError:
                    band = band_lookup[bs.lower()]
                if band < 1 or band > count:
                    raise ValueError(
                        "{} BAND must be between 1 and {}".format(opname, count)
                    )
                bands.add(band)

        # assume all args are float
        args = [float(arg) for arg in args]
        kwargs = dict(zip(opkwargs[opname], args))

        # Create opperation function
        f = _op_factory(
            func=func,
            kwargs=kwargs,
            opname=opname,
            bands=bands,
            rgb_op=(opname in rgb_ops),
        )
        result.append(f)

    return result
<|endoftext|>


<commit_before>import math
import torch
from torch import nn, Tensor

from typing import List, Optional
from .image_list import ImageList

class AnchorGenerator(nn.Module):
    """
    Module that generates anchors for a set of feature maps and
    image sizes.

    The module support computing anchors at multiple sizes and aspect ratios
    per feature map. This module assumes aspect ratio = height / width for
    each anchor.

    sizes and aspect_ratios should have the same number of elements, and it should
    correspond to the number of feature maps.

    sizes[i] and aspect_ratios[i] can have an arbitrary number of elements,
    and AnchorGenerator will output a set of sizes[i] * aspect_ratios[i] anchors
    per spatial location for feature map i.

    Args:
        sizes (Tuple[Tuple[int]]):
        aspect_ratios (Tuple[Tuple[float]]):
    """

    __annotations__ = {
        "cell_anchors": List[torch.Tensor],
    }

    def __init__(
        self,
        sizes=((128, 256, 512),),
        aspect_ratios=((0.5, 1.0, 2.0),),
    ):
        super(AnchorGenerator, self).__init__()

        if not isinstance(sizes[0], (list, tuple)):
            # TODO change this
            sizes = tuple((s,) for s in sizes)
        if not isinstance(aspect_ratios[0], (list, tuple)):
            aspect_ratios = (aspect_ratios,) * len(sizes)

        assert len(sizes) == len(aspect_ratios)

        self.sizes = sizes
        self.aspect_ratios = aspect_ratios
        self.cell_anchors = [self.generate_anchors(size, aspect_ratio)
                             for size, aspect_ratio in zip(sizes, aspect_ratios)]

    # TODO: https://github.com/pytorch/pytorch/issues/26792
    # For every (aspect_ratios, scales) combination, output a zero-centered anchor with those values.
    # (scales, aspect_ratios) are usually an element of zip(self.scales, self.aspect_ratios)
    # This method assumes aspect ratio = height / width for an anchor.
    def generate_anchors(self, scales: List[int], aspect_ratios: List[float], dtype: torch.dtype = torch.float32,
                         device: torch.device = torch.device("cpu")):
        scales = torch.as_tensor(scales, dtype=dtype, device=device)
        aspect_ratios = torch.as_tensor(aspect_ratios, dtype=dtype, device=device)
        h_ratios = torch.sqrt(aspect_ratios)
        w_ratios = 1 / h_ratios

        ws = (w_ratios[:, None] * scales[None, :]).view(-1)
        hs = (h_ratios[:, None] * scales[None, :]).view(-1)

        base_anchors = torch.stack([-ws, -hs, ws, hs], dim=1) / 2
        return base_anchors.round()

    def set_cell_anchors(self, dtype: torch.dtype, device: torch.device):
        self.cell_anchors = [cell_anchor.to(dtype=dtype, device=device)
                             for cell_anchor in self.cell_anchors]

    def num_anchors_per_location(self):
        return [len(s) * len(a) for s, a in zip(self.sizes, self.aspect_ratios)]

    # For every combination of (a, (g, s), i) in (self.cell_anchors, zip(grid_sizes, strides), 0:2),
    # output g[i] anchors that are s[i] distance apart in direction i, with the same dimensions as a.
    def grid_anchors(self, grid_sizes: List[List[int]], strides: List[List[Tensor]]) -> List[Tensor]:
        anchors = []
        cell_anchors = self.cell_anchors
        assert cell_anchors is not None

        if not (len(grid_sizes) == len(strides) == len(cell_anchors)):
            raise ValueError("Anchors should be Tuple[Tuple[int]] because each feature "
                             "map could potentially have different sizes and aspect ratios. "
                             "There needs to be a match between the number of "
                             "feature maps passed and the number of sizes / aspect ratios specified.")

        for size, stride, base_anchors in zip(
            grid_sizes, strides, cell_anchors
        ):
            grid_height, grid_width = size
            stride_height, stride_width = stride
            device = base_anchors.device

            # For output anchor, compute [x_center, y_center, x_center, y_center]
            shifts_x = torch.arange(
                0, grid_width, dtype=torch.float32, device=device
            ) * stride_width
            shifts_y = torch.arange(
                0, grid_height, dtype=torch.float32, device=device
            ) * stride_height
            shift_y, shift_x = torch.meshgrid(shifts_y, shifts_x)
            shift_x = shift_x.reshape(-1)
            shift_y = shift_y.reshape(-1)
            shifts = torch.stack((shift_x, shift_y, shift_x, shift_y), dim=1)

            # For every (base anchor, output anchor) pair,
            # offset each zero-centered base anchor by the center of the output anchor.
            anchors.append(
                (shifts.view(-1, 1, 4) + base_anchors.view(1, -1, 4)).reshape(-1, 4)
            )

        return anchors

    def forward(self, image_list: ImageList, feature_maps: List[Tensor]) -> List[Tensor]:
        grid_sizes = [feature_map.shape[-2:] for feature_map in feature_maps]
        image_size = image_list.tensors.shape[-2:]
        dtype, device = feature_maps[0].dtype, feature_maps[0].device
        strides = [[torch.tensor(image_size[0] // g[0], dtype=torch.int64, device=device),
                    torch.tensor(image_size[1] // g[1], dtype=torch.int64, device=device)] for g in grid_sizes]
        self.set_cell_anchors(dtype, device)
        anchors_over_all_feature_maps = self.grid_anchors(grid_sizes, strides)
        anchors: List[List[torch.Tensor]] = []
        for _ in range(len(image_list.image_sizes)):
            anchors_in_image = [anchors_per_feature_map for anchors_per_feature_map in anchors_over_all_feature_maps]
            anchors.append(anchors_in_image)
        anchors = [torch.cat(anchors_per_image) for anchors_per_image in anchors]
        return anchors

class DefaultBoxGenerator(nn.Module):
    """
    This module generates the default boxes of SSD for a set of feature maps and image sizes.

    Args:
        aspect_ratios (List[List[int]]): A list with all the aspect ratios used in each feature map.
        min_ratio (float): The minimum scale :math:`\text{s}_{\text{min}}` of the default boxes used in the estimation
            of the scales of each feature map. It is used only if the ``scales`` parameter is not provided.
        max_ratio (float): The maximum scale :math:`\text{s}_{\text{max}}`  of the default boxes used in the estimation
            of the scales of each feature map. It is used only if the ``scales`` parameter is not provided.
        scales (List[float]], optional): The scales of the default boxes. If not provided it will be estimated using
            the ``min_ratio`` and ``max_ratio`` parameters.
        steps (List[int]], optional): It's a hyper-parameter that affects the tiling of defalt boxes. If not provided
            it will be estimated from the data.
        clip (bool): Whether the standardized values of default boxes should be clipped between 0 and 1. The clipping
            is applied while the boxes are encoded in format ``(cx, cy, w, h)``.
    """

    def __init__(self, aspect_ratios: List[List[int]], min_ratio: float = 0.15, max_ratio: float = 0.9,
                 scales: Optional[List[float]] = None, steps: Optional[List[int]] = None, clip: bool = True):
        super().__init__()
        if steps is not None:
            assert len(aspect_ratios) == len(steps)
        self.aspect_ratios = aspect_ratios
        self.steps = steps
        self.clip = clip
        num_outputs = len(aspect_ratios)

        # Estimation of default boxes scales
        if scales is None:
            if num_outputs > 1:
                range_ratio = max_ratio - min_ratio
                self.scales = [min_ratio + range_ratio * k / (num_outputs - 1.0) for k in range(num_outputs)]
                self.scales.append(1.0)
            else:
                self.scales = [min_ratio, max_ratio]
        else:
            self.scales = scales

        self._wh_pairs = []
        for k in range(num_outputs):
            # Adding the 2 default width-height pairs for aspect ratio 1 and scale s'k
            s_k = self.scales[k]
            s_prime_k = math.sqrt(self.scales[k] * self.scales[k + 1])
            wh_pairs = [(s_k, s_k), (s_prime_k, s_prime_k)]

            # Adding 2 pairs for each aspect ratio of the feature map k
            for ar in self.aspect_ratios[k]:
                sq_ar = math.sqrt(ar)
                w = self.scales[k] * sq_ar
                h = self.scales[k] / sq_ar
                wh_pairs.extend([(w, h), (h, w)])

            self._wh_pairs.append(wh_pairs)

    def num_anchors_per_location(self):
        # Estimate num of anchors based on aspect ratios: 2 default boxes + 2 * ratios of feaure map.
        return [2 + 2 * len(r) for r in self.aspect_ratios]

    def __repr__(self) -> str:
        s = self.__class__.__name__ + '('
        s += 'aspect_ratios={aspect_ratios}'
        s += ', clip={clip}'
        s += ', scales={scales}'
        s += ', steps={steps}'
        s += ')'
        return s.format(**self.__dict__)

    def forward(self, image_list: ImageList, feature_maps: List[Tensor]) -> List[Tensor]:
        grid_sizes = [feature_map.shape[-2:] for feature_map in feature_maps]
        image_size = image_list.tensors.shape[-2:]
        dtype, device = feature_maps[0].dtype, feature_maps[0].device

        # Default Boxes calculation based on page 6 of SSD paper
        default_boxes: List[List[float]] = []
        for k, f_k in enumerate(grid_sizes):
            # Now add the default boxes for each width-height pair
            for j in range(f_k[0]):
                if self.steps is not None:
                    y_f_k = image_size[1] / self.steps[k]
                else:
                    y_f_k = float(f_k[0])
                cy = (j + 0.5) / y_f_k
                for i in range(f_k[1]):
                    if self.steps is not None:
                        x_f_k = image_size[0] / self.steps[k]
                    else:
                        x_f_k = float(f_k[1])
                    cx = (i + 0.5) / x_f_k
                    default_boxes.extend([[cx, cy, w, h] for w, h in self._wh_pairs[k]])

        dboxes = []
        for _ in image_list.image_sizes:
            dboxes_in_image = torch.tensor(default_boxes, dtype=dtype, device=device)
            if self.clip:
                dboxes_in_image.clamp_(min=0, max=1)
            dboxes_in_image = torch.cat([dboxes_in_image[:, :2] - 0.5 * dboxes_in_image[:, 2:],
                                         dboxes_in_image[:, :2] + 0.5 * dboxes_in_image[:, 2:]], -1)
            dboxes_in_image[:, 0::2] *= image_size[1]
            dboxes_in_image[:, 1::2] *= image_size[0]
            dboxes.append(dboxes_in_image)
        return dboxes
<commit_msg>Refactor grid default boxes with torch meshgrid (#3799)<commit_after>import math
import torch
from torch import nn, Tensor

from typing import List, Optional
from .image_list import ImageList

class AnchorGenerator(nn.Module):
    """
    Module that generates anchors for a set of feature maps and
    image sizes.

    The module support computing anchors at multiple sizes and aspect ratios
    per feature map. This module assumes aspect ratio = height / width for
    each anchor.

    sizes and aspect_ratios should have the same number of elements, and it should
    correspond to the number of feature maps.

    sizes[i] and aspect_ratios[i] can have an arbitrary number of elements,
    and AnchorGenerator will output a set of sizes[i] * aspect_ratios[i] anchors
    per spatial location for feature map i.

    Args:
        sizes (Tuple[Tuple[int]]):
        aspect_ratios (Tuple[Tuple[float]]):
    """

    __annotations__ = {
        "cell_anchors": List[torch.Tensor],
    }

    def __init__(
        self,
        sizes=((128, 256, 512),),
        aspect_ratios=((0.5, 1.0, 2.0),),
    ):
        super(AnchorGenerator, self).__init__()

        if not isinstance(sizes[0], (list, tuple)):
            # TODO change this
            sizes = tuple((s,) for s in sizes)
        if not isinstance(aspect_ratios[0], (list, tuple)):
            aspect_ratios = (aspect_ratios,) * len(sizes)

        assert len(sizes) == len(aspect_ratios)

        self.sizes = sizes
        self.aspect_ratios = aspect_ratios
        self.cell_anchors = [self.generate_anchors(size, aspect_ratio)
                             for size, aspect_ratio in zip(sizes, aspect_ratios)]

    # TODO: https://github.com/pytorch/pytorch/issues/26792
    # For every (aspect_ratios, scales) combination, output a zero-centered anchor with those values.
    # (scales, aspect_ratios) are usually an element of zip(self.scales, self.aspect_ratios)
    # This method assumes aspect ratio = height / width for an anchor.
    def generate_anchors(self, scales: List[int], aspect_ratios: List[float], dtype: torch.dtype = torch.float32,
                         device: torch.device = torch.device("cpu")):
        scales = torch.as_tensor(scales, dtype=dtype, device=device)
        aspect_ratios = torch.as_tensor(aspect_ratios, dtype=dtype, device=device)
        h_ratios = torch.sqrt(aspect_ratios)
        w_ratios = 1 / h_ratios

        ws = (w_ratios[:, None] * scales[None, :]).view(-1)
        hs = (h_ratios[:, None] * scales[None, :]).view(-1)

        base_anchors = torch.stack([-ws, -hs, ws, hs], dim=1) / 2
        return base_anchors.round()

    def set_cell_anchors(self, dtype: torch.dtype, device: torch.device):
        self.cell_anchors = [cell_anchor.to(dtype=dtype, device=device)
                             for cell_anchor in self.cell_anchors]

    def num_anchors_per_location(self):
        return [len(s) * len(a) for s, a in zip(self.sizes, self.aspect_ratios)]

    # For every combination of (a, (g, s), i) in (self.cell_anchors, zip(grid_sizes, strides), 0:2),
    # output g[i] anchors that are s[i] distance apart in direction i, with the same dimensions as a.
    def grid_anchors(self, grid_sizes: List[List[int]], strides: List[List[Tensor]]) -> List[Tensor]:
        anchors = []
        cell_anchors = self.cell_anchors
        assert cell_anchors is not None

        if not (len(grid_sizes) == len(strides) == len(cell_anchors)):
            raise ValueError("Anchors should be Tuple[Tuple[int]] because each feature "
                             "map could potentially have different sizes and aspect ratios. "
                             "There needs to be a match between the number of "
                             "feature maps passed and the number of sizes / aspect ratios specified.")

        for size, stride, base_anchors in zip(
            grid_sizes, strides, cell_anchors
        ):
            grid_height, grid_width = size
            stride_height, stride_width = stride
            device = base_anchors.device

            # For output anchor, compute [x_center, y_center, x_center, y_center]
            shifts_x = torch.arange(
                0, grid_width, dtype=torch.float32, device=device
            ) * stride_width
            shifts_y = torch.arange(
                0, grid_height, dtype=torch.float32, device=device
            ) * stride_height
            shift_y, shift_x = torch.meshgrid(shifts_y, shifts_x)
            shift_x = shift_x.reshape(-1)
            shift_y = shift_y.reshape(-1)
            shifts = torch.stack((shift_x, shift_y, shift_x, shift_y), dim=1)

            # For every (base anchor, output anchor) pair,
            # offset each zero-centered base anchor by the center of the output anchor.
            anchors.append(
                (shifts.view(-1, 1, 4) + base_anchors.view(1, -1, 4)).reshape(-1, 4)
            )

        return anchors

    def forward(self, image_list: ImageList, feature_maps: List[Tensor]) -> List[Tensor]:
        grid_sizes = [feature_map.shape[-2:] for feature_map in feature_maps]
        image_size = image_list.tensors.shape[-2:]
        dtype, device = feature_maps[0].dtype, feature_maps[0].device
        strides = [[torch.tensor(image_size[0] // g[0], dtype=torch.int64, device=device),
                    torch.tensor(image_size[1] // g[1], dtype=torch.int64, device=device)] for g in grid_sizes]
        self.set_cell_anchors(dtype, device)
        anchors_over_all_feature_maps = self.grid_anchors(grid_sizes, strides)
        anchors: List[List[torch.Tensor]] = []
        for _ in range(len(image_list.image_sizes)):
            anchors_in_image = [anchors_per_feature_map for anchors_per_feature_map in anchors_over_all_feature_maps]
            anchors.append(anchors_in_image)
        anchors = [torch.cat(anchors_per_image) for anchors_per_image in anchors]
        return anchors

class DefaultBoxGenerator(nn.Module):
    """
    This module generates the default boxes of SSD for a set of feature maps and image sizes.

    Args:
        aspect_ratios (List[List[int]]): A list with all the aspect ratios used in each feature map.
        min_ratio (float): The minimum scale :math:`\text{s}_{\text{min}}` of the default boxes used in the estimation
            of the scales of each feature map. It is used only if the ``scales`` parameter is not provided.
        max_ratio (float): The maximum scale :math:`\text{s}_{\text{max}}`  of the default boxes used in the estimation
            of the scales of each feature map. It is used only if the ``scales`` parameter is not provided.
        scales (List[float]], optional): The scales of the default boxes. If not provided it will be estimated using
            the ``min_ratio`` and ``max_ratio`` parameters.
        steps (List[int]], optional): It's a hyper-parameter that affects the tiling of defalt boxes. If not provided
            it will be estimated from the data.
        clip (bool): Whether the standardized values of default boxes should be clipped between 0 and 1. The clipping
            is applied while the boxes are encoded in format ``(cx, cy, w, h)``.
    """

    def __init__(self, aspect_ratios: List[List[int]], min_ratio: float = 0.15, max_ratio: float = 0.9,
                 scales: Optional[List[float]] = None, steps: Optional[List[int]] = None, clip: bool = True):
        super().__init__()
        if steps is not None:
            assert len(aspect_ratios) == len(steps)
        self.aspect_ratios = aspect_ratios
        self.steps = steps
        self.clip = clip
        num_outputs = len(aspect_ratios)

        # Estimation of default boxes scales
        if scales is None:
            if num_outputs > 1:
                range_ratio = max_ratio - min_ratio
                self.scales = [min_ratio + range_ratio * k / (num_outputs - 1.0) for k in range(num_outputs)]
                self.scales.append(1.0)
            else:
                self.scales = [min_ratio, max_ratio]
        else:
            self.scales = scales

        self._wh_pairs = self._generate_wh_pairs(num_outputs)

    def _generate_wh_pairs(self, num_outputs: int, dtype: torch.dtype = torch.float32,
                           device: torch.device = torch.device("cpu")) -> List[Tensor]:
        _wh_pairs: List[Tensor] = []
        for k in range(num_outputs):
            # Adding the 2 default width-height pairs for aspect ratio 1 and scale s'k
            s_k = self.scales[k]
            s_prime_k = math.sqrt(self.scales[k] * self.scales[k + 1])
            wh_pairs = [[s_k, s_k], [s_prime_k, s_prime_k]]

            # Adding 2 pairs for each aspect ratio of the feature map k
            for ar in self.aspect_ratios[k]:
                sq_ar = math.sqrt(ar)
                w = self.scales[k] * sq_ar
                h = self.scales[k] / sq_ar
                wh_pairs.extend([[w, h], [h, w]])

            _wh_pairs.append(torch.as_tensor(wh_pairs, dtype=dtype, device=device))
        return _wh_pairs

    def num_anchors_per_location(self):
        # Estimate num of anchors based on aspect ratios: 2 default boxes + 2 * ratios of feaure map.
        return [2 + 2 * len(r) for r in self.aspect_ratios]

    # Default Boxes calculation based on page 6 of SSD paper
    def _grid_default_boxes(self, grid_sizes: List[List[int]], image_size: List[int],
                            dtype: torch.dtype = torch.float32) -> Tensor:
        default_boxes = []
        for k, f_k in enumerate(grid_sizes):
            # Now add the default boxes for each width-height pair
            if self.steps is not None:
                x_f_k, y_f_k = [img_shape / self.steps[k] for img_shape in image_size]
            else:
                y_f_k, x_f_k = f_k

            shifts_x = (torch.arange(0, f_k[1], dtype=dtype) + 0.5) / x_f_k
            shifts_y = (torch.arange(0, f_k[0], dtype=dtype) + 0.5) / y_f_k
            shift_y, shift_x = torch.meshgrid(shifts_y, shifts_x)
            shift_x = shift_x.reshape(-1)
            shift_y = shift_y.reshape(-1)

            shifts = torch.stack((shift_x, shift_y) * len(self._wh_pairs[k]), dim=-1).reshape(-1, 2)
            # Clipping the default boxes while the boxes are encoded in format (cx, cy, w, h)
            _wh_pair = self._wh_pairs[k].clamp(min=0, max=1) if self.clip else self._wh_pairs[k]
            wh_pairs = _wh_pair.repeat((f_k[0] * f_k[1]), 1)

            default_box = torch.cat((shifts, wh_pairs), dim=1)

            default_boxes.append(default_box)

        return torch.cat(default_boxes, dim=0)

    def __repr__(self) -> str:
        s = self.__class__.__name__ + '('
        s += 'aspect_ratios={aspect_ratios}'
        s += ', clip={clip}'
        s += ', scales={scales}'
        s += ', steps={steps}'
        s += ')'
        return s.format(**self.__dict__)

    def forward(self, image_list: ImageList, feature_maps: List[Tensor]) -> List[Tensor]:
        grid_sizes = [feature_map.shape[-2:] for feature_map in feature_maps]
        image_size = image_list.tensors.shape[-2:]
        dtype, device = feature_maps[0].dtype, feature_maps[0].device
        default_boxes = self._grid_default_boxes(grid_sizes, image_size, dtype=dtype)
        default_boxes = default_boxes.to(device)

        dboxes = []
        for _ in image_list.image_sizes:
            dboxes_in_image = default_boxes
            dboxes_in_image = torch.cat([dboxes_in_image[:, :2] - 0.5 * dboxes_in_image[:, 2:],
                                         dboxes_in_image[:, :2] + 0.5 * dboxes_in_image[:, 2:]], -1)
            dboxes_in_image[:, 0::2] *= image_size[1]
            dboxes_in_image[:, 1::2] *= image_size[0]
            dboxes.append(dboxes_in_image)
        return dboxes
<|endoftext|>


<commit_before>"""Maintains the module version."""

# The MIT License (MIT)
#
# Copyright (c) 2015 by Teradata
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
__version__ = "15.10.0.15"
<commit_msg>Incrementing next patch version to 15.10.00.16<commit_after>"""Maintains the module version."""

# The MIT License (MIT)
#
# Copyright (c) 2015 by Teradata
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
__version__ = "15.10.0.16"
<|endoftext|>


<commit_before>
<commit_msg>Update shine.py<commit_after>
import threading, time
from random import randint

def flashLed(e, t):
    """flash the specified led every 2 second"""
    while not e.isSet():
        print('leds on')
        time.sleep(0.5)
        event_is_set = e.wait(t)
        if event_is_set:
            print('stop led from flashing')
            break
        else:
            print('leds off')
            time.sleep(1.5)

#colour = "red"
e = threading.Event()
t = threading.Thread(name='non-block', target=flashLed, args=(e, 1))
t.start()

print('recording started')
for i in range(0, 10):
    print('recording ...')
    time.sleep(10)
print('recording finished')
e.set()
exit()
<|endoftext|>


<commit_before># Copyright 2017 ACSONE SA/NV
# Copyright 2020 Brainbean Apps (https://brainbeanapps.com)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    'name': 'Account Analytic Sequence',
    'summary': """
        Restore the analytic account sequence""",
    'version': '12.0.1.0.0',
    'license': 'AGPL-3',
    'author': 'ACSONE SA/NV,Odoo Community Association (OCA)',
    'website': 'https://github.com/OCA/account-analytic',
    'depends': [
        'analytic',
    ],
    'data': [
        'data/sequence.xml',
    ],
    'post_init_hook': 'post_init_hook',
}
<commit_msg>account_analytic_sequence 12.0.1.1.0<commit_after># Copyright 2017 ACSONE SA/NV
# Copyright 2020 Brainbean Apps (https://brainbeanapps.com)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    'name': 'Account Analytic Sequence',
    'summary': """
        Restore the analytic account sequence""",
    'version': '12.0.1.1.0',
    'license': 'AGPL-3',
    'author': 'ACSONE SA/NV,Odoo Community Association (OCA)',
    'website': 'https://github.com/OCA/account-analytic',
    'depends': [
        'analytic',
    ],
    'data': [
        'data/sequence.xml',
    ],
    'post_init_hook': 'post_init_hook',
}
<|endoftext|>


<commit_before>##############################################################################
# For copyright and license notices, see __manifest__.py file in module root
# directory
##############################################################################
from odoo import api, models, fields, _
from odoo.tools.safe_eval import safe_eval
# from odoo.exceptions import ValidationError

class ResPartner(models.Model):
    _inherit = 'res.partner'

    def _get_debt_report_lines(self):
        # TODO ver si borramos este metodo que no tiene mucho sentido (get_line_vals)
        def get_line_vals(
                date=None, name=None, detail_lines=None, date_maturity=None,
                amount=None, amount_residual=None, balance=None,
                amount_currency=None,
                currency_name=None, move_line=None):
            if not detail_lines:
                detail_lines = []
            return {
                'date': date,
                'name': name,
                'detail_lines': detail_lines,
                'date_maturity': date_maturity,
                'amount': amount,
                'amount_residual': amount_residual,
                'balance': balance,
                'amount_currency': amount_currency,
                'currency_name': currency_name,
                'move_line': move_line,
            }

        self.ensure_one()

        result_selection = self._context.get('result_selection', False)
        from_date = self._context.get('from_date', False)
        to_date = self._context.get('to_date', False)
        historical_full = self._context.get('historical_full', False)
        company_id = self._context.get('company_id', False)
        show_invoice_detail = self._context.get('show_invoice_detail', False)

        domain = []

        if company_id:
            domain += [('company_id', '=', company_id)]
        else:
            domain += [('company_id', 'in', self.env.companies.ids)]

        if not historical_full:
            domain += [('reconciled', '=', False), ('full_reconcile_id', '=', False)]
            # si pide historial completo entonces mostramos los movimientos
            # si no mostramos los saldos
            balance_field = 'amount_residual'
        else:
            balance_field = 'balance'

        if result_selection == 'receivable':
            domain += [('account_internal_type', '=', 'receivable')]
        elif result_selection == 'payable':
            domain += [('account_internal_type', '=', 'payable')]
        else:
            domain += [('account_internal_type', 'in', ['receivable', 'payable'])]

        domain += [('partner_id', '=', self.id), ('parent_state', '=', 'posted')]

        if from_date:
            initial_domain = domain + [('date', '<', from_date)]
            inicial_lines = self.env['account.move.line'].sudo().read_group(
                initial_domain, fields=['balance'], groupby=['partner_id'])
            balance = inicial_lines[0]['balance'] if inicial_lines else 0.0
            res = [get_line_vals(name=_('INITIAL BALANCE'), balance=balance)]
            domain.append(('date', '>=', from_date))
        else:
            balance = 0.0
            res = []

        if to_date:
            final_line = []
            domain.append(('date', '<=', to_date))
        else:
            final_line = []

        records = self.env['account.move.line'].sudo().search(domain, order='date asc, date_maturity asc, name, id')

        grouped = self.env['account.payment']._fields.get('payment_group_id') and safe_eval(
            self.env['ir.config_parameter'].sudo().get_param(
                'account_debt_report.group_payment_group_payments', 'False'))

        last_payment_group_id = False

        # construimos una nueva lista con los valores que queremos y de
        # manera mas facil
        for record in records:
            detail_lines = []
            if show_invoice_detail:
                for inv_line in record.move_id.invoice_line_ids:
                    detail_lines.append(
                        ("* %s x %s %s" % (
                            inv_line.name.replace(
                                '\n', ' ').replace('\r', ''),
                            inv_line.quantity,
                            inv_line.product_uom_id.name)))
            name = record.move_id.name
            # similar to _format_aml_name
            if record.ref and record.ref != '/':
                name += ' - ' + record.ref

            date_maturity = record.date_maturity
            date = record.date
            currency = record.currency_id
            balance += record[balance_field]
            amount = record.balance
            amount_residual = record.amount_residual
            amount_currency = record.amount_currency

            if grouped and record.payment_id and record.payment_id.payment_group_id == last_payment_group_id:
                # si agrupamos pagos y el grupo de pagos coincide con el último, entonces acumulamos en linea anterior
                res[-1].update({
                    'amount': res[-1]['amount'] + record.balance,
                    'amount_residual': res[-1]['amount_residual'] + record.amount_residual,
                    'amount_currency': res[-1]['amount_currency'] + record.amount_currency,
                    'balance': balance,
                })
                continue
            elif grouped and record.payment_id and record.payment_id.payment_group_id != last_payment_group_id:
                # si es un payment pero no es del payment group anterior, seteamos este como ultimo payment group
                last_payment_group_id = record.payment_id.payment_group_id
            elif not grouped and record.payment_id:
                # si no agrupamos y es pago, agregamos nombre de diario para que sea mas claro
                name += ' - ' + record.journal_id.name

            # TODO tal vez la suma podriamos probar hacerla en el xls como hacemos en libro iva v11/v12
            res.append(get_line_vals(
                date=date,
                name=name,
                detail_lines=detail_lines,
                date_maturity=date_maturity,
                amount=amount,
                amount_residual=amount_residual,
                balance=balance,
                amount_currency=amount_currency,
                currency_name=currency.name,
                # move_line=record.move_line_id,
            ))
        res += final_line
        return res
<commit_msg>[FIX] account_debt_report: corregir importes en Detalle de Deuda<commit_after>##############################################################################
# For copyright and license notices, see __manifest__.py file in module root
# directory
##############################################################################
from odoo import api, models, fields, _
from odoo.tools.safe_eval import safe_eval
# from odoo.exceptions import ValidationError

class ResPartner(models.Model):
    _inherit = 'res.partner'

    def _get_debt_report_lines(self):
        # TODO ver si borramos este metodo que no tiene mucho sentido (get_line_vals)
        def get_line_vals(
                date=None, name=None, detail_lines=None, date_maturity=None,
                amount=None, amount_residual=None, balance=None,
                amount_currency=None,
                currency_name=None, move_line=None):
            if not detail_lines:
                detail_lines = []
            return {
                'date': date,
                'name': name,
                'detail_lines': detail_lines,
                'date_maturity': date_maturity,
                'amount': amount,
                'amount_residual': amount_residual,
                'balance': balance,
                'amount_currency': amount_currency,
                'currency_name': currency_name,
                'move_line': move_line,
            }

        self.ensure_one()

        result_selection = self._context.get('result_selection', False)
        from_date = self._context.get('from_date', False)
        to_date = self._context.get('to_date', False)
        historical_full = self._context.get('historical_full', False)
        company_id = self._context.get('company_id', False)
        show_invoice_detail = self._context.get('show_invoice_detail', False)

        domain = []

        if company_id:
            domain += [('company_id', '=', company_id)]
        else:
            domain += [('company_id', 'in', self.env.companies.ids)]

        if not historical_full:
            domain += [('reconciled', '=', False), ('full_reconcile_id', '=', False)]
            # si pide historial completo entonces mostramos los movimientos
            # si no mostramos los saldos
            balance_field = 'amount_residual'
        else:
            balance_field = 'balance'

        if result_selection == 'receivable':
            domain += [('account_internal_type', '=', 'receivable')]
        elif result_selection == 'payable':
            domain += [('account_internal_type', '=', 'payable')]
        else:
            domain += [('account_internal_type', 'in', ['receivable', 'payable'])]

        domain += [('partner_id', '=', self.id), ('parent_state', '=', 'posted')]

        if from_date:
            initial_domain = domain + [('date', '<', from_date)]
            inicial_lines = self.env['account.move.line'].sudo().read_group(
                initial_domain, fields=['balance'], groupby=['partner_id'])
            balance = inicial_lines[0]['balance'] if inicial_lines else 0.0
            res = [get_line_vals(name=_('INITIAL BALANCE'), balance=balance)]
            domain.append(('date', '>=', from_date))
        else:
            balance = 0.0
            res = []

        if to_date:
            final_line = []
            domain.append(('date', '<=', to_date))
        else:
            final_line = []

        records = self.env['account.move.line'].sudo().search(domain, order='date asc, name, move_id desc, date_maturity asc, id')

        grouped = self.env['account.payment']._fields.get('payment_group_id') and safe_eval(
            self.env['ir.config_parameter'].sudo().get_param(
                'account_debt_report.group_payment_group_payments', 'False'))

        last_payment_group_id = False

        # construimos una nueva lista con los valores que queremos y de
        # manera mas facil
        for record in records:
            detail_lines = []
            if show_invoice_detail:
                for inv_line in record.move_id.invoice_line_ids:
                    detail_lines.append(
                        ("* %s x %s %s" % (
                            inv_line.name.replace(
                                '\n', ' ').replace('\r', ''),
                            inv_line.quantity,
                            inv_line.product_uom_id.name)))
            name = record.move_id.name
            # similar to _format_aml_name
            if record.ref and record.ref != '/':
                name += ' - ' + record.ref

            date_maturity = record.date_maturity
            date = record.date
            currency = record.currency_id
            balance += record[balance_field]
            amount = record.balance
            amount_residual = record.amount_residual
            amount_currency = record.amount_currency

            if grouped and record.payment_id and record.payment_id.payment_group_id == last_payment_group_id:
                # si agrupamos pagos y el grupo de pagos coincide con el último, entonces acumulamos en linea anterior
                res[-1].update({
                    'amount': res[-1]['amount'] + record.balance,
                    'amount_residual': res[-1]['amount_residual'] + record.amount_residual,
                    'amount_currency': res[-1]['amount_currency'] + record.amount_currency,
                    'balance': balance,
                })
                continue
            elif grouped and record.payment_id and record.payment_id.payment_group_id != last_payment_group_id:
                # si es un payment pero no es del payment group anterior, seteamos este como ultimo payment group
                last_payment_group_id = record.payment_id.payment_group_id
            elif not grouped and record.payment_id:
                # si no agrupamos y es pago, agregamos nombre de diario para que sea mas claro
                name += ' - ' + record.journal_id.name

            # TODO tal vez la suma podriamos probar hacerla en el xls como hacemos en libro iva v11/v12
            res.append(get_line_vals(
                date=date,
                name=name,
                detail_lines=detail_lines,
                date_maturity=date_maturity,
                amount=amount,
                amount_residual=amount_residual,
                balance=balance,
                amount_currency=amount_currency,
                currency_name=currency.name,
                # move_line=record.move_line_id,
            ))
        res += final_line
        return res
<|endoftext|>


<commit_before># cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Andoni Morales Alastruey <ylatuya@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

from cerbero.commands import Command, register_command
from cerbero.build.cookbook import CookBook
from cerbero.packages.packagesstore import PackagesStore
from cerbero.utils import _, N_, ArgparseArgument, remove_list_duplicates
from cerbero.utils import messages as m

class Fetch(Command):

    def __init__(self, args=[]):
        args.append(ArgparseArgument('--reset-rdeps', action='store_true',
                    default=False, help=_('reset the status of reverse '
                    'dependencies too')))
        args.append(ArgparseArgument('--full-reset', action='store_true',
                    default=False, help=_('reset to extract step if rebuild is needed')))
        Command.__init__(self, args)

    def fetch(self, cookbook, recipes, no_deps, reset_rdeps, full_reset):
        fetch_recipes = []
        if not recipes:
            fetch_recipes = cookbook.get_recipes_list()
        elif no_deps:
            fetch_recipes = [cookbook.get_recipe(x) for x in recipes]
        else:
            for recipe in recipes:
                fetch_recipes += cookbook.list_recipe_deps(recipe)
            fetch_recipes = remove_list_duplicates (fetch_recipes)
        m.message(_("Fetching the following recipes: %s") %
                  ' '.join([x.name for x in fetch_recipes]))
        to_rebuild = []
        for i in range(len(fetch_recipes)):
            recipe = fetch_recipes[i]
            m.build_step(i + 1, len(fetch_recipes), recipe, 'Fetch')
            recipe.fetch()
            bv = cookbook.recipe_built_version(recipe.name)
            cv = recipe.built_version()
            if bv != cv:
                # On different versions, only reset recipe if:
                #  * forced
                #  * OR it was fully built already
                if full_reset or not cookbook.recipe_needs_build(recipe.name):
                    to_rebuild.append(recipe)
                    cookbook.reset_recipe_status(recipe.name)
                    if reset_rdeps:
                        for r in cookbook.list_recipe_reverse_deps(recipe.name):
                            to_rebuild.append(r)
                            cookbook.reset_recipe_status(r.name)

        if to_rebuild:
            to_rebuild = sorted(list(set(to_rebuild)), key=lambda r:r.name)
            m.message(_("These recipes have been updated and will "
                        "be rebuilt:\n%s") %
                        '\n'.join([x.name for x in to_rebuild]))

class FetchRecipes(Fetch):
    doc = N_('Fetch the recipes sources')
    name = 'fetch'

    def __init__(self):
        args = [
                ArgparseArgument('recipes', nargs='*',
                    help=_('list of the recipes to fetch (fetch all if none '
                           'is passed)')),
                ArgparseArgument('--no-deps', action='store_true',
                    default=False, help=_('do not fetch dependencies')),
                ]
        Fetch.__init__(self, args)

    def run(self, config, args):
        cookbook = CookBook(config)
        return self.fetch(cookbook, args.recipes, args.no_deps,
                          args.reset_rdeps, args.full_reset)

class FetchPackage(Fetch):
    doc = N_('Fetch the recipes sources from a package')
    name = 'fetch-package'

    def __init__(self):
        args = [
                ArgparseArgument('package', nargs=1,
                    help=_('package to fetch')),
                ]
        Fetch.__init__(self, args)

    def run(self, config, args):
        store = PackagesStore(config)
        package = store.get_package(args.package[0])
        return self.fetch(store.cookbook, package.recipes_dependencies(),
                          True, args.reset_rdeps, args.full_reset)

register_command(FetchRecipes)
register_command(FetchPackage)
<commit_msg>fetch: Add --deps to also "fetch-package" to also fetch dependencies<commit_after># cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Andoni Morales Alastruey <ylatuya@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

from cerbero.commands import Command, register_command
from cerbero.build.cookbook import CookBook
from cerbero.packages.packagesstore import PackagesStore
from cerbero.utils import _, N_, ArgparseArgument, remove_list_duplicates
from cerbero.utils import messages as m

class Fetch(Command):

    def __init__(self, args=[]):
        args.append(ArgparseArgument('--reset-rdeps', action='store_true',
                    default=False, help=_('reset the status of reverse '
                    'dependencies too')))
        args.append(ArgparseArgument('--full-reset', action='store_true',
                    default=False, help=_('reset to extract step if rebuild is needed')))
        Command.__init__(self, args)

    def fetch(self, cookbook, recipes, no_deps, reset_rdeps, full_reset):
        fetch_recipes = []
        if not recipes:
            fetch_recipes = cookbook.get_recipes_list()
        elif no_deps:
            fetch_recipes = [cookbook.get_recipe(x) for x in recipes]
        else:
            for recipe in recipes:
                fetch_recipes += cookbook.list_recipe_deps(recipe)
            fetch_recipes = remove_list_duplicates (fetch_recipes)
        m.message(_("Fetching the following recipes: %s") %
                  ' '.join([x.name for x in fetch_recipes]))
        to_rebuild = []
        for i in range(len(fetch_recipes)):
            recipe = fetch_recipes[i]
            m.build_step(i + 1, len(fetch_recipes), recipe, 'Fetch')
            recipe.fetch()
            bv = cookbook.recipe_built_version(recipe.name)
            cv = recipe.built_version()
            if bv != cv:
                # On different versions, only reset recipe if:
                #  * forced
                #  * OR it was fully built already
                if full_reset or not cookbook.recipe_needs_build(recipe.name):
                    to_rebuild.append(recipe)
                    cookbook.reset_recipe_status(recipe.name)
                    if reset_rdeps:
                        for r in cookbook.list_recipe_reverse_deps(recipe.name):
                            to_rebuild.append(r)
                            cookbook.reset_recipe_status(r.name)

        if to_rebuild:
            to_rebuild = sorted(list(set(to_rebuild)), key=lambda r:r.name)
            m.message(_("These recipes have been updated and will "
                        "be rebuilt:\n%s") %
                        '\n'.join([x.name for x in to_rebuild]))

class FetchRecipes(Fetch):
    doc = N_('Fetch the recipes sources')
    name = 'fetch'

    def __init__(self):
        args = [
                ArgparseArgument('recipes', nargs='*',
                    help=_('list of the recipes to fetch (fetch all if none '
                           'is passed)')),
                ArgparseArgument('--no-deps', action='store_true',
                    default=False, help=_('do not fetch dependencies')),
                ]
        Fetch.__init__(self, args)

    def run(self, config, args):
        cookbook = CookBook(config)
        return self.fetch(cookbook, args.recipes, args.no_deps,
                          args.reset_rdeps, args.full_reset)

class FetchPackage(Fetch):
    doc = N_('Fetch the recipes sources from a package')
    name = 'fetch-package'

    def __init__(self):
        args = [
                ArgparseArgument('package', nargs=1,
                    help=_('package to fetch')),
                ArgparseArgument('--deps', action='store_false',
                    default=True, help=_('also fetch dependencies')),
                ]
        Fetch.__init__(self, args)

    def run(self, config, args):
        store = PackagesStore(config)
        package = store.get_package(args.package[0])
        return self.fetch(store.cookbook, package.recipes_dependencies(),
                          args.deps, args.reset_rdeps, args.full_reset)

register_command(FetchRecipes)
register_command(FetchPackage)
<|endoftext|>