<commit_before>#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# About this script:
#   This script is intended to simplify creating Subversion releases, by
#   automating as much as is possible.  It works well with our Apache
#   infrastructure, and should make rolling, posting, and announcing
#   releases dirt simple.
#
#   This script may be run on a number of platforms, but it is intended to
#   be run on people.apache.org.  As such, it may have dependencies (such
#   as Python version) which may not be common, but are guaranteed to be
#   available on people.apache.org.

# It'd be kind of nice to use the Subversion python bindings in this script,
# but people.apache.org doesn't currently have them installed

# Stuff we need
import os
import sys
import shutil
import urllib2
import hashlib
import tarfile
import logging
import subprocess
import argparse       # standard in Python 2.7

# Our required / recommended versions
autoconf_ver = '2.68'
libtool_ver = '2.4'
swig_ver = '2.0.4'

# Some constants
repos = 'http://svn.apache.org/repos/asf/subversion'

#----------------------------------------------------------------------
# Utility functions

def get_prefix(base_dir):
    return os.path.join(base_dir, 'prefix')

def get_tempdir(base_dir):
    return os.path.join(base_dir, 'tempdir')

def get_deploydir(base_dir):
    return os.path.join(base_dir, 'deploy')

def get_nullfile():
    # This is certainly not cross platform
    return open('/dev/null', 'w')

def run_script(verbose, script):
    if verbose:
        stdout = None
        stderr = None
    else:
        stdout = get_nullfile()
        stderr = subprocess.STDOUT

    for l in script.split('\n'):
        subprocess.check_call(l.split(), stdout=stdout, stderr=stderr)

def download_file(url, target):
    response = urllib2.urlopen(url)
    target_file = open(target, 'w')
    target_file.write(response.read())

#----------------------------------------------------------------------
# Cleaning up the environment

def cleanup(base_dir, args):
    'Remove generated files and folders.'
    logging.info('Cleaning')

    shutil.rmtree(get_prefix(base_dir), True)
    shutil.rmtree(get_tempdir(base_dir), True)
    shutil.rmtree(get_deploydir(base_dir), True)

#----------------------------------------------------------------------
# Creating an environment to roll the release

class RollDep(object):
    'The super class for each of the build dependencies.'
    def __init__(self, base_dir, use_existing, verbose):
        self._base_dir = base_dir
        self._use_existing = use_existing
        self._verbose = verbose

    def _test_version(self, cmd):
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        rc = proc.wait()
        if rc: return ''

        return stdout.split('\n')

    def build(self):
        if not hasattr(self, '_extra_configure_flags'):
            self._extra_configure_flags = ''
        cwd = os.getcwd()
        tempdir = get_tempdir(self._base_dir)
        tarball = os.path.join(tempdir, self._filebase + '.tar.gz')

        if os.path.exists(tarball):
            if not self._use_existing:
                raise RuntimeError('autoconf tarball "%s" already exists'
                                                                    % tarball)
            logging.info('Using existing %s.tar.gz' % self._filebase)
        else:
            logging.info('Fetching %s' % self._filebase)
            download_file(self._url, tarball)

        # Extract tarball
        tarfile.open(tarball).extractall(tempdir)

        logging.info('Building ' + self.label)
        os.chdir(os.path.join(tempdir, self._filebase))
        run_script(self._verbose,
                   '''./configure --prefix=%s %s
                      make
                      make install''' % (get_prefix(self._base_dir),
                                         self._extra_configure_flags))

        os.chdir(cwd)

class AutoconfDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'autoconf'
        self._filebase = 'autoconf-' + autoconf_ver
        self._url = 'http://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['autoconf', '-V'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == autoconf_ver

    def use_system(self):
        if not self._use_existing: return False
        return self.have_usable()

class LibtoolDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'libtool'
        self._filebase = 'libtool-' + libtool_ver
        self._url = 'http://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['libtool', '--version'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == libtool_ver

    def use_system(self):
        # We unconditionally return False here, to avoid using a borked
        # system libtool (I'm looking at you, Debian).
        return False

class SwigDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose, sf_mirror):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'swig'
        self._filebase = 'swig-' + swig_ver
        self._url = 'http://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \
            { 'swig' : self._filebase,
              'sf_mirror' : sf_mirror }
        self._extra_configure_flags = '--without-pcre'

    def have_usable(self):
        output = self._test_version(['swig', '-version'])
        if not output: return False

        version = output[1].split()[-1:][0]
        return version == swig_ver

    def use_system(self):
        if not self._use_existing: return False
        return have_usable()

def build_env(base_dir, args):
    'Download prerequisites for a release and prepare the environment.'
    logging.info('Creating release environment')

    try:
        os.mkdir(get_prefix(base_dir))
        os.mkdir(get_tempdir(base_dir))
    except OSError:
        if not args.use_existing:
            raise

    autoconf = AutoconfDep(base_dir, args.use_existing, args.verbose)
    libtool = LibtoolDep(base_dir, args.use_existing, args.verbose)
    swig = SwigDep(base_dir, args.use_existing, args.verbose, args.sf_mirror)

    # iterate over our rolling deps, and build them if needed
    for dep in [autoconf, libtool, swig]:
        if dep.use_system():
            logging.info('Using system %s' % dep.label)
        else:
            dep.build()

#----------------------------------------------------------------------
# Create release artifacts

def roll_tarballs(base_dir, args):
    'Create the release artifacts.'
    extns = ['zip', 'tar.gz', 'tar.bz2']
    version_base = args.version.split('-')[0]
    version_extra = args.version.split('-')[1]

    if args.branch:
        branch = args.branch
    else:
        branch = version_base[:-1] + 'x'

    logging.info('Rolling release %s from branch %s@%d' % (args.version,
                                                           branch, args.revnum))

    # Ensure we've got the appropriate rolling dependencies available
    autoconf = AutoconfDep(base_dir, False, args.verbose)
    libtool = LibtoolDep(base_dir, False, args.verbose)
    swig = SwigDep(base_dir, False, args.verbose, None)

    for dep in [autoconf, libtool, swig]:
        if not dep.have_usable():
           raise RuntimeError('Cannot find usable %s' % dep.label)

    # Make sure CHANGES is sync'd
    if branch != 'trunk':
        trunk_CHANGES = '%s/trunk/CHANGES@%d' % (repos, args.revnum)
        branch_CHANGES = '%s/branches/%s/CHANGES@%d' % (repos, branch,
                                                        args.revnum)
        proc = subprocess.Popen(['svn', 'diff', '--summarize', branch_CHANGES,
                                   trunk_CHANGES],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        proc.wait()

        if stdout:
            raise RuntimeError('CHANGES not synced between trunk and branch')

    # Create the output directory
    if not os.path.exists(get_deploydir(base_dir)):
        os.mkdir(get_deploydir(base_dir))

    # For now, just delegate to dist.sh to create the actual artifacts
    if version_extra:
        if version_extra.startswith('alpha'):
            extra_args = '-alpha %s' % version_extra[5:]
        elif version_extra.startswith('beta'):
            extra_args = '-beta %s' % version_extra[4:]
        elif version_extra.startswith('rc'):
            extra_args = '-rc %s' % version_extra[2:]
        elif version_extra.startswith('nightly'):
            extra_args = '-nightly'
        else:
            extra_args = ''
    logging.info('Building UNIX tarballs')
    run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d %s'
                     % (sys.path[0], version_base, branch, args.revnum,
                        extra_args) )
    logging.info('Buildling Windows tarballs')
    run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d -zip %s'
                     % (sys.path[0], version_base, branch, args.revnum,
                        extra_args) )

    # Move the results to the deploy directory
    logging.info('Moving artifacts and calculating checksums')
    for e in extns:
        if version_extra and version_extra.startswith('nightly'):
            filename = 'subversion-trunk.%s' % e
        else:
            filename = 'subversion-%s.%s' % (args.version, e)

        shutil.move(filename, get_deploydir(base_dir))
        filename = os.path.join(get_deploydir(base_dir), filename)
        m = hashlib.sha1()
        m.update(open(filename, 'r').read())
        open(filename + '.sha1', 'w').write(m.hexdigest())

    shutil.move('svn_version.h.dist', get_deploydir(base_dir))

    # And we're done!

def announce(base_dir, args):
    'Write the release announcement.'

#----------------------------------------------------------------------
# Main entry point for argument parsing and handling

def main():
    'Parse arguments, and drive the appropriate subcommand.'

    # Setup our main parser
    parser = argparse.ArgumentParser(
                            description='Create an Apache Subversion release.')
    parser.add_argument('--clean', action='store_true', default=False,
                   help='Remove any directories previously created by %(prog)s')
    parser.add_argument('--verbose', action='store_true', default=False,
                   help='Increase output verbosity')
    parser.add_argument('--base-dir', default=os.getcwd(),
                   help='''The directory in which to create needed files and
                           folders.  The default is the current working
                           directory.''')
    subparsers = parser.add_subparsers(title='subcommands')

    # Setup the parser for the build-env subcommand
    subparser = subparsers.add_parser('build-env',
                    help='''Download release prerequisistes, including autoconf,
                            libtool, and swig.''')
    subparser.set_defaults(func=build_env)
    subparser.add_argument('--sf-mirror', default='softlayer',
                    help='''The mirror to use for downloading files from
                            SourceForge.  If in the EU, you may want to use
                            'kent' for this value.''')
    subparser.add_argument('--use-existing', action='store_true', default=False,
                    help='''Attempt to use existing build dependencies before
                            downloading and building a private set.''')

    # Setup the parser for the roll subcommand
    subparser = subparsers.add_parser('roll',
                    help='''Create the release artifacts.''')
    subparser.set_defaults(func=roll_tarballs)
    subparser.add_argument('version',
                    help='''The release label, such as '1.7.0-alpha1'.''')
    subparser.add_argument('revnum', type=int,
                    help='''The revision number to base the release on.''')
    subparser.add_argument('--branch',
                    help='''The branch to base the release on.''')

    # A meta-target
    subparser = subparsers.add_parser('clean',
                    help='''The same as the '--clean' switch, but as a
                            separate subcommand.''')
    subparser.set_defaults(func=cleanup)

    # Parse the arguments
    args = parser.parse_args()

    # first, process any global operations
    if args.clean:
        cleanup(args.base_dir, args)

    # Set up logging
    logger = logging.getLogger()
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    # Fix up our path so we can use our installed versions
    os.environ['PATH'] = os.path.join(get_prefix(args.base_dir), 'bin') + ':' \
                                                            + os.environ['PATH']

    # finally, run the subcommand, and give it the parsed arguments
    args.func(args.base_dir, args)

if __name__ == '__main__':
    main()
<commit_msg>* tools/dist/release.py   (roll_tarballs): Really ensure we set a variable.<commit_after>#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# About this script:
#   This script is intended to simplify creating Subversion releases, by
#   automating as much as is possible.  It works well with our Apache
#   infrastructure, and should make rolling, posting, and announcing
#   releases dirt simple.
#
#   This script may be run on a number of platforms, but it is intended to
#   be run on people.apache.org.  As such, it may have dependencies (such
#   as Python version) which may not be common, but are guaranteed to be
#   available on people.apache.org.

# It'd be kind of nice to use the Subversion python bindings in this script,
# but people.apache.org doesn't currently have them installed

# Stuff we need
import os
import sys
import shutil
import urllib2
import hashlib
import tarfile
import logging
import subprocess
import argparse       # standard in Python 2.7

# Our required / recommended versions
autoconf_ver = '2.68'
libtool_ver = '2.4'
swig_ver = '2.0.4'

# Some constants
repos = 'http://svn.apache.org/repos/asf/subversion'

#----------------------------------------------------------------------
# Utility functions

def get_prefix(base_dir):
    return os.path.join(base_dir, 'prefix')

def get_tempdir(base_dir):
    return os.path.join(base_dir, 'tempdir')

def get_deploydir(base_dir):
    return os.path.join(base_dir, 'deploy')

def get_nullfile():
    # This is certainly not cross platform
    return open('/dev/null', 'w')

def run_script(verbose, script):
    if verbose:
        stdout = None
        stderr = None
    else:
        stdout = get_nullfile()
        stderr = subprocess.STDOUT

    for l in script.split('\n'):
        subprocess.check_call(l.split(), stdout=stdout, stderr=stderr)

def download_file(url, target):
    response = urllib2.urlopen(url)
    target_file = open(target, 'w')
    target_file.write(response.read())

#----------------------------------------------------------------------
# Cleaning up the environment

def cleanup(base_dir, args):
    'Remove generated files and folders.'
    logging.info('Cleaning')

    shutil.rmtree(get_prefix(base_dir), True)
    shutil.rmtree(get_tempdir(base_dir), True)
    shutil.rmtree(get_deploydir(base_dir), True)

#----------------------------------------------------------------------
# Creating an environment to roll the release

class RollDep(object):
    'The super class for each of the build dependencies.'
    def __init__(self, base_dir, use_existing, verbose):
        self._base_dir = base_dir
        self._use_existing = use_existing
        self._verbose = verbose

    def _test_version(self, cmd):
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        rc = proc.wait()
        if rc: return ''

        return stdout.split('\n')

    def build(self):
        if not hasattr(self, '_extra_configure_flags'):
            self._extra_configure_flags = ''
        cwd = os.getcwd()
        tempdir = get_tempdir(self._base_dir)
        tarball = os.path.join(tempdir, self._filebase + '.tar.gz')

        if os.path.exists(tarball):
            if not self._use_existing:
                raise RuntimeError('autoconf tarball "%s" already exists'
                                                                    % tarball)
            logging.info('Using existing %s.tar.gz' % self._filebase)
        else:
            logging.info('Fetching %s' % self._filebase)
            download_file(self._url, tarball)

        # Extract tarball
        tarfile.open(tarball).extractall(tempdir)

        logging.info('Building ' + self.label)
        os.chdir(os.path.join(tempdir, self._filebase))
        run_script(self._verbose,
                   '''./configure --prefix=%s %s
                      make
                      make install''' % (get_prefix(self._base_dir),
                                         self._extra_configure_flags))

        os.chdir(cwd)

class AutoconfDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'autoconf'
        self._filebase = 'autoconf-' + autoconf_ver
        self._url = 'http://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['autoconf', '-V'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == autoconf_ver

    def use_system(self):
        if not self._use_existing: return False
        return self.have_usable()

class LibtoolDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'libtool'
        self._filebase = 'libtool-' + libtool_ver
        self._url = 'http://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['libtool', '--version'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == libtool_ver

    def use_system(self):
        # We unconditionally return False here, to avoid using a borked
        # system libtool (I'm looking at you, Debian).
        return False

class SwigDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose, sf_mirror):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'swig'
        self._filebase = 'swig-' + swig_ver
        self._url = 'http://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \
            { 'swig' : self._filebase,
              'sf_mirror' : sf_mirror }
        self._extra_configure_flags = '--without-pcre'

    def have_usable(self):
        output = self._test_version(['swig', '-version'])
        if not output: return False

        version = output[1].split()[-1:][0]
        return version == swig_ver

    def use_system(self):
        if not self._use_existing: return False
        return have_usable()

def build_env(base_dir, args):
    'Download prerequisites for a release and prepare the environment.'
    logging.info('Creating release environment')

    try:
        os.mkdir(get_prefix(base_dir))
        os.mkdir(get_tempdir(base_dir))
    except OSError:
        if not args.use_existing:
            raise

    autoconf = AutoconfDep(base_dir, args.use_existing, args.verbose)
    libtool = LibtoolDep(base_dir, args.use_existing, args.verbose)
    swig = SwigDep(base_dir, args.use_existing, args.verbose, args.sf_mirror)

    # iterate over our rolling deps, and build them if needed
    for dep in [autoconf, libtool, swig]:
        if dep.use_system():
            logging.info('Using system %s' % dep.label)
        else:
            dep.build()

#----------------------------------------------------------------------
# Create release artifacts

def roll_tarballs(base_dir, args):
    'Create the release artifacts.'
    extns = ['zip', 'tar.gz', 'tar.bz2']
    version_base = args.version.split('-')[0]
    version_extra = args.version.split('-')[1]

    if args.branch:
        branch = args.branch
    else:
        branch = version_base[:-1] + 'x'

    logging.info('Rolling release %s from branch %s@%d' % (args.version,
                                                           branch, args.revnum))

    # Ensure we've got the appropriate rolling dependencies available
    autoconf = AutoconfDep(base_dir, False, args.verbose)
    libtool = LibtoolDep(base_dir, False, args.verbose)
    swig = SwigDep(base_dir, False, args.verbose, None)

    for dep in [autoconf, libtool, swig]:
        if not dep.have_usable():
           raise RuntimeError('Cannot find usable %s' % dep.label)

    # Make sure CHANGES is sync'd
    if branch != 'trunk':
        trunk_CHANGES = '%s/trunk/CHANGES@%d' % (repos, args.revnum)
        branch_CHANGES = '%s/branches/%s/CHANGES@%d' % (repos, branch,
                                                        args.revnum)
        proc = subprocess.Popen(['svn', 'diff', '--summarize', branch_CHANGES,
                                   trunk_CHANGES],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        proc.wait()

        if stdout:
            raise RuntimeError('CHANGES not synced between trunk and branch')

    # Create the output directory
    if not os.path.exists(get_deploydir(base_dir)):
        os.mkdir(get_deploydir(base_dir))

    # For now, just delegate to dist.sh to create the actual artifacts
    extra_args = ''
    if version_extra:
        if version_extra.startswith('alpha'):
            extra_args = '-alpha %s' % version_extra[5:]
        elif version_extra.startswith('beta'):
            extra_args = '-beta %s' % version_extra[4:]
        elif version_extra.startswith('rc'):
            extra_args = '-rc %s' % version_extra[2:]
        elif version_extra.startswith('nightly'):
            extra_args = '-nightly'
    logging.info('Building UNIX tarballs')
    run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d %s'
                     % (sys.path[0], version_base, branch, args.revnum,
                        extra_args) )
    logging.info('Buildling Windows tarballs')
    run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d -zip %s'
                     % (sys.path[0], version_base, branch, args.revnum,
                        extra_args) )

    # Move the results to the deploy directory
    logging.info('Moving artifacts and calculating checksums')
    for e in extns:
        if version_extra and version_extra.startswith('nightly'):
            filename = 'subversion-trunk.%s' % e
        else:
            filename = 'subversion-%s.%s' % (args.version, e)

        shutil.move(filename, get_deploydir(base_dir))
        filename = os.path.join(get_deploydir(base_dir), filename)
        m = hashlib.sha1()
        m.update(open(filename, 'r').read())
        open(filename + '.sha1', 'w').write(m.hexdigest())

    shutil.move('svn_version.h.dist', get_deploydir(base_dir))

    # And we're done!

def announce(base_dir, args):
    'Write the release announcement.'

#----------------------------------------------------------------------
# Main entry point for argument parsing and handling

def main():
    'Parse arguments, and drive the appropriate subcommand.'

    # Setup our main parser
    parser = argparse.ArgumentParser(
                            description='Create an Apache Subversion release.')
    parser.add_argument('--clean', action='store_true', default=False,
                   help='Remove any directories previously created by %(prog)s')
    parser.add_argument('--verbose', action='store_true', default=False,
                   help='Increase output verbosity')
    parser.add_argument('--base-dir', default=os.getcwd(),
                   help='''The directory in which to create needed files and
                           folders.  The default is the current working
                           directory.''')
    subparsers = parser.add_subparsers(title='subcommands')

    # Setup the parser for the build-env subcommand
    subparser = subparsers.add_parser('build-env',
                    help='''Download release prerequisistes, including autoconf,
                            libtool, and swig.''')
    subparser.set_defaults(func=build_env)
    subparser.add_argument('--sf-mirror', default='softlayer',
                    help='''The mirror to use for downloading files from
                            SourceForge.  If in the EU, you may want to use
                            'kent' for this value.''')
    subparser.add_argument('--use-existing', action='store_true', default=False,
                    help='''Attempt to use existing build dependencies before
                            downloading and building a private set.''')

    # Setup the parser for the roll subcommand
    subparser = subparsers.add_parser('roll',
                    help='''Create the release artifacts.''')
    subparser.set_defaults(func=roll_tarballs)
    subparser.add_argument('version',
                    help='''The release label, such as '1.7.0-alpha1'.''')
    subparser.add_argument('revnum', type=int,
                    help='''The revision number to base the release on.''')
    subparser.add_argument('--branch',
                    help='''The branch to base the release on.''')

    # A meta-target
    subparser = subparsers.add_parser('clean',
                    help='''The same as the '--clean' switch, but as a
                            separate subcommand.''')
    subparser.set_defaults(func=cleanup)

    # Parse the arguments
    args = parser.parse_args()

    # first, process any global operations
    if args.clean:
        cleanup(args.base_dir, args)

    # Set up logging
    logger = logging.getLogger()
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    # Fix up our path so we can use our installed versions
    os.environ['PATH'] = os.path.join(get_prefix(args.base_dir), 'bin') + ':' \
                                                            + os.environ['PATH']

    # finally, run the subcommand, and give it the parsed arguments
    args.func(args.base_dir, args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2017 KMEE
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo import api, fields, models, _, tools
from ..models.financial_move_model import (
    FINANCIAL_MOVE,
    FINANCIAL_STATE,
    FINANCIAL_TYPE
)

class FinancialCashflow(models.Model):

    _name = 'financial.cashflow'
    #_inherit = 'financial.move.model'
    _auto = False

    cumulative_sum = fields.Monetary(
        string=u"Cumulative sum",
    )

    state = fields.Selection(
        selection=FINANCIAL_STATE,
        string='Status',
    )
    company_id = fields.Many2one(
        comodel_name='res.company',
        string=u'Company',
    )
    currency_id = fields.Many2one(
        comodel_name='res.currency',
        string='Currency',
    )
    partner_id = fields.Many2one(
        comodel_name='res.partner',
    )
    document_number = fields.Char(
        string=u"Document Nº",
    )
    document_item = fields.Char(
        string=u"Document item",
    )
    document_date = fields.Date(
        string=u"Document date",
    )
    amount_document = fields.Monetary(
        string=u"Document amount",
    )
    due_date = fields.Date(
        string=u"Due date",
    )
    move_type = fields.Selection(
        selection=FINANCIAL_TYPE,
    )
    business_due_date = fields.Date(
        string='Business due date',
    )
    payment_mode = fields.Many2one(
        comodel_name='payment.mode',  # FIXME:
    )
    payment_term = fields.Many2one(
        comodel_name='payment.term',  # FIXME:
    )

    @api.model_cr
    def init(self):
        tools.drop_view_if_exists(self.env.cr, self._table)
        self.env.cr.execute("""
            CREATE OR REPLACE VIEW financial_cashflow as (
SELECT
                    create_date, id, document_number, document_item, move_type,
                    state,
                    business_due_date,
                    document_date,
                    payment_mode,
                    payment_term,
                    due_date, partner_id, currency_id, amount_document,
                    sum as cumulative_sum

FROM (

                SELECT
                    create_date, id, document_number, document_item, move_type,
                    state,
                    business_due_date,
                    document_date,
                    payment_mode,
                    payment_term,
                    due_date, partner_id, currency_id, amount_document,
                    SUM(amount_document)
                OVER (ORDER BY id)

                FROM (

                    SELECT * FROM (

                        SELECT
                        financial_move.create_date,
                        financial_move.id,
                        financial_move.document_number,
                        financial_move.document_item,
                        financial_move.move_type,

                        financial_move.state,
                        financial_move.business_due_date,
                        financial_move.document_date,
                        financial_move.payment_mode,
                        financial_move.payment_term,

                        financial_move.due_date,
                        financial_move.partner_id,
                        financial_move.currency_id,
                        financial_move.amount_document
                        FROM
                        public.financial_move
                        WHERE
                        financial_move.move_type = 'r'
                        ) r

                    UNION (

                        SELECT
                        financial_move.create_date,
                        financial_move.id,
                        financial_move.document_number,
                        financial_move.document_item,
                        financial_move.move_type,

                        financial_move.state,
                        financial_move.business_due_date,
                        financial_move.document_date,
                        financial_move.payment_mode,
                        financial_move.payment_term,

                        financial_move.due_date,
                        financial_move.partner_id,
                        financial_move.currency_id,
                        (-1) * financial_move.amount_document as amount_document
                        FROM
                        public.financial_move
                        WHERE
                        financial_move.move_type = 'p'
                        )

                ) AS subq

                GROUP BY (create_date, id, document_number,
                    document_item, move_type,
                    state,
                    business_due_date,
                    document_date,
                    payment_mode,
                    payment_term,
                    due_date, partner_id, currency_id, amount_document)
            ) as out
        )""")
<commit_msg>[REF] Divide queries to ease maintenance<commit_after># -*- coding: utf-8 -*-
# Copyright 2017 KMEE
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo import api, fields, models, _, tools
from ..models.financial_move_model import (
    FINANCIAL_MOVE,
    FINANCIAL_STATE,
    FINANCIAL_TYPE
)

class FinancialCashflow(models.Model):

    _name = 'financial.cashflow'
    _auto = False

    cumulative_sum = fields.Monetary(
        string=u"Cumulative sum",
    )

    state = fields.Selection(
        selection=FINANCIAL_STATE,
        string='Status',
    )
    company_id = fields.Many2one(
        comodel_name='res.company',
        string=u'Company',
    )
    currency_id = fields.Many2one(
        comodel_name='res.currency',
        string='Currency',
    )
    partner_id = fields.Many2one(
        comodel_name='res.partner',
    )
    document_number = fields.Char(
        string=u"Document Nº",
    )
    document_item = fields.Char(
        string=u"Document item",
    )
    document_date = fields.Date(
        string=u"Document date",
    )
    amount_document = fields.Monetary(
        string=u"Document amount",
    )
    due_date = fields.Date(
        string=u"Due date",
    )
    move_type = fields.Selection(
        selection=FINANCIAL_TYPE,
    )
    business_due_date = fields.Date(
        string='Business due date',
    )
    payment_mode = fields.Many2one(
        comodel_name='payment.mode',  # FIXME:
    )
    payment_term = fields.Many2one(
        comodel_name='payment.term',  # FIXME:
    )

    @api.model_cr
    def init(self):
        tools.drop_view_if_exists(self.env.cr, self._table)
        self.env.cr.execute("""
            CREATE OR REPLACE VIEW financial_cashflow_credit_type_r as (
                SELECT * FROM (
                    SELECT
                        financial_move.create_date,
                        financial_move.id,
                        financial_move.document_number,
                        financial_move.document_item,
                        financial_move.move_type,

                        financial_move.state,
                        financial_move.business_due_date,
                        financial_move.document_date,
                        financial_move.payment_mode,
                        financial_move.payment_term,

                        financial_move.due_date,
                        financial_move.partner_id,
                        financial_move.currency_id,
                        financial_move.amount_document AS amount_document,
                        financial_move.amount_document AS credit_value,
                        0 AS debit_value
                    FROM
                        public.financial_move
                    WHERE
                        financial_move.move_type = 'r'
                ) r
            )
        """)

        self.env.cr.execute("""
            CREATE OR REPLACE VIEW financial_cashflow_debit_type_p as (
                SELECT * FROM (
                    SELECT
                        financial_move.create_date,
                        financial_move.id,
                        financial_move.document_number,
                        financial_move.document_item,
                        financial_move.move_type,

                        financial_move.state,
                        financial_move.business_due_date,
                        financial_move.document_date,
                        financial_move.payment_mode,
                        financial_move.payment_term,

                        financial_move.due_date,
                        financial_move.partner_id,
                        financial_move.currency_id,
                        (-1) * financial_move.amount_document AS amount_document,
                        0 AS credit_value,
                        (-1) * financial_move.amount_document AS debit_value
                    FROM
                        public.financial_move
                    WHERE
                        financial_move.move_type = 'p'
                ) r
            )
        """)

        self.env.cr.execute("""
            CREATE OR REPLACE VIEW financial_cashflow_main_view AS (
                SELECT
                    create_date, id, document_number, document_item, move_type,
                    state,
                    business_due_date,
                    document_date,
                    payment_mode,
                    payment_term,
                    due_date, partner_id, currency_id, amount_document,
                    SUM(credit_value),
                    SUM(debit_value)
                OVER (ORDER BY business_due_date)

                FROM (
                    SELECT * FROM financial_cashflow_credit_type_r
                    UNION
                    SELECT * FROM financial_cashflow_debit_type_p
                    ORDER BY business_due_date
                ) AS subq

                GROUP BY (create_date, id, document_number,
                    document_item, move_type,
                    state,
                    business_due_date,
                    document_date,
                    payment_mode,
                    payment_term,
                    due_date, partner_id, currency_id,
                    amount_document,
                    credit_value,
                    debit_value
		)
            )
        """)

        self.env.cr.execute("""
            CREATE OR REPLACE VIEW financial_cashflow AS (
                SELECT
                    create_date, id, document_number, document_item, move_type,
                    state,
                    business_due_date,
                    document_date,
                    payment_mode,
                    payment_term,
                    due_date, partner_id, currency_id, amount_document,
                    sum AS cumulative_sum

                    FROM (
                        SELECT * FROM financial_cashflow_main_view
                    ) AS out_format
            )
        """)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#!/usr/bin/python
##-------------------------------------------------------------------
##
## File : elasticsearch_force_merge.py
## Author :
## Description :
##    Run force merge for existing indices, when ratio of deleted count/doc count is over 0.1
## --
## Created : <2017-02-24>
## Updated: Time-stamp: <2017-04-10 17:22:23>
##-------------------------------------------------------------------
import argparse
import requests
import sys
import socket
import json

NAGIOS_OK_ERROR=0
NAGIOS_EXIT_ERROR=2

################################################################################
def print_all_index_summary(es_host, es_port):
    url = "http://%s:%s/_cat/indices?v" % (es_host, es_port)
    r = requests.get(url)
    if r.status_code != 200:
        print "ERROR: fail to run REST API: %s" % (url)
        sys_exit(es_host, es_port)
    print r.content

def print_index_setting(es_host, es_port, index_name):
    url = "http://%s:%s/%s/_stats?pretty" % (es_host, es_port, index_name)
    r = requests.get(url)
    if r.status_code != 200:
        print "ERROR: fail to run REST API: %s" % (url)
        sys_exit(es_host, es_port)
    content_json = json.loads(r.content)
    print "Index setting for %s.\n\tdocs:%s\n\tmerges:%s\n\tsegments:%s\n\n" % \
        (index_name,
         json.dumps(content_json["_all"]["primaries"]["docs"]),
         json.dumps(content_json["_all"]["primaries"]["merges"]),
         json.dumps(content_json["_all"]["primaries"]["segments"]))
################################################################################

def sys_exit(es_host, es_port):
    print "Unexpected error has happened. Current summary of ES indices."
    print_all_index_summary(es_host, es_port)
    sys.exit(NAGIOS_EXIT_ERROR)

def get_es_index_info(es_host, es_port, es_pattern_regexp, \
                      min_deleted_count, min_deleted_ratio):
    index_list = []
    url = "http://%s:%s/_cat/indices?v" % (es_host, es_port)
    # TODO: error handling, if curl requests fails
    r = requests.get(url)
    '''
Sample output:
root@test:/# curl 172.17.0.8:9200/_cat/indices?v
health status index                                          pri rep docs.count docs.deleted store.size pri.store.size
green  open   master-index-098f6bcd4621d373cade4e832627b4f6    1   0          1            0      8.1kb          8.1kb
green  open   master-index-13a1f8adbec032ed68f3d035449ef48d    1   0          1            0     10.6kb         10.6kb
...
...
'''
    if r.status_code != 200:
        print "ERROR: fail to run REST API: %s" % (url)
        sys_exit(es_host, es_port)

    # TODO: use python library for ES
    for line in r.content.split("\n"):
        # remove the header, and skip closed ES indices
        if line == '' or " index " in line  or " close " in line:
            continue
        else:
            line = " ".join(line.split())
            l = line.split()
            index_name = l[2]
            total_doc_count = int(l[5])
            deleted_doc_count = int(l[6])
            if min_deleted_count != 0 and min_deleted_ratio != 0:
                if (deleted_doc_count < min_deleted_count):
                    continue
                if float(deleted_doc_count)/total_doc_count < min_deleted_ratio:
                    continue
            index_list.append([index_name, total_doc_count, deleted_doc_count])
    return index_list

def force_merge_index(es_host, es_port, index_name):
    print_index_setting(es_host, es_port, index_name)

    # TODO: Quit if something wrong; get time performance
    # force-merge is a sync call, and it might take a long time
    url = \
          "http://%s:%s/%s/_forcemerge?pretty&only_expunge_deletes=true" % \
          (es_host, es_port, index_name)
    r = requests.post(url)
    if r.status_code != 200:
        print "ERROR: fail to run REST API: %s" % (url)
        sys_exit(es_host, es_port)
    print "http response: %s" % (r.content)

    # TODO: print the output

    # TODO: get time performance
    print_index_setting(es_host, es_port, index_name)

# Sample:
# python ./elasticsearch_force_merge.py --es_pattern_regexp "master-.*|staging-.*" \
#          --min_deleted_count 1000 \
#          --min_deleted_ratio 0.1
#
# python /tmp/elasticsearch_force_merge.py --min_deleted_count 0  --min_deleted_ratio 0

if __name__ == '__main__':
    # get parameters from users
    parser = argparse.ArgumentParser()
    parser.add_argument('--es_host', required=False, \
                        help="server ip or hostname for elasticsearch instance. Default value is ip of eth0", type=str)
    parser.add_argument('--es_port', default='9200', required=False, \
                        help="server port for elasticsearch instance", type=str)
    parser.add_argument('--es_pattern_regexp', required=False, default='', \
                        help="ES index name pattern. Only ES indices with matched pattern will be examined", type=str)
    parser.add_argument('--min_deleted_count', default=1000, required=False, \
                        help='If indices do not have too many deleted docs, skip the force merge', type=int)
    parser.add_argument('--min_deleted_ratio', default=0.05, required=False, \
                        help='If the ratio of deleted/total doc count is too small, skip the force merge', type=float)
    l = parser.parse_args()

    es_host = l.es_host
    es_port = l.es_port
    es_pattern_regexp = l.es_pattern_regexp
    min_deleted_count = int(l.min_deleted_count)
    min_deleted_ratio = float(l.min_deleted_ratio)

    # get ip of eth0, if es_host is not given
    if es_host is None:
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(("8.8.8.8", 80))
        es_host = s.getsockname()[0]

    es_index_list = get_es_index_info(es_host, es_port, es_pattern_regexp, \
                                      min_deleted_count, min_deleted_ratio)
    if len(es_index_list) == 0:
        print "OK: no indices need to run force-merge."
    else:
        print "Indices summary before force-merge."
        print_all_index_summary(es_host, es_port)

        # TODO: print timestamp
        updated_index_list = []
        for es_index in es_index_list:
            index_name = es_index[0]
            print "Run force-merge for %s" % (index_name)
            force_merge_index(es_host, es_port, index_name)
            updated_index_list.append(index_name)
        print "OK: Run force-merge successfully on below indices: %s" % (','.join(updated_index_list))

        print "Indices summary after force-merge."
        print_all_index_summary(es_host, es_port)
## File : elasticsearch_force_merge.py ends
<commit_msg>python use logger to add timestamp to stdout<commit_after># -*- coding: utf-8 -*-
#!/usr/bin/python
##-------------------------------------------------------------------
##
## File : elasticsearch_force_merge.py
## Author :
## Description :
##    Run force merge for existing indices, when ratio of deleted count/doc count is over 0.1
## --
## Created : <2017-02-24>
## Updated: Time-stamp: <2017-04-10 17:35:19>
##-------------------------------------------------------------------
import argparse
import requests
import sys
import socket
import json

NAGIOS_OK_ERROR=0
NAGIOS_EXIT_ERROR=2

logger = setup_custom_logger('myapp')
################################################################################
def setup_custom_logger(name):
    import logging
    formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s',
                                  datefmt='%Y-%m-%d %H:%M:%S')
    log_fname = "/var/log/%s.log" % (name)
    handler = logging.FileHandler(log_fname, mode='w')
    handler.setFormatter(formatter)
    screen_handler = logging.StreamHandler(stream=sys.stdout)
    screen_handler.setFormatter(formatter)
    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    logger.addHandler(handler)
    logger.addHandler(screen_handler)
    return logger

def print_all_index_summary(es_host, es_port):
    url = "http://%s:%s/_cat/indices?v" % (es_host, es_port)
    r = requests.get(url)
    if r.status_code != 200:
        logger.error("Fail to run REST API: %s" % (url))
        sys_exit(es_host, es_port)
    logger.info(r.content)

def print_index_setting(es_host, es_port, index_name):
    url = "http://%s:%s/%s/_stats?pretty" % (es_host, es_port, index_name)
    r = requests.get(url)
    if r.status_code != 200:
        logger.error("Fail to run REST API: %s" % (url))
        sys_exit(es_host, es_port)
    content_json = json.loads(r.content)
    logger.info("Index setting for %s.\n\tdocs:%s\n\tmerges:%s\n\tsegments:%s\n\n" % \
        (index_name,
         json.dumps(content_json["_all"]["primaries"]["docs"]),
         json.dumps(content_json["_all"]["primaries"]["merges"]),
         json.dumps(content_json["_all"]["primaries"]["segments"])))
################################################################################

def sys_exit(es_host, es_port):
    logger.error("Unexpected error has happened. Current summary of ES indices.")
    print_all_index_summary(es_host, es_port)
    sys.exit(NAGIOS_EXIT_ERROR)

def get_es_index_info(es_host, es_port, es_pattern_regexp, \
                      min_deleted_count, min_deleted_ratio):
    index_list = []
    url = "http://%s:%s/_cat/indices?v" % (es_host, es_port)
    # TODO: error handling, if curl requests fails
    r = requests.get(url)
    '''
Sample output:
root@test:/# curl 172.17.0.8:9200/_cat/indices?v
health status index                                          pri rep docs.count docs.deleted store.size pri.store.size
green  open   master-index-098f6bcd4621d373cade4e832627b4f6    1   0          1            0      8.1kb          8.1kb
green  open   master-index-13a1f8adbec032ed68f3d035449ef48d    1   0          1            0     10.6kb         10.6kb
...
...
'''
    if r.status_code != 200:
        logger.error("Fail to run REST API: %s" % (url))
        sys_exit(es_host, es_port)

    # TODO: use python library for ES
    for line in r.content.split("\n"):
        # remove the header, and skip closed ES indices
        if line == '' or " index " in line  or " close " in line:
            continue
        else:
            line = " ".join(line.split())
            l = line.split()
            index_name = l[2]
            total_doc_count = int(l[5])
            deleted_doc_count = int(l[6])
            if min_deleted_count != 0 and min_deleted_ratio != 0:
                if (deleted_doc_count < min_deleted_count):
                    continue
                if float(deleted_doc_count)/total_doc_count < min_deleted_ratio:
                    continue
            index_list.append([index_name, total_doc_count, deleted_doc_count])
    return index_list

def force_merge_index(es_host, es_port, index_name):
    print_index_setting(es_host, es_port, index_name)

    # TODO: Quit if something wrong; get time performance
    # force-merge is a sync call, and it might take a long time
    url = \
          "http://%s:%s/%s/_forcemerge?pretty&only_expunge_deletes=true" % \
          (es_host, es_port, index_name)
    r = requests.post(url)
    if r.status_code != 200:
        logger.error("Fail to run REST API: %s" % (url))
        sys_exit(es_host, es_port)
    logger.info("http response: %s" % (r.content))

    print_index_setting(es_host, es_port, index_name)

# Sample:
# python ./elasticsearch_force_merge.py --es_pattern_regexp "master-.*|staging-.*" \
#          --min_deleted_count 1000 \
#          --min_deleted_ratio 0.1
#
# python /tmp/elasticsearch_force_merge.py --min_deleted_count 0  --min_deleted_ratio 0

if __name__ == '__main__':
    # get parameters from users
    parser = argparse.ArgumentParser()
    parser.add_argument('--es_host', required=False, \
                        help="server ip or hostname for elasticsearch instance. Default value is ip of eth0", type=str)
    parser.add_argument('--es_port', default='9200', required=False, \
                        help="server port for elasticsearch instance", type=str)
    parser.add_argument('--es_pattern_regexp', required=False, default='', \
                        help="ES index name pattern. Only ES indices with matched pattern will be examined", type=str)
    parser.add_argument('--min_deleted_count', default=1000, required=False, \
                        help='If indices do not have too many deleted docs, skip the force merge', type=int)
    parser.add_argument('--min_deleted_ratio', default=0.05, required=False, \
                        help='If the ratio of deleted/total doc count is too small, skip the force merge', type=float)
    l = parser.parse_args()

    es_host = l.es_host
    es_port = l.es_port
    es_pattern_regexp = l.es_pattern_regexp
    min_deleted_count = int(l.min_deleted_count)
    min_deleted_ratio = float(l.min_deleted_ratio)

    # get ip of eth0, if es_host is not given
    if es_host is None:
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(("8.8.8.8", 80))
        es_host = s.getsockname()[0]

    es_index_list = get_es_index_info(es_host, es_port, es_pattern_regexp, \
                                      min_deleted_count, min_deleted_ratio)
    if len(es_index_list) == 0:
        logger.info("OK: no indices need to run force-merge.")
    else:
        logger.info("Indices summary before force-merge.")
        print_all_index_summary(es_host, es_port)

        updated_index_list = []
        for es_index in es_index_list:
            index_name = es_index[0]
            logger.info("Run force-merge for %s" % (index_name))
            force_merge_index(es_host, es_port, index_name)
            updated_index_list.append(index_name)
        logger.info("OK: Run force-merge successfully on below indices: %s" % (','.join(updated_index_list)))

        logger.info("Indices summary after force-merge.")
        print_all_index_summary(es_host, es_port)
## File : elasticsearch_force_merge.py ends
<|endoftext|>


<commit_before>import numpy as np
from Semibandits import Semibandit
import torch.nn as nn
import torch
import torch.optim as optim
from torch.nn.functional import softmax, mse_loss

class LinearNetwork(nn.Module):
    def __init__(self, input_size):
        super(LinearNetwork, self).__init__()
        self.network = nn.Sequential(
            nn.Linear(input_size,1)
        )

    def forward(self, feat):
        x = self.network(feat)
        return x

class PolicyGradient(Semibandit):

    def __init__(self,B):
        self.B = B

    def init(self, T, params={}):
        """
        Initialize the policy parameters.
        Three parameters here:
        1. epsilon: parameter for uniform exploration, default = 0
        2. delta: softmax parameter, default =1
        3. lr: learning rate
        """

        self.T = T
        self.d = self.B.d

        self.t = 1

        if "delta" in params.keys():
            self.delta = params['delta']
        else:
            self.delta = 1.0

        if "epsilon" in params.keys():
            self.eps = params['epsilon']
        else:
            self.eps = 0

        if "lr" in params.keys():
            self.lr = params['lr']
        else:
            self.lr = 0.01

        self.reward = []
        self.opt_reward = []

        self.model = LinearNetwork(self.d)
        self.optim = optim.Adam(self.model.parameters(),lr=0.0001)

    def get_action(self, x):
        features = np.matrix(x.get_ld_features())
        # print(features, flush=True)
        logits = self.model(torch.Tensor(features)).detach()/self.delta
        logits = logits.T[0,:]
        # print(logits)
        outputs = softmax(logits, dim=0).numpy()
        # print(outputs, flush=True)
        proba = (1-self.eps)*outputs + self.eps*np.ones((x.get_K(),))/x.get_K()
        # print(proba)
        a = torch.multinomial(torch.Tensor(proba), x.get_L(), replacement=False).numpy()
        # print(a)
        self.imp_weight = proba[a]
        # print(self.imp_weight)
        self.action = a
        return a

    def update(self, x, A, y_vec, r):
        self.optim.zero_grad()
        features = np.matrix(x.get_ld_features())
        logits = self.model(torch.Tensor(features))/self.delta
        logits = logits.T[0,:]
        outputs = softmax(logits,dim=0)
        iw_rew = np.zeros((x.get_K(),))
        iw_rew[A] = y_vec/self.imp_weight
        # print(outputs)
        # print(iw_rew)
        loss = -1*torch.dot(torch.Tensor(iw_rew), outputs)
        loss.backward()
        self.optim.step()

class OnlineEpsGreedy(Semibandit):

    def __init__(self,B):
        self.B = B

    def init(self, T, params={}):
        """
        Initialize the policy parameters.
        Three parameters here:
        1. epsilon: parameter for uniform exploration, default = 0
        2. delta: softmax parameter, default =1
        3. lr: learning rate
        """

        self.T = T
        self.d = self.B.d

        self.t = 1

        if "delta" in params.keys():
            self.delta = params['delta']
        else:
            self.delta = 1.0

        if "epsilon" in params.keys():
            self.eps = params['epsilon']
        else:
            self.eps = 0

        if "lr" in params.keys():
            self.lr = params['lr']
        else:
            self.lr = 0.01

        self.reward = []
        self.opt_reward = []

        self.model = LinearNetwork(self.d)
        self.optim = optim.Adam(self.model.parameters(),lr=0.0001)

    def get_action(self, x):
        features = np.matrix(x.get_ld_features())
        # print(features, flush=True)
        logits = self.model(torch.Tensor(features)).detach()
        logits = logits.T[0,:]
        greedy_acts = np.argsort(logits)[-x.get_L():]
        proba = self.eps*np.ones((x.get_K(),))/x.get_K()
        proba[greedy_acts] += (1-self.eps)
        # print(proba)
        a = torch.multinomial(torch.Tensor(proba), x.get_L(), replacement=False).numpy()
        # print(a)
        self.imp_weight = proba[a]
        # print(self.imp_weight)
        self.action = a
        return a

    def update(self, x, A, y_vec, r):
        self.optim.zero_grad()
        features = np.matrix(x.get_ld_features())
        outputs = self.model(torch.Tensor(features))
        outputs = outputs.T[0,:]
        iw_rew = np.zeros((x.get_K(),))
        iw_rew[A] = y_vec/self.imp_weight
        # print(outputs)
        # print(iw_rew)
        loss = mse_loss(torch.Tensor(iw_rew), outputs)
        loss.backward()
        self.optim.step()

if __name__=='__main__':
    import sys, os, argparse, time
    import Simulators, Semibandits

    parser = argparse.ArgumentParser()
    parser.add_argument('--T', action='store', default=50000, help='number of rounds', type=int)
    parser.add_argument('--iters', action='store', default=1, type=int)
    parser.add_argument('--d', action='store', default=20, type=int)
    parser.add_argument('--K', action='store', default=5, type=int)

    parser.add_argument('--alg', action='store', default='all', choices=['pg','linucb','egreedy'])
    parser.add_argument('--param', action='store', default=None)
    parser.add_argument('--lr', action='store', type=float, default=0.1)
    parser.add_argument('--eps', action='store', type=float, default=0)
    parser.add_argument('--noise', action='store', default=None)
    parser.add_argument('--outdir_pref', action='store', type=str, default='./')

    Args = parser.parse_args(sys.argv[1:])
    print(Args,flush=True)
    if Args.noise is not None:
        Args.noise = float(Args.noise)

    outdir = '%spg_resultsT=%d_d=%d_K=%d/' % (Args.outdir_pref,Args.T, Args.d, Args.K)
    if not os.path.isdir(outdir):
        os.makedirs(outdir)

    if Args.param is not None:
        Args.param = float(Args.param)

    rewards = []
    regrets = []
    times = []
    for i in range(Args.iters):
        S = Simulators.LinearBandit(Args.d, 1, Args.K, noise=Args.noise, seed=i)
        if Args.alg == 'linucb':
            Alg = Semibandits.LinUCB(S)
            if Args.param is not None:
                start = time.time()
                (r,reg,val_tmp) = Alg.play(Args.T, verbose=True, params={'delta': Args.param, 'schedule': 1})
                stop = time.time()
        if Args.alg == 'pg':
            Alg = PolicyGradient(S)
            if Args.param is not None:
                start = time.time()
                (r,reg,val_tmp) = Alg.play(Args.T, verbose=True, params={'delta': Args.param, 'eps': Args.eps, 'lr': Args.lr})
                stop = time.time()
        if Args.alg == 'egreedy':
            Alg = OnlineEpsGreedy(S)
            if Args.param is not None:
                start = time.time()
                (r,reg,val_tmp) = Alg.play(Args.T, verbose=True, params={'delta': Args.param, 'eps': Args.eps, 'lr': Args.lr})
                stop = time.time()
        times.append(stop-start)
        rewards.append(r)
        regrets.append(reg)
<commit_msg>minor<commit_after>import numpy as np
from Semibandits import Semibandit
import torch.nn as nn
import torch
import torch.optim as optim
from torch.nn.functional import softmax, mse_loss

class LinearNetwork(nn.Module):
    def __init__(self, input_size):
        super(LinearNetwork, self).__init__()
        self.network = nn.Sequential(
            nn.Linear(input_size,1)
        )

    def forward(self, feat):
        x = self.network(feat)
        return x

class PolicyGradient(Semibandit):

    def __init__(self,B):
        self.B = B
        self.name='pg'

    def init(self, T, params={}):
        """
        Initialize the policy parameters.
        Three parameters here:
        1. epsilon: parameter for uniform exploration, default = 0
        2. delta: softmax parameter, default =1
        3. lr: learning rate
        """

        self.T = T
        self.d = self.B.d

        self.t = 1

        if "delta" in params.keys():
            self.delta = params['delta']
        else:
            self.delta = 1.0

        if "eps" in params.keys():
            self.eps = params['eps']
        else:
            self.eps = 0

        if "lr" in params.keys():
            self.lr = params['lr']
        else:
            self.lr = 0.01

        self.reward = []
        self.opt_reward = []

        self.model = LinearNetwork(self.d)
        self.optim = optim.Adam(self.model.parameters(),lr=0.0001)

    def get_action(self, x):
        features = np.matrix(x.get_ld_features())
        # print(features, flush=True)
        logits = self.model(torch.Tensor(features)).detach()/self.delta
        logits = logits.T[0,:]
        # print(logits)
        outputs = softmax(logits, dim=0).numpy()
        # print(outputs, flush=True)
        proba = (1-self.eps)*outputs + self.eps*np.ones((x.get_K(),))/x.get_K()
        # print(proba)
        a = torch.multinomial(torch.Tensor(proba), x.get_L(), replacement=False).numpy()
        # print(a)
        self.imp_weight = proba[a]
        # print(self.imp_weight)
        self.action = a
        return a

    def update(self, x, A, y_vec, r):
        self.optim.zero_grad()
        features = np.matrix(x.get_ld_features())
        logits = self.model(torch.Tensor(features))/self.delta
        logits = logits.T[0,:]
        outputs = softmax(logits,dim=0)
        iw_rew = np.zeros((x.get_K(),))
        iw_rew[A] = y_vec/self.imp_weight
        # print(outputs)
        # print(iw_rew)
        loss = -1*torch.dot(torch.Tensor(iw_rew), outputs)
        loss.backward()
        self.optim.step()

class OnlineEpsGreedy(Semibandit):

    def __init__(self,B):
        self.B = B
        self.name='eps'

    def init(self, T, params={}):
        """
        Initialize the policy parameters.
        Three parameters here:
        1. epsilon: parameter for uniform exploration, default = 0
        2. delta: softmax parameter, default =1
        3. lr: learning rate
        """

        self.T = T
        self.d = self.B.d

        self.t = 1

        if "delta" in params.keys():
            self.eps = params['delta']
        else:
            self.eps = 0.0

        if "lr" in params.keys():
            self.lr = params['lr']
        else:
            self.lr = 0.01

        self.reward = []
        self.opt_reward = []

        self.model = LinearNetwork(self.d)
        self.optim = optim.Adam(self.model.parameters(),lr=0.0001)

    def get_action(self, x):
        features = np.matrix(x.get_ld_features())
        # print(features, flush=True)
        logits = self.model(torch.Tensor(features)).detach()
        logits = logits.T[0,:]
        greedy_acts = np.argsort(logits)[-x.get_L():]
        proba = self.eps*np.ones((x.get_K(),))/x.get_K()
        proba[greedy_acts] += (1-self.eps)
        # print(proba)
        a = torch.multinomial(torch.Tensor(proba), x.get_L(), replacement=False).numpy()
        # print(a)
        self.imp_weight = proba[a]
        # print(self.imp_weight)
        self.action = a
        return a

    def update(self, x, A, y_vec, r):
        self.optim.zero_grad()
        features = np.matrix(x.get_ld_features())
        outputs = self.model(torch.Tensor(features))
        outputs = outputs.T[0,:]
        iw_rew = np.zeros((x.get_K(),))
        iw_rew[A] = y_vec/self.imp_weight
        # print(outputs)
        # print(iw_rew)
        loss = mse_loss(torch.Tensor(iw_rew), outputs)
        loss.backward()
        self.optim.step()

if __name__=='__main__':
    import sys, os, argparse, time
    import Simulators, Semibandits

    parser = argparse.ArgumentParser()
    parser.add_argument('--T', action='store', default=50000, help='number of rounds', type=int)
    parser.add_argument('--iters', action='store', default=1, type=int)
    parser.add_argument('--d', action='store', default=20, type=int)
    parser.add_argument('--K', action='store', default=5, type=int)

    parser.add_argument('--alg', action='store', default='all', choices=['pg','linucb','egreedy'])
    parser.add_argument('--param', action='store', default=None)
    parser.add_argument('--lr', action='store', type=float, default=0.1)
    parser.add_argument('--eps', action='store', type=float, default=0)
    parser.add_argument('--noise', action='store', default=None)
    parser.add_argument('--outdir_pref', action='store', type=str, default='./')

    Args = parser.parse_args(sys.argv[1:])
    print(Args,flush=True)
    if Args.noise is not None:
        Args.noise = float(Args.noise)

    outdir = '%spg_resultsT=%d_d=%d_K=%d/' % (Args.outdir_pref,Args.T, Args.d, Args.K)
    if not os.path.isdir(outdir):
        os.makedirs(outdir)

    if Args.param is not None:
        Args.param = float(Args.param)

    rewards = []
    regrets = []
    times = []
    for i in range(Args.iters):
        S = Simulators.LinearBandit(Args.d, 1, Args.K, noise=Args.noise, seed=i)
        if Args.alg == 'linucb':
            Alg = Semibandits.LinUCB(S)
            if Args.param is not None:
                start = time.time()
                (r,reg,val_tmp) = Alg.play(Args.T, verbose=True, params={'delta': Args.param, 'schedule': 1})
                stop = time.time()
        if Args.alg == 'pg':
            Alg = PolicyGradient(S)
            if Args.param is not None:
                start = time.time()
                (r,reg,val_tmp) = Alg.play(Args.T, verbose=True, params={'delta': Args.param, 'eps': Args.eps, 'lr': Args.lr})
                stop = time.time()
        if Args.alg == 'egreedy':
            Alg = OnlineEpsGreedy(S)
            if Args.param is not None:
                start = time.time()
                (r,reg,val_tmp) = Alg.play(Args.T, verbose=True, params={'delta': Args.param, 'lr': Args.lr})
                stop = time.time()
        times.append(stop-start)
        rewards.append(r)
        regrets.append(reg)
<|endoftext|>


<commit_before>"""
Init
"""
from __future__ import unicode_literals

import datetime
import os
import subprocess

VERSION = (2, 2, 15, 'final', 0)

def get_version(version=None):
    """
    Returns a PEP 386-compliant version number from VERSION.
    """
    if not version:
        version = VERSION
    else:
        assert len(version) == 5
        assert version[3] in ('alpha', 'beta', 'rc', 'final')

    # Now build the two parts of the version number:
    # main = X.Y[.Z]
    # sub = .devN - for pre-alpha releases
    #     | {a|b|c}N - for alpha, beta and rc releases

    parts = 2 if version[2] == 0 else 3
    main = '.'.join(str(x) for x in version[:parts])

    sub = ''
    if version[3] == 'alpha' and version[4] == 0:
        git_changeset = get_git_changeset()
        if git_changeset:
            sub = '.dev%s' % git_changeset

    elif version[3] != 'final':
        mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'c'}
        sub = mapping[version[3]] + str(version[4])

    return str(main + sub)

def get_git_changeset():
    """
    Returns a numeric identifier of the latest git changeset.

    The result is the UTC timestamp of the changeset in YYYYMMDDHHMMSS format.
    This value isn't guaranteed to be unique, but collisions are very unlikely,
    so it's sufficient for generating the development version numbers.
    """
    repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    git_log = subprocess.Popen('git log --pretty=format:%ct --quiet -1 HEAD',
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               shell=True, cwd=repo_dir,
                               universal_newlines=True)
    timestamp = git_log.communicate()[0]

    try:
        timestamp = datetime.datetime.utcfromtimestamp(int(timestamp))
    except ValueError:
        return None

    return timestamp.strftime('%Y%m%d%H%M%S')
<commit_msg>fixed version<commit_after>"""
Init
"""
from __future__ import unicode_literals

import datetime
import os
import subprocess

VERSION = (2, 3, 1, 'final', 0)

def get_version(version=None):
    """
    Returns a PEP 386-compliant version number from VERSION.
    """
    if not version:
        version = VERSION
    else:
        assert len(version) == 5
        assert version[3] in ('alpha', 'beta', 'rc', 'final')

    # Now build the two parts of the version number:
    # main = X.Y[.Z]
    # sub = .devN - for pre-alpha releases
    #     | {a|b|c}N - for alpha, beta and rc releases

    parts = 2 if version[2] == 0 else 3
    main = '.'.join(str(x) for x in version[:parts])

    sub = ''
    if version[3] == 'alpha' and version[4] == 0:
        git_changeset = get_git_changeset()
        if git_changeset:
            sub = '.dev%s' % git_changeset

    elif version[3] != 'final':
        mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'c'}
        sub = mapping[version[3]] + str(version[4])

    return str(main + sub)

def get_git_changeset():
    """
    Returns a numeric identifier of the latest git changeset.

    The result is the UTC timestamp of the changeset in YYYYMMDDHHMMSS format.
    This value isn't guaranteed to be unique, but collisions are very unlikely,
    so it's sufficient for generating the development version numbers.
    """
    repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    git_log = subprocess.Popen('git log --pretty=format:%ct --quiet -1 HEAD',
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               shell=True, cwd=repo_dir,
                               universal_newlines=True)
    timestamp = git_log.communicate()[0]

    try:
        timestamp = datetime.datetime.utcfromtimestamp(int(timestamp))
    except ValueError:
        return None

    return timestamp.strftime('%Y%m%d%H%M%S')
<|endoftext|>


<commit_before># coding=utf-8
from __future__ import unicode_literals

from django.urls import reverse
from django.utils.functional import cached_property
from datetime import datetime
from django.utils.encoding import python_2_unicode_compatible
import uuid
from django.db import models

from django.conf import settings

from utils.models import CreacionModificacionFechaProfileMixin, CreacionModificacionUserProfileMixin
from authtools.models import User

"""
def useridExist():
    #Verifica la existencia de un registro en CajaCajero
    try:
        userid = User.objects.latest('id')
    except User.DoesNotExist:
        userid = None
        pass
    return userid
"""

class BaseProfile(CreacionModificacionFechaProfileMixin, CreacionModificacionUserProfileMixin, models.Model):
    user = models.OneToOneField(settings.AUTH_USER_MODEL, null=True)
    # user = models.OneToOneField(User, models.DO_NOTHING, db_column='user_id', default=useridExist, null=True)
    slug = models.UUIDField(default=uuid.uuid4, blank=True, editable=False, null=True)
    # Add more user profile fields here. Make sure they are nullable
    # or with default values
    picture = models.ImageField('Profile picture',
                                upload_to='profile_pics/%Y-%m-%d/',
                                null=True,
                                blank=True)
    bio = models.CharField("Short Bio", max_length=200, blank=True, null=True)
    email_verified = models.BooleanField("Email verified", default=False)

    nombre = models.CharField(max_length=50)
    segundo_nombre = models.CharField(max_length=200, blank=True, null=True)
    apellido_pa = models.CharField(max_length=50, null=True, blank=True)
    apellido_ma = models.CharField(max_length=50, blank=True, null=True)
    tipo_documento = models.IntegerField(null=True)
    numero_documento = models.CharField(db_index=True, max_length=15, null=True)
    sexo = models.IntegerField(null=True)
    correo = models.EmailField(db_index=True, max_length=100, blank=True, null=True)
    fecha_nac = models.DateField(null=True)

    # fecha_creacion_persona = models.DateField()
    # fecha_modificacion_persona = models.DateField()
    # usuario_creacion_persona = models.CharField(max_length=10)
    # usuario_modificacion_persona = models.CharField(max_length=10)

    @cached_property
    def getNombreCompleto(self):
        """
        Método que concatena los nombres y apellidos

        :return: Nombre completo de la persona
        """
        cadena = ""
        if self.nombre:
            cadena = self.nombre.capitalize() + " "

        if self.segundo_nombre:
            cadena += self.segundo_nombre.capitalize() + " "

        if self.apellido_pa:
            cadena += self.apellido_pa.capitalize() + " "

        if self.apellido_ma:
            cadena += self.apellido_ma.capitalize()

        # return "{0} {1} {2} {3}".format(self.nombre, self.segundo_nombre, self.apellido_pa, self.apellido_ma)
        return "{0}".format(cadena)

    @cached_property
    def getNombreFormal(self):
        """
        Método que concatena los nombres y apellidos

        :return: Nombre completo de la persona
        """
        cadena = ""
        if self.apellido_pa:
            cadena = self.apellido_pa.capitalize() + " "

        if self.apellido_ma:
            cadena += self.apellido_ma.capitalize() + ", "

        if self.nombre:
            cadena += self.nombre.capitalize() + " "

        if self.segundo_nombre:
            cadena += self.segundo_nombre.capitalize()
        # return "{0} {1} {2} {3}".format(self.nombre, self.segundo_nombre, self.apellido_pa, self.apellido_ma)
        return "{0}".format(cadena)

    @property
    def getEdad(self):
        """
        Método que calcula los años de una persona
        :return: La cantidad de años de la persona
        """

        edad = datetime.now().date().year - self.fecha_nac.year

        años = edad

        return años

    @property
    def getSexo(self):
        """
        Método que retorna la descripción del tipo de sexo, cruzandolo con el catalogo TipoSexo

        :return: Descripción del catalogo TipoSexo
        """
        from utils.models import TipoSexo

        idsexo = self.sexo

        sexo = TipoSexo.objects.get(pk=idsexo)

        return sexo.descripcion

    @property
    def getTipoDocumento(self):
        """
        Método que retorna la descripción del tipo de documento, cruzándolo con el cataloto TipoDocumento
        :return: Descripción del catalogo TipoDocumento
        """

        from utils.models import TipoDocumento

        idtipo = self.tipo_documento

        tipodoc = TipoDocumento.objects.get(pk=idtipo)

        return tipodoc.descripcion

    class Meta:
        abstract = True

        # unique_together = (("tipo_documento", "numero_documento"),)

@python_2_unicode_compatible
class Profile(BaseProfile):
    id_persona = models.AutoField(primary_key=True)

    def get_absolute_url(self):
        """
        Redirecciona las views que usan como modelo esta clase
        :return: url de detalles de la persona
        """
        return reverse('registers:personal_detail', kwargs={'pk': self.pk})

    def full_detail(self):
        """
        Da una descripcion detallada de la informacion del Tipo de Servicio
        :return: lista de todos los atributos de la clase
        """
        detalle_completo = ["Nombres y Apellidos: {0}".format(self.getNombreCompleto),
                            "Tipo Documento: {0}".format(self.getTipoDocumento),
                            "Número Documento: {0}".format(self.numero_documento),
                            "Sexo: {0}".format(self.getSexo),
                            "Correo: {0}".format(self.correo),
                            "Edad: {0}".format(self.getEdad),
                            "Usuario creacion: {0}".format(self.usuario_creacion_persona),
                            "Fecha creacion: {0}".format(self.fecha_creacion_persona),
                            "Usuario modificacion: {0}".format(self.usuario_modificacion_persona),
                            "Fecha modificacion: {0}".format(self.fecha_modificacion_persona)
                            ]
        return detalle_completo

    def profile_with_telefonos(self):
        return self.telefonos.select_related('persona')

    def profile_with_direcciones(self):
        return self.direcciones.select_related('persona')

    def __str__(self):
        return "{}'s profile".format(self.user)

    def save(self, **kwargs):
        super(Profile, self).save(**kwargs)

        if PersonaEmisor.objects.filter(profile_id=self.id_persona).exists():
            pass

        else:
            persona_emisor = PersonaEmisor(profile=self)
            persona_emisor.save()

        if PersonaReceptor.objects.filter(profile_id=self.id_persona).exists():
            pass
        else:
            persona_receptor = PersonaReceptor(profile=self)
            persona_receptor.save()

class PersonaEmisor(models.Model):
    id_persona_emisor = models.AutoField(primary_key=True)
    profile = models.ForeignKey(Profile, models.DO_NOTHING, db_column="id_persona")

    def __str__(self):
        return "{0}".format(self.profile.getNombreCompleto)

    class Meta:
        managed = False
        ordering = ["id_persona_emisor"]
        db_table = 'alerta_personaemisor'  # Verificar luego del migrate, posible error

class PersonaReceptor(models.Model):
    id_persona_receptor = models.AutoField(primary_key=True)
    profile = models.ForeignKey(Profile, models.DO_NOTHING, db_column="id_persona")

    def __str__(self):
        return "{0}".format(self.profile.getNombreCompleto)

    class Meta:
        managed = False
        ordering = ["id_persona_receptor"]
        db_table = 'alerta_personareceptor'  # Verificar luego del migrate, posible error

class TokenFirebase(models.Model):
    id_token = models.AutoField(primary_key=True)
    persona = models.ForeignKey(Profile, models.DO_NOTHING, db_column="id_persona")
    codigo = models.CharField(max_length=1000, blank=True, null=True, unique=True)
    alumno_id = models.IntegerField(null=True)

    def __str__(self):
        return "{0} {1}".format(' Token: ', self.codigo)

    class Meta:
        managed = False
        ordering = ["id_token"]
<commit_msg>document_N_and_imgs<commit_after># coding=utf-8
from __future__ import unicode_literals

from django.urls import reverse
from django.utils.functional import cached_property
from datetime import datetime
from django.utils.encoding import python_2_unicode_compatible
import uuid
from django.db import models

from django.conf import settings

from utils.models import CreacionModificacionFechaProfileMixin, CreacionModificacionUserProfileMixin
from authtools.models import User

"""
def useridExist():
    #Verifica la existencia de un registro en CajaCajero
    try:
        userid = User.objects.latest('id')
    except User.DoesNotExist:
        userid = None
        pass
    return userid
"""

class BaseProfile(CreacionModificacionFechaProfileMixin, CreacionModificacionUserProfileMixin, models.Model):
    user = models.OneToOneField(settings.AUTH_USER_MODEL, null=True)
    # user = models.OneToOneField(User, models.DO_NOTHING, db_column='user_id', default=useridExist, null=True)
    slug = models.UUIDField(default=uuid.uuid4, blank=True, editable=False, null=True)
    # Add more user profile fields here. Make sure they are nullable
    # or with default values
    picture = models.ImageField('Profile picture',
                                upload_to='profile_pics/%Y-%m-%d/',
                                null=True,
                                blank=True)
    bio = models.CharField("Short Bio", max_length=200, blank=True, null=True)
    email_verified = models.BooleanField("Email verified", default=False)

    nombre = models.CharField(max_length=50)
    segundo_nombre = models.CharField(max_length=200, blank=True, null=True)
    apellido_pa = models.CharField(max_length=50, null=True, blank=True)
    apellido_ma = models.CharField(max_length=50, blank=True, null=True)
    tipo_documento = models.IntegerField(null=True)
    numero_documento = models.CharField(db_index=True, max_length=15, null=True)
    sexo = models.IntegerField(null=True)
    correo = models.EmailField(db_index=True, max_length=100, blank=True, null=True)
    fecha_nac = models.DateField(null=True)

    # fecha_creacion_persona = models.DateField()
    # fecha_modificacion_persona = models.DateField()
    # usuario_creacion_persona = models.CharField(max_length=10)
    # usuario_modificacion_persona = models.CharField(max_length=10)

    @cached_property
    def getNombreCompleto(self):
        """
        Método que concatena los nombres y apellidos

        :return: Nombre completo de la persona
        """
        cadena = ""
        if self.nombre:
            cadena = self.nombre.capitalize() + " "

        if self.segundo_nombre:
            cadena += self.segundo_nombre.capitalize() + " "

        if self.apellido_pa:
            cadena += self.apellido_pa.capitalize() + " "

        if self.apellido_ma:
            cadena += self.apellido_ma.capitalize()

        # return "{0} {1} {2} {3}".format(self.nombre, self.segundo_nombre, self.apellido_pa, self.apellido_ma)
        return "{0}".format(cadena)

    @cached_property
    def getNombreFormal(self):
        """
        Método que concatena los nombres y apellidos

        :return: Nombre completo de la persona
        """
        cadena = ""
        if self.apellido_pa:
            cadena = self.apellido_pa.capitalize() + " "

        if self.apellido_ma:
            cadena += self.apellido_ma.capitalize() + ", "

        if self.nombre:
            cadena += self.nombre.capitalize() + " "

        if self.segundo_nombre:
            cadena += self.segundo_nombre.capitalize()
        # return "{0} {1} {2} {3}".format(self.nombre, self.segundo_nombre, self.apellido_pa, self.apellido_ma)
        return "{0}".format(cadena)

    @property
    def getEdad(self):
        """
        Método que calcula los años de una persona
        :return: La cantidad de años de la persona
        """

        edad = datetime.now().date().year - self.fecha_nac.year

        años = edad

        return años

    @property
    def getSexo(self):
        """
        Método que retorna la descripción del tipo de sexo, cruzandolo con el catalogo TipoSexo

        :return: Descripción del catalogo TipoSexo
        """
        from utils.models import TipoSexo

        idsexo = self.sexo

        sexo = TipoSexo.objects.get(pk=idsexo)

        return sexo.descripcion

    @property
    def getTipoDocumento(self):
        """
        Método que retorna la descripción del tipo de documento, cruzándolo con el cataloto TipoDocumento
        :return: Descripción del catalogo TipoDocumento
        """

        from utils.models import TipoDocumento

        idtipo = self.tipo_documento

        tipodoc = TipoDocumento.objects.get(pk=idtipo)

        return tipodoc.descripcion

    class Meta:
        abstract = True

        # unique_together = (("tipo_documento", "numero_documento"),)

@python_2_unicode_compatible
class Profile(BaseProfile):
    id_persona = models.AutoField(primary_key=True)

    def get_absolute_url(self):
        """
        Redirecciona las views que usan como modelo esta clase
        :return: url de detalles de la persona
        """
        return reverse('registers:personal_detail', kwargs={'pk': self.pk})

    def full_detail(self):
        """
        Da una descripcion detallada de la informacion del Tipo de Servicio
        :return: lista de todos los atributos de la clase
        """
        detalle_completo = ["Nombres y Apellidos: {0}".format(self.getNombreCompleto),
                            "Tipo Documento: {0}".format(self.getTipoDocumento),
                            "Número Documento: {0}".format(self.numero_documento),
                            "Sexo: {0}".format(self.getSexo),
                            "Correo: {0}".format(self.correo),
                            "Edad: {0}".format(self.getEdad),
                            "Usuario creacion: {0}".format(self.usuario_creacion_persona),
                            "Fecha creacion: {0}".format(self.fecha_creacion_persona),
                            "Usuario modificacion: {0}".format(self.usuario_modificacion_persona),
                            "Fecha modificacion: {0}".format(self.fecha_modificacion_persona)
                            ]
        return detalle_completo

    def profile_with_telefonos(self):
        return self.telefonos.select_related('persona')

    def profile_with_direcciones(self):
        return self.direcciones.select_related('persona')

    def __str__(self):
        return "{}'s profile".format(self.user)

    def save(self, **kwargs):
        super(Profile, self).save(**kwargs)

        if PersonaEmisor.objects.filter(profile_id=self.id_persona).exists():
            pass

        else:
            persona_emisor = PersonaEmisor(profile=self)
            persona_emisor.save()

        if PersonaReceptor.objects.filter(profile_id=self.id_persona).exists():
            pass
        else:
            persona_receptor = PersonaReceptor(profile=self)
            persona_receptor.save()

class PersonaEmisor(models.Model):
    id_persona_emisor = models.AutoField(primary_key=True)
    profile = models.ForeignKey(Profile, models.DO_NOTHING, db_column="id_persona")

    def __str__(self):
        return "{0}".format(self.profile.getNombreCompleto)

    class Meta:
        managed = False
        ordering = ["id_persona_emisor"]
        db_table = 'alerta_personaemisor'  # Verificar luego del migrate, posible error

class PersonaReceptor(models.Model):
    id_persona_receptor = models.AutoField(primary_key=True)
    profile = models.ForeignKey(Profile, models.DO_NOTHING, db_column="id_persona")

    def __str__(self):
        return "{0}".format(self.profile.getNombreCompleto)

    class Meta:
        managed = False
        ordering = ["id_persona_receptor"]
        db_table = 'alerta_personareceptor'  # Verificar luego del migrate, posible error

class TokenFirebase(models.Model):
    id_token = models.AutoField(primary_key=True)
    persona = models.ForeignKey(Profile, models.DO_NOTHING, db_column="id_persona")
    codigo = models.CharField(max_length=1000, blank=True, null=True, unique=True)
    alumno_id = models.IntegerField(null=True)

    def __str__(self):
        return "{0} {1}".format(' Token: ', self.codigo)

    class Meta:
        managed = False
        ordering = ["id_token"]
<|endoftext|>


<commit_before># Copyright (C) 2009  Renato Lima - Akretion, Gabriel C. Stabel
# Copyright (C) 2012  Raphaël Valyi - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

from odoo import api, fields, models

from ...l10n_br_fiscal.constants.fiscal import TAX_FRAMEWORK

class PurchaseOrderLine(models.Model):
    _name = 'purchase.order.line'
    _inherit = [_name, 'l10n_br_fiscal.document.line.mixin']

    @api.model
    def _default_fiscal_operation(self):
        return self.env.user.company_id.purchase_fiscal_operation_id

    @api.model
    def _fiscal_operation_domain(self):
        domain = [
            ('state', '=', 'approved'),
            ('fiscal_type', 'in', ('purchase', 'other', 'purchase_refund'))]
        return domain

    # Adapt Mixin's fields
    fiscal_operation_id = fields.Many2one(
        comodel_name='l10n_br_fiscal.operation',
        readonly=True,
        states={'draft': [('readonly', False)]},
        default=_default_fiscal_operation,
        domain=lambda self: self._fiscal_operation_domain(),
    )

    fiscal_operation_line_id = fields.Many2one(
        comodel_name="l10n_br_fiscal.operation.line",
        string="Operation Line",
        domain="[('fiscal_operation_id', '=', fiscal_operation_id), "
               "('state', '=', 'approved')]")

    fiscal_tax_ids = fields.Many2many(
        comodel_name='l10n_br_fiscal.tax',
        relation='fiscal_purchase_line_tax_rel',
        column1='document_id',
        column2='fiscal_tax_id',
        string='Fiscal Taxes',
    )

    quantity = fields.Float(
        string='Mixin Quantity',
        related='product_qty',
        depends=['product_qty'],
    )

    uom_id = fields.Many2one(
        string='Mixin UOM',
        related='product_uom',
        depends=['product_uom'],
    )

    tax_framework = fields.Selection(
        selection=TAX_FRAMEWORK,
        related="order_id.company_id.tax_framework",
        string='Tax Framework',
    )

    @api.depends(
        'product_uom_qty',
        'price_unit',
        'fiscal_price',
        'fiscal_quantity',
        'discount_value',
        'freight_value',
        'insurance_value',
        'other_value',
        'taxes_id')
    def _compute_amount(self):
        """Compute the amounts of the PO line."""
        super()._compute_amount()
        for l in self:
            # Update taxes fields
            l._update_taxes()
            # Call mixin compute method
            l._compute_amounts()
            # Update record
            l.update({
                'price_subtotal': l.amount_untaxed,
                'price_tax': l.amount_tax,
                'price_total': l.amount_total,
            })

    @api.onchange('product_qty', 'product_uom')
    def _onchange_quantity(self):
        """To call the method in the mixin to update
        the price and fiscal quantity."""
        super()._onchange_quantity()
        self._onchange_commercial_quantity()

    @api.multi
    def _compute_tax_id(self):
        super()._compute_tax_id()
        for line in self:
            line.taxes_id |= line.fiscal_tax_ids.account_taxes(
                user_type='purchase')

    @api.onchange('fiscal_tax_ids')
    def _onchange_fiscal_tax_ids(self):
        super()._onchange_fiscal_tax_ids()
        self.taxes_id |= self.fiscal_tax_ids.account_taxes(
            user_type='purchase')
<commit_msg>[IMP] Add comments to explain changes<commit_after># Copyright (C) 2009  Renato Lima - Akretion, Gabriel C. Stabel
# Copyright (C) 2012  Raphaël Valyi - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

from odoo import api, fields, models

from ...l10n_br_fiscal.constants.fiscal import TAX_FRAMEWORK

class PurchaseOrderLine(models.Model):
    _name = 'purchase.order.line'
    _inherit = [_name, 'l10n_br_fiscal.document.line.mixin']

    @api.model
    def _default_fiscal_operation(self):
        return self.env.user.company_id.purchase_fiscal_operation_id

    @api.model
    def _fiscal_operation_domain(self):
        domain = [
            ('state', '=', 'approved'),
            ('fiscal_type', 'in', ('purchase', 'other', 'purchase_refund'))]
        return domain

    # Adapt Mixin's fields
    fiscal_operation_id = fields.Many2one(
        comodel_name='l10n_br_fiscal.operation',
        readonly=True,
        states={'draft': [('readonly', False)]},
        default=_default_fiscal_operation,
        domain=lambda self: self._fiscal_operation_domain(),
    )

    # This redundancy is necessary for the system to be able to load the report
    fiscal_operation_line_id = fields.Many2one(
        comodel_name="l10n_br_fiscal.operation.line",
        string="Operation Line",
        domain="[('fiscal_operation_id', '=', fiscal_operation_id), "
               "('state', '=', 'approved')]")

    fiscal_tax_ids = fields.Many2many(
        comodel_name='l10n_br_fiscal.tax',
        relation='fiscal_purchase_line_tax_rel',
        column1='document_id',
        column2='fiscal_tax_id',
        string='Fiscal Taxes',
    )

    quantity = fields.Float(
        string='Mixin Quantity',
        related='product_qty',
        depends=['product_qty'],
    )

    uom_id = fields.Many2one(
        string='Mixin UOM',
        related='product_uom',
        depends=['product_uom'],
    )

    tax_framework = fields.Selection(
        selection=TAX_FRAMEWORK,
        related="order_id.company_id.tax_framework",
        string='Tax Framework',
    )

    @api.depends(
        'product_uom_qty',
        'price_unit',
        'fiscal_price',
        'fiscal_quantity',
        'discount_value',
        'freight_value',
        'insurance_value',
        'other_value',
        'taxes_id')
    def _compute_amount(self):
        """Compute the amounts of the PO line."""
        super()._compute_amount()
        for l in self:
            # Update taxes fields
            l._update_taxes()
            # Call mixin compute method
            l._compute_amounts()
            # Update record
            l.update({
                'price_subtotal': l.amount_untaxed,
                'price_tax': l.amount_tax,
                'price_total': l.amount_total,
            })

    @api.onchange('product_qty', 'product_uom')
    def _onchange_quantity(self):
        """To call the method in the mixin to update
        the price and fiscal quantity."""
        super()._onchange_quantity()
        self._onchange_commercial_quantity()

    @api.multi
    def _compute_tax_id(self):
        super()._compute_tax_id()
        for line in self:
            line.taxes_id |= line.fiscal_tax_ids.account_taxes(
                user_type='purchase')

    @api.onchange('fiscal_tax_ids')
    def _onchange_fiscal_tax_ids(self):
        super()._onchange_fiscal_tax_ids()
        self.taxes_id |= self.fiscal_tax_ids.account_taxes(
            user_type='purchase')
<|endoftext|>