<commit_before>#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# About this script:
#   This script is intended to simplify creating Subversion releases, by
#   automating as much as is possible.  It works well with our Apache
#   infrastructure, and should make rolling, posting, and announcing
#   releases dirt simple.
#
#   This script may be run on a number of platforms, but it is intended to
#   be run on people.apache.org.  As such, it may have dependencies (such
#   as Python version) which may not be common, but are guaranteed to be
#   available on people.apache.org.

# It'd be kind of nice to use the Subversion python bindings in this script,
# but people.apache.org doesn't currently have them installed

# Stuff we need
import os
import sys
import shutil
import urllib2
import hashlib
import tarfile
import logging
import subprocess
import argparse       # standard in Python 2.7

# Our required / recommended versions
autoconf_ver = '2.68'
libtool_ver = '2.4'
swig_ver = '2.0.4'

# Some constants
repos = 'http://svn.apache.org/repos/asf/subversion'

#----------------------------------------------------------------------
# Utility functions

def get_prefix(base_dir):
    return os.path.join(base_dir, 'prefix')

def get_tempdir(base_dir):
    return os.path.join(base_dir, 'tempdir')

def get_deploydir(base_dir):
    return os.path.join(base_dir, 'deploy')

def get_nullfile():
    # This is certainly not cross platform
    return open('/dev/null', 'w')

def run_script(verbose, script):
    if verbose:
        stdout = None
        stderr = None
    else:
        stdout = get_nullfile()
        stderr = subprocess.STDOUT

    for l in script.split('\n'):
        subprocess.check_call(l.split(), stdout=stdout, stderr=stderr)

def download_file(url, target):
    response = urllib2.urlopen(url)
    target_file = open(target, 'w')
    target_file.write(response.read())

#----------------------------------------------------------------------
# Cleaning up the environment

def cleanup(base_dir, args):
    'Remove generated files and folders.'
    logging.info('Cleaning')

    shutil.rmtree(get_prefix(base_dir), True)
    shutil.rmtree(get_tempdir(base_dir), True)
    shutil.rmtree(get_deploydir(base_dir), True)

#----------------------------------------------------------------------
# Creating an environment to roll the release

class RollDep(object):
    'The super class for each of the build dependencies.'
    def __init__(self, base_dir, use_existing, verbose):
        self._base_dir = base_dir
        self._use_existing = use_existing
        self._verbose = verbose

    def _test_version(self, cmd):
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        rc = proc.wait()
        if rc: return ''

        return stdout.split('\n')

    def build(self):
        if not hasattr(self, '_extra_configure_flags'):
            self._extra_configure_flags = ''
        cwd = os.getcwd()
        tempdir = get_tempdir(self._base_dir)
        tarball = os.path.join(tempdir, self._filebase + '.tar.gz')

        if os.path.exists(tarball):
            if not self._use_existing:
                raise RuntimeError('autoconf tarball "%s" already exists'
                                                                    % tarball)
            logging.info('Using existing %s.tar.gz' % self._filebase)
        else:
            logging.info('Fetching %s' % self._filebase)
            download_file(self._url, tarball)

        # Extract tarball
        tarfile.open(tarball).extractall(tempdir)

        logging.info('Building ' + self.label)
        os.chdir(os.path.join(tempdir, self._filebase))
        run_script(self._verbose,
                   '''./configure --prefix=%s %s
                      make
                      make install''' % (get_prefix(self._base_dir),
                                         self._extra_configure_flags))

        os.chdir(cwd)

class AutoconfDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'autoconf'
        self._filebase = 'autoconf-' + autoconf_ver
        self._url = 'http://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['autoconf', '-V'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == autoconf_ver

    def use_system(self):
        if not self._use_existing: return False
        return self.have_usable()

class LibtoolDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'libtool'
        self._filebase = 'libtool-' + libtool_ver
        self._url = 'http://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['libtool', '--version'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == libtool_ver

    def use_system(self):
        # We unconditionally return False here, to avoid using a borked
        # system libtool (I'm looking at you, Debian).
        return False

class SwigDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose, sf_mirror):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'swig'
        self._filebase = 'swig-' + swig_ver
        self._url = 'http://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \
            { 'swig' : self._filebase,
              'sf_mirror' : sf_mirror }
        self._extra_configure_flags = '--without-pcre'

    def have_usable(self):
        output = self._test_version(['swig', '-version'])
        if not output: return False

        version = output[1].split()[-1:][0]
        return version == swig_ver

    def use_system(self):
        if not self._use_existing: return False
        return have_usable()

def build_env(base_dir, args):
    'Download prerequisites for a release and prepare the environment.'
    logging.info('Creating release environment')

    try:
        os.mkdir(get_prefix(base_dir))
        os.mkdir(get_tempdir(base_dir))
    except OSError:
        if not args.use_existing:
            raise

    autoconf = AutoconfDep(base_dir, args.use_existing, args.verbose)
    libtool = LibtoolDep(base_dir, args.use_existing, args.verbose)
    swig = SwigDep(base_dir, args.use_existing, args.verbose, args.sf_mirror)

    # iterate over our rolling deps, and build them if needed
    for dep in [autoconf, libtool, swig]:
        if dep.use_system():
            logging.info('Using system %s' % dep.label)
        else:
            dep.build()

#----------------------------------------------------------------------
# Create release artifacts

def roll_tarballs(base_dir, args):
    'Create the release artifacts.'
    extns = ['zip', 'tar.gz', 'tar.bz2']
    version_base = args.version.split('-')[0]
    version_extra = args.version.split('-')[1]

    if args.branch:
        branch = args.branch
    else:
        branch = version_base[:-1] + 'x'

    logging.info('Rolling release %s from branch %s@%d' % (args.version,
                                                           branch, args.revnum))

    # Ensure we've got the appropriate rolling dependencies available
    autoconf = AutoconfDep(base_dir, False, args.verbose)
    libtool = LibtoolDep(base_dir, False, args.verbose)
    swig = SwigDep(base_dir, False, args.verbose, None)

    for dep in [autoconf, libtool, swig]:
        if not dep.have_usable():
           raise RuntimeError('Cannot find usable %s' % dep.label)

    # Make sure CHANGES is sync'd
    if branch != 'trunk':
        trunk_CHANGES = '%s/trunk/CHANGES@%d' % (repos, args.revnum)
        branch_CHANGES = '%s/branches/%s/CHANGES@%d' % (repos, branch,
                                                        args.revnum)
        proc = subprocess.Popen(['svn', 'diff', '--summarize', branch_CHANGES,
                                   trunk_CHANGES],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        proc.wait()

        if stdout:
            raise RuntimeError('CHANGES not synced between trunk and branch')

    # Create the output directory
    if not os.path.exists(get_deploydir(base_dir)):
        os.mkdir(get_deploydir(base_dir))

    # For now, just delegate to dist.sh to create the actual artifacts
    if version_extra:
        if version_extra.startswith('alpha'):
            extra_args = '-alpha %s' % version_extra[5:]
        elif version_extra.startswith('beta'):
            extra_args = '-beta %s' % version_extra[4:]
        elif version_extra.startswith('rc'):
            extra_args = '-rc %s' % version_extra[2:]
        elif version_extra.startswith('nightly'):
            extra_args = '-nightly'
        else:
            extra_args = ''
    logging.info('Building UNIX tarballs')
    run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d %s'
                     % (sys.path[0], version_base, branch, args.revnum,
                        extra_args) )
    logging.info('Buildling Windows tarballs')
    run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d -zip %s'
                     % (sys.path[0], version_base, branch, args.revnum,
                        extra_args) )

    # Move the results to the deploy directory
    logging.info('Moving artifacts and calculating checksums')
    for e in extns:
        if version_extra and version_extra.startswith('nightly'):
            filename = 'subversion-trunk.%s' % e
        else:
            filename = 'subversion-%s.%s' % (args.version, e)

        shutil.move(filename, get_deploydir(base_dir))
        filename = os.path.join(get_deploydir(base_dir), filename)
        m = hashlib.sha1()
        m.update(open(filename, 'r').read())
        open(filename + '.sha1', 'w').write(m.hexdigest())

    shutil.move('svn_version.h.dist', get_deploydir(base_dir))

    # And we're done!

def announce(base_dir, args):
    'Write the release announcement.'

#----------------------------------------------------------------------
# Main entry point for argument parsing and handling

def main():
    'Parse arguments, and drive the appropriate subcommand.'

    # Setup our main parser
    parser = argparse.ArgumentParser(
                            description='Create an Apache Subversion release.')
    parser.add_argument('--clean', action='store_true', default=False,
                   help='Remove any directories previously created by %(prog)s')
    parser.add_argument('--verbose', action='store_true', default=False,
                   help='Increase output verbosity')
    parser.add_argument('--base-dir', default=os.getcwd(),
                   help='''The directory in which to create needed files and
                           folders.  The default is the current working
                           directory.''')
    subparsers = parser.add_subparsers(title='subcommands')

    # Setup the parser for the build-env subcommand
    subparser = subparsers.add_parser('build-env',
                    help='''Download release prerequisistes, including autoconf,
                            libtool, and swig.''')
    subparser.set_defaults(func=build_env)
    subparser.add_argument('--sf-mirror', default='softlayer',
                    help='''The mirror to use for downloading files from
                            SourceForge.  If in the EU, you may want to use
                            'kent' for this value.''')
    subparser.add_argument('--use-existing', action='store_true', default=False,
                    help='''Attempt to use existing build dependencies before
                            downloading and building a private set.''')

    # Setup the parser for the roll subcommand
    subparser = subparsers.add_parser('roll',
                    help='''Create the release artifacts.''')
    subparser.set_defaults(func=roll_tarballs)
    subparser.add_argument('version',
                    help='''The release label, such as '1.7.0-alpha1'.''')
    subparser.add_argument('revnum', type=int,
                    help='''The revision number to base the release on.''')
    subparser.add_argument('--branch',
                    help='''The branch to base the release on.''')

    # A meta-target
    subparser = subparsers.add_parser('clean',
                    help='''The same as the '--clean' switch, but as a
                            separate subcommand.''')
    subparser.set_defaults(func=cleanup)

    # Parse the arguments
    args = parser.parse_args()

    # first, process any global operations
    if args.clean:
        cleanup(args.base_dir, args)

    # Set up logging
    logger = logging.getLogger()
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    # Fix up our path so we can use our installed versions
    os.environ['PATH'] = os.path.join(get_prefix(args.base_dir), 'bin') + ':' \
                                                            + os.environ['PATH']

    # finally, run the subcommand, and give it the parsed arguments
    args.func(args.base_dir, args)

if __name__ == '__main__':
    main()
<commit_msg>* tools/dist/release.py   (roll_tarballs): Really ensure we set a variable.<commit_after>#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# About this script:
#   This script is intended to simplify creating Subversion releases, by
#   automating as much as is possible.  It works well with our Apache
#   infrastructure, and should make rolling, posting, and announcing
#   releases dirt simple.
#
#   This script may be run on a number of platforms, but it is intended to
#   be run on people.apache.org.  As such, it may have dependencies (such
#   as Python version) which may not be common, but are guaranteed to be
#   available on people.apache.org.

# It'd be kind of nice to use the Subversion python bindings in this script,
# but people.apache.org doesn't currently have them installed

# Stuff we need
import os
import sys
import shutil
import urllib2
import hashlib
import tarfile
import logging
import subprocess
import argparse       # standard in Python 2.7

# Our required / recommended versions
autoconf_ver = '2.68'
libtool_ver = '2.4'
swig_ver = '2.0.4'

# Some constants
repos = 'http://svn.apache.org/repos/asf/subversion'

#----------------------------------------------------------------------
# Utility functions

def get_prefix(base_dir):
    return os.path.join(base_dir, 'prefix')

def get_tempdir(base_dir):
    return os.path.join(base_dir, 'tempdir')

def get_deploydir(base_dir):
    return os.path.join(base_dir, 'deploy')

def get_nullfile():
    # This is certainly not cross platform
    return open('/dev/null', 'w')

def run_script(verbose, script):
    if verbose:
        stdout = None
        stderr = None
    else:
        stdout = get_nullfile()
        stderr = subprocess.STDOUT

    for l in script.split('\n'):
        subprocess.check_call(l.split(), stdout=stdout, stderr=stderr)

def download_file(url, target):
    response = urllib2.urlopen(url)
    target_file = open(target, 'w')
    target_file.write(response.read())

#----------------------------------------------------------------------
# Cleaning up the environment

def cleanup(base_dir, args):
    'Remove generated files and folders.'
    logging.info('Cleaning')

    shutil.rmtree(get_prefix(base_dir), True)
    shutil.rmtree(get_tempdir(base_dir), True)
    shutil.rmtree(get_deploydir(base_dir), True)

#----------------------------------------------------------------------
# Creating an environment to roll the release

class RollDep(object):
    'The super class for each of the build dependencies.'
    def __init__(self, base_dir, use_existing, verbose):
        self._base_dir = base_dir
        self._use_existing = use_existing
        self._verbose = verbose

    def _test_version(self, cmd):
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        rc = proc.wait()
        if rc: return ''

        return stdout.split('\n')

    def build(self):
        if not hasattr(self, '_extra_configure_flags'):
            self._extra_configure_flags = ''
        cwd = os.getcwd()
        tempdir = get_tempdir(self._base_dir)
        tarball = os.path.join(tempdir, self._filebase + '.tar.gz')

        if os.path.exists(tarball):
            if not self._use_existing:
                raise RuntimeError('autoconf tarball "%s" already exists'
                                                                    % tarball)
            logging.info('Using existing %s.tar.gz' % self._filebase)
        else:
            logging.info('Fetching %s' % self._filebase)
            download_file(self._url, tarball)

        # Extract tarball
        tarfile.open(tarball).extractall(tempdir)

        logging.info('Building ' + self.label)
        os.chdir(os.path.join(tempdir, self._filebase))
        run_script(self._verbose,
                   '''./configure --prefix=%s %s
                      make
                      make install''' % (get_prefix(self._base_dir),
                                         self._extra_configure_flags))

        os.chdir(cwd)

class AutoconfDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'autoconf'
        self._filebase = 'autoconf-' + autoconf_ver
        self._url = 'http://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['autoconf', '-V'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == autoconf_ver

    def use_system(self):
        if not self._use_existing: return False
        return self.have_usable()

class LibtoolDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'libtool'
        self._filebase = 'libtool-' + libtool_ver
        self._url = 'http://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['libtool', '--version'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == libtool_ver

    def use_system(self):
        # We unconditionally return False here, to avoid using a borked
        # system libtool (I'm looking at you, Debian).
        return False

class SwigDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose, sf_mirror):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'swig'
        self._filebase = 'swig-' + swig_ver
        self._url = 'http://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \
            { 'swig' : self._filebase,
              'sf_mirror' : sf_mirror }
        self._extra_configure_flags = '--without-pcre'

    def have_usable(self):
        output = self._test_version(['swig', '-version'])
        if not output: return False

        version = output[1].split()[-1:][0]
        return version == swig_ver

    def use_system(self):
        if not self._use_existing: return False
        return have_usable()

def build_env(base_dir, args):
    'Download prerequisites for a release and prepare the environment.'
    logging.info('Creating release environment')

    try:
        os.mkdir(get_prefix(base_dir))
        os.mkdir(get_tempdir(base_dir))
    except OSError:
        if not args.use_existing:
            raise

    autoconf = AutoconfDep(base_dir, args.use_existing, args.verbose)
    libtool = LibtoolDep(base_dir, args.use_existing, args.verbose)
    swig = SwigDep(base_dir, args.use_existing, args.verbose, args.sf_mirror)

    # iterate over our rolling deps, and build them if needed
    for dep in [autoconf, libtool, swig]:
        if dep.use_system():
            logging.info('Using system %s' % dep.label)
        else:
            dep.build()

#----------------------------------------------------------------------
# Create release artifacts

def roll_tarballs(base_dir, args):
    'Create the release artifacts.'
    extns = ['zip', 'tar.gz', 'tar.bz2']
    version_base = args.version.split('-')[0]
    version_extra = args.version.split('-')[1]

    if args.branch:
        branch = args.branch
    else:
        branch = version_base[:-1] + 'x'

    logging.info('Rolling release %s from branch %s@%d' % (args.version,
                                                           branch, args.revnum))

    # Ensure we've got the appropriate rolling dependencies available
    autoconf = AutoconfDep(base_dir, False, args.verbose)
    libtool = LibtoolDep(base_dir, False, args.verbose)
    swig = SwigDep(base_dir, False, args.verbose, None)

    for dep in [autoconf, libtool, swig]:
        if not dep.have_usable():
           raise RuntimeError('Cannot find usable %s' % dep.label)

    # Make sure CHANGES is sync'd
    if branch != 'trunk':
        trunk_CHANGES = '%s/trunk/CHANGES@%d' % (repos, args.revnum)
        branch_CHANGES = '%s/branches/%s/CHANGES@%d' % (repos, branch,
                                                        args.revnum)
        proc = subprocess.Popen(['svn', 'diff', '--summarize', branch_CHANGES,
                                   trunk_CHANGES],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        proc.wait()

        if stdout:
            raise RuntimeError('CHANGES not synced between trunk and branch')

    # Create the output directory
    if not os.path.exists(get_deploydir(base_dir)):
        os.mkdir(get_deploydir(base_dir))

    # For now, just delegate to dist.sh to create the actual artifacts
    extra_args = ''
    if version_extra:
        if version_extra.startswith('alpha'):
            extra_args = '-alpha %s' % version_extra[5:]
        elif version_extra.startswith('beta'):
            extra_args = '-beta %s' % version_extra[4:]
        elif version_extra.startswith('rc'):
            extra_args = '-rc %s' % version_extra[2:]
        elif version_extra.startswith('nightly'):
            extra_args = '-nightly'
    logging.info('Building UNIX tarballs')
    run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d %s'
                     % (sys.path[0], version_base, branch, args.revnum,
                        extra_args) )
    logging.info('Buildling Windows tarballs')
    run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d -zip %s'
                     % (sys.path[0], version_base, branch, args.revnum,
                        extra_args) )

    # Move the results to the deploy directory
    logging.info('Moving artifacts and calculating checksums')
    for e in extns:
        if version_extra and version_extra.startswith('nightly'):
            filename = 'subversion-trunk.%s' % e
        else:
            filename = 'subversion-%s.%s' % (args.version, e)

        shutil.move(filename, get_deploydir(base_dir))
        filename = os.path.join(get_deploydir(base_dir), filename)
        m = hashlib.sha1()
        m.update(open(filename, 'r').read())
        open(filename + '.sha1', 'w').write(m.hexdigest())

    shutil.move('svn_version.h.dist', get_deploydir(base_dir))

    # And we're done!

def announce(base_dir, args):
    'Write the release announcement.'

#----------------------------------------------------------------------
# Main entry point for argument parsing and handling

def main():
    'Parse arguments, and drive the appropriate subcommand.'

    # Setup our main parser
    parser = argparse.ArgumentParser(
                            description='Create an Apache Subversion release.')
    parser.add_argument('--clean', action='store_true', default=False,
                   help='Remove any directories previously created by %(prog)s')
    parser.add_argument('--verbose', action='store_true', default=False,
                   help='Increase output verbosity')
    parser.add_argument('--base-dir', default=os.getcwd(),
                   help='''The directory in which to create needed files and
                           folders.  The default is the current working
                           directory.''')
    subparsers = parser.add_subparsers(title='subcommands')

    # Setup the parser for the build-env subcommand
    subparser = subparsers.add_parser('build-env',
                    help='''Download release prerequisistes, including autoconf,
                            libtool, and swig.''')
    subparser.set_defaults(func=build_env)
    subparser.add_argument('--sf-mirror', default='softlayer',
                    help='''The mirror to use for downloading files from
                            SourceForge.  If in the EU, you may want to use
                            'kent' for this value.''')
    subparser.add_argument('--use-existing', action='store_true', default=False,
                    help='''Attempt to use existing build dependencies before
                            downloading and building a private set.''')

    # Setup the parser for the roll subcommand
    subparser = subparsers.add_parser('roll',
                    help='''Create the release artifacts.''')
    subparser.set_defaults(func=roll_tarballs)
    subparser.add_argument('version',
                    help='''The release label, such as '1.7.0-alpha1'.''')
    subparser.add_argument('revnum', type=int,
                    help='''The revision number to base the release on.''')
    subparser.add_argument('--branch',
                    help='''The branch to base the release on.''')

    # A meta-target
    subparser = subparsers.add_parser('clean',
                    help='''The same as the '--clean' switch, but as a
                            separate subcommand.''')
    subparser.set_defaults(func=cleanup)

    # Parse the arguments
    args = parser.parse_args()

    # first, process any global operations
    if args.clean:
        cleanup(args.base_dir, args)

    # Set up logging
    logger = logging.getLogger()
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    # Fix up our path so we can use our installed versions
    os.environ['PATH'] = os.path.join(get_prefix(args.base_dir), 'bin') + ':' \
                                                            + os.environ['PATH']

    # finally, run the subcommand, and give it the parsed arguments
    args.func(args.base_dir, args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.9.2 on 2017-04-25 11:36
from __future__ import unicode_literals

import os
import uuid
import django.db.models.deletion
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
from django.core import management
from arches.app.models.models import GraphModel
from arches.app.models.system_settings import settings
from arches.app.search.mappings import prepare_search_index, delete_search_index

def forwards_func(apps, schema_editor):
    # We get the model from the versioned app registry;
    # if we directly import it, it'll be the wrong version

    delete_search_index()
    for graphid in GraphModel.objects.filter(isresource=True).values_list('graphid', flat=True):
        prepare_search_index(str(graphid), create=True)

    settings_data_file = os.path.join(settings.ROOT_DIR, 'db', 'system_settings', 'Arches_System_Settings.json')
    local_settings_available = os.path.isfile(os.path.join(settings.SYSTEM_SETTINGS_LOCAL_PATH))

    if local_settings_available == True:
        settings_data_file = settings.SYSTEM_SETTINGS_LOCAL_PATH

    management.call_command('es', operation='index_resources')
    management.call_command('packages', operation='import_graphs', source=os.path.join(settings.ROOT_DIR, 'db', 'system_settings', 'Arches_System_Settings_Model.json'))
    management.call_command('packages', operation='import_business_data', source=settings_data_file, overwrite='overwrite')

def reverse_func(apps, schema_editor):
    GraphModel.objects.get(graphid=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID).delete()

def add_permissions(apps, schema_editor, with_create_permissions=True):
    db_alias = schema_editor.connection.alias
    Group = apps.get_model("auth", "Group")
    Permission = apps.get_model("auth", "Permission")

    read_nodegroup = Permission.objects.get(codename='read_nodegroup', content_type__app_label='models', content_type__model='nodegroup')

    resource_editor_group = Group.objects.using(db_alias).get(name='Resource Editor')
    resource_editor_group.permissions.add(read_nodegroup)
    rdm_admin_group = Group.objects.using(db_alias).get(name='RDM Administrator')
    rdm_admin_group.permissions.add(read_nodegroup)
    app_admin_group = Group.objects.using(db_alias).get(name='Application Administrator')
    app_admin_group.permissions.add(read_nodegroup)
    sys_admin_group = Group.objects.using(db_alias).get(name='System Administrator')
    sys_admin_group.permissions.add(read_nodegroup)
    mobile_project_admin_group = Group.objects.using(db_alias).get(name='Mobile Project Administrator')
    mobile_project_admin_group.permissions.add(read_nodegroup)
    crowdsource_editor_group = Group.objects.using(db_alias).get(name='Crowdsource Editor')
    crowdsource_editor_group.permissions.add(read_nodegroup)
    guest_group = Group.objects.using(db_alias).get(name='Guest')
    guest_group.permissions.add(read_nodegroup)

class Migration(migrations.Migration):

    dependencies = [
        ('models', '0002_40b4'),
    ]

    operations = [
        migrations.CreateModel(
            name='Geocoder',
            fields=[
                ('geocoderid', models.UUIDField(default=uuid.uuid1, primary_key=True, serialize=False)),
                ('name', models.TextField()),
                ('component', models.TextField()),
                ('api_key', models.TextField()),
            ],
            options={
                'db_table': 'geocoders',
                'managed': True,
            },
        ),
        migrations.RunSQL("""
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/string',
                    configname = 'string-datatype-config'
                WHERE datatype = 'string';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/number',
                    configname = 'number-datatype-config'
                WHERE datatype = 'number';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/boolean',
                    configname = 'boolean-datatype-config'
                WHERE datatype = 'boolean';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/domain-value',
                    configname = 'domain-value-datatype-config'
                WHERE datatype = 'domain-value';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/concept',
                    configname = 'concept-datatype-config'
                WHERE datatype = 'concept';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/date',
                    configname = 'date-datatype-config'
                WHERE datatype = 'date';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/concept',
                    configname = 'concept-datatype-config'
                WHERE datatype = 'concept-list';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/domain-value',
                    configname = 'domain-value-datatype-config'
                WHERE datatype = 'domain-value-list';
        """, """
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'string';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'number';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'boolean';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'domain-value';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'concept';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'date';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'concept-list';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'domain-value-list';

        """),

        migrations.RunSQL("""
            INSERT INTO iiif_manifests(id, url) VALUES (public.uuid_generate_v1mc(), 'https://data.getty.edu/museum/api/iiif/249995/manifest.json');
        """, """
            DELETE FROM public.iiif_manifests WHERE url = 'https://data.getty.edu/museum/api/iiif/249995/manifest.json';
        """),
        migrations.RunSQL("""
            INSERT INTO public.relations(relationid, conceptidfrom, conceptidto, relationtype) VALUES (public.uuid_generate_v1mc(), '00000000-0000-0000-0000-000000000004', '00000000-0000-0000-0000-000000000007', 'narrower') ON CONFLICT DO NOTHING;
        """,
        """
            DELETE FROM public.relations WHERE conceptidfrom = '00000000-0000-0000-0000-000000000004' AND conceptidto = '00000000-0000-0000-0000-000000000007' AND relationtype = 'narrower';
        """),
        migrations.RunSQL("""
            INSERT INTO widgets(widgetid, name, component, datatype, defaultconfig)
                VALUES ('10000000-0000-0000-0000-000000000022', 'iiif-widget', 'views/components/widgets/iiif', 'iiif-drawing', '{
                        "placeholder": "",
                        "options": [],
                        "nameLabel": "Name",
                        "typeLabel": "Type"
                    }'
                );
            INSERT INTO widgets(widgetid, name, component, datatype, defaultconfig) VALUES ('10000000-0000-0000-0000-000000000020', 'csv-chart-widget', 'views/components/widgets/csv-chart', 'csv-chart-json', '{"acceptedFiles": "", "maxFilesize": "200"}');
            INSERT INTO d_data_types VALUES ('csv-chart-json', 'fa fa-line-chart', 'datatypes.py', 'CSVChartJsonDataType', null, null, null, FALSE, '10000000-0000-0000-0000-000000000020');
            INSERT INTO d_data_types VALUES ('iiif-drawing', 'fa fa-file-code-o', 'datatypes.py', 'IIIFDrawingDataType', '{"rdmCollection": null}', 'views/graph/datatypes/concept', 'concept-datatype-config', FALSE, '10000000-0000-0000-0000-000000000022');
            UPDATE d_data_types SET (modulename, classname) = ('datatypes.py', 'DomainDataType') WHERE datatype = 'domain-value';
            UPDATE d_data_types SET (modulename, classname) = ('datatypes.py', 'DomainListDataType') WHERE datatype = 'domain-value-list';
            """,
            """
            DELETE FROM d_data_types WHERE datatype = 'iiif-drawing';
            DELETE FROM d_data_types WHERE datatype = 'csv-chart-json';
            DELETE FROM widgets WHERE widgetid = '10000000-0000-0000-0000-000000000020';
            DELETE from widgets WHERE widgetid = '10000000-0000-0000-0000-000000000022';
            UPDATE d_data_types SET (modulename, classname) = ('concept_types.py', 'ConceptDataType') WHERE datatype = 'domain-value';
            UPDATE d_data_types SET (modulename, classname) = ('concept_types.py', 'ConceptListDataType') WHERE datatype = 'domain-value-list';
        """),
        migrations.RunSQL("""
            INSERT INTO public.geocoders(geocoderid, name, component, api_key) VALUES ('10000000-0000-0000-0000-010000000000', 'Mapbox', 'views/components/geocoders/mapbox', '');
        """,
        """
            DELETE FROM public.geocoders WHERE geocoderid = '10000000-0000-0000-0000-010000000000';
        """),
        ## the following command has to be run after the previous RunSQL commands that update the domain datatype values
        migrations.RunPython(forwards_func, reverse_func),
        migrations.RunPython(add_permissions,reverse_code=lambda *args,**kwargs: True),
    ]
<commit_msg>Added updates to widgets and reports that used 'MapzenGeocoder' as their default<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.9.2 on 2017-04-25 11:36
from __future__ import unicode_literals

import os
import uuid
import django.db.models.deletion
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
from django.core import management
from arches.app.models.models import GraphModel
from arches.app.models.system_settings import settings
from arches.app.search.mappings import prepare_search_index, delete_search_index

def forwards_func(apps, schema_editor):
    # We get the model from the versioned app registry;
    # if we directly import it, it'll be the wrong version

    delete_search_index()
    for graphid in GraphModel.objects.filter(isresource=True).values_list('graphid', flat=True):
        prepare_search_index(str(graphid), create=True)

    settings_data_file = os.path.join(settings.ROOT_DIR, 'db', 'system_settings', 'Arches_System_Settings.json')
    local_settings_available = os.path.isfile(os.path.join(settings.SYSTEM_SETTINGS_LOCAL_PATH))

    if local_settings_available == True:
        settings_data_file = settings.SYSTEM_SETTINGS_LOCAL_PATH

    management.call_command('es', operation='index_resources')
    management.call_command('packages', operation='import_graphs', source=os.path.join(settings.ROOT_DIR, 'db', 'system_settings', 'Arches_System_Settings_Model.json'))
    management.call_command('packages', operation='import_business_data', source=settings_data_file, overwrite='overwrite')

def reverse_func(apps, schema_editor):
    GraphModel.objects.get(graphid=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID).delete()

def add_permissions(apps, schema_editor, with_create_permissions=True):
    db_alias = schema_editor.connection.alias
    Group = apps.get_model("auth", "Group")
    Permission = apps.get_model("auth", "Permission")

    read_nodegroup = Permission.objects.get(codename='read_nodegroup', content_type__app_label='models', content_type__model='nodegroup')

    resource_editor_group = Group.objects.using(db_alias).get(name='Resource Editor')
    resource_editor_group.permissions.add(read_nodegroup)
    rdm_admin_group = Group.objects.using(db_alias).get(name='RDM Administrator')
    rdm_admin_group.permissions.add(read_nodegroup)
    app_admin_group = Group.objects.using(db_alias).get(name='Application Administrator')
    app_admin_group.permissions.add(read_nodegroup)
    sys_admin_group = Group.objects.using(db_alias).get(name='System Administrator')
    sys_admin_group.permissions.add(read_nodegroup)
    mobile_project_admin_group = Group.objects.using(db_alias).get(name='Mobile Project Administrator')
    mobile_project_admin_group.permissions.add(read_nodegroup)
    crowdsource_editor_group = Group.objects.using(db_alias).get(name='Crowdsource Editor')
    crowdsource_editor_group.permissions.add(read_nodegroup)
    guest_group = Group.objects.using(db_alias).get(name='Guest')
    guest_group.permissions.add(read_nodegroup)

class Migration(migrations.Migration):

    dependencies = [
        ('models', '0002_40b4'),
    ]

    operations = [
        migrations.CreateModel(
            name='Geocoder',
            fields=[
                ('geocoderid', models.UUIDField(default=uuid.uuid1, primary_key=True, serialize=False)),
                ('name', models.TextField()),
                ('component', models.TextField()),
                ('api_key', models.TextField()),
            ],
            options={
                'db_table': 'geocoders',
                'managed': True,
            },
        ),
        migrations.RunSQL("""
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/string',
                    configname = 'string-datatype-config'
                WHERE datatype = 'string';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/number',
                    configname = 'number-datatype-config'
                WHERE datatype = 'number';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/boolean',
                    configname = 'boolean-datatype-config'
                WHERE datatype = 'boolean';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/domain-value',
                    configname = 'domain-value-datatype-config'
                WHERE datatype = 'domain-value';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/concept',
                    configname = 'concept-datatype-config'
                WHERE datatype = 'concept';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/date',
                    configname = 'date-datatype-config'
                WHERE datatype = 'date';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/concept',
                    configname = 'concept-datatype-config'
                WHERE datatype = 'concept-list';
            UPDATE d_data_types
                SET issearchable = true,
                    configcomponent = 'views/graph/datatypes/domain-value',
                    configname = 'domain-value-datatype-config'
                WHERE datatype = 'domain-value-list';
        """, """
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'string';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'number';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'boolean';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'domain-value';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'concept';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'date';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'concept-list';
            UPDATE d_data_types
                SET issearchable = false,
                    configcomponent = NULL,
                    configname = NULL
                WHERE datatype = 'domain-value-list';

        """),

        migrations.RunSQL("""
            INSERT INTO iiif_manifests(id, url) VALUES (public.uuid_generate_v1mc(), 'https://data.getty.edu/museum/api/iiif/249995/manifest.json');
        """, """
            DELETE FROM public.iiif_manifests WHERE url = 'https://data.getty.edu/museum/api/iiif/249995/manifest.json';
        """),
        migrations.RunSQL("""
            INSERT INTO public.relations(relationid, conceptidfrom, conceptidto, relationtype) VALUES (public.uuid_generate_v1mc(), '00000000-0000-0000-0000-000000000004', '00000000-0000-0000-0000-000000000007', 'narrower') ON CONFLICT DO NOTHING;
        """,
        """
            DELETE FROM public.relations WHERE conceptidfrom = '00000000-0000-0000-0000-000000000004' AND conceptidto = '00000000-0000-0000-0000-000000000007' AND relationtype = 'narrower';
        """),
        migrations.RunSQL("""
            INSERT INTO widgets(widgetid, name, component, datatype, defaultconfig)
                VALUES ('10000000-0000-0000-0000-000000000022', 'iiif-widget', 'views/components/widgets/iiif', 'iiif-drawing', '{
                        "placeholder": "",
                        "options": [],
                        "nameLabel": "Name",
                        "typeLabel": "Type"
                    }'
                );
            INSERT INTO widgets(widgetid, name, component, datatype, defaultconfig) VALUES ('10000000-0000-0000-0000-000000000020', 'csv-chart-widget', 'views/components/widgets/csv-chart', 'csv-chart-json', '{"acceptedFiles": "", "maxFilesize": "200"}');
            INSERT INTO d_data_types VALUES ('csv-chart-json', 'fa fa-line-chart', 'datatypes.py', 'CSVChartJsonDataType', null, null, null, FALSE, '10000000-0000-0000-0000-000000000020');
            INSERT INTO d_data_types VALUES ('iiif-drawing', 'fa fa-file-code-o', 'datatypes.py', 'IIIFDrawingDataType', '{"rdmCollection": null}', 'views/graph/datatypes/concept', 'concept-datatype-config', FALSE, '10000000-0000-0000-0000-000000000022');
            UPDATE d_data_types SET (modulename, classname) = ('datatypes.py', 'DomainDataType') WHERE datatype = 'domain-value';
            UPDATE d_data_types SET (modulename, classname) = ('datatypes.py', 'DomainListDataType') WHERE datatype = 'domain-value-list';
            """,
            """
            DELETE FROM d_data_types WHERE datatype = 'iiif-drawing';
            DELETE FROM d_data_types WHERE datatype = 'csv-chart-json';
            DELETE FROM widgets WHERE widgetid = '10000000-0000-0000-0000-000000000020';
            DELETE from widgets WHERE widgetid = '10000000-0000-0000-0000-000000000022';
            UPDATE d_data_types SET (modulename, classname) = ('concept_types.py', 'ConceptDataType') WHERE datatype = 'domain-value';
            UPDATE d_data_types SET (modulename, classname) = ('concept_types.py', 'ConceptListDataType') WHERE datatype = 'domain-value-list';
        """),
        migrations.RunSQL("""
            INSERT INTO public.geocoders(geocoderid, name, component, api_key) VALUES ('10000000-0000-0000-0000-010000000000', 'Mapbox', 'views/components/geocoders/mapbox', '');
        """,
        """
            DELETE FROM public.geocoders WHERE geocoderid = '10000000-0000-0000-0000-010000000000';
        """),
        migrations.RunSQL("""
            UPDATE report_templates SET (defaultconfig) = ('{
                "basemap": "streets",
                "geometryTypes": [{"text":"Point", "id":"Point"}, {"text":"Line", "id":"Line"}, {"text":"Polygon", "id":"Polygon"}],
                "overlayConfigs": [],
                "overlayOpacity": 0.0,
                "geocodeProvider": "10000000-0000-0000-0000-010000000000",
                "zoom": 10,
                "maxZoom": 20,
                "minZoom": 0,
                "centerX": -122.3979693,
                "centerY": 37.79,
                "pitch": 0.0,
                "bearing": 0.0,
                "geocodePlaceholder": "Search",
                "geocoderVisible": true,
                "featureColor": null,
                "featureLineWidth": null,
                "featurePointSize": null,
                "featureEditingDisabled": true,
                "mapControlsHidden": false
            }') WHERE templateid = '50000000-0000-0000-0000-000000000002';

            UPDATE widgets SET (defaultconfig) = ('{
                    "basemap": "streets",
                    "geometryTypes": [{"text":"Point", "id":"Point"}, {"text":"Line", "id":"Line"}, {"text":"Polygon", "id":"Polygon"}],
                    "overlayConfigs": [],
                    "overlayOpacity": 0.0,
                    "geocodeProvider": "10000000-0000-0000-0000-010000000000",
                    "zoom": 0,
                    "maxZoom": 20,
                    "minZoom": 0,
                    "centerX": 0,
                    "centerY": 0,
                    "pitch": 0.0,
                    "bearing": 0.0,
                    "geocodePlaceholder": "Search",
                    "geocoderVisible": true,
                    "featureColor": null,
                    "featureLineWidth": null,
                    "featurePointSize": null
                }') WHERE widgetid = '10000000-0000-0000-0000-000000000007';
        """,
        """
            UPDATE report_templates SET (defaultconfig) = ('{
                "basemap": "streets",
                "geometryTypes": [{"text":"Point", "id":"Point"}, {"text":"Line", "id":"Line"}, {"text":"Polygon", "id":"Polygon"}],
                "overlayConfigs": [],
                "overlayOpacity": 0.0,
                "geocodeProvider": "MapzenGeocoder",
                "zoom": 10,
                "maxZoom": 20,
                "minZoom": 0,
                "centerX": -122.3979693,
                "centerY": 37.79,
                "pitch": 0.0,
                "bearing": 0.0,
                "geocodePlaceholder": "Search",
                "geocoderVisible": true,
                "featureColor": null,
                "featureLineWidth": null,
                "featurePointSize": null,
                "featureEditingDisabled": true,
                "mapControlsHidden": false
            }') WHERE templateid = '50000000-0000-0000-0000-000000000002';
            UPDATE widgets SET (defaultconfig) = ('{
                "basemap": "streets",
                "geometryTypes": [{"text":"Point", "id":"Point"}, {"text":"Line", "id":"Line"}, {"text":"Polygon", "id":"Polygon"}],
                "overlayConfigs": [],
                "overlayOpacity": 0.0,
                "geocodeProvider": "MapzenGeocoder",
                "zoom": 0,
                "maxZoom": 20,
                "minZoom": 0,
                "centerX": 0,
                "centerY": 0,
                "pitch": 0.0,
                "bearing": 0.0,
                "geocodePlaceholder": "Search",
                "geocoderVisible": true,
                "featureColor": null,
                "featureLineWidth": null,
                "featurePointSize": null
            }') WHERE widgetid = '10000000-0000-0000-0000-000000000007';
        """),

        ## the following command has to be run after the previous RunSQL commands that update the domain datatype values
        migrations.RunPython(forwards_func, reverse_func),
        migrations.RunPython(add_permissions,reverse_code=lambda *args,**kwargs: True),
    ]
<|endoftext|>


<commit_before>from __future__ import division

import numpy, scipy
import time
import math

from nltk import cluster
from nltk.cluster import euclidean_distance, cosine_distance
from scipy import spatial
from scipy.sparse import csr_matrix
import scipy.cluster.hierarchy as hier
import scipy.spatial.distance as dist
import fastcluster

import nimfa

def cluster_nmf(vectors, rank):

    """ Takes in vectors and clusters them using Non Negative Matrix Factorization.

    Inputs:
    vectors -- matrix containing rows of vectors
    rank -- number of clusters to create

    """

    print "Starting NMF clustering"

    start_time = time.time()

    # Run NMF.
    # Change this later and see which is best
    vectors_matrix = numpy.matrix(vectors)
    vectors_matrix = vectors_matrix.transpose()

    # Generate random matrix factors which we will pass as fixed factors to Nimfa.
    init_W = numpy.random.rand(vectors_matrix.shape[0], rank)
    init_H = numpy.random.rand(rank, vectors_matrix.shape[1])

    fctr = nimfa.mf(vectors_matrix, method = "nmf", seed = "fixed", W = init_W, H = init_H, rank = rank)
    fctr_res = nimfa.mf_run(fctr)

    # Basis matrix
    W = fctr_res.basis()

    # Mixture matrix
    H = fctr_res.coef()

    # get assignments
    assignment = []
    for index in range(H.shape[1]):
        column = list(H[:, index])
        assignment.append(column.index(max(column)))

    # Print the loss function (Euclidean distance between target matrix and its estimate).
    print "Euclidean distance: %5.3e" % fctr_res.distance(metric = "euclidean")

    end_time = time.time()
    print "Clustering required", (end_time-start_time),"seconds"

    return assignment

def cluster_gaac(vectors, num_clusters):

    """ Takes in vectors and clusters them using Group Average Agglomerative clustering with cosine distance.

    Inputs:
    vectors -- matrix containing rows of vectors
    num_clusters -- number of clusters to create

    """

    print "Starting GAAC clustering"

    start_time = time.time()

    distance = spatial.distance.pdist(vectors, 'cosine')

    linkage = fastcluster.linkage(distance, method="weighted")

    clustdict = {i:[i] for i in xrange(len(linkage)+1)}
    for i in xrange(len(linkage)-num_clusters+1):
        clust1= int(linkage[i][0])
        clust2= int(linkage[i][1])
        clustdict[max(clustdict)+1] = clustdict[clust1] + clustdict[clust2]
        del clustdict[clust1], clustdict[clust2]

    # generate the assignment list (vector -> cluster id)
    assignment = [-1]*len(vectors)

    count = 0
    for key in clustdict:
        value = clustdict[key]
        for item in value:
            assignment[item] = count
        count = count + 1

    end_time = time.time()
    print "Clustering required", (end_time-start_time),"seconds"

    return assignment

def cluster_kmeans(vectors, num_clusters, distance_metric = "cosine"):

    """ Takes in vectors and clusters them using KMeans clustering.

    Inputs:
    vectors -- matrix containing rows of vectors
    num_clusters -- number of clusters to create
    distance_metric -- distance measure between vectors (default "cosine")

    """

    print "Starting KMeans clustering"

    start_time = time.time()

    # initialize
    if distance_metric == "euclidean":
        clusterer = cluster.KMeansClusterer(num_clusters, euclidean_distance)
    elif distance_metric == "cosine":
        clusterer = cluster.KMeansClusterer(num_clusters, cosine_distance)

    assignment = clusterer.cluster(vectors, True)

    end_time = time.time()
    print "Clustering required", (end_time-start_time),"seconds"

    return assignment

<commit_msg>Debug print statements for NMF<commit_after>from __future__ import division

import numpy, scipy
import time
import math

from nltk import cluster
from nltk.cluster import euclidean_distance, cosine_distance
from scipy import spatial
from scipy.sparse import csr_matrix
import scipy.cluster.hierarchy as hier
import scipy.spatial.distance as dist
import fastcluster

import nimfa

def cluster_nmf(vectors, rank):

    """ Takes in vectors and clusters them using Non Negative Matrix Factorization.

    Inputs:
    vectors -- matrix containing rows of vectors
    rank -- number of clusters to create

    """

    print "Starting NMF clustering"

    start_time = time.time()

    # Run NMF.
    # Change this later and see which is best
    vectors_matrix = numpy.matrix(vectors)
    vectors_matrix = vectors_matrix.transpose()
    print "Created vectors_matrix"

    # Generate random matrix factors which we will pass as fixed factors to Nimfa.
    init_W = numpy.random.rand(vectors_matrix.shape[0], rank)
    init_H = numpy.random.rand(rank, vectors_matrix.shape[1])
    print "Generated random matrix factors"

    fctr = nimfa.mf(vectors_matrix, method = "nmf", seed = "fixed", W = init_W, H = init_H, rank = rank)
    fctr_res = nimfa.mf_run(fctr)
    print "NIMFA"

    # Basis matrix
    W = fctr_res.basis()

    # Mixture matrix
    H = fctr_res.coef()
    print "Extracted Basis and Mixture matrices"

    # get assignments
    assignment = []
    for index in range(H.shape[1]):
        column = list(H[:, index])
        assignment.append(column.index(max(column)))

    print "Assignments extracted"

    # Print the loss function (Euclidean distance between target matrix and its estimate).
    print "Euclidean distance: %5.3e" % fctr_res.distance(metric = "euclidean")

    end_time = time.time()
    print "Clustering required", (end_time-start_time),"seconds"

    return assignment

def cluster_gaac(vectors, num_clusters):

    """ Takes in vectors and clusters them using Group Average Agglomerative clustering with cosine distance.

    Inputs:
    vectors -- matrix containing rows of vectors
    num_clusters -- number of clusters to create

    """

    print "Starting GAAC clustering"

    start_time = time.time()

    distance = spatial.distance.pdist(vectors, 'cosine')

    linkage = fastcluster.linkage(distance, method="weighted")

    clustdict = {i:[i] for i in xrange(len(linkage)+1)}
    for i in xrange(len(linkage)-num_clusters+1):
        clust1= int(linkage[i][0])
        clust2= int(linkage[i][1])
        clustdict[max(clustdict)+1] = clustdict[clust1] + clustdict[clust2]
        del clustdict[clust1], clustdict[clust2]

    # generate the assignment list (vector -> cluster id)
    assignment = [-1]*len(vectors)

    count = 0
    for key in clustdict:
        value = clustdict[key]
        for item in value:
            assignment[item] = count
        count = count + 1

    end_time = time.time()
    print "Clustering required", (end_time-start_time),"seconds"

    return assignment

def cluster_kmeans(vectors, num_clusters, distance_metric = "cosine"):

    """ Takes in vectors and clusters them using KMeans clustering.

    Inputs:
    vectors -- matrix containing rows of vectors
    num_clusters -- number of clusters to create
    distance_metric -- distance measure between vectors (default "cosine")

    """

    print "Starting KMeans clustering"

    start_time = time.time()

    # initialize
    if distance_metric == "euclidean":
        clusterer = cluster.KMeansClusterer(num_clusters, euclidean_distance)
    elif distance_metric == "cosine":
        clusterer = cluster.KMeansClusterer(num_clusters, cosine_distance)

    assignment = clusterer.cluster(vectors, True)

    end_time = time.time()
    print "Clustering required", (end_time-start_time),"seconds"

    return assignment

<|endoftext|>


<commit_before># Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Copyright 2010 OpenStack Foundation
# Copyright 2013 IBM Corp.
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import ConfigParser
import os
import sys

import install_venv_common as install_venv

def print_help(project, venv, root):
    help = """
    %(project)s development environment setup is complete.

    %(project)s development uses virtualenv to track and manage Python
    dependencies while in development and testing.

    To activate the %(project)s virtualenv for the extent of your current
    shell session you can run:

    $ source %(venv)s/bin/activate

    Or, if you prefer, you can run commands in the virtualenv on a case by
    case basis by running:

    $ %(root)s/tools/with_venv.sh <your command>
    """
    print (help % dict(project=project, venv=venv, root=root))

def main(argv):
    root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

    if os.environ.get('tools_path'):
        root = os.environ['tools_path']
    venv = os.path.join(root, '.venv')
    if os.environ.get('venv'):
        venv = os.environ['venv']

    pip_requires = os.path.join(root, 'requirements.txt')
    test_requires = os.path.join(root, 'test-requirements.txt')
    py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
    setup_cfg = ConfigParser.ConfigParser()
    setup_cfg.read('setup.cfg')
    project = setup_cfg.get('metadata', 'name')

    install = install_venv.InstallVenv(
        root, venv, pip_requires, test_requires, py_version, project)
    options = install.parse_args(argv)
    install.check_python_version()
    install.check_dependencies()
    install.create_virtualenv(no_site_packages=options.no_site_packages)
    install.install_dependencies()
    print_help(project, venv, root)

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>Replace ConfigParser with six.moves.configparser<commit_after># Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Copyright 2010 OpenStack Foundation
# Copyright 2013 IBM Corp.
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import os
import sys

from six.moves import configparser

import install_venv_common as install_venv

def print_help(project, venv, root):
    help = """
    %(project)s development environment setup is complete.

    %(project)s development uses virtualenv to track and manage Python
    dependencies while in development and testing.

    To activate the %(project)s virtualenv for the extent of your current
    shell session you can run:

    $ source %(venv)s/bin/activate

    Or, if you prefer, you can run commands in the virtualenv on a case by
    case basis by running:

    $ %(root)s/tools/with_venv.sh <your command>
    """
    print (help % dict(project=project, venv=venv, root=root))

def main(argv):
    root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

    if os.environ.get('tools_path'):
        root = os.environ['tools_path']
    venv = os.path.join(root, '.venv')
    if os.environ.get('venv'):
        venv = os.environ['venv']

    pip_requires = os.path.join(root, 'requirements.txt')
    test_requires = os.path.join(root, 'test-requirements.txt')
    py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
    setup_cfg = configparser.ConfigParser()
    setup_cfg.read('setup.cfg')
    project = setup_cfg.get('metadata', 'name')

    install = install_venv.InstallVenv(
        root, venv, pip_requires, test_requires, py_version, project)
    options = install.parse_args(argv)
    install.check_python_version()
    install.check_dependencies()
    install.create_virtualenv(no_site_packages=options.no_site_packages)
    install.install_dependencies()
    print_help(project, venv, root)

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>#
#	\namespace	blur3d.api.softimage.softimagesceneobject
#
#	\remarks	The SoftimageSceneModel class provides the implementation of the AbstractSceneModel class as it applies
#				to Softimage scenes
#
#	\author		douglas@blur.com
#	\author		Blur Studio
#	\date		04/04/11
#

import os
import copy

from blur3d.api import application
from win32com.client import Dispatch
from win32com.client import constants
from PySoftimage import xsi, xsiFactory
from blurdev.decorators import pendingdeprecation
from blur3d.api.abstract.abstractscenemodel import AbstractSceneModel

xsiUIToolkit = Dispatch("XSI.UIToolkit")

#------------------------------------------------------------------------

class SoftimageSceneModel(AbstractSceneModel):

	def _setNativeParent(self, nativeParent):
		xsi.Application.ParentObj(nativeParent, self._nativePointer)
		return True

	def _nativeGroups(self, wildcard='*'):
		return self._nativePointer.Groups.Filter('', '', wildcard)

	def isReferenced(self):
		return self._nativePointer.ModelKind == constants.siModelKind_Reference

	def update(self):
		xsi.UpdateReferencedModel(self._nativePointer)
		return True

	def offloaded(self):
		return not bool(self._nativePointer.Parameters('active_resolution').Value)

	def offload(self):
		xsi.SetResolutionOfRefModels(self._nativePointer, 0)
		return True

	def addResolution(self, name='', path='', load=False):
		if self.isReferenced():
			xsi.AddRefModelResolution(self._nativePointer, name, path)
			if load:
				self.setResolution(name)
			return True
		return False

	def removeResolution(self, name):
		if self.isReferenced():

			# Softimage does not allow to remove active resolutions.
			if self.resolution() == name:
				self.offload()

			xsi.RemoveRefModelResolution(self._nativePointer, name)
			return True
		return False

	def resolutionsPaths(self):
		return [self.resolutionPath(resolution) for resolution in self.resolutions() if resolution != 'Offloaded']

	def resolutionPath(self, resolution=''):
		if self.isReferenced():
			if not resolution:
				resolution = self.resolution()
			resolutions = self.resolutions()
			if resolution in resolutions:
				return xsi.GetValue("%s.resolutions.res%i.file" % (self.name(), resolutions.index(resolution)))
		return ''

	def setResolutionPath(self, path, resolution=''):
		if self.isReferenced():
			if not resolution:
				resolution = self.resolution()
			resolutions = self.resolutions()
			if resolution in resolutions:
				xsi.SetValue("%s.resolutions.res%i.file" % (self.name(), resolutions.index(resolution)), path)
				return True
		return False

	def setResolution(self, resolution):

		if self.isReferenced():
			# handles qstrings
			resolution = unicode(resolution)
			# If I dont re-initialize a model object it does not return me the right resolutions.
			resolutions = self._scene.findObject(self.name()).resolutions()
			for res in resolutions:

				# Handles cases sensitivity (crappy, but needed because resolution
				# names often come from filenames).
				if res.lower() != resolution.lower():
					continue

				# Storing the user props that might be compromised when switching the reference model.
				userProps = self.userProps()
				oldUserProps = userProps.lookupProps()

				# Setting the resolution.
				xsi.SetValue("%s.Delta.persist_mixer_modifications"%self._nativePointer.FullName, 13, "")
				xsi.SetResolutionOfRefModels(self._nativePointer, resolutions.index(res))
				xsi.SetValue("%s.Delta.persist_mixer_modifications"%self._nativePointer.FullName, 15, "")

				# Making sure all the keys that where on the old resolution are re-assigned to the new one.
				for key in oldUserProps:
					if oldUserProps.get(key) and not userProps.get(key):
						userProps[key] = oldUserProps[key]

				# If the resolution did change.
				if self.resolution() == res:
					return True
		else:
			self.userProps()['resolution'] = resolution

		return False

	def resolution(self):
		if self.isReferenced():

			# If I dont re-initialize a model object it does not return me the right resolutions.
			obj = self._scene.findObject(self.name())
			if obj:
				resolutions = obj.resolutions()
				if resolutions:
					active_res = obj._nativePointer.Parameters('active_resolution')
					if active_res is not None:
						return resolutions[active_res.Value]

		return self.userProps().get('resolution', '')

	def resolutions(self):
		resolutions = []
		# Resolutions don't always return correctly if the object isn't re-initialized
		obj = self._scene.findObject(self.name())
		if obj:
			for parameter in obj._nativePointer.Parameters:
				if 'resolutions' in parameter.fullname:
					resolutions.append(parameter.Value)
		return resolutions

	def export(self, fileName):
		xsi.ExportModel(self._nativePointer, fileName, True, False)
		return True

	def export(self, fileName):
		xsi.ExportModel(self._nativePointer, fileName, True, False)
		return True

	@pendingdeprecation('Use loadAnimation instead with mixer flag true.')
	def addAnimationClip(self, path, name=None):
		self.loadAnimationInMixer(path, name)

	def storePose(self, name='', objects=[]):
		if not objects:
			objects = self.objects()

		controllers = xsiFactory.CreateObject('XSI.Collection')
		controllers.AddItems([obj.nativePointer() for obj in objects])
		parameters = controllers.FindObjectsByMarkingAndCapabilities(None, 2048)

		# Creating the action.
		return xsi.SIStoreAction(self._nativePointer, parameters, 1, False, '', '', '', '', True)

	def storeAnimation(self, name='', objects=[]):
		if not objects:
			objects = self.objects()

		controllers = xsiFactory.CreateObject('XSI.Collection')
		controllers.AddItems([obj.nativePointer() for obj in objects])
		parameters = controllers.FindObjectsByMarkingAndCapabilities(None, 2048)

		# Creating the action.
		return xsi.SIStoreAction(self._nativePointer, parameters, 2, name, False, '', '', '', '', True)

	def savePose(self, basePath, name='', objects=[]):
		xsi.ExportAction(self.storePose(name, objects), os.path.join(basePath, name + '.eani'))
		return True

	def saveAnimation(self, basePath, name='', objects=[]):
		xsi.ExportAction(self.storeAnimation(name, objects), os.path.join(basePath, name + '.eani'))
		return True

	def loadPose(self, path):
		self.loadAnimation(path)
		# TODO: Set key if auto-key is on for relevant objects.

	def loadAnimation(self, path):
		if os.path.exists(path):
			xsi.ImportActionAndApply(self._nativePointer, path)

	def loadAnimationInMixer(self, path, name=None):
		native_model = self.nativePointer()
		native_mixer = native_model.mixer

		base_track_name = 'BlurAnimTrack'

		if name is None:
			name = os.path.basename(path).replace('.', '_') + '_Clip'

		# Each clip should be loaded onto its own track.
		# Look at the mixer on the Asset to find an empty track, or create a new
		# empty track to add the clip to.

		# If an asset doesn't have a mixer, it means it doesn't have any tracks
		# either. Create a new track, which will create the mixer.
		if not native_mixer:
			track_name = '{}0'.format(base_track_name)
			native_track = xsi.AddTrack(native_model, '', 0, track_name)
			native_mixer = native_model.mixer

		# If a mixer does exist, uses sequential numbering to create a new track,
		# or if that track already exists, test if it's empty.
		else:
			i = 0
			native_track = None
			while native_track is None or native_track.clips.count > 0:
				track_name = '{}{}'.format(base_track_name, i)
				native_track = native_mixer.tracks(track_name)
				if not native_track:
					native_track = xsi.AddTrack(native_model, native_model, 0, track_name)
					break
				i += 1

		# Adding clips is a 2-step process.  First, import the animation as an
		# Action.  Then create an "instance" of that animation as a Clip on one
		# of the tracks (a single action can be added as a clip to multiple tracks
		# on multiple assets).
		# Also, by Convention, clips should be offset to their start frame. This
		# will be true unless Layout changes the way they export things.
		native_action = xsi.ImportAction(native_model, path, 0)
		native_clip = xsi.AddClip(native_model, native_action, "", native_track.FullName, native_action.FrameStart.Value, name)

		#TODO: wrap native clip (no specific blur3d object for it yet) and return it.
		return True

	def matchPose(self, model, objects=[]):
		action = model.storePose('Match', objects)
		xsi.ApplyAction(action, self._nativePointer)

		for obj in objects:
			objectToMatch = self.findChild(obj.displayName(), recursive=True)
			if objectToMatch:
				objectToMatch.matchTransforms(obj)

		return True

	def matchAnimation(self, model, objects=[]):

		action = model.storeAnimation('Match', objects)
		xsi.ApplyAction(action, self._nativePointer)
		frames = self._scene.objectsKeyedFrames(objects)

		progressBar = xsiUIToolkit.ProgressBar
		progressBar.Caption = 'Matching...'
		progressBar.Maximum = len(frames)
		progressBar.Step = 1
		progressBar.Visible = True
		progressBar.CancelEnabled = False

		# Constraining the controllers.
		objectsToMatch = []
		for obj in objects:
			objectToMatch = self.findChild(obj.displayName(), recursive=True)
			if objectToMatch:
				objectsToMatch.append(objectToMatch)
				objectToMatch.nativePointer().Kinematics.AddConstraint("Pose", obj.nativePointer(), False)

		for frame in frames:

			self._scene.setCurrentFrame(frame)
			application.refresh()

			for obj in objectsToMatch:
				obj.key()

			progressBar.Increment()

		progressBar.Visible = False

		# Removing the constaints.
		xsi.RemoveCnsType("Pose", [obj.nativePointer() for obj in objectsToMatch]);

		return True

# register the symbol
from blur3d import api
api.registerSymbol('SceneModel', SoftimageSceneModel)
<commit_msg>setDisplayName now renames the constraints attached to this model.<commit_after>#
#	\namespace	blur3d.api.softimage.softimagesceneobject
#
#	\remarks	The SoftimageSceneModel class provides the implementation of the AbstractSceneModel class as it applies
#				to Softimage scenes
#
#	\author		douglas@blur.com
#	\author		Blur Studio
#	\date		04/04/11
#

import os, re
import copy

from blur3d.api import application
from win32com.client import Dispatch
from win32com.client import constants
from PySoftimage import xsi, xsiFactory
from blurdev.decorators import pendingdeprecation
from blur3d.api.abstract.abstractscenemodel import AbstractSceneModel

xsiUIToolkit = Dispatch("XSI.UIToolkit")

#------------------------------------------------------------------------

class SoftimageSceneModel(AbstractSceneModel):

	def _setNativeParent(self, nativeParent):
		xsi.Application.ParentObj(nativeParent, self._nativePointer)
		return True

	def _nativeGroups(self, wildcard='*'):
		return self._nativePointer.Groups.Filter('', '', wildcard)

	def isReferenced(self):
		return self._nativePointer.ModelKind == constants.siModelKind_Reference

	def update(self):
		xsi.UpdateReferencedModel(self._nativePointer)
		return True

	def offloaded(self):
		return not bool(self._nativePointer.Parameters('active_resolution').Value)

	def offload(self):
		xsi.SetResolutionOfRefModels(self._nativePointer, 0)
		return True

	def addResolution(self, name='', path='', load=False):
		if self.isReferenced():
			xsi.AddRefModelResolution(self._nativePointer, name, path)
			if load:
				self.setResolution(name)
			return True
		return False

	def removeResolution(self, name):
		if self.isReferenced():

			# Softimage does not allow to remove active resolutions.
			if self.resolution() == name:
				self.offload()

			xsi.RemoveRefModelResolution(self._nativePointer, name)
			return True
		return False

	def resolutionsPaths(self):
		return [self.resolutionPath(resolution) for resolution in self.resolutions() if resolution != 'Offloaded']

	def resolutionPath(self, resolution=''):
		if self.isReferenced():
			if not resolution:
				resolution = self.resolution()
			resolutions = self.resolutions()
			if resolution in resolutions:
				return xsi.GetValue("%s.resolutions.res%i.file" % (self.name(), resolutions.index(resolution)))
		return ''

	def setResolutionPath(self, path, resolution=''):
		if self.isReferenced():
			if not resolution:
				resolution = self.resolution()
			resolutions = self.resolutions()
			if resolution in resolutions:
				xsi.SetValue("%s.resolutions.res%i.file" % (self.name(), resolutions.index(resolution)), path)
				return True
		return False

	def setResolution(self, resolution):

		if self.isReferenced():
			# handles qstrings
			resolution = unicode(resolution)
			# If I dont re-initialize a model object it does not return me the right resolutions.
			resolutions = self._scene.findObject(self.name()).resolutions()
			for res in resolutions:

				# Handles cases sensitivity (crappy, but needed because resolution
				# names often come from filenames).
				if res.lower() != resolution.lower():
					continue

				# Storing the user props that might be compromised when switching the reference model.
				userProps = self.userProps()
				oldUserProps = userProps.lookupProps()

				# Setting the resolution.
				xsi.SetValue("%s.Delta.persist_mixer_modifications"%self._nativePointer.FullName, 13, "")
				xsi.SetResolutionOfRefModels(self._nativePointer, resolutions.index(res))
				xsi.SetValue("%s.Delta.persist_mixer_modifications"%self._nativePointer.FullName, 15, "")

				# Making sure all the keys that where on the old resolution are re-assigned to the new one.
				for key in oldUserProps:
					if oldUserProps.get(key) and not userProps.get(key):
						userProps[key] = oldUserProps[key]

				# If the resolution did change.
				if self.resolution() == res:
					return True
		else:
			self.userProps()['resolution'] = resolution

		return False

	def resolution(self):
		if self.isReferenced():

			# If I dont re-initialize a model object it does not return me the right resolutions.
			obj = self._scene.findObject(self.name())
			if obj:
				resolutions = obj.resolutions()
				if resolutions:
					active_res = obj._nativePointer.Parameters('active_resolution')
					if active_res is not None:
						return resolutions[active_res.Value]

		return self.userProps().get('resolution', '')

	def resolutions(self):
		resolutions = []
		# Resolutions don't always return correctly if the object isn't re-initialized
		obj = self._scene.findObject(self.name())
		if obj:
			for parameter in obj._nativePointer.Parameters:
				if 'resolutions' in parameter.fullname:
					resolutions.append(parameter.Value)
		return resolutions

	def export(self, fileName):
		xsi.ExportModel(self._nativePointer, fileName, True, False)
		return True

	def export(self, fileName):
		xsi.ExportModel(self._nativePointer, fileName, True, False)
		return True

	def setDisplayName(self, name):

		oldName = self.displayName()

		super(SoftimageSceneModel, self).setDisplayName(name)

		# We need to add a fix to all the constrains pointing to that model.
		regex = re.compile( "(?P<model>[A-Za-z0-9_-]+).(?P<object>[A-Za-z0-9_-]+)" )

		# Get all the deltas of the scene
		deltas = xsi.FindObjects(None, "{6D906778-A713-40E3-A5FB-0F5CA3361370}")
		for delta in deltas:

			cns = delta.ActionDeltas("StoredConstraints")
			if not cns:
				continue

			for item in cns.Items:
				match = regex.match(item.Value)
				if not match:
					continue
				tokens = match.groupdict()
				if tokens["model"] == oldName:
					item.Value = item.Value.replace(tokens["model"], name)

	@pendingdeprecation('Use loadAnimation instead with mixer flag true.')
	def addAnimationClip(self, path, name=None):
		self.loadAnimationInMixer(path, name)

	def storePose(self, name='', objects=[]):
		if not objects:
			objects = self.objects()

		controllers = xsiFactory.CreateObject('XSI.Collection')
		controllers.AddItems([obj.nativePointer() for obj in objects])
		parameters = controllers.FindObjectsByMarkingAndCapabilities(None, 2048)

		# Creating the action.
		return xsi.SIStoreAction(self._nativePointer, parameters, 1, False, '', '', '', '', True)

	def storeAnimation(self, name='', objects=[]):
		if not objects:
			objects = self.objects()

		controllers = xsiFactory.CreateObject('XSI.Collection')
		controllers.AddItems([obj.nativePointer() for obj in objects])
		parameters = controllers.FindObjectsByMarkingAndCapabilities(None, 2048)

		# Creating the action.
		return xsi.SIStoreAction(self._nativePointer, parameters, 2, name, False, '', '', '', '', True)

	def savePose(self, basePath, name='', objects=[]):
		xsi.ExportAction(self.storePose(name, objects), os.path.join(basePath, name + '.eani'))
		return True

	def saveAnimation(self, basePath, name='', objects=[]):
		xsi.ExportAction(self.storeAnimation(name, objects), os.path.join(basePath, name + '.eani'))
		return True

	def loadPose(self, path):
		self.loadAnimation(path)
		# TODO: Set key if auto-key is on for relevant objects.

	def loadAnimation(self, path):
		if os.path.exists(path):
			xsi.ImportActionAndApply(self._nativePointer, path)

	def loadAnimationInMixer(self, path, name=None):
		native_model = self.nativePointer()
		native_mixer = native_model.mixer

		base_track_name = 'BlurAnimTrack'

		if name is None:
			name = os.path.basename(path).replace('.', '_') + '_Clip'

		# Each clip should be loaded onto its own track.
		# Look at the mixer on the Asset to find an empty track, or create a new
		# empty track to add the clip to.

		# If an asset doesn't have a mixer, it means it doesn't have any tracks
		# either. Create a new track, which will create the mixer.
		if not native_mixer:
			track_name = '{}0'.format(base_track_name)
			native_track = xsi.AddTrack(native_model, '', 0, track_name)
			native_mixer = native_model.mixer

		# If a mixer does exist, uses sequential numbering to create a new track,
		# or if that track already exists, test if it's empty.
		else:
			i = 0
			native_track = None
			while native_track is None or native_track.clips.count > 0:
				track_name = '{}{}'.format(base_track_name, i)
				native_track = native_mixer.tracks(track_name)
				if not native_track:
					native_track = xsi.AddTrack(native_model, native_model, 0, track_name)
					break
				i += 1

		# Adding clips is a 2-step process.  First, import the animation as an
		# Action.  Then create an "instance" of that animation as a Clip on one
		# of the tracks (a single action can be added as a clip to multiple tracks
		# on multiple assets).
		# Also, by Convention, clips should be offset to their start frame. This
		# will be true unless Layout changes the way they export things.
		native_action = xsi.ImportAction(native_model, path, 0)
		native_clip = xsi.AddClip(native_model, native_action, "", native_track.FullName, native_action.FrameStart.Value, name)

		#TODO: wrap native clip (no specific blur3d object for it yet) and return it.
		return True

	def matchPose(self, model, objects=[]):
		action = model.storePose('Match', objects)
		xsi.ApplyAction(action, self._nativePointer)

		for obj in objects:
			objectToMatch = self.findChild(obj.displayName(), recursive=True)
			if objectToMatch:
				objectToMatch.matchTransforms(obj)

		return True

	def matchAnimation(self, model, objects=[]):

		action = model.storeAnimation('Match', objects)
		xsi.ApplyAction(action, self._nativePointer)
		frames = self._scene.objectsKeyedFrames(objects)

		progressBar = xsiUIToolkit.ProgressBar
		progressBar.Caption = 'Matching...'
		progressBar.Maximum = len(frames)
		progressBar.Step = 1
		progressBar.Visible = True
		progressBar.CancelEnabled = False

		# Constraining the controllers.
		objectsToMatch = []
		for obj in objects:
			objectToMatch = self.findChild(obj.displayName(), recursive=True)
			if objectToMatch:
				objectsToMatch.append(objectToMatch)
				objectToMatch.nativePointer().Kinematics.AddConstraint("Pose", obj.nativePointer(), False)

		for frame in frames:

			self._scene.setCurrentFrame(frame)
			application.refresh()

			for obj in objectsToMatch:
				obj.key()

			progressBar.Increment()

		progressBar.Visible = False

		# Removing the constaints.
		xsi.RemoveCnsType("Pose", [obj.nativePointer() for obj in objectsToMatch]);

		return True

# register the symbol
from blur3d import api
api.registerSymbol('SceneModel', SoftimageSceneModel)
<|endoftext|>