<commit_before>#!/usr/bin/python

import argparse
import hashlib
import json
import logging
import os
import sys

READ_SIZE = 1024 * 4

class Error(Exception):
    pass

class BadRoot(Error):
    pass

class CannotCreateCacheFile(Error):
    def __init__(self, cache_path):
        self.cache_path = cache_path

    def __str__(self):
        return 'Cannot create cache file "%s"' % self.cache_path

class CacheItem(object):
    """Represents a file on disk with properties evaluated lazily."""

    def __init__(self, abspath, mtime=None, size=None, checksum=None):
        self.abspath = abspath
        self._mtime = mtime
        self._size = size
        self._checksum = checksum

    def __str__(self):
        return '%(abspath)s: mtime=%(_mtime)s size=%(_size)s checksum=%(_checksum)s' % (self.__dict__)

    def __repr__(self):
        return str(self)

    @property
    def checksum(self):
        if self._checksum:
            return self._checksum

        logging.info('calculating checksum for %s', self.abspath)
        m = hashlib.sha1()
        with open(self.abspath, 'r') as f:
            while True:
                b = f.read(READ_SIZE)
                if b == '':
                    break
                m.update(b)
        self._checksum = m.hexdigest()
        return self._checksum

    def _stat(self):
        stat = os.stat(self.abspath)
        logging.debug('statting %s', self.abspath)
        self._size = stat.st_size
        self._mtime = int(stat.st_mtime)

    def verify(self):
        """Verify the cache item hasn't changed.

        For speed, only looks at mtime and size."""
        if self._size is None or self._mtime is None:
            logging.debug('%s does not have stats in the cache', self.abspath)
            return False
        logging.debug('checking if %s has changed', self.abspath)
        stat = os.stat(self.abspath)
        return self._size == stat.st_size and self._mtime == int(stat.st_mtime)

    @property
    def size(self):
        if self._size is not None:
            return self._size
        self._stat()
        return self._size

    @property
    def mtime(self):
        if self._mtime is not None:
            return self._mtime
        self._stat()
        return self._mtime

class CacheItemEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, CacheItem):
            # Return the serialized item, and avoid evaluation of properties.
            return [obj.abspath, obj._mtime, obj._size, obj._checksum]
        return json.JSONEncoder.default(self, obj)

def hashwalk(root, cache=None):
    """Walk a tree and create CacheItems for files.

    Args:
        cache: optional dict keyed by abspath, containing CacheItems. Will be
               modified.
    Yields:
        CacheItems found.
    """
    if not os.path.exists(root):
        raise BadRoot('%s does not exist' % root)
    if not os.path.isdir(root):
        raise BadRoot('%s is not a directory' % root)

    logging.debug('walking %s', root)
    for (dirpath, dirnames, filenames) in os.walk(root):
        for filename in filenames:
            abspath = os.path.join(dirpath, filename)
            if os.path.islink(abspath):
                continue
            # If the item is in the cache and it hasn't changed.
            if cache is not None:
                if abspath in cache and cache[abspath].verify():
                    logging.debug('cache hit for %s', abspath)
                    yield cache[abspath]
                else:
                    logging.debug('cache miss for %s', abspath)
                    cache[abspath] = CacheItem(abspath)
                    yield cache[abspath]
            else:
                logging.debug('returning cacheitem for %s', abspath)
                yield CacheItem(abspath)

def get_tree_filesizes(
        root, cache, min_filesize=1024*1024*1024, show_source_dupes=True):
    """Walk a tree, get CacheItems.

    Args:
        root: root directory path
        cache: optional dict keyed by abspath, containing CacheItems. Will be
               modified.
        min_filesize: minimum filesize in bytes to look at (default: 1GB)
        show_source_dupes: report dupes in the source tree (default: True)
    Returns:
        Dictionary keyed by file size, containing lists of CacheItem objects.
    """
    logging.info('walking %s...', root)
    sizes = {}
    num_items = 0
    for cacheitem in hashwalk(root, cache=cache):
        if cacheitem.size < min_filesize:
            logging.info('skipping small file %s (size=%s < min=%s)', cacheitem.abspath, cacheitem.size, min_filesize)
            continue
        num_items += 1
        if show_source_dupes and cacheitem.size in sizes:
            for other in sizes[cacheitem.size]:
                logging.debug('found files with same size: %s and %s',
                        cacheitem.abspath, other.abspath)
                if cacheitem.checksum == other.checksum:
                    # Print collisions in the tree
                    logging.warning('%s: %s == %s',
                            root,
                            cacheitem.abspath.replace(root, '', 1).lstrip('/'),
                            other.abspath.replace(root, '', 1).lstrip('/'))
        sizes.setdefault(cacheitem.size, []).append(cacheitem)

    logging.info('saw %d files and %d sizes in %s.', num_items, len(sizes), root)
    return sizes

def print_cleanup_command(other, cleanup):
    print 'ln -sf %s %s' % (other, cleanup)

def create_symlink(other, cleanup):
    logging.info('creating link %s -> %s', cleanup, other)
    os.unlink(cleanup)
    os.symlink(other, cleanup)

def prompt_before_symlinking(other, cleanup):
    sys.stdout.write('Create symlink %s -> %s? [Y/n] ' % (cleanup, other))
    choice = raw_input().lower()
    if choice.startswith('n'):
        logging.info('Not creating symlink for %s', cleanup)
    else:
        create_symlink(other, cleanup)

def cleanup_tree(root, sizes, callback):
    logging.info('attempting to clean up %s', root)
    num_deduped = 0

    for cacheitem in hashwalk(root):
        if cacheitem.size not in sizes:
            logging.debug('%s is unique', cacheitem)
            return
        for other in sizes[cacheitem.size]:
            if other.checksum != cacheitem.checksum:
                continue
            if os.path.realpath(other.abspath) == os.path.realpath(cacheitem.abspath):
                logging.error("not symlinking %s to %s: they're the same file",
                        other.abspath, cacheitem.abspath)
            else:
                callback(other.abspath, cacheitem.abspath)
                num_deduped += 1
                break

    logging.info('deduped %d files', num_deduped)

def load_cache(cache_path):
    """Read a dict of CacheItems from a JSON-encoded file.

    Args:
        cache_path: file to write
    Returns:
        dictionary keyed by abspath of CacheItems.
    """
    cache = {}
    if os.path.exists(cache_path):
        with open(cache_path, 'r') as f:
            for line in f:
                values = json.loads(line)
                cache_item = CacheItem(
                        values[0], mtime=values[1], size=values[2],
                        checksum=values[3])
                cache[cache_item.abspath] = cache_item
    else:
        if not os.access(os.path.dirname(cache_path), os.W_OK):
            raise CannotCreateCacheFile(cache_path)
        logging.warning('cache file %s does not exist' % cache_path)
    return cache

def write_cache(cache_path, cache):
    """Write a dictionary of CacheItems to a JSON-encoded file.

    Args:
        cache_path: file to write
        cache: dictionary of CacheItems
    """
    # Write to a temp file, then swap to the final destination.
    tmp_output = cache_path + '.tmp'
    with open(tmp_output, 'w') as f:
        for cache_key in cache:
            assert cache[cache_key].abspath == cache_key
            print >>f, json.dumps(cache[cache_key], cls=CacheItemEncoder)
    os.rename(tmp_output, cache_path)
    logging.info('wrote cache to %s', cache_path)

def main():
    parser = argparse.ArgumentParser(
            'Find duplicate files and turn half of them into symlinks.')

    parser.add_argument(
            'source', help='Directory root containing original files.')
    parser.add_argument(
            'cleanup',
            help='Directory root containing duplicate files '
                 'to be turned into symlinks.')

    parser.add_argument(
            '--cache_file',
            default=os.path.expanduser('~/.dedooper.cache'),
            help='where to store the cache')

    parser.add_argument(
            '--show_source_dupes',
            dest='show_source_dupes',
            action='store_true',
            help='Show duplicate files in the source directory')
    parser.add_argument(
            '--noshow_source_dupes',
            dest='show_source_dupes',
            action='store_false',
            help="Don't show duplicate files in the source directory")
    parser.set_defaults(show_source_dupes=False)

    parser.add_argument(
            '--dry_run',
            dest='dry_run',
            action='store_true',
            help="Print commands, don't execute them")
    parser.add_argument(
            '--nodry_run',
            dest='dry_run',
            action='store_false',
            help='Execute commands')
    parser.set_defaults(dry_run=True)

    parser.add_argument(
            '--prompt',
            dest='prompt',
            action='store_true',
            help='Prompt before creating symlinks')
    parser.add_argument(
            '--noprompt',
            dest='prompt',
            action='store_false',
            help="Don't prompt before creating symlinks")
    parser.set_defaults(prompt=True)

    parser.add_argument(
            '--log_level',
            dest='log_level',
            default=logging.INFO,
            help='DEBUG, INFO, WARNING, ERROR, or CRITICAL (default: INFO)')
    parser.add_argument(
            '--min_filesize',
            dest='min_filesize',
            type=int,
            default=1024*1024*1024,
            help='minimum filesize in bytes to look at (default: 1GB)')

    args = parser.parse_args()

    logging.basicConfig(
            level=args.log_level,
            format='%(levelname)-8s: %(message)s')

    if os.path.realpath(args.source) == os.path.realpath(args.cleanup):
        logging.critical('source dir is the same as cleanup dir')
        sys.exit(1)

    # Find out what's in the source tree.
    cache = {}
    try:
        cache = load_cache(args.cache_file)
    except CannotCreateCacheFile as e:
        logging.critical(e)
        sys.exit(1)

    try:
        src_filesizes = get_tree_filesizes(
                os.path.abspath(args.source),
                cache,
                min_filesize=args.min_filesize,
                show_source_dupes=args.show_source_dupes)
    except BadRoot as e:
        logging.critical(e)
        sys.exit(1)
    finally:
        write_cache(args.cache_file, cache)

    # Pick a cleanup strategy.
    callback = print_cleanup_command
    if not args.dry_run:
        if args.prompt:
            callback = prompt_before_symlinking
        else:
            callback = create_symlink

    # Clean up the cleanup tree.
    try:
        cleanup_tree(
                os.path.abspath(args.cleanup), src_filesizes, callback=callback)
    except BadRoot as e:
        logging.critical(e)
        sys.exit(1)

if __name__ == '__main__':
    main()
<commit_msg>bugfix: continue instead of return<commit_after>#!/usr/bin/python

import argparse
import hashlib
import json
import logging
import os
import sys

READ_SIZE = 1024 * 4

class Error(Exception):
    pass

class BadRoot(Error):
    pass

class CannotCreateCacheFile(Error):
    def __init__(self, cache_path):
        self.cache_path = cache_path

    def __str__(self):
        return 'Cannot create cache file "%s"' % self.cache_path

class CacheItem(object):
    """Represents a file on disk with properties evaluated lazily."""

    def __init__(self, abspath, mtime=None, size=None, checksum=None):
        self.abspath = abspath
        self._mtime = mtime
        self._size = size
        self._checksum = checksum

    def __str__(self):
        return '%(abspath)s: mtime=%(_mtime)s size=%(_size)s checksum=%(_checksum)s' % (self.__dict__)

    def __repr__(self):
        return str(self)

    @property
    def checksum(self):
        if self._checksum:
            return self._checksum

        logging.info('calculating checksum for %s', self.abspath)
        m = hashlib.sha1()
        with open(self.abspath, 'r') as f:
            while True:
                b = f.read(READ_SIZE)
                if b == '':
                    break
                m.update(b)
        self._checksum = m.hexdigest()
        return self._checksum

    def _stat(self):
        stat = os.stat(self.abspath)
        logging.debug('statting %s', self.abspath)
        self._size = stat.st_size
        self._mtime = int(stat.st_mtime)

    def verify(self):
        """Verify the cache item hasn't changed.

        For speed, only looks at mtime and size."""
        if self._size is None or self._mtime is None:
            logging.debug('%s does not have stats in the cache', self.abspath)
            return False
        logging.debug('checking if %s has changed', self.abspath)
        stat = os.stat(self.abspath)
        return self._size == stat.st_size and self._mtime == int(stat.st_mtime)

    @property
    def size(self):
        if self._size is not None:
            return self._size
        self._stat()
        return self._size

    @property
    def mtime(self):
        if self._mtime is not None:
            return self._mtime
        self._stat()
        return self._mtime

class CacheItemEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, CacheItem):
            # Return the serialized item, and avoid evaluation of properties.
            return [obj.abspath, obj._mtime, obj._size, obj._checksum]
        return json.JSONEncoder.default(self, obj)

def hashwalk(root, cache=None):
    """Walk a tree and create CacheItems for files.

    Args:
        cache: optional dict keyed by abspath, containing CacheItems. Will be
               modified.
    Yields:
        CacheItems found.
    """
    if not os.path.exists(root):
        raise BadRoot('%s does not exist' % root)
    if not os.path.isdir(root):
        raise BadRoot('%s is not a directory' % root)

    logging.debug('walking %s', root)
    for (dirpath, dirnames, filenames) in os.walk(root):
        for filename in filenames:
            abspath = os.path.join(dirpath, filename)
            if os.path.islink(abspath):
                continue
            # If the item is in the cache and it hasn't changed.
            if cache is not None:
                if abspath in cache and cache[abspath].verify():
                    logging.debug('cache hit for %s', abspath)
                    yield cache[abspath]
                else:
                    logging.debug('cache miss for %s', abspath)
                    cache[abspath] = CacheItem(abspath)
                    yield cache[abspath]
            else:
                yield CacheItem(abspath)

def get_tree_filesizes(
        root, cache, min_filesize=1024*1024*1024, show_source_dupes=True):
    """Walk a tree, get CacheItems.

    Args:
        root: root directory path
        cache: optional dict keyed by abspath, containing CacheItems. Will be
               modified.
        min_filesize: minimum filesize in bytes to look at (default: 1GB)
        show_source_dupes: report dupes in the source tree (default: True)
    Returns:
        Dictionary keyed by file size, containing lists of CacheItem objects.
    """
    logging.info('walking %s...', root)
    sizes = {}
    num_items = 0
    for cacheitem in hashwalk(root, cache=cache):
        if cacheitem.size < min_filesize:
            logging.info('skipping small file %s (size=%s < min=%s)', cacheitem.abspath, cacheitem.size, min_filesize)
            continue
        num_items += 1
        if show_source_dupes and cacheitem.size in sizes:
            for other in sizes[cacheitem.size]:
                logging.debug('found files with same size: %s and %s',
                        cacheitem.abspath, other.abspath)
                if cacheitem.checksum == other.checksum:
                    # Print collisions in the tree
                    logging.warning('%s: %s == %s',
                            root,
                            cacheitem.abspath.replace(root, '', 1).lstrip('/'),
                            other.abspath.replace(root, '', 1).lstrip('/'))
        sizes.setdefault(cacheitem.size, []).append(cacheitem)

    logging.info('saw %d files and %d sizes in %s.', num_items, len(sizes), root)
    return sizes

def print_cleanup_command(other, cleanup):
    print 'ln -sf %s %s' % (other, cleanup)

def create_symlink(other, cleanup):
    logging.info('creating link %s -> %s', cleanup, other)
    os.unlink(cleanup)
    os.symlink(other, cleanup)

def prompt_before_symlinking(other, cleanup):
    sys.stdout.write('Create symlink %s -> %s? [Y/n] ' % (cleanup, other))
    choice = raw_input().lower()
    if choice.startswith('n'):
        logging.info('Not creating symlink for %s', cleanup)
    else:
        create_symlink(other, cleanup)

def cleanup_tree(root, sizes, callback):
    logging.info('attempting to clean up %s', root)
    num_deduped = 0

    for cacheitem in hashwalk(root):
        if cacheitem.size not in sizes:
            logging.debug('%s is unique', cacheitem)
            continue
        for other in sizes[cacheitem.size]:
            if other.checksum != cacheitem.checksum:
                continue
            if os.path.realpath(other.abspath) == os.path.realpath(cacheitem.abspath):
                logging.error("not symlinking %s to %s: they're the same file",
                        other.abspath, cacheitem.abspath)
            else:
                callback(other.abspath, cacheitem.abspath)
                num_deduped += 1
                break

    logging.info('deduped %d files', num_deduped)

def load_cache(cache_path):
    """Read a dict of CacheItems from a JSON-encoded file.

    Args:
        cache_path: file to write
    Returns:
        dictionary keyed by abspath of CacheItems.
    """
    cache = {}
    if os.path.exists(cache_path):
        with open(cache_path, 'r') as f:
            for line in f:
                values = json.loads(line)
                cache_item = CacheItem(
                        values[0], mtime=values[1], size=values[2],
                        checksum=values[3])
                cache[cache_item.abspath] = cache_item
    else:
        if not os.access(os.path.dirname(cache_path), os.W_OK):
            raise CannotCreateCacheFile(cache_path)
        logging.warning('cache file %s does not exist' % cache_path)
    return cache

def write_cache(cache_path, cache):
    """Write a dictionary of CacheItems to a JSON-encoded file.

    Args:
        cache_path: file to write
        cache: dictionary of CacheItems
    """
    # Write to a temp file, then swap to the final destination.
    tmp_output = cache_path + '.tmp'
    with open(tmp_output, 'w') as f:
        for cache_key in cache:
            assert cache[cache_key].abspath == cache_key
            print >>f, json.dumps(cache[cache_key], cls=CacheItemEncoder)
    os.rename(tmp_output, cache_path)
    logging.info('wrote cache to %s', cache_path)

def main():
    parser = argparse.ArgumentParser(
            'Find duplicate files and turn half of them into symlinks.')

    parser.add_argument(
            'source', help='Directory root containing original files.')
    parser.add_argument(
            'cleanup',
            help='Directory root containing duplicate files '
                 'to be turned into symlinks.')

    parser.add_argument(
            '--cache_file',
            default=os.path.expanduser('~/.dedooper.cache'),
            help='where to store the cache')

    parser.add_argument(
            '--show_source_dupes',
            dest='show_source_dupes',
            action='store_true',
            help='Show duplicate files in the source directory')
    parser.add_argument(
            '--noshow_source_dupes',
            dest='show_source_dupes',
            action='store_false',
            help="Don't show duplicate files in the source directory")
    parser.set_defaults(show_source_dupes=False)

    parser.add_argument(
            '--dry_run',
            dest='dry_run',
            action='store_true',
            help="Print commands, don't execute them")
    parser.add_argument(
            '--nodry_run',
            dest='dry_run',
            action='store_false',
            help='Execute commands')
    parser.set_defaults(dry_run=True)

    parser.add_argument(
            '--prompt',
            dest='prompt',
            action='store_true',
            help='Prompt before creating symlinks')
    parser.add_argument(
            '--noprompt',
            dest='prompt',
            action='store_false',
            help="Don't prompt before creating symlinks")
    parser.set_defaults(prompt=True)

    parser.add_argument(
            '--log_level',
            dest='log_level',
            default=logging.INFO,
            help='DEBUG, INFO, WARNING, ERROR, or CRITICAL (default: INFO)')
    parser.add_argument(
            '--min_filesize',
            dest='min_filesize',
            type=int,
            default=1024*1024*1024,
            help='minimum filesize in bytes to look at (default: 1GB)')

    args = parser.parse_args()

    logging.basicConfig(
            level=args.log_level,
            format='%(levelname)-8s: %(message)s')

    if os.path.realpath(args.source) == os.path.realpath(args.cleanup):
        logging.critical('source dir is the same as cleanup dir')
        sys.exit(1)

    # Find out what's in the source tree.
    cache = {}
    try:
        cache = load_cache(args.cache_file)
    except CannotCreateCacheFile as e:
        logging.critical(e)
        sys.exit(1)

    try:
        src_filesizes = get_tree_filesizes(
                os.path.abspath(args.source),
                cache,
                min_filesize=args.min_filesize,
                show_source_dupes=args.show_source_dupes)
    except BadRoot as e:
        logging.critical(e)
        sys.exit(1)
    finally:
        write_cache(args.cache_file, cache)

    # Pick a cleanup strategy.
    callback = print_cleanup_command
    if not args.dry_run:
        if args.prompt:
            callback = prompt_before_symlinking
        else:
            callback = create_symlink

    # Clean up the cleanup tree.
    try:
        cleanup_tree(
                os.path.abspath(args.cleanup), src_filesizes, callback=callback)
    except BadRoot as e:
        logging.critical(e)
        sys.exit(1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding:utf-8

import os
import sys
import glob
import locale
import shutil
import argparse
import subprocess

__author__ = 'coderzh'

GIT_REPO = [
    ['origin',  'gh-pages', 'git@github.com:coderzh/hugo-blog-deployed.git'],
    # ['gitcafe', 'gh-pages', 'git@gitcafe.com:coderzh/coderzh-hugo-blog.git'],
    ['coding',  'gh-pages', 'git@git.coding.net:coderzh/hugo-blog-deployed.git'],
]

DEPLOY_DIR = 'gh-pages'

class ChDir:
    """Context manager for changing the current working directory"""
    def __init__(self, new_path):
        self.newPath = os.path.expanduser(new_path)

    def __enter__(self):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, exception_type, exception_value, traceback):
        os.chdir(self.savedPath)

def deploy(args):
    current_dir = os.path.dirname(os.path.abspath(__file__))
    parent_dir = os.path.abspath(os.path.join(current_dir, '..'))
    public_dir = os.path.join(current_dir, 'public')
    commit_msg = ''

    with ChDir(current_dir):
        # step1 clean
        if os.path.exists(public_dir):
            shutil.rmtree(public_dir)

        if args.type == 'auto':
            subprocess.call('git fetch origin', shell=True)
            subprocess.call('git checkout master', shell=True)
            subprocess.call('git reset --hard origin/master', shell=True)
            subprocess.call('git clean -fdx', shell=True)
            subprocess.call('git submodule init', shell=True)
            subprocess.call('git submodule update', shell=True)
            subprocess.call('git submodule foreach git pull --rebase origin master', shell=True)

        # on windows set TERM=msys
        s = subprocess.Popen('git log -1 --pretty=format:"%s"',
                             shell=True, stdout=subprocess.PIPE)
        commit_msg = s.communicate()[0].decode('utf-8').encode(locale.getpreferredencoding())
        # step2 build
        subprocess.call('hugo -v --cacheDir="./cache"', shell=True)

    deploy_dir = os.path.join(parent_dir, DEPLOY_DIR)

    # step3 clone if not exists
    if not os.path.exists(deploy_dir):
        os.makedirs(deploy_dir)
        with ChDir(deploy_dir):
            subprocess.call('git init', shell=True)
            for repo in GIT_REPO:
                subprocess.call('git remote add {0} {1}'.format(repo[0], repo[2]), shell=True)
    elif args.type == 'first':
        with ChDir(deploy_dir):
            subprocess.call('git init', shell=True)
            for repo in GIT_REPO:
                subprocess.call('git remote add {0} {1}'.format(repo[0], repo[2]), shell=True)

    with ChDir(deploy_dir):
        # step4 clean and pull
        if len(GIT_REPO) > 0:
            subprocess.call('git fetch {0}'.format(GIT_REPO[0][0]), shell=True)
            if args.type == 'first':
                subprocess.call('git checkout --orphan temp', shell=True)
                subprocess.call('git rm --cached -r .', shell=True)
                subprocess.call('git clean -fdx', shell=True)
                subprocess.call('git branch -D {0}'.format(GIT_REPO[0][1]), shell=True)
                subprocess.call('git checkout -b {0}'.format(GIT_REPO[0][1]), shell=True)
            else:
                subprocess.call('git checkout {0}'.format(GIT_REPO[0][1]), shell=True)

            subprocess.call('git reset --hard {0}/{1}'.format(GIT_REPO[0][0], GIT_REPO[0][1]), shell=True)
            subprocess.call('git clean -fdx', shell=True)

        # step5 remove all files
        for f in os.listdir('.'):
            if f != '.git':
                if os.path.isfile(f):
                    os.remove(f)
                elif os.path.isdir(f):
                    shutil.rmtree(f)

        # step6 copy new files
        for f in os.listdir(public_dir):
            file_path = os.path.join(public_dir, f)
            if os.path.isfile(file_path):
                shutil.copy(file_path, '.')
            elif os.path.isdir(file_path):
                shutil.copytree(file_path, f)

        # step7 commit and push
        if len(GIT_REPO) > 0:
            subprocess.call('git add --all', shell=True)
            subprocess.call('git commit -a -m "{0}"'.format(commit_msg), shell=True)
            for repo in GIT_REPO:
                if args.test:
                    print('git push {0} {1}:{2} -u'.format(repo[0], GIT_REPO[0][1], repo[1]))
                else:
                    subprocess.call('git push {0} {1}:{2} -u'.format(repo[0], GIT_REPO[0][1], repo[1]), shell=True)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='deploy hugo')
    parser.add_argument('type', help='auto or manual')
    parser.add_argument('-t', dest='test', action='store_true', help='for test')
    args = parser.parse_args()

    if args.type in ['auto', 'manual', 'first']:
        deploy(args)

<commit_msg>update deploy.py<commit_after>#!/usr/bin/env python
# coding:utf-8

import os
import sys
import glob
import locale
import shutil
import argparse
import subprocess

__author__ = 'coderzh'

GIT_REPO = [
    ['origin',  'gh-pages', 'git@github.com:coderzh/hugo-blog-deployed.git'],
    # ['gitcafe', 'gh-pages', 'git@gitcafe.com:coderzh/coderzh-hugo-blog.git'],
    ['coding',  'gh-pages', 'git@git.coding.net:coderzh/hugo-blog-deployed.git'],
]

DEPLOY_DIR = 'gh-pages'

class ChDir:
    """Context manager for changing the current working directory"""
    def __init__(self, new_path):
        self.newPath = os.path.expanduser(new_path)

    def __enter__(self):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, exception_type, exception_value, traceback):
        os.chdir(self.savedPath)

def deploy(args):
    current_dir = os.path.dirname(os.path.abspath(__file__))
    parent_dir = os.path.abspath(os.path.join(current_dir, '..'))
    public_dir = os.path.join(current_dir, 'public')
    commit_msg = ''

    with ChDir(current_dir):
        # step1 clean
        if os.path.exists(public_dir):
            shutil.rmtree(public_dir)

        if args.type == 'auto':
            subprocess.call('git fetch origin', shell=True)
            subprocess.call('git checkout master', shell=True)
            subprocess.call('git reset --hard origin/master', shell=True)
            subprocess.call('git clean -fdx', shell=True)
            subprocess.call('git submodule init', shell=True)
            subprocess.call('git submodule update', shell=True)
            subprocess.call('git submodule foreach git pull --rebase origin master', shell=True)

        # on windows set TERM=msys
        s = subprocess.Popen('git log -1 --pretty=format:"%s"',
                             shell=True, stdout=subprocess.PIPE)
        commit_msg = s.communicate()[0].decode('utf-8').encode(locale.getpreferredencoding())
        # step2 build
        subprocess.call('hugo -v', shell=True)

    deploy_dir = os.path.join(parent_dir, DEPLOY_DIR)

    # step3 clone if not exists
    if not os.path.exists(deploy_dir):
        os.makedirs(deploy_dir)
        with ChDir(deploy_dir):
            subprocess.call('git init', shell=True)
            for repo in GIT_REPO:
                subprocess.call('git remote add {0} {1}'.format(repo[0], repo[2]), shell=True)
    elif args.type == 'first':
        with ChDir(deploy_dir):
            subprocess.call('git init', shell=True)
            for repo in GIT_REPO:
                subprocess.call('git remote add {0} {1}'.format(repo[0], repo[2]), shell=True)

    with ChDir(deploy_dir):
        # step4 clean and pull
        if len(GIT_REPO) > 0:
            subprocess.call('git fetch {0}'.format(GIT_REPO[0][0]), shell=True)
            if args.type == 'first':
                subprocess.call('git checkout --orphan temp', shell=True)
                subprocess.call('git rm --cached -r .', shell=True)
                subprocess.call('git clean -fdx', shell=True)
                subprocess.call('git branch -D {0}'.format(GIT_REPO[0][1]), shell=True)
                subprocess.call('git checkout -b {0}'.format(GIT_REPO[0][1]), shell=True)
            else:
                subprocess.call('git checkout {0}'.format(GIT_REPO[0][1]), shell=True)

            subprocess.call('git reset --hard {0}/{1}'.format(GIT_REPO[0][0], GIT_REPO[0][1]), shell=True)
            subprocess.call('git clean -fdx', shell=True)

        # step5 remove all files
        for f in os.listdir('.'):
            if f != '.git':
                if os.path.isfile(f):
                    os.remove(f)
                elif os.path.isdir(f):
                    shutil.rmtree(f)

        # step6 copy new files
        for f in os.listdir(public_dir):
            file_path = os.path.join(public_dir, f)
            if os.path.isfile(file_path):
                shutil.copy(file_path, '.')
            elif os.path.isdir(file_path):
                shutil.copytree(file_path, f)

        # step7 commit and push
        if len(GIT_REPO) > 0:
            subprocess.call('git add --all', shell=True)
            subprocess.call('git commit -a -m "{0}"'.format(commit_msg), shell=True)
            for repo in GIT_REPO:
                if args.test:
                    print('git push {0} {1}:{2} -u'.format(repo[0], GIT_REPO[0][1], repo[1]))
                else:
                    subprocess.call('git push {0} {1}:{2} -u'.format(repo[0], GIT_REPO[0][1], repo[1]), shell=True)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='deploy hugo')
    parser.add_argument('type', help='auto or manual')
    parser.add_argument('-t', dest='test', action='store_true', help='for test')
    args = parser.parse_args()

    if args.type in ['auto', 'manual', 'first']:
        deploy(args)

<|endoftext|>


<commit_before>
__author__ = 'Alban Diquet'
__license__ = 'GPLv2'
__version__ = '0.14.0'
__email__ = 'nabla.c0d3@gmail.com'
PROJECT_URL = 'https://github.com/nabla-c0d3/sslyze'
PROJECT_DESC = 'Fast and full-featured SSL scanner.'
<commit_msg>Bump version number<commit_after>
__author__ = 'Alban Diquet'
__license__ = 'GPLv2'
__version__ = '0.14.1'
__email__ = 'nabla.c0d3@gmail.com'
PROJECT_URL = 'https://github.com/nabla-c0d3/sslyze'
PROJECT_DESC = 'Fast and full-featured SSL scanner.'
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright 2020 The StackStorm Authors.
# Copyright 2019 Extreme Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os.path

from setuptools import setup, find_packages

from dist_utils import check_pip_version
from dist_utils import fetch_requirements
from dist_utils import apply_vagrant_workaround

from st2client import __version__

check_pip_version()

ST2_COMPONENT = "st2client"
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
REQUIREMENTS_FILE = os.path.join(BASE_DIR, "requirements.txt")
README_FILE = os.path.join(BASE_DIR, "README.rst")

install_reqs, dep_links = fetch_requirements(REQUIREMENTS_FILE)
apply_vagrant_workaround()

with open(README_FILE) as f:
    readme = f.read()

setup(
    name=ST2_COMPONENT,
    version=__version__,
    description=(
        "Python client library and CLI for the StackStorm (st2) event-driven "
        "automation platform."
    ),
    long_description=readme,
    author="StackStorm",
    author_email="info@stackstorm.com",
    url="https://stackstorm.com/",
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Information Technology",
        "Intended Audience :: Developers",
        "Intended Audience :: System Administrators",
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: POSIX :: Linux",
        "Programming Language :: Python",
        "Programming Language :: Python :: 2",
        "Programming Language :: Python :: 2.7",
    ],
    install_requires=install_reqs,
    dependency_links=dep_links,
    test_suite=ST2_COMPONENT,
    zip_safe=False,
    include_package_data=True,
    packages=find_packages(exclude=["setuptools", "tests"]),
    entry_points={"console_scripts": ["st2 = st2client.shell:main"]},
    project_urls={
        "Pack Exchange": "https://exchange.stackstorm.org",
        "Repository": "https://github.com/StackStorm/st2",
        "Documentation": "https://docs.stackstorm.com",
        "Community": "https://stackstorm.com/community-signup",
        "Questions": "https://forum.stackstorm.com/",
        "Donate": "https://funding.communitybridge.org/projects/stackstorm",
        "News/Blog": "https://stackstorm.com/blog",
        "Security": "https://docs.stackstorm.com/latest/security.html",
        "Bug Reports": "https://github.com/StackStorm/st2/issues",
    },
)
<commit_msg>Update out of date st2client setup.py metadata.<commit_after>#!/usr/bin/env python3
# Copyright 2020 The StackStorm Authors.
# Copyright 2019 Extreme Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os.path

from setuptools import setup, find_packages

from dist_utils import check_pip_version
from dist_utils import fetch_requirements
from dist_utils import apply_vagrant_workaround

from st2client import __version__

check_pip_version()

ST2_COMPONENT = "st2client"
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
REQUIREMENTS_FILE = os.path.join(BASE_DIR, "requirements.txt")
README_FILE = os.path.join(BASE_DIR, "README.rst")

install_reqs, dep_links = fetch_requirements(REQUIREMENTS_FILE)
apply_vagrant_workaround()

with open(README_FILE) as f:
    readme = f.read()

setup(
    name=ST2_COMPONENT,
    version=__version__,
    description=(
        "Python client library and CLI for the StackStorm (st2) event-driven "
        "automation platform."
    ),
    long_description=readme,
    author="StackStorm",
    author_email="info@stackstorm.com",
    url="https://stackstorm.com/",
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Information Technology",
        "Intended Audience :: Developers",
        "Intended Audience :: System Administrators",
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: POSIX :: Linux",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
    ],
    install_requires=install_reqs,
    dependency_links=dep_links,
    test_suite=ST2_COMPONENT,
    zip_safe=False,
    include_package_data=True,
    packages=find_packages(exclude=["setuptools", "tests"]),
    entry_points={"console_scripts": ["st2 = st2client.shell:main"]},
    project_urls={
        "Pack Exchange": "https://exchange.stackstorm.org",
        "Repository": "https://github.com/StackStorm/st2",
        "Documentation": "https://docs.stackstorm.com",
        "Community": "https://stackstorm.com/community-signup",
        "Questions": "https://forum.stackstorm.com/",
        "Donate": "https://funding.communitybridge.org/projects/stackstorm",
        "News/Blog": "https://stackstorm.com/blog",
        "Security": "https://docs.stackstorm.com/latest/security.html",
        "Bug Reports": "https://github.com/StackStorm/st2/issues",
    },
)
<|endoftext|>


<commit_before>"""
per-test stdout/stderr capturing mechanism.

"""
from __future__ import with_statement

import sys
import os
from tempfile import TemporaryFile

import py
import pytest

from py.io import TextIO
unicode = py.builtin.text

patchsysdict = {0: 'stdin', 1: 'stdout', 2: 'stderr'}

def pytest_addoption(parser):
    group = parser.getgroup("general")
    group._addoption(
        '--capture', action="store",
        default="fd" if hasattr(os, "dup") else "sys",
        metavar="method", choices=['fd', 'sys', 'no'],
        help="per-test capturing method: one of fd|sys|no.")
    group._addoption(
        '-s', action="store_const", const="no", dest="capture",
        help="shortcut for --capture=no.")

@pytest.mark.tryfirst
def pytest_load_initial_conftests(early_config, parser, args, __multicall__):
    ns = early_config.known_args_namespace
    pluginmanager = early_config.pluginmanager
    capman = CaptureManager(ns.capture)
    pluginmanager.register(capman, "capturemanager")

    # make sure that capturemanager is properly reset at final shutdown
    pluginmanager.add_shutdown(capman.reset_capturings)

    # make sure logging does not raise exceptions at the end
    def silence_logging_at_shutdown():
        if "logging" in sys.modules:
            sys.modules["logging"].raiseExceptions = False
    pluginmanager.add_shutdown(silence_logging_at_shutdown)

    # finally trigger conftest loading but while capturing (issue93)
    capman.init_capturings()
    try:
        try:
            return __multicall__.execute()
        finally:
            out, err = capman.suspendcapture()
    except:
        sys.stdout.write(out)
        sys.stderr.write(err)
        raise

class CaptureManager:
    def __init__(self, method):
        self._method = method

    def _getcapture(self, method):
        if method == "fd":
            return MultiCapture(out=True, err=True, Capture=FDCapture)
        elif method == "sys":
            return MultiCapture(out=True, err=True, Capture=SysCapture)
        elif method == "no":
            return MultiCapture(out=False, err=False, in_=False)
        else:
            raise ValueError("unknown capturing method: %r" % method)

    def init_capturings(self):
        assert not hasattr(self, "_capturing")
        self._capturing = self._getcapture(self._method)
        self._capturing.start_capturing()

    def reset_capturings(self):
        cap = self.__dict__.pop("_capturing", None)
        if cap is not None:
            cap.pop_outerr_to_orig()
            cap.stop_capturing()

    def resumecapture(self):
        self._capturing.resume_capturing()

    def suspendcapture(self, in_=False):
        self.deactivate_funcargs()
        cap = getattr(self, "_capturing", None)
        if cap is not None:
            outerr = cap.readouterr()
            cap.suspend_capturing(in_=in_)
            return outerr

    def activate_funcargs(self, pyfuncitem):
        capfuncarg = pyfuncitem.__dict__.pop("_capfuncarg", None)
        if capfuncarg is not None:
            capfuncarg._start()
            self._capfuncarg = capfuncarg

    def deactivate_funcargs(self):
        capfuncarg = self.__dict__.pop("_capfuncarg", None)
        if capfuncarg is not None:
            capfuncarg.close()

    @pytest.mark.tryfirst
    def pytest_make_collect_report(self, __multicall__, collector):
        if not isinstance(collector, pytest.File):
            return
        self.resumecapture()
        try:
            rep = __multicall__.execute()
        finally:
            out, err = self.suspendcapture()
        if out:
            rep.sections.append(("Captured stdout", out))
        if err:
            rep.sections.append(("Captured stderr", err))
        return rep

    @pytest.mark.hookwrapper
    def pytest_runtest_setup(self, item):
        self.resumecapture()
        yield
        self.suspendcapture_item(item, "setup")

    @pytest.mark.hookwrapper
    def pytest_runtest_call(self, item):
        self.resumecapture()
        self.activate_funcargs(item)
        yield
        #self.deactivate_funcargs() called from suspendcapture()
        self.suspendcapture_item(item, "call")

    @pytest.mark.hookwrapper
    def pytest_runtest_teardown(self, item):
        self.resumecapture()
        yield
        self.suspendcapture_item(item, "teardown")

    @pytest.mark.tryfirst
    def pytest_keyboard_interrupt(self, excinfo):
        self.reset_capturings()

    @pytest.mark.tryfirst
    def pytest_internalerror(self, excinfo):
        self.reset_capturings()

    def suspendcapture_item(self, item, when):
        out, err = self.suspendcapture()
        item.add_report_section(when, "out", out)
        item.add_report_section(when, "err", err)

error_capsysfderror = "cannot use capsys and capfd at the same time"

@pytest.fixture
def capsys(request):
    """enables capturing of writes to sys.stdout/sys.stderr and makes
    captured output available via ``capsys.readouterr()`` method calls
    which return a ``(out, err)`` tuple.
    """
    if "capfd" in request._funcargs:
        raise request.raiseerror(error_capsysfderror)
    request.node._capfuncarg = c = CaptureFixture(SysCapture)
    return c

@pytest.fixture
def capfd(request):
    """enables capturing of writes to file descriptors 1 and 2 and makes
    captured output available via ``capfd.readouterr()`` method calls
    which return a ``(out, err)`` tuple.
    """
    if "capsys" in request._funcargs:
        request.raiseerror(error_capsysfderror)
    if not hasattr(os, 'dup'):
        pytest.skip("capfd funcarg needs os.dup")
    request.node._capfuncarg = c = CaptureFixture(FDCapture)
    return c

class CaptureFixture:
    def __init__(self, captureclass):
        self.captureclass = captureclass

    def _start(self):
        self._capture = MultiCapture(out=True, err=True, in_=False,
                                       Capture=self.captureclass)
        self._capture.start_capturing()

    def close(self):
        cap = self.__dict__.pop("_capture", None)
        if cap is not None:
            self._outerr = cap.pop_outerr_to_orig()
            cap.stop_capturing()

    def readouterr(self):
        try:
            return self._capture.readouterr()
        except AttributeError:
            return self._outerr

def safe_text_dupfile(f, mode, default_encoding="UTF8"):
    """ return a open text file object that's a duplicate of f on the
        FD-level if possible.
    """
    encoding = getattr(f, "encoding", None)
    try:
        fd = f.fileno()
    except Exception:
        if "b" not in getattr(f, "mode", "") and hasattr(f, "encoding"):
            # we seem to have a text stream, let's just use it
            return f
    else:
        newfd = os.dup(fd)
        if "b" not in mode:
            mode += "b"
        f = os.fdopen(newfd, mode, 0)  # no buffering
    return EncodedFile(f, encoding or default_encoding)

class EncodedFile(object):
    errors = "strict"  # possibly needed by py3 code (issue555)
    def __init__(self, buffer, encoding):
        self.buffer = buffer
        self.encoding = encoding

    def write(self, obj):
        if isinstance(obj, unicode):
            obj = obj.encode(self.encoding, "replace")
        self.buffer.write(obj)

    def writelines(self, linelist):
        data = ''.join(linelist)
        self.write(data)

    def __getattr__(self, name):
        return getattr(self.buffer, name)

class MultiCapture(object):
    out = err = in_ = None

    def __init__(self, out=True, err=True, in_=True, Capture=None):
        if in_:
            self.in_ = Capture(0)
        if out:
            self.out = Capture(1)
        if err:
            self.err = Capture(2)

    def start_capturing(self):
        if self.in_:
            self.in_.start()
        if self.out:
            self.out.start()
        if self.err:
            self.err.start()

    def pop_outerr_to_orig(self):
        """ pop current snapshot out/err capture and flush to orig streams. """
        out, err = self.readouterr()
        if out:
            self.out.writeorg(out)
        if err:
            self.err.writeorg(err)
        return out, err

    def suspend_capturing(self, in_=False):
        if self.out:
            self.out.suspend()
        if self.err:
            self.err.suspend()
        if in_ and self.in_:
            self.in_.suspend()
            self._in_suspended = True

    def resume_capturing(self):
        if self.out:
            self.out.resume()
        if self.err:
            self.err.resume()
        if hasattr(self, "_in_suspended"):
            self.in_.resume()
            del self._in_suspended

    def stop_capturing(self):
        """ stop capturing and reset capturing streams """
        if hasattr(self, '_reset'):
            raise ValueError("was already stopped")
        self._reset = True
        if self.out:
            self.out.done()
        if self.err:
            self.err.done()
        if self.in_:
            self.in_.done()

    def readouterr(self):
        """ return snapshot unicode value of stdout/stderr capturings. """
        return (self.out.snap() if self.out is not None else "",
                self.err.snap() if self.err is not None else "")

class NoCapture:
    __init__ = start = done = suspend = resume = lambda *args: None

class FDCapture:
    """ Capture IO to/from a given os-level filedescriptor. """

    def __init__(self, targetfd, tmpfile=None):
        self.targetfd = targetfd
        try:
            self.targetfd_save = os.dup(self.targetfd)
        except OSError:
            self.start = lambda: None
            self.done = lambda: None
        else:
            if targetfd == 0:
                assert not tmpfile, "cannot set tmpfile with stdin"
                tmpfile = open(os.devnull, "r")
                self.syscapture = SysCapture(targetfd)
            else:
                if tmpfile is None:
                    f = TemporaryFile()
                    with f:
                        tmpfile = safe_text_dupfile(f, mode="wb+")
                if targetfd in patchsysdict:
                    self.syscapture = SysCapture(targetfd, tmpfile)
                else:
                    self.syscapture = NoCapture()
            self.tmpfile = tmpfile
            self.tmpfile_fd = tmpfile.fileno()

    def __repr__(self):
        return "<FDCapture %s oldfd=%s>" % (self.targetfd, self.targetfd_save)

    def start(self):
        """ Start capturing on targetfd using memorized tmpfile. """
        try:
            os.fstat(self.targetfd_save)
        except (AttributeError, OSError):
            raise ValueError("saved filedescriptor not valid anymore")
        os.dup2(self.tmpfile_fd, self.targetfd)
        self.syscapture.start()

    def snap(self):
        f = self.tmpfile
        f.seek(0)
        res = f.read()
        if res:
            enc = getattr(f, "encoding", None)
            if enc and isinstance(res, bytes):
                res = py.builtin._totext(res, enc, "replace")
            f.truncate(0)
            f.seek(0)
            return res
        return ''

    def done(self):
        """ stop capturing, restore streams, return original capture file,
        seeked to position zero. """
        targetfd_save = self.__dict__.pop("targetfd_save")
        os.dup2(targetfd_save, self.targetfd)
        os.close(targetfd_save)
        self.syscapture.done()
        self.tmpfile.close()

    def suspend(self):
        self.syscapture.suspend()
        os.dup2(self.targetfd_save, self.targetfd)

    def resume(self):
        self.syscapture.resume()
        os.dup2(self.tmpfile_fd, self.targetfd)

    def writeorg(self, data):
        """ write to original file descriptor. """
        if py.builtin._istext(data):
            data = data.encode("utf8") # XXX use encoding of original stream
        os.write(self.targetfd_save, data)

class SysCapture:
    def __init__(self, fd, tmpfile=None):
        name = patchsysdict[fd]
        self._old = getattr(sys, name)
        self.name = name
        if tmpfile is None:
            if name == "stdin":
                tmpfile = DontReadFromInput()
            else:
                tmpfile = TextIO()
        self.tmpfile = tmpfile

    def start(self):
        setattr(sys, self.name, self.tmpfile)

    def snap(self):
        f = self.tmpfile
        res = f.getvalue()
        f.truncate(0)
        f.seek(0)
        return res

    def done(self):
        setattr(sys, self.name, self._old)
        del self._old
        self.tmpfile.close()

    def suspend(self):
        setattr(sys, self.name, self._old)

    def resume(self):
        setattr(sys, self.name, self.tmpfile)

    def writeorg(self, data):
        self._old.write(data)
        self._old.flush()

class DontReadFromInput:
    """Temporary stub class.  Ideally when stdin is accessed, the
    capturing should be turned off, with possibly all data captured
    so far sent to the screen.  This should be configurable, though,
    because in automated test runs it is better to crash than
    hang indefinitely.
    """
    def read(self, *args):
        raise IOError("reading from stdin while output is captured")
    readline = read
    readlines = read
    __iter__ = read

    def fileno(self):
        raise ValueError("redirected Stdin is pseudofile, has no fileno()")

    def isatty(self):
        return False

    def close(self):
        pass
<commit_msg>fixed strange infinite recursion bug<commit_after>"""
per-test stdout/stderr capturing mechanism.

"""
from __future__ import with_statement

import sys
import os
from tempfile import TemporaryFile

import py
import pytest

from py.io import TextIO
unicode = py.builtin.text

patchsysdict = {0: 'stdin', 1: 'stdout', 2: 'stderr'}

def pytest_addoption(parser):
    group = parser.getgroup("general")
    group._addoption(
        '--capture', action="store",
        default="fd" if hasattr(os, "dup") else "sys",
        metavar="method", choices=['fd', 'sys', 'no'],
        help="per-test capturing method: one of fd|sys|no.")
    group._addoption(
        '-s', action="store_const", const="no", dest="capture",
        help="shortcut for --capture=no.")

@pytest.mark.tryfirst
def pytest_load_initial_conftests(early_config, parser, args, __multicall__):
    ns = early_config.known_args_namespace
    pluginmanager = early_config.pluginmanager
    capman = CaptureManager(ns.capture)
    pluginmanager.register(capman, "capturemanager")

    # make sure that capturemanager is properly reset at final shutdown
    pluginmanager.add_shutdown(capman.reset_capturings)

    # make sure logging does not raise exceptions at the end
    def silence_logging_at_shutdown():
        if "logging" in sys.modules:
            sys.modules["logging"].raiseExceptions = False
    pluginmanager.add_shutdown(silence_logging_at_shutdown)

    # finally trigger conftest loading but while capturing (issue93)
    capman.init_capturings()
    try:
        try:
            return __multicall__.execute()
        finally:
            out, err = capman.suspendcapture()
    except:
        sys.stdout.write(out)
        sys.stderr.write(err)
        raise

class CaptureManager:
    def __init__(self, method):
        self._method = method

    def _getcapture(self, method):
        if method == "fd":
            return MultiCapture(out=True, err=True, Capture=FDCapture)
        elif method == "sys":
            return MultiCapture(out=True, err=True, Capture=SysCapture)
        elif method == "no":
            return MultiCapture(out=False, err=False, in_=False)
        else:
            raise ValueError("unknown capturing method: %r" % method)

    def init_capturings(self):
        assert not hasattr(self, "_capturing")
        self._capturing = self._getcapture(self._method)
        self._capturing.start_capturing()

    def reset_capturings(self):
        cap = self.__dict__.pop("_capturing", None)
        if cap is not None:
            cap.pop_outerr_to_orig()
            cap.stop_capturing()

    def resumecapture(self):
        self._capturing.resume_capturing()

    def suspendcapture(self, in_=False):
        self.deactivate_funcargs()
        cap = getattr(self, "_capturing", None)
        if cap is not None:
            outerr = cap.readouterr()
            cap.suspend_capturing(in_=in_)
            return outerr

    def activate_funcargs(self, pyfuncitem):
        capfuncarg = pyfuncitem.__dict__.pop("_capfuncarg", None)
        if capfuncarg is not None:
            capfuncarg._start()
            self._capfuncarg = capfuncarg

    def deactivate_funcargs(self):
        capfuncarg = self.__dict__.pop("_capfuncarg", None)
        if capfuncarg is not None:
            capfuncarg.close()

    @pytest.mark.tryfirst
    def pytest_make_collect_report(self, __multicall__, collector):
        if not isinstance(collector, pytest.File):
            return
        self.resumecapture()
        try:
            rep = __multicall__.execute()
        finally:
            out, err = self.suspendcapture()
        if out:
            rep.sections.append(("Captured stdout", out))
        if err:
            rep.sections.append(("Captured stderr", err))
        return rep

    @pytest.mark.hookwrapper
    def pytest_runtest_setup(self, item):
        self.resumecapture()
        yield
        self.suspendcapture_item(item, "setup")

    @pytest.mark.hookwrapper
    def pytest_runtest_call(self, item):
        self.resumecapture()
        self.activate_funcargs(item)
        yield
        #self.deactivate_funcargs() called from suspendcapture()
        self.suspendcapture_item(item, "call")

    @pytest.mark.hookwrapper
    def pytest_runtest_teardown(self, item):
        self.resumecapture()
        yield
        self.suspendcapture_item(item, "teardown")

    @pytest.mark.tryfirst
    def pytest_keyboard_interrupt(self, excinfo):
        self.reset_capturings()

    @pytest.mark.tryfirst
    def pytest_internalerror(self, excinfo):
        self.reset_capturings()

    def suspendcapture_item(self, item, when):
        out, err = self.suspendcapture()
        item.add_report_section(when, "out", out)
        item.add_report_section(when, "err", err)

error_capsysfderror = "cannot use capsys and capfd at the same time"

@pytest.fixture
def capsys(request):
    """enables capturing of writes to sys.stdout/sys.stderr and makes
    captured output available via ``capsys.readouterr()`` method calls
    which return a ``(out, err)`` tuple.
    """
    if "capfd" in request._funcargs:
        raise request.raiseerror(error_capsysfderror)
    request.node._capfuncarg = c = CaptureFixture(SysCapture)
    return c

@pytest.fixture
def capfd(request):
    """enables capturing of writes to file descriptors 1 and 2 and makes
    captured output available via ``capfd.readouterr()`` method calls
    which return a ``(out, err)`` tuple.
    """
    if "capsys" in request._funcargs:
        request.raiseerror(error_capsysfderror)
    if not hasattr(os, 'dup'):
        pytest.skip("capfd funcarg needs os.dup")
    request.node._capfuncarg = c = CaptureFixture(FDCapture)
    return c

class CaptureFixture:
    def __init__(self, captureclass):
        self.captureclass = captureclass

    def _start(self):
        self._capture = MultiCapture(out=True, err=True, in_=False,
                                       Capture=self.captureclass)
        self._capture.start_capturing()

    def close(self):
        cap = self.__dict__.pop("_capture", None)
        if cap is not None:
            self._outerr = cap.pop_outerr_to_orig()
            cap.stop_capturing()

    def readouterr(self):
        try:
            return self._capture.readouterr()
        except AttributeError:
            return self._outerr

def safe_text_dupfile(f, mode, default_encoding="UTF8"):
    """ return a open text file object that's a duplicate of f on the
        FD-level if possible.
    """
    encoding = getattr(f, "encoding", None)
    try:
        fd = f.fileno()
    except Exception:
        if "b" not in getattr(f, "mode", "") and hasattr(f, "encoding"):
            # we seem to have a text stream, let's just use it
            return f
    else:
        newfd = os.dup(fd)
        if "b" not in mode:
            mode += "b"
        f = os.fdopen(newfd, mode, 0)  # no buffering
    return EncodedFile(f, encoding or default_encoding)

class EncodedFile(object):
    errors = "strict"  # possibly needed by py3 code (issue555)
    def __init__(self, buffer, encoding):
        self.buffer = buffer
        self.encoding = encoding

    def write(self, obj):
        if isinstance(obj, unicode):
            obj = obj.encode(self.encoding, "replace")
        self.buffer.write(obj)

    def writelines(self, linelist):
        data = ''.join(linelist)
        self.write(data)

    def __getattr__(self, name):
        if name != "buffer":
            return getattr(self.buffer, name)

class MultiCapture(object):
    out = err = in_ = None

    def __init__(self, out=True, err=True, in_=True, Capture=None):
        if in_:
            self.in_ = Capture(0)
        if out:
            self.out = Capture(1)
        if err:
            self.err = Capture(2)

    def start_capturing(self):
        if self.in_:
            self.in_.start()
        if self.out:
            self.out.start()
        if self.err:
            self.err.start()

    def pop_outerr_to_orig(self):
        """ pop current snapshot out/err capture and flush to orig streams. """
        out, err = self.readouterr()
        if out:
            self.out.writeorg(out)
        if err:
            self.err.writeorg(err)
        return out, err

    def suspend_capturing(self, in_=False):
        if self.out:
            self.out.suspend()
        if self.err:
            self.err.suspend()
        if in_ and self.in_:
            self.in_.suspend()
            self._in_suspended = True

    def resume_capturing(self):
        if self.out:
            self.out.resume()
        if self.err:
            self.err.resume()
        if hasattr(self, "_in_suspended"):
            self.in_.resume()
            del self._in_suspended

    def stop_capturing(self):
        """ stop capturing and reset capturing streams """
        if hasattr(self, '_reset'):
            raise ValueError("was already stopped")
        self._reset = True
        if self.out:
            self.out.done()
        if self.err:
            self.err.done()
        if self.in_:
            self.in_.done()

    def readouterr(self):
        """ return snapshot unicode value of stdout/stderr capturings. """
        return (self.out.snap() if self.out is not None else "",
                self.err.snap() if self.err is not None else "")

class NoCapture:
    __init__ = start = done = suspend = resume = lambda *args: None

class FDCapture:
    """ Capture IO to/from a given os-level filedescriptor. """

    def __init__(self, targetfd, tmpfile=None):
        self.targetfd = targetfd
        try:
            self.targetfd_save = os.dup(self.targetfd)
        except OSError:
            self.start = lambda: None
            self.done = lambda: None
        else:
            if targetfd == 0:
                assert not tmpfile, "cannot set tmpfile with stdin"
                tmpfile = open(os.devnull, "r")
                self.syscapture = SysCapture(targetfd)
            else:
                if tmpfile is None:
                    f = TemporaryFile()
                    with f:
                        tmpfile = safe_text_dupfile(f, mode="wb+")
                if targetfd in patchsysdict:
                    self.syscapture = SysCapture(targetfd, tmpfile)
                else:
                    self.syscapture = NoCapture()
            self.tmpfile = tmpfile
            self.tmpfile_fd = tmpfile.fileno()

    def __repr__(self):
        return "<FDCapture %s oldfd=%s>" % (self.targetfd, self.targetfd_save)

    def start(self):
        """ Start capturing on targetfd using memorized tmpfile. """
        try:
            os.fstat(self.targetfd_save)
        except (AttributeError, OSError):
            raise ValueError("saved filedescriptor not valid anymore")
        os.dup2(self.tmpfile_fd, self.targetfd)
        self.syscapture.start()

    def snap(self):
        f = self.tmpfile
        f.seek(0)
        res = f.read()
        if res:
            enc = getattr(f, "encoding", None)
            if enc and isinstance(res, bytes):
                res = py.builtin._totext(res, enc, "replace")
            f.truncate(0)
            f.seek(0)
            return res
        return ''

    def done(self):
        """ stop capturing, restore streams, return original capture file,
        seeked to position zero. """
        targetfd_save = self.__dict__.pop("targetfd_save")
        os.dup2(targetfd_save, self.targetfd)
        os.close(targetfd_save)
        self.syscapture.done()
        self.tmpfile.close()

    def suspend(self):
        self.syscapture.suspend()
        os.dup2(self.targetfd_save, self.targetfd)

    def resume(self):
        self.syscapture.resume()
        os.dup2(self.tmpfile_fd, self.targetfd)

    def writeorg(self, data):
        """ write to original file descriptor. """
        if py.builtin._istext(data):
            data = data.encode("utf8") # XXX use encoding of original stream
        os.write(self.targetfd_save, data)

class SysCapture:
    def __init__(self, fd, tmpfile=None):
        name = patchsysdict[fd]
        self._old = getattr(sys, name)
        self.name = name
        if tmpfile is None:
            if name == "stdin":
                tmpfile = DontReadFromInput()
            else:
                tmpfile = TextIO()
        self.tmpfile = tmpfile

    def start(self):
        setattr(sys, self.name, self.tmpfile)

    def snap(self):
        f = self.tmpfile
        res = f.getvalue()
        f.truncate(0)
        f.seek(0)
        return res

    def done(self):
        setattr(sys, self.name, self._old)
        del self._old
        self.tmpfile.close()

    def suspend(self):
        setattr(sys, self.name, self._old)

    def resume(self):
        setattr(sys, self.name, self.tmpfile)

    def writeorg(self, data):
        self._old.write(data)
        self._old.flush()

class DontReadFromInput:
    """Temporary stub class.  Ideally when stdin is accessed, the
    capturing should be turned off, with possibly all data captured
    so far sent to the screen.  This should be configurable, though,
    because in automated test runs it is better to crash than
    hang indefinitely.
    """
    def read(self, *args):
        raise IOError("reading from stdin while output is captured")
    readline = read
    readlines = read
    __iter__ = read

    def fileno(self):
        raise ValueError("redirected Stdin is pseudofile, has no fileno()")

    def isatty(self):
        return False

    def close(self):
        pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from setup.create_teams import migrate_teams
from setup.create_divisions import create_divisions

if __name__ == '__main__':
    # migrating teams from json file to database
    migrate_teams(simulation=True)
    # creating divisions from division configuration file
    create_divisions(simulation=True)
<commit_msg>Include player data migration in setup<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from setup.create_teams import migrate_teams
from setup.create_divisions import create_divisions
from setup.create_players import migrate_players

if __name__ == '__main__':
    # migrating teams from json file to database
    migrate_teams(simulation=True)
    # creating divisions from division configuration file
    create_divisions(simulation=True)
    # migrating players from json file to database
    migrate_players(simulation=True)
<|endoftext|>


<commit_before>####################################################################################################
#
# These are the reference implementations for the node functions of the standard node types.
# Node net engine implementations performing node calculations in Python are free to use
# these implementations directly.
# Non-python engine implementations, or performance-oriented python implementations,
# can use these implementations as definitions (or for equivalence tests) and implement them
# differently.
#
####################################################################################################

def register(netapi, node=None, **params):
    activation = node.get_slot('gen').activation
    node.activation = activation
    node.get_gate('gen').gate_function(activation)

def sensor(netapi, node=None, datasource=None, **params):
    datasource_value = netapi.world.get_datasource(netapi.uid, datasource)
    node.activation = datasource_value
    node.get_gate('gen').gate_function(datasource_value)

def actor(netapi, node=None, datatarget=None, **params):
    if not netapi.world:
        return
    activation_to_set = node.get_slot("gen").activation
    netapi.world.set_datatarget(netapi.uid, datatarget, activation_to_set)
    # if activation_to_set > 0:
        # node.activation = 1
    feedback = netapi.world.get_datatarget_feedback(netapi.uid, datatarget)
    if feedback is not None:
        node.get_gate('gen').gate_function(feedback)

def concept(netapi, node=None, **params):
    activation = node.get_slot('gen').activation
    node.activation = activation
    for gate_type in node.get_gate_types():
        node.get_gate(gate_type).gate_function(activation)

def script(netapi, node=None, **params):
    """ Script nodes are state machines that use the node activation for determining their behavior.
        They form hierarchical scripts that are started via sub-activating their top-node. If the sur-activation
        is turned off, the script stops executing.
        At the bottom, the scripts must connect to activation sources (that yield positive sur activation), otherwise
        they will fail.
        The links have the following semantics:
            - sub determines sub-actions. Inactive sub-actions will turn "ready" if their parent action turns
              "requesting", by sending activation via sub.
            - por determines successors. If an action is neither inactive nor confirmed, it will inhibit its successors
              from requesting. Successors must wait for their predecessors to stop sending por-activation. The first
              action in a por-chain receives no such inhibition, it may turn from "ready" to "requesting". After
              an action has become "requesting", it turns "pending" and continues to request its sub-actions.
              All other "ready" actions will switch to "waiting" until their predecessors stop inhibiting them.
            - ret determines predecessors. Successors inhibit their predecessors from telling their parents when they
              confirm. (Once an action without sucessor confirms, we are done.)
            - sur has two functions. A high level of sur activation confirms the parent action. A low level of sur
              tells the parent action that at least one of its sub-actions has not given up yet. If no more sur
              activation is received by a pending parent action, it will fail.
            The distinction between "ready" and "waiting", and "requesting and "pending" is necessary to bridge the
            time until neighboring nodes have had time to make themselves heard.

        Currently, the node states correspond to the following activation levels:
        < 0     failed (will stick until requesting ends)
        < 0.01  inactive (will change to prepared when requested)
        < 0.3   preparing (inhibits neighbors  and changes to suppressed)
        < 0.5   suppressed (inhibits neighbors and goes to requesting when no longer inhibited)
        < 0.7   requesting (starts requesting and changes to pending)
        < 1     pending (keeps requesting, will either change to confirmed or failed)
        >=1     confirmed (will stick until requesting ends)

    """

    if node.get_slot("sub").activation < 0.01:  # node is not requested and is turned off
        node.activation = 0.0
        node.get_gate("por").gate_function(0.0)
        node.get_gate("ret").gate_function(0.0)
        node.get_gate("sub").gate_function(0.0)
        node.get_gate("sur").gate_function(0.0)
        return

    node.activation = (
        node.activation if node.activation < -0.01 else  # failed -> failed
        0.2 if node.activation < 0.01 else  # (we already tested that sub is positive): inactive -> preparing
        0.4 if node.activation < 0.5 and node.get_slot("por").activation < 0 else  # preparing -> supressed
        0.6 if node.activation < 0.5 else  # preparing/supressed -> requesting
        0.8 if node.activation < 0.7 else  # requesting -> pending
        1.0 if node.get_slot("sur").activation >= 1 else  # pending -> confirmed
        -1. if node.get_slot("sur").activation <= 0 else  # pending -> failed
        node.activation
    )

    # always inhibit successor, except when confirmed
    node.get_gate("por").gate_function(-1.0 if node.activation < 1 else 1.0)
    # inhibit confirmation of predecessor, and tell it to stop once successor is requested
    node.get_gate("ret").gate_function(-1.0 if 0.1 < node.activation < 1 else 1.0)
    # request children when becoming requesting
    node.get_gate("sub").gate_function(1.0 if 0.5 < node.activation else 0)
    # keep parent from failing while pending or processing, confirm parent when confirmed
    node.get_gate("sur").gate_function(
        0 if node.activation < 0.01 or node.get_slot("ret").activation > 0 else
        0.01 if node.activation < 1 else
        0.01 if node.get_slot("ret").activation < 0 else
        1)

def pipe(netapi, node=None, sheaf="default", **params):
    gen = 0.0
    por = 0.0
    ret = 0.0
    sub = 0.0
    sur = 0.0
    cat = 0.0
    exp = 0.0

    gen += node.get_slot("gen").get_activation(sheaf)
    if gen < 0.1: gen = 0                                                   # cut off gen loop at lower threshold
    gen += node.get_slot("sur").get_activation(sheaf)
    gen += node.get_slot("exp").get_activation(sheaf)
    if gen > 1: gen = 1

    sub += max(node.get_slot("sur").get_activation(sheaf), 0)
    sub += node.get_slot("sub").get_activation(sheaf)
    sub *= node.get_slot("por").get_activation(sheaf) if not node.get_slot("por").empty else 1
    sub *= 0 if node.get_slot("gen").get_activation(sheaf) > 0 else 1
    if sub > 0: sub = 1

    sur += node.get_slot("sur").get_activation(sheaf)
    if sur == 0: sur += node.get_slot("sur").get_activation("default")      # no activation in our sheaf, maybe from sensors?
    sur += 0 if node.get_slot("gen").get_activation(sheaf) < 0.2 else 1     # cut off sur-reports from gen looping before the loop fades away
    sur += node.get_slot("exp").get_activation(sheaf)
    if sur > 0 and len(node.get_slot('ret').get_links()) == 1:     # else: always propagate failure
        sur = 0
    sur *= node.get_slot("por").get_activation(sheaf) if not node.get_slot("por").empty else 1

    if node.get_slot('por').empty and node.get_slot('ret').empty:           # both empty
        classifierelements = 0
        if len(node.get_gate("sur").get_links()) == 1:
            surnode = node.get_gate("sur").get_links()[0].target_node
            classifierelements = len(surnode.get_gate("sub").get_links())
        sur /= classifierelements if classifierelements > 1 else 1          # classifier case

    por += node.get_slot("sur").get_activation(sheaf)
    por *= node.get_slot("por").get_activation(sheaf) if not node.get_slot("por").empty else 1
    por += (0 if node.get_slot("gen").get_activation(sheaf) < 0.1 else 1) * \
           (1+node.get_slot("por").get_activation(sheaf))
    por += node.get_slot("por").get_activation(sheaf) if node.get_slot("sub").get_activation(sheaf) == 0 and node.get_slot("sur").get_activation(sheaf) == 0 else 0
    if por > 0: por = 1
    if por < 0: por = 0

    ret += node.get_slot("ret").get_activation(sheaf) if node.get_slot("sub").get_activation(sheaf) == 0 and node.get_slot("sur").get_activation(sheaf) == 0 else 0

    cat = sub
    if cat == 0: cat += node.get_slot("cat").get_activation(sheaf)
    if cat < 0: cat = 0

    exp += node.get_slot("sur").get_activation(sheaf)
    exp += node.get_slot("exp").get_activation(sheaf) * 0.1                 # magic priming number
    if exp == 0: exp += node.get_slot("sur").get_activation("default")      # no activation in our sheaf, maybe from sensors?
    if exp > 1: exp = 1

    # handle locking if configured for this node
    sub_lock_needed = node.get_parameter('sublock')
    if sub_lock_needed is not None:
        surinput = node.get_slot("sur").get_activation(sheaf)
        if sub > 0 and surinput < 1:
            # we want to go sub, but we need to acquire a lock for that
            if netapi.is_locked(sub_lock_needed):
                if not netapi.is_locked_by(sub_lock_needed, node.uid+sheaf):
                    # it's locked and not by us, so we need to pace ourselves and wait
                    sub = 0
            else:
                # we can proceed, but we need to lock
                netapi.lock(sub_lock_needed, node.uid+sheaf)

        if surinput >= 1:
            # we can clear a lock if it's us who had acquired it
            if netapi.is_locked_by(sub_lock_needed, node.uid+sheaf):
                netapi.unlock(sub_lock_needed)

    # set gates
    node.set_sheaf_activation(gen, sheaf)
    node.get_gate("gen").gate_function(gen, sheaf)
    node.get_gate("por").gate_function(por, sheaf)
    node.get_gate("ret").gate_function(ret, sheaf)
    node.get_gate("sub").gate_function(sub, sheaf)
    node.get_gate("sur").gate_function(sur, sheaf)
    node.get_gate("exp").gate_function(exp, sheaf)
    if cat > 0 and sub > 0:     # cats will be checked in their own sheaf
        node.get_gate("cat").open_sheaf(cat, sheaf)
        node.get_gate("cat").gate_function(0, sheaf)
    else:
        node.get_gate("cat").gate_function(cat, sheaf)

def activator(netapi, node, **params):
    node.activation = node.get_slot("gen").activation
    netapi.get_nodespace(node.parent_nodespace).set_activator_value(node.get_parameter('type'), node.activation)
<commit_msg>Less link counting, relying on ret activation for non-reporting sur in scripts<commit_after>####################################################################################################
#
# These are the reference implementations for the node functions of the standard node types.
# Node net engine implementations performing node calculations in Python are free to use
# these implementations directly.
# Non-python engine implementations, or performance-oriented python implementations,
# can use these implementations as definitions (or for equivalence tests) and implement them
# differently.
#
####################################################################################################

def register(netapi, node=None, **params):
    activation = node.get_slot('gen').activation
    node.activation = activation
    node.get_gate('gen').gate_function(activation)

def sensor(netapi, node=None, datasource=None, **params):
    datasource_value = netapi.world.get_datasource(netapi.uid, datasource)
    node.activation = datasource_value
    node.get_gate('gen').gate_function(datasource_value)

def actor(netapi, node=None, datatarget=None, **params):
    if not netapi.world:
        return
    activation_to_set = node.get_slot("gen").activation
    netapi.world.set_datatarget(netapi.uid, datatarget, activation_to_set)
    # if activation_to_set > 0:
        # node.activation = 1
    feedback = netapi.world.get_datatarget_feedback(netapi.uid, datatarget)
    if feedback is not None:
        node.get_gate('gen').gate_function(feedback)

def concept(netapi, node=None, **params):
    activation = node.get_slot('gen').activation
    node.activation = activation
    for gate_type in node.get_gate_types():
        node.get_gate(gate_type).gate_function(activation)

def script(netapi, node=None, **params):
    """ Script nodes are state machines that use the node activation for determining their behavior.
        They form hierarchical scripts that are started via sub-activating their top-node. If the sur-activation
        is turned off, the script stops executing.
        At the bottom, the scripts must connect to activation sources (that yield positive sur activation), otherwise
        they will fail.
        The links have the following semantics:
            - sub determines sub-actions. Inactive sub-actions will turn "ready" if their parent action turns
              "requesting", by sending activation via sub.
            - por determines successors. If an action is neither inactive nor confirmed, it will inhibit its successors
              from requesting. Successors must wait for their predecessors to stop sending por-activation. The first
              action in a por-chain receives no such inhibition, it may turn from "ready" to "requesting". After
              an action has become "requesting", it turns "pending" and continues to request its sub-actions.
              All other "ready" actions will switch to "waiting" until their predecessors stop inhibiting them.
            - ret determines predecessors. Successors inhibit their predecessors from telling their parents when they
              confirm. (Once an action without sucessor confirms, we are done.)
            - sur has two functions. A high level of sur activation confirms the parent action. A low level of sur
              tells the parent action that at least one of its sub-actions has not given up yet. If no more sur
              activation is received by a pending parent action, it will fail.
            The distinction between "ready" and "waiting", and "requesting and "pending" is necessary to bridge the
            time until neighboring nodes have had time to make themselves heard.

        Currently, the node states correspond to the following activation levels:
        < 0     failed (will stick until requesting ends)
        < 0.01  inactive (will change to prepared when requested)
        < 0.3   preparing (inhibits neighbors  and changes to suppressed)
        < 0.5   suppressed (inhibits neighbors and goes to requesting when no longer inhibited)
        < 0.7   requesting (starts requesting and changes to pending)
        < 1     pending (keeps requesting, will either change to confirmed or failed)
        >=1     confirmed (will stick until requesting ends)

    """

    if node.get_slot("sub").activation < 0.01:  # node is not requested and is turned off
        node.activation = 0.0
        node.get_gate("por").gate_function(0.0)
        node.get_gate("ret").gate_function(0.0)
        node.get_gate("sub").gate_function(0.0)
        node.get_gate("sur").gate_function(0.0)
        return

    node.activation = (
        node.activation if node.activation < -0.01 else  # failed -> failed
        0.2 if node.activation < 0.01 else  # (we already tested that sub is positive): inactive -> preparing
        0.4 if node.activation < 0.5 and node.get_slot("por").activation < 0 else  # preparing -> supressed
        0.6 if node.activation < 0.5 else  # preparing/supressed -> requesting
        0.8 if node.activation < 0.7 else  # requesting -> pending
        1.0 if node.get_slot("sur").activation >= 1 else  # pending -> confirmed
        -1. if node.get_slot("sur").activation <= 0 else  # pending -> failed
        node.activation
    )

    # always inhibit successor, except when confirmed
    node.get_gate("por").gate_function(-1.0 if node.activation < 1 else 1.0)
    # inhibit confirmation of predecessor, and tell it to stop once successor is requested
    node.get_gate("ret").gate_function(-1.0 if 0.1 < node.activation < 1 else 1.0)
    # request children when becoming requesting
    node.get_gate("sub").gate_function(1.0 if 0.5 < node.activation else 0)
    # keep parent from failing while pending or processing, confirm parent when confirmed
    node.get_gate("sur").gate_function(
        0 if node.activation < 0.01 or node.get_slot("ret").activation > 0 else
        0.01 if node.activation < 1 else
        0.01 if node.get_slot("ret").activation < 0 else
        1)

def pipe(netapi, node=None, sheaf="default", **params):
    gen = 0.0
    por = 0.0
    ret = 0.0
    sub = 0.0
    sur = 0.0
    cat = 0.0
    exp = 0.0

    gen += node.get_slot("gen").get_activation(sheaf)
    if gen < 0.1: gen = 0                                                   # cut off gen loop at lower threshold
    gen += node.get_slot("sur").get_activation(sheaf)
    gen += node.get_slot("exp").get_activation(sheaf)
    if gen > 1: gen = 1

    sub += max(node.get_slot("sur").get_activation(sheaf), 0)
    sub += node.get_slot("sub").get_activation(sheaf)
    sub *= node.get_slot("por").get_activation(sheaf) if not node.get_slot("por").empty else 1
    sub *= 0 if node.get_slot("gen").get_activation(sheaf) > 0 else 1
    if sub > 0: sub = 1

    sur += node.get_slot("sur").get_activation(sheaf)
    if sur == 0: sur += node.get_slot("sur").get_activation("default")      # no activation in our sheaf, maybe from sensors?
    sur += 0 if node.get_slot("gen").get_activation(sheaf) < 0.2 else 1     # cut off sur-reports from gen looping before the loop fades away
    sur += node.get_slot("exp").get_activation(sheaf)

    if sur > 0 and node.get_slot("ret").get_activation(sheaf) < 0:
        sur = 0

    if sur > 0 and not node.get_slot("por").empty:
        sur *= node.get_slot("por").get_activation(sheaf)

    if node.get_slot('por').empty and node.get_slot('ret').empty:           # both empty
        classifierelements = 0
        if len(node.get_gate("sur").get_links()) == 1:
            surnode = node.get_gate("sur").get_links()[0].target_node
            classifierelements = len(surnode.get_gate("sub").get_links())
        sur /= classifierelements if classifierelements > 1 else 1          # classifier case

    por += node.get_slot("sur").get_activation(sheaf)
    por *= node.get_slot("por").get_activation(sheaf) if not node.get_slot("por").empty else 1
    por += (0 if node.get_slot("gen").get_activation(sheaf) < 0.1 else 1) * \
           (1+node.get_slot("por").get_activation(sheaf))
    por += node.get_slot("por").get_activation(sheaf) if node.get_slot("sub").get_activation(sheaf) == 0 and node.get_slot("sur").get_activation(sheaf) == 0 else 0
    if por > 0: por = 1
    if por < 0: por = 0

    ret += node.get_slot("ret").get_activation(sheaf) if node.get_slot("sub").get_activation(sheaf) == 0 and node.get_slot("sur").get_activation(sheaf) == 0 else 0
    ret -= node.get_slot("sub").get_activation(sheaf)
    if ret > 1:
        ret = 1

    cat = sub
    if cat == 0: cat += node.get_slot("cat").get_activation(sheaf)
    if cat < 0: cat = 0

    exp += node.get_slot("sur").get_activation(sheaf)
    exp += node.get_slot("exp").get_activation(sheaf) * 0.1                 # magic priming number
    if exp == 0: exp += node.get_slot("sur").get_activation("default")      # no activation in our sheaf, maybe from sensors?
    if exp > 1: exp = 1

    # handle locking if configured for this node
    sub_lock_needed = node.get_parameter('sublock')
    if sub_lock_needed is not None:
        surinput = node.get_slot("sur").get_activation(sheaf)
        if sub > 0 and surinput < 1:
            # we want to go sub, but we need to acquire a lock for that
            if netapi.is_locked(sub_lock_needed):
                if not netapi.is_locked_by(sub_lock_needed, node.uid+sheaf):
                    # it's locked and not by us, so we need to pace ourselves and wait
                    sub = 0
            else:
                # we can proceed, but we need to lock
                netapi.lock(sub_lock_needed, node.uid+sheaf)

        if surinput >= 1:
            # we can clear a lock if it's us who had acquired it
            if netapi.is_locked_by(sub_lock_needed, node.uid+sheaf):
                netapi.unlock(sub_lock_needed)

    # set gates
    node.set_sheaf_activation(gen, sheaf)
    node.get_gate("gen").gate_function(gen, sheaf)
    node.get_gate("por").gate_function(por, sheaf)
    node.get_gate("ret").gate_function(ret, sheaf)
    node.get_gate("sub").gate_function(sub, sheaf)
    node.get_gate("sur").gate_function(sur, sheaf)
    node.get_gate("exp").gate_function(exp, sheaf)
    if cat > 0 and sub > 0:     # cats will be checked in their own sheaf
        node.get_gate("cat").open_sheaf(cat, sheaf)
        node.get_gate("cat").gate_function(0, sheaf)
    else:
        node.get_gate("cat").gate_function(cat, sheaf)

def activator(netapi, node, **params):
    node.activation = node.get_slot("gen").activation
    netapi.get_nodespace(node.parent_nodespace).set_activator_value(node.get_parameter('type'), node.activation)
<|endoftext|>


<commit_before>import os
from os.path import join

from django.core.management.base import copy_helper, CommandError, LabelCommand
from django.utils.importlib import import_module

class Command(LabelCommand):
    directory_sufix = 'apps/'
    help = "Creates a Django app directory structure for the given app name in the " + directory_sufix + " directory."
    args = "[appname]"
    label = 'application name'

    requires_model_validation = False
    # Can't import settings during this command, because they haven't
    # necessarily been created.
    can_import_settings = False

    def handle_label(self, app_name, directory=None, **options):
        print("Calling me")
        if directory is None:
            directory = os.getcwd()

        # Determine the project_name by using the basename of directory,
        # which should be the full path of the project directory (or the
        # current directory if no directory was passed).
        project_name = os.path.basename(directory)
        if app_name == project_name:
            raise CommandError("You cannot create an app with the same name"
                               " (%r) as your project." % app_name)

        # Check that the app_name cannot be imported.
        try:
            import_module(app_name)
        except ImportError:
            pass
        else:
            raise CommandError("%r conflicts with the name of an existing Python module and cannot be used as an app name. Please try another name." % app_name)

        copy_helper(self.style, 'app', app_name, join(directory, self.directory_sufix), project_name)

class ProjectCommand(Command):
    help = ("Creates a Django app directory structure for the given app name"
            " in this project's directory.")

    def __init__(self, project_directory):
        super(ProjectCommand, self).__init__()
        self.project_directory = project_directory

    def handle_label(self, app_name, **options):
        super(ProjectCommand, self).handle_label(app_name, self.project_directory, **options)
<commit_msg>remvoe debug print<commit_after>import os
from os.path import join

from django.core.management.base import copy_helper, CommandError, LabelCommand
from django.utils.importlib import import_module

class Command(LabelCommand):
    directory_sufix = 'apps/'
    help = "Creates a Django app directory structure for the given app name in the " + directory_sufix + " directory."
    args = "[appname]"
    label = 'application name'

    requires_model_validation = False
    # Can't import settings during this command, because they haven't
    # necessarily been created.
    can_import_settings = False

    def handle_label(self, app_name, directory=None, **options):
        if directory is None:
            directory = os.getcwd()

        # Determine the project_name by using the basename of directory,
        # which should be the full path of the project directory (or the
        # current directory if no directory was passed).
        project_name = os.path.basename(directory)
        if app_name == project_name:
            raise CommandError("You cannot create an app with the same name"
                               " (%r) as your project." % app_name)

        # Check that the app_name cannot be imported.
        try:
            import_module(app_name)
        except ImportError:
            pass
        else:
            raise CommandError("%r conflicts with the name of an existing Python module and cannot be used as an app name. Please try another name." % app_name)

        copy_helper(self.style, 'app', app_name, join(directory, self.directory_sufix), project_name)

class ProjectCommand(Command):
    help = ("Creates a Django app directory structure for the given app name"
            " in this project's directory.")

    def __init__(self, project_directory):
        super(ProjectCommand, self).__init__()
        self.project_directory = project_directory

    def handle_label(self, app_name, **options):
        super(ProjectCommand, self).handle_label(app_name, self.project_directory, **options)
<|endoftext|>


<commit_before>from django.contrib import admin

from academics.models import Dorm, AcademicYear

class DormAdmin(admin.ModelAdmin):
    filter_horizontal = ['heads']

class AcademicYearAdmin(admin.ModelAdmin):
    fields = ['year', 'current']
    list_display = ['year', 'current']
    readonly_fields = ['year']

# Register your models here.
admin.site.register(Dorm, DormAdmin)
admin.site.register(AcademicYear, AcademicYearAdmin)<commit_msg>Admin for all of academics, read only for most of it<commit_after>from django.contrib import admin

from academics.models import Dorm, AcademicYear, Student, Teacher, Enrollment, Course, Section

class ReadOnlyAdmin(admin.ModelAdmin):
    def get_readonly_fields(self, request, obj=None):
        if self.declared_fieldsets:
            return flatten_fieldsets(self.declared_fieldsets)
        else:
            return list(set(
                [field.name for field in self.opts.local_fields] +
                [field.name for field in self.opts.local_many_to_many]
            ))

class AcademicYearAdmin(ReadOnlyAdmin):
    fields = ['year', 'current']
    list_display = ['year', 'current']
    readonly_fields = ['year']

class StudentAdmin(ReadOnlyAdmin):
    list_display = ['student_id', 'first_name', 'last_name', 'email', 'current']
    list_filter = ['current']

class TeacherAdmin(ReadOnlyAdmin):
    list_display = ['teacher_id', 'first_name', 'last_name', 'email', 'active']
    list_filter = ['active']

class DormAdmin(admin.ModelAdmin):
    filter_horizontal = ['heads']

class EnrollmentAdmin(ReadOnlyAdmin):
    list_display = ['student', 'academic_year']
    list_filter = ['academic_year__current']

class CourseAdmin(ReadOnlyAdmin):
    list_display = ['number', 'course_name']

class SectionAdmin(ReadOnlyAdmin):
    list_display = ['csn', 'course', 'academic_year', 'teacher']

# Register your models here.
admin.site.register(Dorm, DormAdmin)
admin.site.register(AcademicYear, AcademicYearAdmin)
admin.site.register(Student, StudentAdmin)
admin.site.register(Teacher, TeacherAdmin)
admin.site.register(Enrollment, EnrollmentAdmin)
admin.site.register(Course, CourseAdmin)
admin.site.register(Section, SectionAdmin)
<|endoftext|>