<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Provide file objects for training and testing part-of-speech taggers."""

import codecs

from . import ten

def to_unicode_or_bust(obj, encoding='utf-8'):
    ## function written by Kumar McMillan ( http://farmdev.com/talks/unicode )
    """Ensure that an object is unicode."""
    if isinstance(obj, basestring):
        if not isinstance(obj, unicode):
            obj = unicode(obj, encoding)
    return obj

class BaseFile(object):
    """Base file type from which training and testing files are derived."""

    def __init__(self, file_name, separator="_", ws_delim=True):
        """Initialize the file object.

           Parameters
           ----------
             file_name (str) : name of the file, with extension
             separator (str) : character used in the file to separate
               words from their POS tags, e.g.:
                   'table/NN' -- separator is '/'
                   'table_NN' -- separator is '_'
             ws_delim (boolean) : is the file already whitespace-delimited?
               if yes, then True. Example:
                   Tursun_Npr ._PUNCT
               if no, then False. Example:
                   Tursun_Npr._PUNCT
        """
        self.file_name = file_name
        self.sep = separator
        self.ws_delim = ws_delim
        ## dict to organize files used for training and testing
        self.all_files = {}

    def contents(self):
        """Create a list of sentences from the provided file.

           The result list is structured as follows:
                [
                    (0, u'Sen_PN2si sening_PN2si.Gen ma...'),
                    (1, u'Sen_PN2si mantini_N-ACC yedim...'),
                    (2, u'Sen_PN2si manta_N yegenliking...'),
                ]
        """
        with open(self.file_name) as f:
            self.raw_content = [to_unicode_or_bust(l) for l in f.readlines()]

        return [(idx, ln[:-1]) for idx, ln in enumerate(self.raw_content)]

    def to_string(self):
        """Output the BaseFile object as a single unicode string."""
        return u"\n".join([i[1] for i in self.contents()])

    def write(self, save_name=None):
        """Write the training file to disk."""
        if save_name == None:
            save_name = self.file_name
        with codecs.open(save_name, mode='w+', encoding='utf-8') as stream:
            stream.write(self.to_string())

    def groups(self, num_of_groups=10):
        """Split the file into ten randomly assigned groups.

           Parameters
           ----------
             num_of_groups (int) : the number of groups to be formed
        """
        return [g for g in ten.create_groups(len(self.contents()),
                                             n=num_of_groups)]

    def split_groups(self, num_of_groups=10):
        """Split the file into training and test files."""
        # test_01 is the test file containing group 01
        # train_01 is the training file containing all groups EXCEPT 1
        i = 1
        group_list = self.groups()
        while i <= num_of_groups:
            idx = str(i).rjust(2, '0')
            test_name = "test_{}.txt".format(idx)
            train_name = "train_{}.train".format(idx)

            # form lists (remember that they're zero-indexed!)
            test_list = group_list[i-1]
            # first produce a nested list of non-i lists
            train_list_nested = [group_list[j] for j in range(
                len(group_list)) if j != i-1]
            # then collapse nested list into flat list
            train_list = [k for l in train_list_nested for k in l]

            test_sentences = []
            train_sentences = []
            for position, sentence in self.contents():
                if position in test_list:
                    test_sentences.append((position, sentence))
                else:
                    train_sentences.append((position, sentence))
            with codecs.open(test_name, 'w+', encoding='utf-8') as test:
                test.write(u"\n".join([s[1] for s in test_sentences]))
            with codecs.open(train_name, 'w+', encoding='utf-8') as train:
                train.write(u"\n".join([s[1] for s in train_sentences]))
            # add index and matching filenames to all_files dict
            self.all_files[idx] = (test_name, train_name)
            i += 1

class TrainingFile(BaseFile):
    """Training file consisting of hand-tagged sentences."""

    def __init__(self, file_name, separator='_', idx=1):
        """Initialize the TrainingFile object.

           Parameters
           ----------
             idx (str) : index used to identify the subset of the original
               BaseFile.
        """
        BaseFile.__init__(self, file_name, separator)
        # one-digit numbers should be prefaced with a leading zero
        self.idx = str(idx).rjust(2, '0')

    def __str__(self):
        """Provide a human-readable representation of the object.

           Returns two measurements of the size of the training file:
               number of sentences and total number of tokens.
        """
        summary = 'TrainingFile with {} total sentences and {} total tokens'
        return summary.format(len(self.contents()),
                sum([len(i[1].split(' ')) for i in self.contents()]))

class TestingOutputFile(BaseFile):
    """POS Tagging Results file, the accuracy of which we want to measure."""

    def __init__(self, file_name, separator='_', idx=1):
        """Initialize the TestingOutputFile object."""
        BaseFile.__init__(self, file_name, separator)
        # one-digit numbers should be prefaced with a leading zero
        self.idx = str(idx).rjust(2, '0')

<commit_msg>simplify group numbering in files.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Provide file objects for training and testing part-of-speech taggers."""

import codecs

from . import ten

def to_unicode_or_bust(obj, encoding='utf-8'):
    ## function written by Kumar McMillan ( http://farmdev.com/talks/unicode )
    """Ensure that an object is unicode."""
    if isinstance(obj, basestring):
        if not isinstance(obj, unicode):
            obj = unicode(obj, encoding)
    return obj

class BaseFile(object):
    """Base file type from which training and testing files are derived."""

    def __init__(self, file_name, separator="_", ws_delim=True,
                 number_of_groups=10):
        """Initialize the file object.

           Parameters
           ----------
             file_name (str) : name of the file, with extension
             separator (str) : character used in the file to separate
               words from their POS tags, e.g.:
                   'table/NN' -- separator is '/'
                   'table_NN' -- separator is '_'
             ws_delim (boolean) : is the file already whitespace-delimited?
               if yes, then True. Example:
                   Tursun_Npr ._PUNCT
               if no, then False. Example:
                   Tursun_Npr._PUNCT
             number_of_groups (int) : the number of groups that the file
               will be split into (for cross-validation)
        """
        self.file_name = file_name
        self.sep = separator
        self.ws_delim = ws_delim
        self.num_groups = number_of_groups
        ## dict to organize files used for training and testing
        self.all_files = {}

    def contents(self):
        """Create a list of sentences from the provided file.

           The result list is structured as follows:
                [
                    (0, u'Sen_PN2si sening_PN2si.Gen ma...'),
                    (1, u'Sen_PN2si mantini_N-ACC yedim...'),
                    (2, u'Sen_PN2si manta_N yegenliking...'),
                ]
        """
        with open(self.file_name) as f:
            self.raw_content = [to_unicode_or_bust(l) for l in f.readlines()]

        return [(idx, ln[:-1]) for idx, ln in enumerate(self.raw_content)]

    def to_string(self):
        """Output the BaseFile object as a single unicode string."""
        return u"\n".join([i[1] for i in self.contents()])

    def write(self, save_name=None):
        """Write the training file to disk."""
        if save_name == None:
            save_name = self.file_name
        with codecs.open(save_name, mode='w+', encoding='utf-8') as stream:
            stream.write(self.to_string())

    def groups(self, num_of_groups=None):
        """Split the file into ten randomly assigned groups.

           Parameters
           ----------
             num_of_groups (int) : the number of groups to be formed
        """
        if num_of_groups == None:
            num_of_groups = self.num_groups
        return [g for g in ten.create_groups(len(self.contents()),
                                             n=num_of_groups)]

    def split_groups(self, num_of_groups=None, verbose=True):
        """Split the file into training and test files.

           Parameters
           ----------
             num_of_groups (int) : the number of groups to be formed
             verbose (Boolean) : print the contents of each file
               to the console (if True), or not (if False)
        """
        if num_of_groups == None:
            num_of_groups = self.num_groups
        # test_01 is the test file containing group 01
        # train_01 is the training file containing all groups EXCEPT 1
        i = 1
        group_list = self.groups()
        while i <= num_of_groups:
            idx = str(i).rjust(2, '0')
            test_name = "test_{}.txt".format(idx)
            train_name = "train_{}.train".format(idx)

            # form lists (remember that they're zero-indexed!)
            test_list = group_list[i-1]
            # first produce a nested list of non-i lists
            train_list_nested = [group_list[j] for j in range(
                len(group_list)) if j != i-1]
            # then collapse nested list into flat list
            train_list = [k for l in train_list_nested for k in l]

            test_sentences = []
            train_sentences = []
            for position, sentence in self.contents():
                if position in test_list:
                    test_sentences.append((position, sentence))
                else:
                    train_sentences.append((position, sentence))

            if verbose == True:
                print u"TEST SENTENCES:"
                for s in test_sentences:
                    print u"\t{}\t{}".format(s[0], s[1])
                print u"\nTRAINING SENTENCES:"
                for s in train_sentences:
                    print u"\t{}\t{}".format(s[0], s[1])

            with codecs.open(test_name, 'w+', encoding='utf-8') as test:
                test.write(u"\n".join([s[1] for s in test_sentences]))
            with codecs.open(train_name, 'w+', encoding='utf-8') as train:
                train.write(u"\n".join([s[1] for s in train_sentences]))
            # add index and matching filenames to all_files dict
            self.all_files[idx] = (test_name, train_name)
            i += 1

class TrainingFile(BaseFile):
    """Training file consisting of hand-tagged sentences."""

    def __init__(self, file_name, separator='_', idx=1):
        """Initialize the TrainingFile object.

           Parameters
           ----------
             idx (str) : index used to identify the subset of the original
               BaseFile.
        """
        BaseFile.__init__(self, file_name, separator)
        # one-digit numbers should be prefaced with a leading zero
        self.idx = str(idx).rjust(2, '0')

    def __str__(self):
        """Provide a human-readable representation of the object.

           Returns two measurements of the size of the training file:
               number of sentences and total number of tokens.
        """
        summary = 'TrainingFile with {} total sentences and {} total tokens'
        return summary.format(len(self.contents()),
                sum([len(i[1].split(' ')) for i in self.contents()]))

class TestingOutputFile(BaseFile):
    """POS Tagging Results file, the accuracy of which we want to measure."""

    def __init__(self, file_name, separator='_', idx=1):
        """Initialize the TestingOutputFile object."""
        BaseFile.__init__(self, file_name, separator)
        # one-digit numbers should be prefaced with a leading zero
        self.idx = str(idx).rjust(2, '0')

<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
# -*- coding: utf-8 -*-
"""
hpsspy.validate
~~~~~~~~~~~~~~~

Functions for testing the validity of backup configurations before
actually running the backup.
"""
#
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

def main():
    """Entry-point for command-line scripts.

    Returns
    -------
    :class:`int`
        An integer suitable for passing to :func:`sys.exit`.
    """
    import logging
    import re
    import json
    from os import environ, stat
    from os.path import basename, exists, isdir, join, splitext
    from sys import argv
    from argparse import ArgumentParser
    from . import __version__ as hpsspyVersion
    from .scan import compile_map, process_missing, scan_disk
    desc = 'Validate HPSS backup configuration.'
    parser = ArgumentParser(prog=basename(argv[0]), description=desc)
    parser.add_argument('-c', '--cache-dir', action='store', dest='cache',
                        metavar='DIR',
                        default=join(environ['HOME'], 'scratch'),
                        help=('Write cache files to DIR (Default: ' +
                              '%(default)s).'))
    parser.add_argument('-D', '--clobber-disk', action='store_true',
                        dest='clobber_disk',
                        help='Ignore any existing disk cache files.')
    parser.add_argument('-l', '--size-limit', action='store', type=int,
                        dest='limit', metavar='N', default=1024,
                        help=("Do not allow archive files larger than " +
                              "N GB (Default: %(default)s GB)."))
    parser.add_argument('-r', '--report', action='store', type=int,
                        dest='report', metavar='N', default=10000,
                        help=("Print an informational message after " +
                              "every N files (Default: %(default)s)."))
    parser.add_argument('-s', '--scan-files', action='store',
                        metavar='RELEASE',
                        dest='release',
                        help=('Validate the configuration file against ' +
                              'files on disk (might be slow!).'))
    parser.add_argument('-v', '--verbose', action='store_true',
                        dest='verbose',
                        help="Increase verbosity.")
    parser.add_argument('-V', '--version', action='version',
                        version="%(prog)s " + hpsspyVersion)
    parser.add_argument('config', metavar='FILE',
                        help="Read configuration from FILE.")
    options = parser.parse_args()
    #
    # Logging
    #
    ll = logging.INFO
    # if options.test or options.verbose:
    if options.verbose:
        ll = logging.DEBUG
    log_format = '%(asctime)s %(name)s %(levelname)s: %(message)s'
    logging.basicConfig(level=ll, format=log_format,
                        datefmt='%Y-%m-%dT%H:%M:%S')
    logger = logging.getLogger(__name__)
    #
    # Test validity of JSON file.
    #
    foo, xtn = splitext(basename(options.config))
    if xtn != '.json':
        logger.warn("%s might not be a JSON file!", options.config)
    try:
        with open(options.config) as fp:
            try:
                json_data = json.load(fp)
            except json.JSONDecodeError:
                logger.critical("%s is not valid JSON.", options.config)
                return 1
    except FileNotFoundError:
        logger.critical("%s does not exist. Try again.", options.config)
        return 1
    if 'config' in json_data:
        for k in ('root', 'hpss_root', 'physical_disks'):
            if k not in json_data['config']:
                logger.warning("%s 'config' section does not contain an " +
                               "entry for '%s'.", options.config, k)
    else:
        logger.error("%s does not contain a 'config' section.", options.config)
    for k in json_data:
        if k == 'config':
            continue
        if 'exclude' not in json_data[k]:
            logger.warning("Section '%s' should at least have an " +
                           "'exclude' entry.", k)
        try:
            new_map = compile_map(json_data, k)
        except re.error:
            logger.critical("Regular Expression error detected in " +
                            "section '%s'!", k)
            return 1
    #
    # Now check the JSON file against files on disk.
    #
    if options.release:
        logger.info("You have chosen to validate the configuration " +
                    "of '%s' against files on disk.  This could be slow.",
                    options.release)
        hpss_map = compile_map(json_data, options.release)
        config = json_data['config']
        release_root = join(config['root'], options.release)
        hpss_release_root = join(config['hpss_root'], options.release)
        if isdir(release_root):
            #
            # Read disk files and cache.
            #
            disk_files_cache = join(options.cache,
                                    ('disk_files_' +
                                     '{0}.txt').format(options.release))
            logger.debug("disk_files_cache = '%s'", disk_files_cache)
            disk_roots = [release_root.replace(basename(config['root']), d)
                          for d in config['physical_disks']]
            status = scan_disk(disk_roots, disk_files_cache,
                               clobber=options.clobber_disk)
            if not status:
                return 1
            #
            # Now that we've got the disk cache, assume HPSS is empty, and
            # make sure all files get backed up.
            #
            nfiles = 0
            nmissing = 0
            nmultiple = 0
            mapped_to_hpss = dict()
            hpss_size = dict()
            with open(disk_files_cache) as t:
                for l in t:
                    f = l.strip()
                    if f in hpss_map["exclude"]:
                        logger.info("%s skipped.", f)
                    else:
                        section = f.split('/')[0]
                        try:
                            s = hpss_map[section]
                        except KeyError:
                            #
                            # If the section is not described, that's not
                            # good, but continue.
                            #
                            logger.error("%s is in a directory not " +
                                         "described in the configuration!",
                                         f)
                            continue
                        #
                        # If the section is blank, that's OK.
                        #
                        if not s:
                            logger.info("%s is in a directory not yet " +
                                        "configured.",
                                        f)
                            continue
                        #
                        # Now check if it is mapped.
                        #
                        mapped = 0
                        f_size = stat(join(release_root, f)).st_size
                        for r in s:
                            m = r[0].match(f)
                            if m is not None:
                                reName = r[0].sub(r[1], f)
                                if reName in mapped_to_hpss:
                                    mapped_to_hpss[reName].append(f)
                                    hpss_size[reName] += f_size
                                else:
                                    mapped_to_hpss[reName] = [f]
                                    hpss_size[reName] = f_size
                                mapped += 1
                                logger.debug("%s in %s.", f, reName)
                        if mapped == 0:
                            logger.error("%s is not mapped to any file on " +
                                         "HPSS!", f)
                            nmissing += 1
                        if mapped > 1:
                            logger.error("%s is mapped to multiple files on " +
                                         "HPSS!", f)
                            nmultiple += 1
                    nfiles += 1
                    if (nfiles % options.report) == 0:
                        logger.info("%9d files scanned.", nfiles)
            missing_files_cache = join(options.cache,
                                       ('missing_files_{0}' +
                                        '.json').format(options.release))
            logger.debug("missing_files_cache = '%s'", missing_files_cache)
            with open(missing_files_cache, 'w') as fp:
                json.dump(mapped_to_hpss, fp, indent=2, separators=(',', ': '))
            if nmissing > 0:
                logger.critical("Not all files would be backed up with " +
                                "this configuration!")
                return 1
            if nmultiple > 0:
                logger.critical("Some files would be backed up more than " +
                                "once with this configuration!")
                return 1
            for k in hpss_size:
                logger.info('%k is %d bytes.', k, hpss_size[k])
                if hpss_size[k]/1024/1024/1024 > options.limit:
                    logger.critical("HPSS file %s would be too large!", k)
            #
            # All files map to a file on HPSS, so print out the commands
            # that would do a full backup.
            #
            logger.setLevel(logging.DEBUG)
            process_missing(missing_files_cache, release_root,
                            hpss_release_root, test=True)
        else:
            logger.critical("%s does not exist!", release_root)
            return 1
    return 0
<commit_msg>fix log message [skip ci]<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
# -*- coding: utf-8 -*-
"""
hpsspy.validate
~~~~~~~~~~~~~~~

Functions for testing the validity of backup configurations before
actually running the backup.
"""
#
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

def main():
    """Entry-point for command-line scripts.

    Returns
    -------
    :class:`int`
        An integer suitable for passing to :func:`sys.exit`.
    """
    import logging
    import re
    import json
    from os import environ, stat
    from os.path import basename, exists, isdir, join, splitext
    from sys import argv
    from argparse import ArgumentParser
    from . import __version__ as hpsspyVersion
    from .scan import compile_map, process_missing, scan_disk
    desc = 'Validate HPSS backup configuration.'
    parser = ArgumentParser(prog=basename(argv[0]), description=desc)
    parser.add_argument('-c', '--cache-dir', action='store', dest='cache',
                        metavar='DIR',
                        default=join(environ['HOME'], 'scratch'),
                        help=('Write cache files to DIR (Default: ' +
                              '%(default)s).'))
    parser.add_argument('-D', '--clobber-disk', action='store_true',
                        dest='clobber_disk',
                        help='Ignore any existing disk cache files.')
    parser.add_argument('-l', '--size-limit', action='store', type=int,
                        dest='limit', metavar='N', default=1024,
                        help=("Do not allow archive files larger than " +
                              "N GB (Default: %(default)s GB)."))
    parser.add_argument('-r', '--report', action='store', type=int,
                        dest='report', metavar='N', default=10000,
                        help=("Print an informational message after " +
                              "every N files (Default: %(default)s)."))
    parser.add_argument('-s', '--scan-files', action='store',
                        metavar='RELEASE',
                        dest='release',
                        help=('Validate the configuration file against ' +
                              'files on disk (might be slow!).'))
    parser.add_argument('-v', '--verbose', action='store_true',
                        dest='verbose',
                        help="Increase verbosity.")
    parser.add_argument('-V', '--version', action='version',
                        version="%(prog)s " + hpsspyVersion)
    parser.add_argument('config', metavar='FILE',
                        help="Read configuration from FILE.")
    options = parser.parse_args()
    #
    # Logging
    #
    ll = logging.INFO
    # if options.test or options.verbose:
    if options.verbose:
        ll = logging.DEBUG
    log_format = '%(asctime)s %(name)s %(levelname)s: %(message)s'
    logging.basicConfig(level=ll, format=log_format,
                        datefmt='%Y-%m-%dT%H:%M:%S')
    logger = logging.getLogger(__name__)
    #
    # Test validity of JSON file.
    #
    foo, xtn = splitext(basename(options.config))
    if xtn != '.json':
        logger.warn("%s might not be a JSON file!", options.config)
    try:
        with open(options.config) as fp:
            try:
                json_data = json.load(fp)
            except json.JSONDecodeError:
                logger.critical("%s is not valid JSON.", options.config)
                return 1
    except FileNotFoundError:
        logger.critical("%s does not exist. Try again.", options.config)
        return 1
    if 'config' in json_data:
        for k in ('root', 'hpss_root', 'physical_disks'):
            if k not in json_data['config']:
                logger.warning("%s 'config' section does not contain an " +
                               "entry for '%s'.", options.config, k)
    else:
        logger.error("%s does not contain a 'config' section.", options.config)
    for k in json_data:
        if k == 'config':
            continue
        if 'exclude' not in json_data[k]:
            logger.warning("Section '%s' should at least have an " +
                           "'exclude' entry.", k)
        try:
            new_map = compile_map(json_data, k)
        except re.error:
            logger.critical("Regular Expression error detected in " +
                            "section '%s'!", k)
            return 1
    #
    # Now check the JSON file against files on disk.
    #
    if options.release:
        logger.info("You have chosen to validate the configuration " +
                    "of '%s' against files on disk.  This could be slow.",
                    options.release)
        hpss_map = compile_map(json_data, options.release)
        config = json_data['config']
        release_root = join(config['root'], options.release)
        hpss_release_root = join(config['hpss_root'], options.release)
        if isdir(release_root):
            #
            # Read disk files and cache.
            #
            disk_files_cache = join(options.cache,
                                    ('disk_files_' +
                                     '{0}.txt').format(options.release))
            logger.debug("disk_files_cache = '%s'", disk_files_cache)
            disk_roots = [release_root.replace(basename(config['root']), d)
                          for d in config['physical_disks']]
            status = scan_disk(disk_roots, disk_files_cache,
                               clobber=options.clobber_disk)
            if not status:
                return 1
            #
            # Now that we've got the disk cache, assume HPSS is empty, and
            # make sure all files get backed up.
            #
            nfiles = 0
            nmissing = 0
            nmultiple = 0
            mapped_to_hpss = dict()
            hpss_size = dict()
            with open(disk_files_cache) as t:
                for l in t:
                    f = l.strip()
                    if f in hpss_map["exclude"]:
                        logger.info("%s skipped.", f)
                    else:
                        section = f.split('/')[0]
                        try:
                            s = hpss_map[section]
                        except KeyError:
                            #
                            # If the section is not described, that's not
                            # good, but continue.
                            #
                            logger.error("%s is in a directory not " +
                                         "described in the configuration!",
                                         f)
                            continue
                        #
                        # If the section is blank, that's OK.
                        #
                        if not s:
                            logger.info("%s is in a directory not yet " +
                                        "configured.",
                                        f)
                            continue
                        #
                        # Now check if it is mapped.
                        #
                        mapped = 0
                        f_size = stat(join(release_root, f)).st_size
                        for r in s:
                            m = r[0].match(f)
                            if m is not None:
                                reName = r[0].sub(r[1], f)
                                if reName in mapped_to_hpss:
                                    mapped_to_hpss[reName].append(f)
                                    hpss_size[reName] += f_size
                                else:
                                    mapped_to_hpss[reName] = [f]
                                    hpss_size[reName] = f_size
                                mapped += 1
                                logger.debug("%s in %s.", f, reName)
                        if mapped == 0:
                            logger.error("%s is not mapped to any file on " +
                                         "HPSS!", f)
                            nmissing += 1
                        if mapped > 1:
                            logger.error("%s is mapped to multiple files on " +
                                         "HPSS!", f)
                            nmultiple += 1
                    nfiles += 1
                    if (nfiles % options.report) == 0:
                        logger.info("%9d files scanned.", nfiles)
            missing_files_cache = join(options.cache,
                                       ('missing_files_{0}' +
                                        '.json').format(options.release))
            logger.debug("missing_files_cache = '%s'", missing_files_cache)
            with open(missing_files_cache, 'w') as fp:
                json.dump(mapped_to_hpss, fp, indent=2, separators=(',', ': '))
            if nmissing > 0:
                logger.critical("Not all files would be backed up with " +
                                "this configuration!")
                return 1
            if nmultiple > 0:
                logger.critical("Some files would be backed up more than " +
                                "once with this configuration!")
                return 1
            for k in hpss_size:
                logger.info('%s is %d bytes.', k, hpss_size[k])
                if hpss_size[k]/1024/1024/1024 > options.limit:
                    logger.critical("HPSS file %s would be too large!", k)
                    return 1
            #
            # All files map to a file on HPSS, so print out the commands
            # that would do a full backup.
            #
            logger.setLevel(logging.DEBUG)
            process_missing(missing_files_cache, release_root,
                            hpss_release_root, test=True)
        else:
            logger.critical("%s does not exist!", release_root)
            return 1
    return 0
<|endoftext|>


<commit_before>metadata = {
    "netParams": {
        "label": "Net Params",
        "suggestions": "",
        "help": "",
        "hintText": "",
        "children": {
            "popParams": {
                "label": "Population Params",
                "suggestions": "",
                "help": "",
                "hintText": "",
                "children": {
                    "cellType": {
                        "label": "Cell Type",
                        "suggestions": "",
                        "help": "Arbitrary cell type attribute/tag assigned to all cells in this population; can be used as condition to apply specific cell properties. e.g. 'Pyr' (for pyramidal neurons) or 'FS' (for fast-spiking interneurons)",
                        "hintText": "",
                        "type": "str"
                    },
                    "numCells": {
                        "label": "Population Dimensions",
                        "suggestions": "",
                        "help": "The total number of cells in this population, the density in neurons/mm3, or the fixed grid spacing (only one of the three is required). The volume occupied by each population can be customized (see xRange, yRange and zRange); otherwise the full network volume will be used (defined in netParams: sizeX, sizeY, sizeZ). density can be expressed as a function of normalized location (xnorm, ynorm or znorm), by providing a string with the variable and any common Python mathematical operators/functions. e.g. '1e5 * exp(-ynorm/2)'. gridSpacing is the spacing between cells (in um). The total number of cells will be determined based on spacing and sizeX, sizeY, sizeZ. e.g. 10.",
                        "hintText": "number of cells",
                        "type": "int"
                    },
                    "density": {
                        "label": "Density or Grid Spacing",
                        "suggestions": "",
                        "help": "The total number of cells in this population, the density in neurons/mm3, or the fixed grid spacing (only one of the three is required). The volume occupied by each population can be customized (see xRange, yRange and zRange); otherwise the full network volume will be used (defined in netParams: sizeX, sizeY, sizeZ). density can be expressed as a function of normalized location (xnorm, ynorm or znorm), by providing a string with the variable and any common Python mathematical operators/functions. e.g. '1e5 * exp(-ynorm/2)'. gridSpacing is the spacing between cells (in um). The total number of cells will be determined based on spacing and sizeX, sizeY, sizeZ. e.g. 10.",
                        "hintText": "density in neurons/mm3",
                        "type": "str"
                    },
                    "gridSpacing": {
                        "label": "Density or Grid Spacing",
                        "suggestions": "",
                        "help": "The total number of cells in this population, the density in neurons/mm3, or the fixed grid spacing (only one of the three is required). The volume occupied by each population can be customized (see xRange, yRange and zRange); otherwise the full network volume will be used (defined in netParams: sizeX, sizeY, sizeZ). density can be expressed as a function of normalized location (xnorm, ynorm or znorm), by providing a string with the variable and any common Python mathematical operators/functions. e.g. '1e5 * exp(-ynorm/2)'. gridSpacing is the spacing between cells (in um). The total number of cells will be determined based on spacing and sizeX, sizeY, sizeZ. e.g. 10.",
                        "hintText": "fixed grid spacing",
                        "type": "int"
                    },
                    "cellModel": {
                        "label": "Cell Model",
                        "help": "Arbitrary cell model attribute/tag assigned to all cells in this population; can be used as condition to apply specific cell properties. e.g. 'HH' (standard Hodkgin-Huxley type cell model) or 'Izhi2007' (Izhikevich2007 point neuron model).",
                        "suggestions": ['VecStim', 'NetStim', 'IntFire1'],
                        "type": "str"
                    },
                    "xRange": {
                        "label": "X Range",
                        "help": "Range of neuron positions in x-axis (horizontal length), specified2-element list[min, max]. xRange for absolute value in um (e.g.[100, 200]), or xnormRange for normalized value between0 and1 as fraction of sizeX (e.g.[0.1, 0.2]).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "xnormRange": {
                        "label": "X Norm Range",
                        "help": "Range of neuron positions in x-axis (horizontal length), specified2-element list[min, max]. xRange for absolute value in um (e.g.[100, 200]), or xnormRange for normalized value between0 and1 as fraction of sizeX (e.g.[0.1,0.2]).",
                        "suggestions": "",
                        "hintText": "",
                        "default": [0, 1],
                        "type": "list(float)"
                    },
                    "yRange": {
                        "label": "Y Range",
                        "help": "Range of neuron positions in y-axis (vertical height=cortical depth), specified2-element list[min, max].yRange for absolute value in um (e.g.[100,200]), or ynormRange for normalized value between0 and1 as fraction of sizeY (e.g.[0.1,0.2]).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "ynormRange": {
                        "label": "Y Norm Range",
                        "help": "Range of neuron positions in y-axis (vertical height=cortical depth), specified2-element list[min, max]. yRange for absolute value in um (e.g.[100,200]), or ynormRange for normalized value between0 and1 as fraction of sizeY (e.g.[0.1,0.2]).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "zRange": {
                        "label": "Z Range",
                        "help": "Range of neuron positions in z-axis (horizontal depth), specified2-element list[min, max]. zRange for absolute value in um (e.g.[100,200]), or znormRange for normalized value between0 and1 as fraction of sizeZ (e.g.[0.1,0.2]).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "znormRange": {
                        "label": "Z Norm Range",
                        "help": "Range of neuron positions in z-axis (horizontal depth), specified2-element list[min, max]. zRange for absolute value in um (e.g.[100,200]), or znormRange for normalized value between0 and1 as fraction of sizeZ (e.g.[0.1,0.2]).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "interval": {
                        "label": "Spike Interval",
                        "help": "Spike interval in ms.",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "rate": {
                        "label": "Rate",
                        "help": "Firing rate in Hz (note this is the inverse of the NetStim interval property).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "noise": {
                        "label": "Noise",
                        "help": "Fraction of noise in NetStim (0 = deterministic; 1 = completely random).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "start": {
                        "label": "Start",
                        "help": "Time of first spike in ms (default = 0).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "number": {
                        "label": "Number",
                        "help": "Max number of spikes generated (default = 1e12).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "seed": {
                        "label": "Seed",
                        "help": " Seed for randomizer (optional; defaults to value set in simConfig.seeds['stim'])",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "spkTimes": {
                        "label": "Spike Times",
                        "help": "Spike Times(only for ‘VecStim’) - List of spike times (e.g. [1, 10, 40, 50], range(1,500,10), or any variable containing a Python list).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "pulses": {
                        "label": "Pulses",
                        "help": "(only for ‘VecStim’) - List of spiking pulses; each item includes the start (ms), end (ms), rate (Hz), and noise (0 to 1) pulse parameters. ",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    }
                }
            },
            "cellParams": {
                "label": "Cell Params",
                "suggestions": "",
                "help": "",
                "hintText": "",
                "children": {
                    "conds": {
                        "label": "Conds",
                        "suggestions": "",
                        "help": "",
                        "hintText": "",
                        "children": {
                            "cellType": {
                                "label": "Cell Type",
                                "suggestions": "",
                                "help": "",
                                "hintText": "",
                            },
                            "cellModel": {
                                "label": "Cell Model",
                                "suggestions": "",
                                "help": "",
                                "hintText": "",
                            },
                        }
                    },
                    "secs": {
                        "label": "Secs",
                        "suggestions": "",
                        "help": "",
                        "hintText": "",
                        "children": {
                            "geom": {
                                "label": "Cell Type",
                                "suggestions": "",
                                "help": "",
                                "hintText": "",
                                "children": {
                                    "diam": {
                                        "label": "Diameter",
                                        "suggestions": "",
                                        "help": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "L": {
                                        "label": "Length",
                                        "suggestions": "",
                                        "help": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "Ra": {
                                        "label": "Ra",
                                        "suggestions": "",
                                        "help": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "cm": {
                                        "label": "cm",
                                        "suggestions": "",
                                        "help": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "pt3d": {
                                        "label": "pt3d",
                                        "suggestions": "",
                                        "help": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "nseg": {
                                        "label": "nseg",
                                        "suggestions": "",
                                        "help": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                },
                            "topol": {
                                "label": "Topology",
                                "help": "Dictionary with topology properties.Includes parentSec (label of parent section), parentX (parent location where to make connection) and childX (current section –child– location where to make connection).",
                                "suggestions": "",
                                "hintText": "",
                                },
                            "mechs": {
                                "label": "Mechanisms",
                                "help": "Dictionary of density/distributed mechanisms.The key contains the name of the mechanism (e.g. hh or pas) The value contains a dictionary with the properties of the mechanism (e.g. {'g': 0.003, 'e': -70}).",
                                "suggestions": "",
                                "hintText": "",
                                },
                            "ions": {
                                "label": "Ions",
                                "help": "Dictionary of ions.he key contains the name of the ion (e.g. na or k) The value contains a dictionary with the properties of the ion (e.g. {'e': -70}).",
                                "suggestions": "",
                                "hintText": "",
                                },
                            "mechs": {
                                "label": "Mechanisms",
                                "help": "Dictionary of density/distributed mechanisms.The key contains the name of the mechanism (e.g. hh or pas) The value contains a dictionary with the properties of the mechanism (e.g. {'g': 0.003, 'e': -70}).",
                                "suggestions": "",
                                "hintText": "",
                                },
                            "pointps": {
                                "label": "Point processes",
                                "help": "Dictionary of point processes (excluding synaptic mechanisms). The key contains an arbitrary label (e.g. ‘Izhi’) The value contains a dictionary with the point process properties (e.g. {'mod':'Izhi2007a', 'a':0.03, 'b':-2, 'c':-50, 'd':100, 'celltype':1}).",
                                "suggestions": "",
                                "hintText": "",
                                "children": {
                                    "mod": {
                                        "label": "mod",
                                        "help": "the name of the NEURON mechanism, e.g. 'Izhi2007a'",
                                        "suggestions": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "loc": {
                                        "label": "Length",
                                        "help": "section location where to place synaptic mechanism, e.g. 1.0, default=0.5.",
                                        "suggestions": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "vref": {
                                        "label": "vref (optional)",
                                        "help": "internal mechanism variable containing the cell membrane voltage, e.g. 'V'.",
                                        "suggestions": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "synList": {
                                        "label": "synList (optional)",
                                        "help": "list of internal mechanism synaptic mechanism labels, e.g. [‘AMPA’, ‘NMDA’, ‘GABAB’].",
                                        "suggestions": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                },
                            "vinit": {
                                "label": "vinit",
                                "help": "(optional) Initial membrane voltage (in mV) of the section (default: -65).e.g. cellRule['secs']['soma']['vinit'] = -72",
                                "suggestions": "",
                                "hintText": "",
                                },
                            "spikeGenLoc": {
                                "label": "spikeGenLoc",
                                "help": "(optional) Indicates that this section is responsible for spike generation (instead of the default ‘soma’), and provides the location (segment) where spikes are generated.e.g. cellRule['secs']['axon']['spikeGenLoc'] = 1.0.",
                                "suggestions": "",
                                "hintText": "",
                                },
                            "threshold": {
                                "label": "threshold",
                                "help": "(optional) Threshold voltage (in mV) used to detect a spike originating in this section of the cell. If omitted, defaults to netParams.defaultThreshold = 10.0.e.g. cellRule['secs']['soma']['threshold'] = 5.0.",
                                "suggestions": "",
                                "hintText": "",
                                },
                            },
                        "secLists": {
                            "label": "secLists - (optional) ",
                            "help": "Dictionary of sections lists (e.g. {‘all’: [‘soma’, ‘dend’]})",
                            "suggestions": "",
                            "hintText": "",
                            },
                        }
                    }
                }
            }
        }
    },
    "simConfig": {
        "label": "Sim Config",
        "suggestions": "",
        "help": "",
        "hintText": "",
        "children": {
            "simLabel":{
                "label": "Name of Simulation"
            },
            "duration": {
                "label": "Duration",
            },
            "dt": {
                "label": "Dt",
            },
            "seeds": {
                "label": "Seeds",
            },
            "addSynMechs": {
                "label": "Add Syn Mechs",
            },
            "includeParamsLabel": {
                "label": "Include Params Label",
            },
            "timing": {
                "label": "Timing",
            },
            "verbose": {
                "label": "Verbose",
            },
            "saveFolder": {
                "label": "Save Folder",
            },
            "filename": {
                "label": "File Name",
            },
            "saveDataInclude": {
                "label": "Save Data Include",
            },
            "timestampFilename": {
                "label": "Timestamp File Name",
            },
            "savePickle": {
                "label": "Save Pickle",
            },
            "saveJson": {
                "label": "Save Json",
            },
            "saveMat": {
                "label": "Save Mat",
            },
            "saveHDF5": {
                "label": "Save HDF5",
            },
            "saveDpk": {
                "label": "Save Dpk",
            },
            "saveDat": {
                "label": "Save Dat",
            },
            "saveCSV": {
                "label": "Save Csv",
            },
            "saveCellSecs": {
                "label": "Save Cell Secs",
            },
            "saveCellConns": {
                "label": "Save Cell Conns",
            },
            "checkErrors": {
                "label": "Check Errors",
            },
            "checkErrorsVerbose": {
                "label": "Check Errors Verbose",
            },
            "backupCfgFile": {
                "label": "Copy of CFG file"
            }

        }
    }
}
<commit_msg>removed special character backtick<commit_after>metadata = {
    "netParams": {
        "label": "Net Params",
        "suggestions": "",
        "help": "",
        "hintText": "",
        "children": {
            "popParams": {
                "label": "Population Params",
                "suggestions": "",
                "help": "",
                "hintText": "",
                "children": {
                    "cellType": {
                        "label": "Cell Type",
                        "suggestions": "",
                        "help": "Arbitrary cell type attribute/tag assigned to all cells in this population; can be used as condition to apply specific cell properties. e.g. 'Pyr' (for pyramidal neurons) or 'FS' (for fast-spiking interneurons)",
                        "hintText": "",
                        "type": "str"
                    },
                    "numCells": {
                        "label": "Population Dimensions",
                        "suggestions": "",
                        "help": "The total number of cells in this population, the density in neurons/mm3, or the fixed grid spacing (only one of the three is required). The volume occupied by each population can be customized (see xRange, yRange and zRange); otherwise the full network volume will be used (defined in netParams: sizeX, sizeY, sizeZ). density can be expressed as a function of normalized location (xnorm, ynorm or znorm), by providing a string with the variable and any common Python mathematical operators/functions. e.g. '1e5 * exp(-ynorm/2)'. gridSpacing is the spacing between cells (in um). The total number of cells will be determined based on spacing and sizeX, sizeY, sizeZ. e.g. 10.",
                        "hintText": "number of cells",
                        "type": "int"
                    },
                    "density": {
                        "label": "Density or Grid Spacing",
                        "suggestions": "",
                        "help": "The total number of cells in this population, the density in neurons/mm3, or the fixed grid spacing (only one of the three is required). The volume occupied by each population can be customized (see xRange, yRange and zRange); otherwise the full network volume will be used (defined in netParams: sizeX, sizeY, sizeZ). density can be expressed as a function of normalized location (xnorm, ynorm or znorm), by providing a string with the variable and any common Python mathematical operators/functions. e.g. '1e5 * exp(-ynorm/2)'. gridSpacing is the spacing between cells (in um). The total number of cells will be determined based on spacing and sizeX, sizeY, sizeZ. e.g. 10.",
                        "hintText": "density in neurons/mm3",
                        "type": "str"
                    },
                    "gridSpacing": {
                        "label": "Density or Grid Spacing",
                        "suggestions": "",
                        "help": "The total number of cells in this population, the density in neurons/mm3, or the fixed grid spacing (only one of the three is required). The volume occupied by each population can be customized (see xRange, yRange and zRange); otherwise the full network volume will be used (defined in netParams: sizeX, sizeY, sizeZ). density can be expressed as a function of normalized location (xnorm, ynorm or znorm), by providing a string with the variable and any common Python mathematical operators/functions. e.g. '1e5 * exp(-ynorm/2)'. gridSpacing is the spacing between cells (in um). The total number of cells will be determined based on spacing and sizeX, sizeY, sizeZ. e.g. 10.",
                        "hintText": "fixed grid spacing",
                        "type": "int"
                    },
                    "cellModel": {
                        "label": "Cell Model",
                        "help": "Arbitrary cell model attribute/tag assigned to all cells in this population; can be used as condition to apply specific cell properties. e.g. 'HH' (standard Hodkgin-Huxley type cell model) or 'Izhi2007' (Izhikevich2007 point neuron model).",
                        "suggestions": ['VecStim', 'NetStim', 'IntFire1'],
                        "type": "str"
                    },
                    "xRange": {
                        "label": "X Range",
                        "help": "Range of neuron positions in x-axis (horizontal length), specified2-element list[min, max]. xRange for absolute value in um (e.g.[100, 200]), or xnormRange for normalized value between0 and1 as fraction of sizeX (e.g.[0.1, 0.2]).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "xnormRange": {
                        "label": "X Norm Range",
                        "help": "Range of neuron positions in x-axis (horizontal length), specified2-element list[min, max]. xRange for absolute value in um (e.g.[100, 200]), or xnormRange for normalized value between0 and1 as fraction of sizeX (e.g.[0.1,0.2]).",
                        "suggestions": "",
                        "hintText": "",
                        "default": [0, 1],
                        "type": "list(float)"
                    },
                    "yRange": {
                        "label": "Y Range",
                        "help": "Range of neuron positions in y-axis (vertical height=cortical depth), specified2-element list[min, max].yRange for absolute value in um (e.g.[100,200]), or ynormRange for normalized value between0 and1 as fraction of sizeY (e.g.[0.1,0.2]).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "ynormRange": {
                        "label": "Y Norm Range",
                        "help": "Range of neuron positions in y-axis (vertical height=cortical depth), specified2-element list[min, max]. yRange for absolute value in um (e.g.[100,200]), or ynormRange for normalized value between0 and1 as fraction of sizeY (e.g.[0.1,0.2]).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "zRange": {
                        "label": "Z Range",
                        "help": "Range of neuron positions in z-axis (horizontal depth), specified2-element list[min, max]. zRange for absolute value in um (e.g.[100,200]), or znormRange for normalized value between0 and1 as fraction of sizeZ (e.g.[0.1,0.2]).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "znormRange": {
                        "label": "Z Norm Range",
                        "help": "Range of neuron positions in z-axis (horizontal depth), specified2-element list[min, max]. zRange for absolute value in um (e.g.[100,200]), or znormRange for normalized value between0 and1 as fraction of sizeZ (e.g.[0.1,0.2]).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "interval": {
                        "label": "Spike Interval",
                        "help": "Spike interval in ms.",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "rate": {
                        "label": "Rate",
                        "help": "Firing rate in Hz (note this is the inverse of the NetStim interval property).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "noise": {
                        "label": "Noise",
                        "help": "Fraction of noise in NetStim (0 = deterministic; 1 = completely random).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "start": {
                        "label": "Start",
                        "help": "Time of first spike in ms (default = 0).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "number": {
                        "label": "Number",
                        "help": "Max number of spikes generated (default = 1e12).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "seed": {
                        "label": "Seed",
                        "help": " Seed for randomizer (optional; defaults to value set in simConfig.seeds['stim'])",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "spkTimes": {
                        "label": "Spike Times",
                        "help": "Spike Times(only for 'VecStim') - List of spike times (e.g. [1, 10, 40, 50], range(1,500,10), or any variable containing a Python list).",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    },
                    "pulses": {
                        "label": "Pulses",
                        "help": "(only for 'VecStim') - List of spiking pulses; each item includes the start (ms), end (ms), rate (Hz), and noise (0 to 1) pulse parameters. ",
                        "suggestions": "",
                        "hintText": "",
                        "type": "list(float)"
                    }
                }
            },
            "cellParams": {
                "label": "Cell Params",
                "suggestions": "",
                "help": "",
                "hintText": "",
                "children": {
                    "conds": {
                        "label": "Conds",
                        "suggestions": "",
                        "help": "",
                        "hintText": "",
                        "children": {
                            "cellType": {
                                "label": "Cell Type",
                                "suggestions": "",
                                "help": "",
                                "hintText": "",
                            },
                            "cellModel": {
                                "label": "Cell Model",
                                "suggestions": "",
                                "help": "",
                                "hintText": "",
                            },
                        }
                    },
                    "secs": {
                        "label": "Secs",
                        "suggestions": "",
                        "help": "",
                        "hintText": "",
                        "children": {
                            "geom": {
                                "label": "Cell Type",
                                "suggestions": "",
                                "help": "",
                                "hintText": "",
                                "children": {
                                    "diam": {
                                        "label": "Diameter",
                                        "suggestions": "",
                                        "help": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "L": {
                                        "label": "Length",
                                        "suggestions": "",
                                        "help": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "Ra": {
                                        "label": "Ra",
                                        "suggestions": "",
                                        "help": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "cm": {
                                        "label": "cm",
                                        "suggestions": "",
                                        "help": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "pt3d": {
                                        "label": "pt3d",
                                        "suggestions": "",
                                        "help": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "nseg": {
                                        "label": "nseg",
                                        "suggestions": "",
                                        "help": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                },
                            "topol": {
                                "label": "Topology",
                                "help": "Dictionary with topology properties.Includes parentSec (label of parent section), parentX (parent location where to make connection) and childX (current section child location where to make connection).",
                                "suggestions": "",
                                "hintText": "",
                                },
                            "mechs": {
                                "label": "Mechanisms",
                                "help": "Dictionary of density/distributed mechanisms.The key contains the name of the mechanism (e.g. hh or pas) The value contains a dictionary with the properties of the mechanism (e.g. {'g': 0.003, 'e': -70}).",
                                "suggestions": "",
                                "hintText": "",
                                },
                            "ions": {
                                "label": "Ions",
                                "help": "Dictionary of ions.he key contains the name of the ion (e.g. na or k) The value contains a dictionary with the properties of the ion (e.g. {'e': -70}).",
                                "suggestions": "",
                                "hintText": "",
                                },
                            "mechs": {
                                "label": "Mechanisms",
                                "help": "Dictionary of density/distributed mechanisms.The key contains the name of the mechanism (e.g. hh or pas) The value contains a dictionary with the properties of the mechanism (e.g. {'g': 0.003, 'e': -70}).",
                                "suggestions": "",
                                "hintText": "",
                                },
                            "pointps": {
                                "label": "Point processes",
                                "help": "Dictionary of point processes (excluding synaptic mechanisms). The key contains an arbitrary label (e.g. ‘Izhi’) The value contains a dictionary with the point process properties (e.g. {'mod':'Izhi2007a', 'a':0.03, 'b':-2, 'c':-50, 'd':100, 'celltype':1}).",
                                "suggestions": "",
                                "hintText": "",
                                "children": {
                                    "mod": {
                                        "label": "mod",
                                        "help": "the name of the NEURON mechanism, e.g. 'Izhi2007a'",
                                        "suggestions": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "loc": {
                                        "label": "Length",
                                        "help": "section location where to place synaptic mechanism, e.g. 1.0, default=0.5.",
                                        "suggestions": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "vref": {
                                        "label": "vref (optional)",
                                        "help": "internal mechanism variable containing the cell membrane voltage, e.g. 'V'.",
                                        "suggestions": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                    "synList": {
                                        "label": "synList (optional)",
                                        "help": "list of internal mechanism synaptic mechanism labels, e.g. [‘AMPA’, ‘NMDA’, ‘GABAB’].",
                                        "suggestions": "",
                                        "hintText": "",
                                        "type": "float"
                                    },
                                },
                            "vinit": {
                                "label": "vinit",
                                "help": "(optional) Initial membrane voltage (in mV) of the section (default: -65).e.g. cellRule['secs']['soma']['vinit'] = -72",
                                "suggestions": "",
                                "hintText": "",
                                },
                            "spikeGenLoc": {
                                "label": "spikeGenLoc",
                                "help": "(optional) Indicates that this section is responsible for spike generation (instead of the default ‘soma’), and provides the location (segment) where spikes are generated.e.g. cellRule['secs']['axon']['spikeGenLoc'] = 1.0.",
                                "suggestions": "",
                                "hintText": "",
                                },
                            "threshold": {
                                "label": "threshold",
                                "help": "(optional) Threshold voltage (in mV) used to detect a spike originating in this section of the cell. If omitted, defaults to netParams.defaultThreshold = 10.0.e.g. cellRule['secs']['soma']['threshold'] = 5.0.",
                                "suggestions": "",
                                "hintText": "",
                                },
                            },
                        "secLists": {
                            "label": "secLists - (optional) ",
                            "help": "Dictionary of sections lists (e.g. {‘all’: [‘soma’, ‘dend’]})",
                            "suggestions": "",
                            "hintText": "",
                            },
                        }
                    }
                }
            }
        }
    },
    "simConfig": {
        "label": "Sim Config",
        "suggestions": "",
        "help": "",
        "hintText": "",
        "children": {
            "simLabel":{
                "label": "Name of Simulation"
            },
            "duration": {
                "label": "Duration",
            },
            "dt": {
                "label": "Dt",
            },
            "seeds": {
                "label": "Seeds",
            },
            "addSynMechs": {
                "label": "Add Syn Mechs",
            },
            "includeParamsLabel": {
                "label": "Include Params Label",
            },
            "timing": {
                "label": "Timing",
            },
            "verbose": {
                "label": "Verbose",
            },
            "saveFolder": {
                "label": "Save Folder",
            },
            "filename": {
                "label": "File Name",
            },
            "saveDataInclude": {
                "label": "Save Data Include",
            },
            "timestampFilename": {
                "label": "Timestamp File Name",
            },
            "savePickle": {
                "label": "Save Pickle",
            },
            "saveJson": {
                "label": "Save Json",
            },
            "saveMat": {
                "label": "Save Mat",
            },
            "saveHDF5": {
                "label": "Save HDF5",
            },
            "saveDpk": {
                "label": "Save Dpk",
            },
            "saveDat": {
                "label": "Save Dat",
            },
            "saveCSV": {
                "label": "Save Csv",
            },
            "saveCellSecs": {
                "label": "Save Cell Secs",
            },
            "saveCellConns": {
                "label": "Save Cell Conns",
            },
            "checkErrors": {
                "label": "Check Errors",
            },
            "checkErrorsVerbose": {
                "label": "Check Errors Verbose",
            },
            "backupCfgFile": {
                "label": "Copy of CFG file"
            }

        }
    }
}
<|endoftext|>


<commit_before>import json, SocketServer

class RequestHandler(SocketServer.StreamRequestHandler):

    def handle(self):
        request_message = json.loads( self.rfile.readline().rstrip() )

        self.command = request_message['cmd']
        self.window = int(request_message['prm'])
        self.start_frame = int(request_message['frm'])

        if self.command == "Start":

            for offset in range(window):
                response = self.server.movie_data[start_frame + offset]
                self.wfile.write(response)

def seed_movie():
    movie_data = {}
    with open('movie_data.txt', 'rb') as f:
        lines = f.readlines()
        for line in lines:
            clean_line = line.rstrip()
            frame_no = int(line[:5])
            movie_data[frame_no] = line

    return movie_data

if __name__ == "__main__":
    HOST, PORT = "0.0.0.0", 5000
    server = SocketServer.TCPServer((HOST, PORT), RequestHandler)

    print "Seeding movie data..."
    server.movie_data = seed_movie()

    print "Listening for requests..."
    server.serve_forever()<commit_msg>refactored server api a bit<commit_after>import json, SocketServer

class RequestHandler(SocketServer.StreamRequestHandler):

    def handle(self):
        request_message = json.loads( self.rfile.readline().rstrip() )

        self.command = request_message['cmd']
        self.window = int(request_message['wnd'])
        self.start_frame = int(request_message['frm'])

        if self.command == "Start":

            for offset in range(window):
                response = self.server.movie_data[start_frame + offset]
                self.wfile.write(response)

def seed_movie():
    movie_data = {}
    with open('movie_data.txt', 'rb') as f:
        lines = f.readlines()
        for line in lines:
            clean_line = line.rstrip()
            frame_no = int(line[:5])
            movie_data[frame_no] = line

    return movie_data

if __name__ == "__main__":
    HOST, PORT = "0.0.0.0", 5000
    server = SocketServer.TCPServer((HOST, PORT), RequestHandler)

    print "Seeding movie data..."
    server.movie_data = seed_movie()

    print "Listening for requests..."
    server.serve_forever()<|endoftext|>


<commit_before>import logging

from datetime import date
from functools import partial

from django.core.urlresolvers import reverse
from django.http.response import Http404
from django.http import HttpResponseRedirect
from django.shortcuts import render

from talks.api_ox.query import get_info, get_oxford_date
from talks.api_ox.api import ApiException
from .models import Event, EventGroup, Speaker
from .forms import EventForm, EventGroupForm, SpeakerQuickAdd

logger = logging.getLogger(__name__)

def homepage(request):
    # TODO needs to clarify the difference
    # between homepage and upcoming_events?
    return upcoming_events(request)

def upcoming_events(request):
    today = date.today()
    events = Event.objects.filter(start__gte=today).order_by('start')
    return _events_list(request, events)

def events_for_year(request, year):
    events = Event.objects.filter(start__year=year)
    return _events_list(request, events)

def events_for_month(request, year, month):
    events = Event.objects.filter(start__year=year,
                                  start__month=month)
    return _events_list(request, events)

def events_for_day(request, year, month, day):
    events = Event.objects.filter(start__year=year,
                                  start__month=month,
                                  start__day=day)
    return _events_list(request, events)

def _events_list(request, events):
    context = {'events': events}
    return render(request, 'events/events.html', context)

def event(request, event_id):
    ev = Event.objects.get(id=event_id)
    if ev:
        context = {'event': ev}
        if ev.location:
            context['location_name'] = ev.location.name
            try:
                location = get_info(ev.location.identifier)
            except ApiException:
                location = None
            if location:
                context['location'] = location
        formatted = None
        if ev.start:
            try:
                oxford_date = get_oxford_date(ev.start)
                formatted = oxford_date['formatted'] if 'formatted' in oxford_date else None
            except ApiException:
                logger.warn('Unable to reach API', exc_info=True)
        context['oxford_date'] = formatted
    else:
        raise Http404
    return render(request, 'events/event.html', context)

def create_event(request, group_id=None):
    initial = dict()
    if group_id:
        try:
            initial['event_group_select'] = EventGroup.objects.get(id=group_id)
            initial['enabled'] = True
        except EventGroup.DoesNotExist:
            logger.warning("Tried to create new Event in nonexistant group ID: %s" % (group_id,))
            raise Http404("Group does not exist")

    PrefixedEventForm = partial(EventForm, prefix='event')
    PrefixedEventGroupForm = partial(EventGroupForm, prefix='event-group', initial=initial)

    if request.method == 'POST':
        context = {
            'event_form': PrefixedEventForm(request.POST),
            'event_group_form': PrefixedEventGroupForm(request.POST),
            'speaker_form': SpeakerQuickAdd(),
        }
        forms_valid = context['event_form'].is_valid() and context['event_group_form'].is_valid()
        if forms_valid:
            event_group = context['event_group_form'].get_event_group()
            event = context['event_form'].save(commit=False)
            if event_group:
                event_group.save()
                event.group = event_group
            event.save()
            # *Now* we can save the many2many relations
            context['event_form'].save_m2m()
            if 'another' in request.POST:
                # Adding more events, redirect to the create event in existing group form
                return HttpResponseRedirect(reverse('create-event-in-group', args=(event_group.id,)))
            else:
                return HttpResponseRedirect(reverse('event', args=(event.id,)))
        else:
            if 'speakers' in context['event_form'].cleaned_data:
                context['selected_speakers'] = Speaker.objects.filter(
                    id__in=context['event_form'].cleaned_data['speakers'])
    else:
        context = {
            'event_form': PrefixedEventForm(),
            'event_group_form': PrefixedEventGroupForm(),
            'speaker_form': SpeakerQuickAdd(),
        }
    return render(request, 'events/create_event.html', context)
<commit_msg>homepage only shows todays events by default<commit_after>import logging

from datetime import date, timedelta
from functools import partial

from django.core.urlresolvers import reverse
from django.http.response import Http404
from django.http import HttpResponseRedirect
from django.shortcuts import render

from talks.api_ox.query import get_info, get_oxford_date
from talks.api_ox.api import ApiException
from .models import Event, EventGroup, Speaker
from .forms import EventForm, EventGroupForm, SpeakerQuickAdd

logger = logging.getLogger(__name__)

def homepage(request):
    today = date.today()
    tomorrow = today + timedelta(days=1)
    events = Event.objects.filter(start__gte=today, start__lt=tomorrow)
    events = events.order_by('start')
    context = {'events': events}
    return render(request, 'front.html', context)

def upcoming_events(request):
    today = date.today()
    events = Event.objects.filter(start__gte=today).order_by('start')
    return _events_list(request, events)

def events_for_year(request, year):
    events = Event.objects.filter(start__year=year)
    return _events_list(request, events)

def events_for_month(request, year, month):
    events = Event.objects.filter(start__year=year,
                                  start__month=month)
    return _events_list(request, events)

def events_for_day(request, year, month, day):
    events = Event.objects.filter(start__year=year,
                                  start__month=month,
                                  start__day=day)
    return _events_list(request, events)

def _events_list(request, events):
    context = {'events': events}
    return render(request, 'events/events.html', context)

def event(request, event_id):
    ev = Event.objects.get(id=event_id)
    if ev:
        context = {'event': ev}
        if ev.location:
            context['location_name'] = ev.location.name
            try:
                location = get_info(ev.location.identifier)
            except ApiException:
                location = None
            if location:
                context['location'] = location
        formatted = None
        if ev.start:
            try:
                oxford_date = get_oxford_date(ev.start)
                formatted = oxford_date['formatted'] if 'formatted' in oxford_date else None
            except ApiException:
                logger.warn('Unable to reach API', exc_info=True)
        context['oxford_date'] = formatted
    else:
        raise Http404
    return render(request, 'events/event.html', context)

def create_event(request, group_id=None):
    initial = dict()
    if group_id:
        try:
            initial['event_group_select'] = EventGroup.objects.get(id=group_id)
            initial['enabled'] = True
        except EventGroup.DoesNotExist:
            logger.warning("Tried to create new Event in nonexistant group ID: %s" % (group_id,))
            raise Http404("Group does not exist")

    PrefixedEventForm = partial(EventForm, prefix='event')
    PrefixedEventGroupForm = partial(EventGroupForm, prefix='event-group', initial=initial)

    if request.method == 'POST':
        context = {
            'event_form': PrefixedEventForm(request.POST),
            'event_group_form': PrefixedEventGroupForm(request.POST),
            'speaker_form': SpeakerQuickAdd(),
        }
        forms_valid = context['event_form'].is_valid() and context['event_group_form'].is_valid()
        if forms_valid:
            event_group = context['event_group_form'].get_event_group()
            event = context['event_form'].save(commit=False)
            if event_group:
                event_group.save()
                event.group = event_group
            event.save()
            # *Now* we can save the many2many relations
            context['event_form'].save_m2m()
            if 'another' in request.POST:
                # Adding more events, redirect to the create event in existing group form
                return HttpResponseRedirect(reverse('create-event-in-group', args=(event_group.id,)))
            else:
                return HttpResponseRedirect(reverse('event', args=(event.id,)))
        else:
            if 'speakers' in context['event_form'].cleaned_data:
                context['selected_speakers'] = Speaker.objects.filter(
                    id__in=context['event_form'].cleaned_data['speakers'])
    else:
        context = {
            'event_form': PrefixedEventForm(),
            'event_group_form': PrefixedEventGroupForm(),
            'speaker_form': SpeakerQuickAdd(),
        }
    return render(request, 'events/create_event.html', context)
<|endoftext|>


<commit_before>
#
# spyne - Copyright (C) Spyne contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

"""The ``spyne.error`` module contains various common exceptions that the user
code can throw.
"""

from spyne.model.fault import Fault
from spyne.const import MAX_STRING_FIELD_LENGTH

class ResourceNotFoundError(Fault):
    """Raised when requested resource is not found."""

    def __init__(self, fault_object,
                                fault_string="Requested resource %r not found"):
        Fault.__init__(self, 'Client.ResourceNotFound',
                                                    fault_string % fault_object)

class RequestTooLongError(Fault):
    """Raised when request is too long."""

    def __init__(self, faultstring="Request too long"):
        Fault.__init__(self, 'Client.RequestTooLong', faultstring)

class RequestNotAllowed(Fault):
    """Raised when request is incomplete."""

    def __init__(self, faultstring=""):
        Fault.__init__(self, 'Client.RequestNotAllowed', faultstring)

class ArgumentError(Fault):
    """Raised when there is a general problem with input data."""

    def __init__(self, faultstring=""):
        Fault.__init__(self, 'Client.ArgumentError', faultstring)

class ValidationError(Fault):
    """Raised when the input stream does not adhere to type constraints."""

    def __init__(self, obj, custom_msg='The value %r could not be validated.'):
        s = repr(obj)

        if len(s) > MAX_STRING_FIELD_LENGTH:
            s = s[:MAX_STRING_FIELD_LENGTH] + "(...)"

        Fault.__init__(self, 'Client.ValidationError', custom_msg % s)

class InternalError(Fault):
    """Raised to communicate server-side errors."""
    def __init__(self, error):
        Fault.__init__(self, 'Server', "InternalError: An unknown error has occured.")
<commit_msg>add InvalidInput error. raises Http 400<commit_after>
#
# spyne - Copyright (C) Spyne contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

"""The ``spyne.error`` module contains various common exceptions that the user
code can throw.
"""

from spyne.model.fault import Fault
from spyne.const import MAX_STRING_FIELD_LENGTH

class ResourceNotFoundError(Fault):
    """Raised when requested resource is not found."""

    def __init__(self, fault_object,
                                fault_string="Requested resource %r not found"):
        Fault.__init__(self, 'Client.ResourceNotFound',
                                                    fault_string % fault_object)

class RequestTooLongError(Fault):
    """Raised when request is too long."""

    def __init__(self, faultstring="Request too long"):
        Fault.__init__(self, 'Client.RequestTooLong', faultstring)

class RequestNotAllowed(Fault):
    """Raised when request is incomplete."""

    def __init__(self, faultstring=""):
        Fault.__init__(self, 'Client.RequestNotAllowed', faultstring)

class ArgumentError(Fault):
    """Raised when there is a general problem with input data."""

    def __init__(self, faultstring=""):
        Fault.__init__(self, 'Client.ArgumentError', faultstring)

class InvalidInput(Fault):
    """Raised when there is a general problem with input data."""

    def __init__(self, faultstring=""):
        Fault.__init__(self, 'Client.InvalidInput', faultstring)

class ValidationError(Fault):
    """Raised when the input stream does not adhere to type constraints."""

    def __init__(self, obj, custom_msg='The value %r could not be validated.'):
        s = repr(obj)

        if len(s) > MAX_STRING_FIELD_LENGTH:
            s = s[:MAX_STRING_FIELD_LENGTH] + "(...)"

        Fault.__init__(self, 'Client.ValidationError', custom_msg % s)

class InternalError(Fault):
    """Raised to communicate server-side errors."""
    def __init__(self, error):
        Fault.__init__(self, 'Server', "InternalError: An unknown error has occured.")
<|endoftext|>


<commit_before>"""Notes:
* 'test' command: nox will use poetry.lock to determine dependency versions
* 'lint' command: tools and environments are managed by pre-commit
* All other commands: the current environment will be used instead of creating new ones
"""
from os.path import join
from shutil import rmtree

import nox
from nox_poetry import session

nox.options.reuse_existing_virtualenvs = True
nox.options.sessions = ['lint', 'cov']

LIVE_DOCS_PORT = 8181
LIVE_DOCS_IGNORE = ['*.pyc', '*.tmp', join('**', 'modules', '*')]
LIVE_DOCS_WATCH = ['requests_cache', 'examples']
CLEAN_DIRS = ['dist', 'build', join('docs', '_build'), join('docs', 'modules')]

UNIT_TESTS = join('tests', 'unit')
INTEGRATION_TESTS = join('tests', 'integration')
COVERAGE_ARGS = '--cov --cov-report=term --cov-report=html'  # Generate HTML + stdout coverage report
XDIST_ARGS = '--numprocesses=auto --dist=loadfile'  # Run tests in parallel, grouped by test module

@session(python=['3.7', '3.8', '3.9', '3.10'])
def test(session):
    """Run tests for a specific python version"""
    test_paths = session.posargs or [UNIT_TESTS]
    session.install('.', 'pytest', 'pytest-xdist', 'requests-mock', 'timeout-decorator')

    cmd = f'pytest -rs {XDIST_ARGS}'
    session.run(*cmd.split(' '), *test_paths)

@session(python=False)
def clean(session):
    """Clean up temporary build + documentation files"""
    for dir in CLEAN_DIRS:
        print(f'Removing {dir}')
        rmtree(dir, ignore_errors=True)

@session(python=False, name='cov')
def coverage(session):
    """Run tests and generate coverage report"""
    cmd_1 = f'pytest {UNIT_TESTS} -rs {XDIST_ARGS} {COVERAGE_ARGS}'
    cmd_2 = f'pytest {INTEGRATION_TESTS} -rs {XDIST_ARGS} {COVERAGE_ARGS} --cov-append'
    session.run(*cmd_1.split(' '))
    session.run(*cmd_2.split(' '))

@session(python=False)
def docs(session):
    """Build Sphinx documentation"""
    cmd = 'sphinx-build docs docs/_build/html -j auto'
    session.run(*cmd.split(' '))

@session(python=False)
def livedocs(session):
    """Auto-build docs with live reload in browser.
    Add `--open` to also open the browser after starting.
    """
    args = ['-a']
    args += [f'--watch {pattern}' for pattern in LIVE_DOCS_WATCH]
    args += [f'--ignore {pattern}' for pattern in LIVE_DOCS_IGNORE]
    args += [f'--port {LIVE_DOCS_PORT}', '-j auto']
    if session.posargs == ['open']:
        args.append('--open-browser')

    clean(session)
    cmd = 'sphinx-autobuild docs docs/_build/html ' + ' '.join(args)
    session.run(*cmd.split(' '))

@session(python=False)
def lint(session):
    """Run linters and code formatters via pre-commit"""
    cmd = 'pre-commit run --all-files'
    session.run(*cmd.split(' '))
<commit_msg>Add mypy to 'lint' shortcut for nox<commit_after>"""Notes:
* 'test' command: nox will use poetry.lock to determine dependency versions
* 'lint' command: tools and environments are managed by pre-commit
* All other commands: the current environment will be used instead of creating new ones
"""
from os.path import join
from shutil import rmtree

import nox
from nox_poetry import session

nox.options.reuse_existing_virtualenvs = True
nox.options.sessions = ['lint', 'cov']

LIVE_DOCS_PORT = 8181
LIVE_DOCS_IGNORE = ['*.pyc', '*.tmp', join('**', 'modules', '*')]
LIVE_DOCS_WATCH = ['requests_cache', 'examples']
CLEAN_DIRS = ['dist', 'build', join('docs', '_build'), join('docs', 'modules')]

UNIT_TESTS = join('tests', 'unit')
INTEGRATION_TESTS = join('tests', 'integration')
COVERAGE_ARGS = '--cov --cov-report=term --cov-report=html'  # Generate HTML + stdout coverage report
XDIST_ARGS = '--numprocesses=auto --dist=loadfile'  # Run tests in parallel, grouped by test module

@session(python=['3.7', '3.8', '3.9', '3.10'])
def test(session):
    """Run tests for a specific python version"""
    test_paths = session.posargs or [UNIT_TESTS]
    session.install('.', 'pytest', 'pytest-xdist', 'requests-mock', 'timeout-decorator')

    cmd = f'pytest -rs {XDIST_ARGS}'
    session.run(*cmd.split(' '), *test_paths)

@session(python=False)
def clean(session):
    """Clean up temporary build + documentation files"""
    for dir in CLEAN_DIRS:
        print(f'Removing {dir}')
        rmtree(dir, ignore_errors=True)

@session(python=False, name='cov')
def coverage(session):
    """Run tests and generate coverage report"""
    cmd_1 = f'pytest {UNIT_TESTS} -rs {XDIST_ARGS} {COVERAGE_ARGS}'
    cmd_2 = f'pytest {INTEGRATION_TESTS} -rs {XDIST_ARGS} {COVERAGE_ARGS} --cov-append'
    session.run(*cmd_1.split(' '))
    session.run(*cmd_2.split(' '))

@session(python=False)
def docs(session):
    """Build Sphinx documentation"""
    cmd = 'sphinx-build docs docs/_build/html -j auto'
    session.run(*cmd.split(' '))

@session(python=False)
def livedocs(session):
    """Auto-build docs with live reload in browser.
    Add `--open` to also open the browser after starting.
    """
    args = ['-a']
    args += [f'--watch {pattern}' for pattern in LIVE_DOCS_WATCH]
    args += [f'--ignore {pattern}' for pattern in LIVE_DOCS_IGNORE]
    args += [f'--port {LIVE_DOCS_PORT}', '-j auto']
    if session.posargs == ['open']:
        args.append('--open-browser')

    clean(session)
    cmd = 'sphinx-autobuild docs docs/_build/html ' + ' '.join(args)
    session.run(*cmd.split(' '))

@session(python=False)
def lint(session):
    """Run linters and code formatters via pre-commit"""
    cmd_1 = 'pre-commit run --all-files'
    cmd_2 = 'mypy --install-types --non-interactive requests_cache'
    session.run(*cmd_1.split(' '))
    session.run(*cmd_2.split(' '))
<|endoftext|>