<commit_before>""" Set of interfaces that allow interaction with data. Currently
    available interfaces are:

    DataSource: Generic nifti to named Nifti interface
    DataSink: Generic named output from interfaces to data store

    To come :
    XNATSource, XNATSink

"""
from nipype.interfaces.base import Interface, CommandLine, Bunch, InterfaceResult
from copy import deepcopy
from nipype.utils.filemanip import copyfiles, list_to_filename
import glob
import os

class DataSource(Interface):
    """ Generic datasource module that takes a directory containing a
        list of nifti files and provides a set of structured output
        fields.
    """

    def __init__(self, *args, **inputs):
        self._populate_inputs()
        self.inputs.update(**inputs)

    def inputs_help(self):
        """
            Parameters
            --------------------
            (all default to None)

            base_directory : /path/to/dir
                Basedirectory consisting of subject data
            subject_template : string
                Template encoding the subject directory name, indexed
                by subject id. For example, the default
            subject_id: string or int
                Subject identifier
            subject_directory : /path/to/dir
                Path to subject directory
            subject_info : dict
                Provides information about how to map subject run
                numbers to output fields

                subject_id are keys and values are a list of tuples.
                info[subject_id] = [([runnos],fieldname),...]

                Example:
                --------

                info['s1'] = [([4,5],'anat'),([6,7],'bold'),([8],'dti')]
                info[1] = [([3,4],'struct'),([6,7],'bold'),([8],'dti')]

                In the above examples, the output fields of this
                object will be the names 'anat', 'bold', etc.,.
            """
        print self.inputs_help.__doc__

    def _populate_inputs(self):
        self.inputs = Bunch(base_directory=None,
                            subject_template=None,
                            subject_id=None,
                            subject_directory=None,
                            subject_info=None)

    def outputs_help(self):
        """
            Parameters
            ----------

            (all default to None)

            subject_id : string
                Subject identifier
            subject_directory: /path/to/dir
                Location of subject directory containing nifti files

            remaining fields are defined by user. See subject_info in
            inputs_help() for description of how to specify output
            fields
            """
        print self.outputs_help.__doc__

    def aggregate_outputs(self):
        outputs = Bunch(subject_id=None)
        outputs.subject_id = self.inputs.subject_id
        subjdir = self.inputs.subject_directory
        if subjdir is None:
            #print self.inputs['subj_template'],self.inputs['subj_id']
            if self.inputs.subject_template is not None:
                subjdir = self.inputs.subject_template % (self.inputs.subject_id)
            else:
                subjdir = self.inputs.subject_id
            subjdir = os.path.join(self.inputs.base_directory,subjdir)
        if subjdir is None:
            raise Exception('Subject directory not provided')
        outputs.subject_directory = subjdir
        info = self.inputs.subject_info[self.inputs.subject_id]
        for idx,type in info:
            outputs[type] = []
            for i in idx:
                files = '*-%d-*.nii' % i
                path = os.path.abspath(os.path.join(subjdir,files))
                outputs[type].extend(glob.glob(path))
        return outputs

    def run(self):
        """Execute this module.
        """
        runtime = Bunch(returncode=0,
                        stdout=None,
                        stderr=None)
        outputs=self.aggregate_outputs()
        return InterfaceResult(deepcopy(self), runtime, outputs=outputs)

class DataSink(Interface):
    """ Generic datasink module that takes a directory containing a
        list of nifti files and provides a set of structured output
        fields.
    """

    def __init__(self, *args, **inputs):
        self._populate_inputs()
        self.inputs.update(**inputs)

    def inputs_help(self):
        """
            Parameters
            ----------
            (all default to None)

            base_directory : /path/to/dir
                Basedirectory consisting of subject data
            subject_id: string or int
                Subject identifier
            subject_directory : /path/to/dir
                Path to subject directory

            Any fields that are set as lists will be copied to a
            directory under subject_directory with the fieldname as a
            new directory.

        """
        print self.inputs_help.__doc__

    def _populate_inputs(self):
        self.inputs = Bunch(base_directory=None,
                            subject_directory=None,
                            subject_template=None,
                            subject_id=None)
        self.input_keys = self.inputs.__dict__.keys()

    def outputs_help(self):
        """
            No outputs
        """
        print self.outputs_help.__doc__

    def aggregate_outputs(self):
        outputs = Bunch()
        return Bunch()

    def run(self):
        """Execute this module.
        """
        subjdir = self.inputs.subject_directory
        if subjdir is None:
            #print self.inputs['subj_template'],self.inputs['subj_id']
            if self.inputs.subject_template is not None:
                subjdir = self.inputs.subject_template % (self.inputs.subject_id)
            else:
                subjdir = self.inputs.subject_id
            subjdir = os.path.join(self.inputs.base_directory,subjdir)
        if subjdir is None:
            raise Exception('Subject directory not provided')
        outdir = subjdir
        if not os.path.exists(outdir):
            os.mkdir(outdir)
        for k,v in self.inputs.iteritems():
            if k not in self.input_keys:
                if v is not None:
                    tempoutdir = outdir
                    for d in k.split('.'):
                        if d[0] == '@':
                            continue
                        tempoutdir = os.path.join(tempoutdir,d)
                        if not os.path.exists(tempoutdir):
                            os.mkdir(tempoutdir)
                    copyfiles(self.inputs[k],tempoutdir,copy=True)
        runtime = Bunch(returncode=0,
                        stdout=None,
                        stderr=None)
        outputs=self.aggregate_outputs()
        return InterfaceResult(deepcopy(self), runtime, outputs=outputs)

class DataGrabber(Interface):
    """ Generic datagrabber module that wraps around glob in an
        intelligent way for neuroimaging tasks
    """

    def __init__(self, *args, **inputs):
        self._populate_inputs()
        self.inputs.update(**inputs)

    def inputs_help(self):
        """
            Parameters
            --------------------
            (all default to None)

            file_template : string
                template for filename
            template_argtuple: tuple of arguments
                arguments that fit into file_template

            Alternatively can provide upto 3 additional arguments
            to use as iterables
            template_arg1: argument 1
            template_arg2: argument 2
            template_arg3: argument 3
            """
        print self.inputs_help.__doc__

    def _populate_inputs(self):
        self.inputs = Bunch(file_template=None,
                            template_argtuple=None,
                            template_arg1=None,
                            template_arg2=None,
                            template_arg3=None
                            )

    def outputs_help(self):
        """
            Parameters
            ----------

            (all default to None)

            file_list : list
                list of files picked up by the grabber
            """
        print self.outputs_help.__doc__

    def aggregate_outputs(self):
        outputs = Bunch(file_list=None)
        args = []
        if self.inputs.template_argtuple is not None:
            args.extend(list(self.inputs.template_argtuple))

        for i in range(3):
            arg = self.inputs['template_arg%d'%(i+1)]
            if arg is not None:
                args.append(arg)
        template = self.inputs.file_template
        if len(args)>0:
            template = template%tuple(args)
        outputs.file_list = list_to_filename(glob.glob(template))
        return outputs

    def run(self):
        """Execute this module.
        """
        runtime = Bunch(returncode=0,
                        stdout=None,
                        stderr=None)
        outputs=self.aggregate_outputs()
        return InterfaceResult(deepcopy(self), runtime, outputs=outputs)
<commit_msg>added some exception handling<commit_after>""" Set of interfaces that allow interaction with data. Currently
    available interfaces are:

    DataSource: Generic nifti to named Nifti interface
    DataSink: Generic named output from interfaces to data store

    To come :
    XNATSource, XNATSink

"""
from nipype.interfaces.base import Interface, CommandLine, Bunch, InterfaceResult
from copy import deepcopy
from nipype.utils.filemanip import copyfiles, list_to_filename
import glob
import os

class DataSource(Interface):
    """ Generic datasource module that takes a directory containing a
        list of nifti files and provides a set of structured output
        fields.
    """

    def __init__(self, *args, **inputs):
        self._populate_inputs()
        self.inputs.update(**inputs)

    def inputs_help(self):
        """
            Parameters
            --------------------
            (all default to None)

            base_directory : /path/to/dir
                Basedirectory consisting of subject data
            subject_template : string
                Template encoding the subject directory name, indexed
                by subject id. For example, the default
            subject_id: string or int
                Subject identifier
            subject_directory : /path/to/dir
                Path to subject directory
            subject_info : dict
                Provides information about how to map subject run
                numbers to output fields

                subject_id are keys and values are a list of tuples.
                info[subject_id] = [([runnos],fieldname),...]

                Example:
                --------

                info['s1'] = [([4,5],'anat'),([6,7],'bold'),([8],'dti')]
                info[1] = [([3,4],'struct'),([6,7],'bold'),([8],'dti')]

                In the above examples, the output fields of this
                object will be the names 'anat', 'bold', etc.,.
            """
        print self.inputs_help.__doc__

    def _populate_inputs(self):
        self.inputs = Bunch(base_directory=None,
                            subject_template=None,
                            subject_id=None,
                            subject_directory=None,
                            subject_info=None)

    def outputs_help(self):
        """
            Parameters
            ----------

            (all default to None)

            subject_id : string
                Subject identifier
            subject_directory: /path/to/dir
                Location of subject directory containing nifti files

            remaining fields are defined by user. See subject_info in
            inputs_help() for description of how to specify output
            fields
            """
        print self.outputs_help.__doc__

    def aggregate_outputs(self):
        outputs = Bunch(subject_id=None)
        outputs.subject_id = self.inputs.subject_id
        subjdir = self.inputs.subject_directory
        if subjdir is None:
            #print self.inputs['subj_template'],self.inputs['subj_id']
            if self.inputs.subject_template is not None:
                subjdir = self.inputs.subject_template % (self.inputs.subject_id)
            else:
                subjdir = self.inputs.subject_id
            subjdir = os.path.join(self.inputs.base_directory,subjdir)
        if subjdir is None:
            raise Exception('Subject directory not provided')
        outputs.subject_directory = subjdir
        if self.inputs.subject_info is None:
            raise Exception('Subject info not provided')
        try:
            info = self.inputs.subject_info[self.inputs.subject_id]
        except KeyError:
            raise KeyError("Key [%s] does not exist in subject_info dictionary"%self.inputs.subject_id)
        for idx,type in info:
            outputs[type] = []
            for i in idx:
                files = '*-%d-*.nii' % i
                path = os.path.abspath(os.path.join(subjdir,files))
                outputs[type].extend(glob.glob(path))
        return outputs

    def run(self):
        """Execute this module.
        """
        runtime = Bunch(returncode=0,
                        stdout=None,
                        stderr=None)
        outputs=self.aggregate_outputs()
        return InterfaceResult(deepcopy(self), runtime, outputs=outputs)

class DataSink(Interface):
    """ Generic datasink module that takes a directory containing a
        list of nifti files and provides a set of structured output
        fields.
    """

    def __init__(self, *args, **inputs):
        self._populate_inputs()
        self.inputs.update(**inputs)

    def inputs_help(self):
        """
            Parameters
            ----------
            (all default to None)

            base_directory : /path/to/dir
                Basedirectory consisting of subject data
            subject_id: string or int
                Subject identifier
            subject_directory : /path/to/dir
                Path to subject directory

            Any fields that are set as lists will be copied to a
            directory under subject_directory with the fieldname as a
            new directory.

        """
        print self.inputs_help.__doc__

    def _populate_inputs(self):
        self.inputs = Bunch(base_directory=None,
                            subject_directory=None,
                            subject_template=None,
                            subject_id=None)
        self.input_keys = self.inputs.__dict__.keys()

    def outputs_help(self):
        """
            No outputs
        """
        print self.outputs_help.__doc__

    def aggregate_outputs(self):
        outputs = Bunch()
        return Bunch()

    def run(self):
        """Execute this module.
        """
        subjdir = self.inputs.subject_directory
        if subjdir is None:
            #print self.inputs['subj_template'],self.inputs['subj_id']
            if self.inputs.subject_template is not None:
                subjdir = self.inputs.subject_template % (self.inputs.subject_id)
            else:
                subjdir = self.inputs.subject_id
            subjdir = os.path.join(self.inputs.base_directory,subjdir)
        if subjdir is None:
            raise Exception('Subject directory not provided')
        outdir = subjdir
        if not os.path.exists(outdir):
            os.mkdir(outdir)
        for k,v in self.inputs.iteritems():
            if k not in self.input_keys:
                if v is not None:
                    tempoutdir = outdir
                    for d in k.split('.'):
                        if d[0] == '@':
                            continue
                        tempoutdir = os.path.join(tempoutdir,d)
                        if not os.path.exists(tempoutdir):
                            os.mkdir(tempoutdir)
                    copyfiles(self.inputs[k],tempoutdir,copy=True)
        runtime = Bunch(returncode=0,
                        stdout=None,
                        stderr=None)
        outputs=self.aggregate_outputs()
        return InterfaceResult(deepcopy(self), runtime, outputs=outputs)

class DataGrabber(Interface):
    """ Generic datagrabber module that wraps around glob in an
        intelligent way for neuroimaging tasks
    """

    def __init__(self, *args, **inputs):
        self._populate_inputs()
        self.inputs.update(**inputs)

    def inputs_help(self):
        """
            Parameters
            --------------------
            (all default to None)

            file_template : string
                template for filename
            template_argtuple: tuple of arguments
                arguments that fit into file_template

            Alternatively can provide upto 3 additional arguments
            to use as iterables
            template_arg1: argument 1
            template_arg2: argument 2
            template_arg3: argument 3
            """
        print self.inputs_help.__doc__

    def _populate_inputs(self):
        self.inputs = Bunch(file_template=None,
                            template_argtuple=None,
                            template_arg1=None,
                            template_arg2=None,
                            template_arg3=None
                            )

    def outputs_help(self):
        """
            Parameters
            ----------

            (all default to None)

            file_list : list
                list of files picked up by the grabber
            """
        print self.outputs_help.__doc__

    def aggregate_outputs(self):
        outputs = Bunch(file_list=None)
        args = []
        if self.inputs.template_argtuple is not None:
            args.extend(list(self.inputs.template_argtuple))

        for i in range(3):
            arg = self.inputs['template_arg%d'%(i+1)]
            if arg is not None:
                args.append(arg)
        template = self.inputs.file_template
        if len(args)>0:
            template = template%tuple(args)
        outputs.file_list = list_to_filename(glob.glob(template))
        return outputs

    def run(self):
        """Execute this module.
        """
        runtime = Bunch(returncode=0,
                        stdout=None,
                        stderr=None)
        outputs=self.aggregate_outputs()
        return InterfaceResult(deepcopy(self), runtime, outputs=outputs)
<|endoftext|>


<commit_before># Natural Language Toolkit: Corpus Readers
#
# Copyright (C) 2001-2010 NLTK Project
# Author: Edward Loper <edloper@gradient.cis.upenn.edu>
# URL: <http://www.nltk.org/>
# For license information, see LICENSE.TXT

# [xx] this docstring isnt' up-to-date!
"""
NLTK corpus readers.  The modules in this package provide functions
that can be used to read corpus files in a variety of formats.  These
functions can be used to read both the corpus files that are
distributed in the NLTK corpus package, and corpus files that are part
of external corpora.

Available Corpora
=================

Please see http://nltk.googlecode.com/svn/trunk/nltk_data/index.xml
for a complete list.  Install corpora using nltk.download().

Corpus Reader Functions
=======================
Each corpus module defines one or more X{corpus reader functions},
which can be used to read documents from that corpus.  These functions
take an argument, C{item}, which is used to indicate which document
should be read from the corpus:

  - If C{item} is one of the unique identifiers listed in the corpus
    module's C{items} variable, then the corresponding document will
    be loaded from the NLTK corpus package.

  - If C{item} is a filename, then that file will be read.

Additionally, corpus reader functions can be given lists of item
names; in which case, they will return a concatenation of the
corresponding documents.

Corpus reader functions are named based on the type of information
they return.  Some common examples, and their return types, are:

  - I{corpus}.words(): list of str
  - I{corpus}.sents(): list of (list of str)
  - I{corpus}.paras(): list of (list of (list of str))
  - I{corpus}.tagged_words(): list of (str,str) tuple
  - I{corpus}.tagged_sents(): list of (list of (str,str))
  - I{corpus}.tagged_paras(): list of (list of (list of (str,str)))
  - I{corpus}.chunked_sents(): list of (Tree w/ (str,str) leaves)
  - I{corpus}.parsed_sents(): list of (Tree with str leaves)
  - I{corpus}.parsed_paras(): list of (list of (Tree with str leaves))
  - I{corpus}.xml(): A single xml ElementTree
  - I{corpus}.raw(): unprocessed corpus contents

For example, to read a list of the words in the Brown Corpus, use
C{nltk.corpus.brown.words()}:

    >>> from nltk.corpus import brown
    >>> print brown.words()
    ['The', 'Fulton', 'County', 'Grand', 'Jury', 'said', ...]

Corpus Metadata
===============
Metadata about the NLTK corpora, and their individual documents, is
stored using U{Open Language Archives Community (OLAC)
<http://www.language-archives.org/>} metadata records.  These records
can be accessed using C{nltk.corpus.I{corpus}.olac()}.
"""

import re

from nltk.tokenize import RegexpTokenizer
from nltk.tag import simplify_brown_tag, simplify_wsj_tag,\
                     simplify_alpino_tag, simplify_indian_tag,\
                     simplify_tag

from util import LazyCorpusLoader
from reader import *

abc = LazyCorpusLoader(
    'abc', PlaintextCorpusReader, r'(?!\.).*\.txt')
alpino = LazyCorpusLoader(
    'alpino', AlpinoCorpusReader, tag_mapping_function=simplify_alpino_tag)
brown = LazyCorpusLoader(
    'brown', CategorizedTaggedCorpusReader, r'c[a-z]\d\d',
    cat_file='cats.txt', tag_mapping_function=simplify_brown_tag)
cess_cat = LazyCorpusLoader(
    'cess_cat', BracketParseCorpusReader, r'(?!\.).*\.tbf',
    tag_mapping_function=simplify_tag)
cess_esp = LazyCorpusLoader(
    'cess_esp', BracketParseCorpusReader, r'(?!\.).*\.tbf',
    tag_mapping_function=simplify_tag)
cmudict = LazyCorpusLoader(
    'cmudict', CMUDictCorpusReader, ['cmudict'])
conll2000 = LazyCorpusLoader(
    'conll2000', ConllChunkCorpusReader,
    ['train.txt', 'test.txt'], ('NP','VP','PP'))
conll2002 = LazyCorpusLoader(
    'conll2002', ConllChunkCorpusReader, '.*\.(test|train).*',
    ('LOC', 'PER', 'ORG', 'MISC'), encoding='utf-8')
conll2007 = LazyCorpusLoader(
    'conll2007', DependencyCorpusReader, '.*\.(test|train).*',
    encoding='utf-8')
dependency_treebank = LazyCorpusLoader(
    'dependency_treebank', DependencyCorpusReader, '.*\.dp')
floresta = LazyCorpusLoader(
    'floresta', BracketParseCorpusReader, r'(?!\.).*\.ptb', '#',
    tag_mapping_function=simplify_tag)
gazetteers = LazyCorpusLoader(
    'gazetteers', WordListCorpusReader, r'(?!LICENSE|\.).*\.txt')
genesis = LazyCorpusLoader(
    'genesis', PlaintextCorpusReader, r'(?!\.).*\.txt', encoding='utf-8')
gutenberg = LazyCorpusLoader(
    'gutenberg', PlaintextCorpusReader, r'(?!\.).*\.txt')
# corpus not available with NLTK; these lines caused help(nltk.corpus) to break
#hebrew_treebank = LazyCorpusLoader(
#    'hebrew_treebank', BracketParseCorpusReader, r'.*\.txt')
ieer = LazyCorpusLoader(
    'ieer', IEERCorpusReader, r'(?!README|\.).*')
inaugural = LazyCorpusLoader(
    'inaugural', PlaintextCorpusReader, r'(?!\.).*\.txt')
# [XX] This should probably just use TaggedCorpusReader:
indian = LazyCorpusLoader(
    'indian', IndianCorpusReader, r'(?!\.).*\.pos',
    tag_mapping_function=simplify_indian_tag)
ipipan = LazyCorpusLoader(
    'ipipan', IPIPANCorpusReader, r'(?!\.).*morph\.xml')
mac_morpho = LazyCorpusLoader(
    'mac_morpho', MacMorphoCorpusReader, r'(?!\.).*\.txt',
    tag_mapping_function=simplify_tag, encoding='latin-1')
machado = LazyCorpusLoader(
    'machado', PortugueseCategorizedPlaintextCorpusReader,
    r'(?!\.).*\.txt', cat_pattern=r'([a-z]*)/.*', encoding='latin-1')
movie_reviews = LazyCorpusLoader(
    'movie_reviews', CategorizedPlaintextCorpusReader,
    r'(?!\.).*\.txt', cat_pattern=r'(neg|pos)/.*')
names = LazyCorpusLoader(
    'names', WordListCorpusReader, r'(?!\.).*\.txt')
nps_chat = LazyCorpusLoader(
    'nps_chat', NPSChatCorpusReader, r'(?!README|\.).*\.xml',
    tag_mapping_function=simplify_wsj_tag)
pl196x = LazyCorpusLoader(
    'pl196x', Pl196xCorpusReader, r'[a-z]-.*\.xml',
    cat_file='cats.txt', textid_file='textids.txt')
ppattach = LazyCorpusLoader(
    'ppattach', PPAttachmentCorpusReader, ['training', 'test', 'devset'])
qc = LazyCorpusLoader(
    'qc', StringCategoryCorpusReader, ['train.txt', 'test.txt'])
reuters = LazyCorpusLoader(
    'reuters', CategorizedPlaintextCorpusReader, '(training|test).*',
    cat_file='cats.txt')
rte = LazyCorpusLoader(
    'rte', RTECorpusReader, r'(?!\.).*\.xml')
senseval = LazyCorpusLoader(
    'senseval', SensevalCorpusReader, r'(?!\.).*\.pos')
shakespeare = LazyCorpusLoader(
    'shakespeare', XMLCorpusReader, r'(?!\.).*\.xml')
sinica_treebank = LazyCorpusLoader(
    'sinica_treebank', SinicaTreebankCorpusReader, ['parsed'],
    tag_mapping_function=simplify_tag)
state_union = LazyCorpusLoader(
    'state_union', PlaintextCorpusReader, r'(?!\.).*\.txt')
stopwords = LazyCorpusLoader(
    'stopwords', WordListCorpusReader, r'(?!README|\.).*')
swadesh = LazyCorpusLoader(
    'swadesh', SwadeshCorpusReader, r'(?!README|\.).*')
switchboard = LazyCorpusLoader(
    'switchboard', SwitchboardCorpusReader)
timit = LazyCorpusLoader(
    'timit', TimitCorpusReader)
toolbox = LazyCorpusLoader(
    'toolbox', ToolboxCorpusReader, r'(?!.*(README|\.)).*\.(dic|txt)')
treebank = LazyCorpusLoader(
    'treebank/combined', BracketParseCorpusReader, r'wsj_.*\.mrg',
    tag_mapping_function=simplify_wsj_tag)
treebank_chunk = LazyCorpusLoader(
    'treebank/tagged', ChunkedCorpusReader, r'wsj_.*\.pos',
    sent_tokenizer=RegexpTokenizer(r'(?<=/\.)\s*(?![^\[]*\])', gaps=True),
    para_block_reader=tagged_treebank_para_block_reader)
treebank_raw = LazyCorpusLoader(
    'treebank/raw', PlaintextCorpusReader, r'wsj_.*')
udhr = LazyCorpusLoader(
    'udhr', PlaintextCorpusReader, r'(?!README|\.).*',
    # Encodings specified in filenames but not mapped to anything:
    # DallakHelv, VIQR, Cyrillic+Abkh, WinResearcher, font,
    # Afenegus6..60375, VG2Main, VPS, Turkish, TCVN, Az.Times.Lat0117,
    # EUC, Baltic, err, Az.Times.Cyr.Normal0117, T61, Amahuaca, Agra
    encoding=[('.*-UTF8$', 'utf-8'), ('.*-Latin1$', 'latin-1'),
              ('.*-Hebrew$', 'hebrew'), ('.*-Arabic$', 'arabic'),
              ('.*-Cyrillic$', 'cyrillic'), ('.*-SJIS$', 'SJIS'),
              ('.*-GB2312$', 'GB2312'), ('.*-Latin2$', 'ISO-8859-2'),
              ('.*-Greek$', 'greek'), ('.*-UFT8$', 'utf-8'),
              ('Hungarian_Magyar-Unicode', 'utf-16-le')]
    )
verbnet = LazyCorpusLoader(
    'verbnet', VerbnetCorpusReader, r'(?!\.).*\.xml')
webtext = LazyCorpusLoader(
    'webtext', PlaintextCorpusReader, r'(?!README|\.).*\.txt')
wordnet = LazyCorpusLoader(
    'wordnet', WordNetCorpusReader)
wordnet_ic = LazyCorpusLoader(
    'wordnet_ic', WordNetICCorpusReader, '.*\.dat')
words = LazyCorpusLoader(
    'words', WordListCorpusReader, r'(?!README|\.).*')
ycoe = LazyCorpusLoader(
    'ycoe', YCOECorpusReader)
# defined after treebank
propbank = LazyCorpusLoader(
    'propbank', PropbankCorpusReader,
    'prop.txt', 'frames/.*\.xml', 'verbs.txt',
    lambda filename: re.sub(r'^wsj/\d\d/', '', filename),
    treebank) # Must be defined *after* treebank corpus.
nombank = LazyCorpusLoader(
    'nombank.1.0', NombankCorpusReader,
    'nombank.1.0', 'frames/.*\.xml', 'nombank.1.0.words',
    lambda filename: re.sub(r'^wsj/\d\d/', '', filename),
    treebank) # Must be defined *after* treebank corpus.

def demo():
    # This is out-of-date:
    abc.demo()
    brown.demo()
#    chat80.demo()
    cmudict.demo()
    conll2000.demo()
    conll2002.demo()
    genesis.demo()
    gutenberg.demo()
    ieer.demo()
    inaugural.demo()
    indian.demo()
    names.demo()
    ppattach.demo()
    senseval.demo()
    shakespeare.demo()
    sinica_treebank.demo()
    state_union.demo()
    stopwords.demo()
    timit.demo()
    toolbox.demo()
    treebank.demo()
    udhr.demo()
    webtext.demo()
    words.demo()
#    ycoe.demo()

if __name__ == '__main__':
    #demo()
    pass

<commit_msg>Specify encodings for each file in genesis. I determined the encodings with the Unix "file" utility, which gives:<commit_after># Natural Language Toolkit: Corpus Readers
#
# Copyright (C) 2001-2010 NLTK Project
# Author: Edward Loper <edloper@gradient.cis.upenn.edu>
# URL: <http://www.nltk.org/>
# For license information, see LICENSE.TXT

# [xx] this docstring isnt' up-to-date!
"""
NLTK corpus readers.  The modules in this package provide functions
that can be used to read corpus files in a variety of formats.  These
functions can be used to read both the corpus files that are
distributed in the NLTK corpus package, and corpus files that are part
of external corpora.

Available Corpora
=================

Please see http://nltk.googlecode.com/svn/trunk/nltk_data/index.xml
for a complete list.  Install corpora using nltk.download().

Corpus Reader Functions
=======================
Each corpus module defines one or more X{corpus reader functions},
which can be used to read documents from that corpus.  These functions
take an argument, C{item}, which is used to indicate which document
should be read from the corpus:

  - If C{item} is one of the unique identifiers listed in the corpus
    module's C{items} variable, then the corresponding document will
    be loaded from the NLTK corpus package.

  - If C{item} is a filename, then that file will be read.

Additionally, corpus reader functions can be given lists of item
names; in which case, they will return a concatenation of the
corresponding documents.

Corpus reader functions are named based on the type of information
they return.  Some common examples, and their return types, are:

  - I{corpus}.words(): list of str
  - I{corpus}.sents(): list of (list of str)
  - I{corpus}.paras(): list of (list of (list of str))
  - I{corpus}.tagged_words(): list of (str,str) tuple
  - I{corpus}.tagged_sents(): list of (list of (str,str))
  - I{corpus}.tagged_paras(): list of (list of (list of (str,str)))
  - I{corpus}.chunked_sents(): list of (Tree w/ (str,str) leaves)
  - I{corpus}.parsed_sents(): list of (Tree with str leaves)
  - I{corpus}.parsed_paras(): list of (list of (Tree with str leaves))
  - I{corpus}.xml(): A single xml ElementTree
  - I{corpus}.raw(): unprocessed corpus contents

For example, to read a list of the words in the Brown Corpus, use
C{nltk.corpus.brown.words()}:

    >>> from nltk.corpus import brown
    >>> print brown.words()
    ['The', 'Fulton', 'County', 'Grand', 'Jury', 'said', ...]

Corpus Metadata
===============
Metadata about the NLTK corpora, and their individual documents, is
stored using U{Open Language Archives Community (OLAC)
<http://www.language-archives.org/>} metadata records.  These records
can be accessed using C{nltk.corpus.I{corpus}.olac()}.
"""

import re

from nltk.tokenize import RegexpTokenizer
from nltk.tag import simplify_brown_tag, simplify_wsj_tag,\
                     simplify_alpino_tag, simplify_indian_tag,\
                     simplify_tag

from util import LazyCorpusLoader
from reader import *

abc = LazyCorpusLoader(
    'abc', PlaintextCorpusReader, r'(?!\.).*\.txt')
alpino = LazyCorpusLoader(
    'alpino', AlpinoCorpusReader, tag_mapping_function=simplify_alpino_tag)
brown = LazyCorpusLoader(
    'brown', CategorizedTaggedCorpusReader, r'c[a-z]\d\d',
    cat_file='cats.txt', tag_mapping_function=simplify_brown_tag)
cess_cat = LazyCorpusLoader(
    'cess_cat', BracketParseCorpusReader, r'(?!\.).*\.tbf',
    tag_mapping_function=simplify_tag)
cess_esp = LazyCorpusLoader(
    'cess_esp', BracketParseCorpusReader, r'(?!\.).*\.tbf',
    tag_mapping_function=simplify_tag)
cmudict = LazyCorpusLoader(
    'cmudict', CMUDictCorpusReader, ['cmudict'])
conll2000 = LazyCorpusLoader(
    'conll2000', ConllChunkCorpusReader,
    ['train.txt', 'test.txt'], ('NP','VP','PP'))
conll2002 = LazyCorpusLoader(
    'conll2002', ConllChunkCorpusReader, '.*\.(test|train).*',
    ('LOC', 'PER', 'ORG', 'MISC'), encoding='utf-8')
conll2007 = LazyCorpusLoader(
    'conll2007', DependencyCorpusReader, '.*\.(test|train).*',
    encoding='utf-8')
dependency_treebank = LazyCorpusLoader(
    'dependency_treebank', DependencyCorpusReader, '.*\.dp')
floresta = LazyCorpusLoader(
    'floresta', BracketParseCorpusReader, r'(?!\.).*\.ptb', '#',
    tag_mapping_function=simplify_tag)
gazetteers = LazyCorpusLoader(
    'gazetteers', WordListCorpusReader, r'(?!LICENSE|\.).*\.txt')
genesis = LazyCorpusLoader(
    'genesis', PlaintextCorpusReader, r'(?!\.).*\.txt', encoding=[
        ('finnish|french|german', 'latin_1'),
        ('swedish', 'cp865'),
        ('.*', 'utf_8')])
gutenberg = LazyCorpusLoader(
    'gutenberg', PlaintextCorpusReader, r'(?!\.).*\.txt')
# corpus not available with NLTK; these lines caused help(nltk.corpus) to break
#hebrew_treebank = LazyCorpusLoader(
#    'hebrew_treebank', BracketParseCorpusReader, r'.*\.txt')
ieer = LazyCorpusLoader(
    'ieer', IEERCorpusReader, r'(?!README|\.).*')
inaugural = LazyCorpusLoader(
    'inaugural', PlaintextCorpusReader, r'(?!\.).*\.txt')
# [XX] This should probably just use TaggedCorpusReader:
indian = LazyCorpusLoader(
    'indian', IndianCorpusReader, r'(?!\.).*\.pos',
    tag_mapping_function=simplify_indian_tag)
ipipan = LazyCorpusLoader(
    'ipipan', IPIPANCorpusReader, r'(?!\.).*morph\.xml')
mac_morpho = LazyCorpusLoader(
    'mac_morpho', MacMorphoCorpusReader, r'(?!\.).*\.txt',
    tag_mapping_function=simplify_tag, encoding='latin-1')
machado = LazyCorpusLoader(
    'machado', PortugueseCategorizedPlaintextCorpusReader,
    r'(?!\.).*\.txt', cat_pattern=r'([a-z]*)/.*', encoding='latin-1')
movie_reviews = LazyCorpusLoader(
    'movie_reviews', CategorizedPlaintextCorpusReader,
    r'(?!\.).*\.txt', cat_pattern=r'(neg|pos)/.*')
names = LazyCorpusLoader(
    'names', WordListCorpusReader, r'(?!\.).*\.txt')
nps_chat = LazyCorpusLoader(
    'nps_chat', NPSChatCorpusReader, r'(?!README|\.).*\.xml',
    tag_mapping_function=simplify_wsj_tag)
pl196x = LazyCorpusLoader(
    'pl196x', Pl196xCorpusReader, r'[a-z]-.*\.xml',
    cat_file='cats.txt', textid_file='textids.txt')
ppattach = LazyCorpusLoader(
    'ppattach', PPAttachmentCorpusReader, ['training', 'test', 'devset'])
qc = LazyCorpusLoader(
    'qc', StringCategoryCorpusReader, ['train.txt', 'test.txt'])
reuters = LazyCorpusLoader(
    'reuters', CategorizedPlaintextCorpusReader, '(training|test).*',
    cat_file='cats.txt')
rte = LazyCorpusLoader(
    'rte', RTECorpusReader, r'(?!\.).*\.xml')
senseval = LazyCorpusLoader(
    'senseval', SensevalCorpusReader, r'(?!\.).*\.pos')
shakespeare = LazyCorpusLoader(
    'shakespeare', XMLCorpusReader, r'(?!\.).*\.xml')
sinica_treebank = LazyCorpusLoader(
    'sinica_treebank', SinicaTreebankCorpusReader, ['parsed'],
    tag_mapping_function=simplify_tag)
state_union = LazyCorpusLoader(
    'state_union', PlaintextCorpusReader, r'(?!\.).*\.txt')
stopwords = LazyCorpusLoader(
    'stopwords', WordListCorpusReader, r'(?!README|\.).*')
swadesh = LazyCorpusLoader(
    'swadesh', SwadeshCorpusReader, r'(?!README|\.).*')
switchboard = LazyCorpusLoader(
    'switchboard', SwitchboardCorpusReader)
timit = LazyCorpusLoader(
    'timit', TimitCorpusReader)
toolbox = LazyCorpusLoader(
    'toolbox', ToolboxCorpusReader, r'(?!.*(README|\.)).*\.(dic|txt)')
treebank = LazyCorpusLoader(
    'treebank/combined', BracketParseCorpusReader, r'wsj_.*\.mrg',
    tag_mapping_function=simplify_wsj_tag)
treebank_chunk = LazyCorpusLoader(
    'treebank/tagged', ChunkedCorpusReader, r'wsj_.*\.pos',
    sent_tokenizer=RegexpTokenizer(r'(?<=/\.)\s*(?![^\[]*\])', gaps=True),
    para_block_reader=tagged_treebank_para_block_reader)
treebank_raw = LazyCorpusLoader(
    'treebank/raw', PlaintextCorpusReader, r'wsj_.*')
udhr = LazyCorpusLoader(
    'udhr', PlaintextCorpusReader, r'(?!README|\.).*',
    # Encodings specified in filenames but not mapped to anything:
    # DallakHelv, VIQR, Cyrillic+Abkh, WinResearcher, font,
    # Afenegus6..60375, VG2Main, VPS, Turkish, TCVN, Az.Times.Lat0117,
    # EUC, Baltic, err, Az.Times.Cyr.Normal0117, T61, Amahuaca, Agra
    encoding=[('.*-UTF8$', 'utf-8'), ('.*-Latin1$', 'latin-1'),
              ('.*-Hebrew$', 'hebrew'), ('.*-Arabic$', 'arabic'),
              ('.*-Cyrillic$', 'cyrillic'), ('.*-SJIS$', 'SJIS'),
              ('.*-GB2312$', 'GB2312'), ('.*-Latin2$', 'ISO-8859-2'),
              ('.*-Greek$', 'greek'), ('.*-UFT8$', 'utf-8'),
              ('Hungarian_Magyar-Unicode', 'utf-16-le')]
    )
verbnet = LazyCorpusLoader(
    'verbnet', VerbnetCorpusReader, r'(?!\.).*\.xml')
webtext = LazyCorpusLoader(
    'webtext', PlaintextCorpusReader, r'(?!README|\.).*\.txt')
wordnet = LazyCorpusLoader(
    'wordnet', WordNetCorpusReader)
wordnet_ic = LazyCorpusLoader(
    'wordnet_ic', WordNetICCorpusReader, '.*\.dat')
words = LazyCorpusLoader(
    'words', WordListCorpusReader, r'(?!README|\.).*')
ycoe = LazyCorpusLoader(
    'ycoe', YCOECorpusReader)
# defined after treebank
propbank = LazyCorpusLoader(
    'propbank', PropbankCorpusReader,
    'prop.txt', 'frames/.*\.xml', 'verbs.txt',
    lambda filename: re.sub(r'^wsj/\d\d/', '', filename),
    treebank) # Must be defined *after* treebank corpus.
nombank = LazyCorpusLoader(
    'nombank.1.0', NombankCorpusReader,
    'nombank.1.0', 'frames/.*\.xml', 'nombank.1.0.words',
    lambda filename: re.sub(r'^wsj/\d\d/', '', filename),
    treebank) # Must be defined *after* treebank corpus.

def demo():
    # This is out-of-date:
    abc.demo()
    brown.demo()
#    chat80.demo()
    cmudict.demo()
    conll2000.demo()
    conll2002.demo()
    genesis.demo()
    gutenberg.demo()
    ieer.demo()
    inaugural.demo()
    indian.demo()
    names.demo()
    ppattach.demo()
    senseval.demo()
    shakespeare.demo()
    sinica_treebank.demo()
    state_union.demo()
    stopwords.demo()
    timit.demo()
    toolbox.demo()
    treebank.demo()
    udhr.demo()
    webtext.demo()
    words.demo()
#    ycoe.demo()

if __name__ == '__main__':
    #demo()
    pass

<|endoftext|>


<commit_before># Natural Language Toolkit: Linear Logic
#
# Author: Dan Garrette <dhgarrette@gmail.com>
#
# Copyright (C) 2001-2010 NLTK Project
# URL: <http://www.nltk.org/>
# For license information, see LICENSE.TXT

from nltk.internals import Counter
from logic import LogicParser

_counter = Counter()

class Expression(object):
    def applyto(self, other, other_indices=None):
        return ApplicationExpression(self, other, other_indices)

    def __call__(self, other):
        return self.applyto(other)

    def __repr__(self):
        return '<' + self.__class__.__name__ + ' ' + str(self) + '>'

class AtomicExpression(Expression):
    def __init__(self, name, dependencies=None):
        """
        @param name: C{str} for the constant name
        @param dependencies: C{list} of C{int} for the indices on which this atom is dependent
        """
        assert isinstance(name, str)
        self.name = name

        if not dependencies:
            dependencies = []
        self.dependencies = dependencies

    def simplify(self, bindings=None):
        """
        If 'self' is bound by 'bindings', return the atomic to which it is bound.
        Otherwise, return self.

        @param bindings: C{BindingDict} A dictionary of bindings used to simplify
        @return: C{AtomicExpression}
        """
        if bindings and self in bindings:
            return bindings[self]
        else:
            return self

    def compile_pos(self, index_counter, glueFormulaFactory):
        """
        From Iddo Lev's PhD Dissertation p108-109

        @param index_counter: C{Counter} for unique indices
        @param glueFormulaFactory: C{GlueFormula} for creating new glue formulas
        @return: (C{Expression},C{set}) for the compiled linear logic and any newly created glue formulas
        """
        self.dependencies = []
        return (self, [])

    def compile_neg(self, index_counter, glueFormulaFactory):
        """
        From Iddo Lev's PhD Dissertation p108-109

        @param index_counter: C{Counter} for unique indices
        @param glueFormulaFactory: C{GlueFormula} for creating new glue formulas
        @return: (C{Expression},C{set}) for the compiled linear logic and any newly created glue formulas
        """
        self.dependencies = []
        return (self, [])

    def initialize_labels(self, fstruct):
        self.name = fstruct.initialize_label(self.name.lower())

    def __eq__(self, other):
        return self.__class__ == other.__class__ and self.name == other.name

    def __str__(self):
        accum = self.name
        if self.dependencies:
            accum += str(self.dependencies)
        return accum

    def __hash__(self):
        return hash(self.name)

class ConstantExpression(AtomicExpression):
    def unify(self, other, bindings):
        """
        If 'other' is a constant, then it must be equal to 'self'.  If 'other' is a variable,
        then it must not be bound to anything other than 'self'.

        @param other: C{Expression}
        @param bindings: C{BindingDict} A dictionary of all current bindings
        @return: C{BindingDict} A new combined dictionary of of 'bindings' and any new binding
        @raise UnificationException: If 'self' and 'other' cannot be unified in the context of 'bindings'
        """
        assert isinstance(other, Expression)
        if isinstance(other, VariableExpression):
            try:
                return bindings + BindingDict([(other, self)])
            except VariableBindingException:
                pass
        elif self == other:
                return bindings
        raise UnificationException(self, other, bindings)

class VariableExpression(AtomicExpression):
    def unify(self, other, bindings):
        """
        'self' must not be bound to anything other than 'other'.

        @param other: C{Expression}
        @param bindings: C{BindingDict} A dictionary of all current bindings
        @return: C{BindingDict} A new combined dictionary of of 'bindings' and the new binding
        @raise UnificationException: If 'self' and 'other' cannot be unified in the context of 'bindings'
        """
        assert isinstance(other, Expression)
        try:
            if self == other:
                return bindings
            else:
                return bindings + BindingDict([(self, other)])
        except VariableBindingException:
            raise UnificationException(self, other, bindings)

class ImpExpression(Expression):
    def __init__(self, antecedent, consequent):
        """
        @param antecedent: C{Expression} for the antecedent
        @param consequent: C{Expression} for the consequent
        """
        assert isinstance(antecedent, Expression)
        assert isinstance(consequent, Expression)
        self.antecedent = antecedent
        self.consequent = consequent

    def simplify(self, bindings=None):
        return self.__class__(self.antecedent.simplify(bindings), self.consequent.simplify(bindings))

    def unify(self, other, bindings):
        """
        Both the antecedent and consequent of 'self' and 'other' must unify.

        @param other: C{ImpExpression}
        @param bindings: C{BindingDict} A dictionary of all current bindings
        @return: C{BindingDict} A new combined dictionary of of 'bindings' and any new bindings
        @raise UnificationException: If 'self' and 'other' cannot be unified in the context of 'bindings'
        """
        assert isinstance(other, ImpExpression)
        try:
            return bindings + self.antecedent.unify(other.antecedent, bindings) + self.consequent.unify(other.consequent, bindings)
        except VariableBindingException:
            raise UnificationException(self, other, bindings)

    def compile_pos(self, index_counter, glueFormulaFactory):
        """
        From Iddo Lev's PhD Dissertation p108-109

        @param index_counter: C{Counter} for unique indices
        @param glueFormulaFactory: C{GlueFormula} for creating new glue formulas
        @return: (C{Expression},C{set}) for the compiled linear logic and any newly created glue formulas
        """
        (a, a_new) = self.antecedent.compile_neg(index_counter, glueFormulaFactory)
        (c, c_new) = self.consequent.compile_pos(index_counter, glueFormulaFactory)
        return (ImpExpression(a,c), a_new + c_new)

    def compile_neg(self, index_counter, glueFormulaFactory):
        """
        From Iddo Lev's PhD Dissertation p108-109

        @param index_counter: C{Counter} for unique indices
        @param glueFormulaFactory: C{GlueFormula} for creating new glue formulas
        @return: (C{Expression},C{list} of C{GlueFormula}) for the compiled linear logic and any newly created glue formulas
        """
        (a, a_new) = self.antecedent.compile_pos(index_counter, glueFormulaFactory)
        (c, c_new) = self.consequent.compile_neg(index_counter, glueFormulaFactory)
        fresh_index = index_counter.get()
        c.dependencies.append(fresh_index)
        new_v = glueFormulaFactory('v%s' % fresh_index, a, set([fresh_index]))
        return (c, a_new + c_new + [new_v])

    def initialize_labels(self, fstruct):
        self.antecedent.initialize_labels(fstruct)
        self.consequent.initialize_labels(fstruct)

    def __eq__(self, other):
        return self.__class__ == other.__class__ and \
                self.antecedent == other.antecedent and self.consequent == other.consequent

    def __str__(self):
        return Tokens.OPEN + str(self.antecedent) + ' ' + Tokens.IMP + \
               ' ' + str(self.consequent) + Tokens.CLOSE

    def __hash__(self):
        return hash('%s%s%s' % (hash(self.antecedent), Tokens.IMP, hash(self.consequent)))

class ApplicationExpression(Expression):
    def __init__(self, function, argument, argument_indices=None):
        """
        @param function: C{Expression} for the function
        @param argument: C{Expression} for the argument
        @param argument_indices: C{set} for the indices of the glue formula from which the argument came
        @raise LinearLogicApplicationException: If 'function' cannot be applied to 'argument' given 'argument_indices'.
        """
        function_simp = function.simplify()
        argument_simp = argument.simplify()

        assert isinstance(function_simp, ImpExpression)
        assert isinstance(argument_simp, Expression)

        bindings = BindingDict()

        try:
            if isinstance(function, ApplicationExpression):
                bindings += function.bindings
            if isinstance(argument, ApplicationExpression):
                bindings += argument.bindings
            bindings += function_simp.antecedent.unify(argument_simp, bindings)
        except UnificationException, e:
            raise LinearLogicApplicationException, 'Cannot apply %s to %s. %s' % (function_simp, argument_simp, e)

        # If you are running it on complied premises, more conditions apply
        if argument_indices:
            # A.dependencies of (A -o (B -o C)) must be a proper subset of argument_indices
            if not set(function_simp.antecedent.dependencies) < argument_indices:
                raise LinearLogicApplicationException, 'Dependencies unfulfilled when attempting to apply Linear Logic formula %s to %s' % (function_simp, argument_simp)
            if set(function_simp.antecedent.dependencies) == argument_indices:
                raise LinearLogicApplicationException, 'Dependencies not a proper subset of indices when attempting to apply Linear Logic formula %s to %s' % (function_simp, argument_simp)

        self.function = function
        self.argument = argument
        self.bindings = bindings

    def simplify(self, bindings=None):
        """
        Since function is an implication, return its consequent.  There should be
        no need to check that the application is valid since the checking is done
        by the constructor.

        @param bindings: C{BindingDict} A dictionary of bindings used to simplify
        @return: C{Expression}
        """
        if not bindings:
            bindings = self.bindings

        return self.function.simplify(bindings).consequent

    def __eq__(self, other):
        return self.__class__ == other.__class__ and \
                self.function == other.function and self.argument == other.argument

    def __str__(self):
        return str(self.function) + Tokens.OPEN + str(self.argument) + Tokens.CLOSE

    def __hash__(self):
        return hash('%s%s%s' % (hash(self.antecedent), Tokens.OPEN, hash(self.consequent)))

class BindingDict(object):
    def __init__(self, binding_list=None):
        """
        @param binding_list: C{list} of (C{VariableExpression}, C{AtomicExpression}) to initialize the dictionary
        """
        self.d = {}

        if binding_list:
            for (v, b) in binding_list:
                self[v] = b

    def __setitem__(self, variable, binding):
        """
        A binding is consistent with the dict if its variable is not already bound, OR if its
        variable is already bound to its argument.

        @param variable: C{VariableExpression} The variable bind
        @param binding: C{Expression} The expression to which 'variable' should be bound
        @raise VariableBindingException: If the variable cannot be bound in this dictionary
        """
        assert isinstance(variable, VariableExpression)
        assert isinstance(binding, Expression)

        assert variable != binding

        try:
            existing = self.d[variable]
        except KeyError:
            existing = None

        if not existing or binding == existing:
            self.d[variable] = binding
        else:
            raise VariableBindingException, 'Variable %s already bound to another value' % (variable)

    def __getitem__(self, variable):
        """
        Return the expression to which 'variable' is bound
        """
        assert isinstance(variable, VariableExpression)

        intermediate = self.d[variable]
        while intermediate:
            try:
                intermediate = self.d[intermediate]
            except KeyError:
                return intermediate

    def __contains__(self, item):
        return item in self.d

    def __add__(self, other):
        """
        @param other: C{BindingDict} The dict with which to combine self
        @return: C{BindingDict} A new dict containing all the elements of both parameters
        @raise VariableBindingException: If the parameter dictionaries are not consistent with each other
        """
        try:
            combined = BindingDict()
            for v in self.d:
                combined[v] = self.d[v]
            for v in other.d:
                combined[v] = other.d[v]
            return combined
        except VariableBindingException:
            raise VariableBindingException, 'Attempting to add two contradicting'\
                        ' VariableBindingsLists: %s, %s' % (self, other)

    def __str__(self):
        return '{' + ', '.join(['%s: %s' % (v, self.d[v]) for v in self.d]) + '}'

    def __repr__(self):
        return 'BindingDict: ' + str(self)

class VariableBindingException(Exception): pass

class UnificationException(Exception):
    def __init__(self, a, b, bindings):
        Exception.__init__(self, 'Cannot unify %s with %s given %s' % (a, b, bindings))

class LinearLogicApplicationException(Exception): pass

class Tokens(object):
    #Punctuation
    OPEN = '('
    CLOSE = ')'

    #Operations
    IMP = '-o'

    PUNCT = [OPEN, CLOSE]
    TOKENS = PUNCT + [IMP]

class LinearLogicParser(LogicParser):
    """A linear logic expression parser."""
    def __init__(self):
        LogicParser.__init__(self)

        self.order_of_operations = {'APP': 1, Tokens.IMP: 2, None: 3}
        self.right_associated_operations += [Tokens.IMP]

    def get_all_symbols(self):
        return Tokens.TOKENS

    def handle(self, tok, context):
        if tok not in Tokens.TOKENS:
            return self.handle_variable(tok, context)
        elif tok == Tokens.OPEN:
            return self.handle_open(tok, context)

    def get_BooleanExpression_factory(self, tok):
        if tok == Tokens.IMP:
            return ImpExpression
        else:
            return None

    def make_BooleanExpression(self, factory, first, second):
        return factory(first, second)

    def attempt_ApplicationExpression(self, expression, context):
        """Attempt to make an application expression.  If the next tokens
        are an argument in parens, then the argument expression is a
        function being applied to the arguments.  Otherwise, return the
        argument expression."""
        if self.has_priority('APP', context):
            if self.inRange(0) and self.token(0) == Tokens.OPEN:
                self.token() #swallow then open paren
                argument = self.parse_Expression('APP')
                self.assertNextToken(Tokens.CLOSE)
                expression = ApplicationExpression(expression, argument, None)
        return expression

    def make_VariableExpression(self, name):
        if name[0].isupper():
            return VariableExpression(name)
        else:
            return ConstantExpression(name)

def demo():
    llp = LinearLogicParser()

    print llp.parse(r'f')
    print llp.parse(r'(g -o f)')
    print llp.parse(r'((g -o G) -o G)')
    print llp.parse(r'g -o h -o f')
    print llp.parse(r'(g -o f)(g)').simplify()
    print llp.parse(r'(H -o f)(g)').simplify()
    print llp.parse(r'((g -o G) -o G)((g -o f))').simplify()
    print llp.parse(r'(H -o H)((g -o f))').simplify()

if __name__ == '__main__':
    demo()
<commit_msg>renamed LogicParser's order_of_operations field to operator_precedence<commit_after># Natural Language Toolkit: Linear Logic
#
# Author: Dan Garrette <dhgarrette@gmail.com>
#
# Copyright (C) 2001-2010 NLTK Project
# URL: <http://www.nltk.org/>
# For license information, see LICENSE.TXT

from nltk.internals import Counter
from logic import LogicParser

_counter = Counter()

class Expression(object):
    def applyto(self, other, other_indices=None):
        return ApplicationExpression(self, other, other_indices)

    def __call__(self, other):
        return self.applyto(other)

    def __repr__(self):
        return '<' + self.__class__.__name__ + ' ' + str(self) + '>'

class AtomicExpression(Expression):
    def __init__(self, name, dependencies=None):
        """
        @param name: C{str} for the constant name
        @param dependencies: C{list} of C{int} for the indices on which this atom is dependent
        """
        assert isinstance(name, str)
        self.name = name

        if not dependencies:
            dependencies = []
        self.dependencies = dependencies

    def simplify(self, bindings=None):
        """
        If 'self' is bound by 'bindings', return the atomic to which it is bound.
        Otherwise, return self.

        @param bindings: C{BindingDict} A dictionary of bindings used to simplify
        @return: C{AtomicExpression}
        """
        if bindings and self in bindings:
            return bindings[self]
        else:
            return self

    def compile_pos(self, index_counter, glueFormulaFactory):
        """
        From Iddo Lev's PhD Dissertation p108-109

        @param index_counter: C{Counter} for unique indices
        @param glueFormulaFactory: C{GlueFormula} for creating new glue formulas
        @return: (C{Expression},C{set}) for the compiled linear logic and any newly created glue formulas
        """
        self.dependencies = []
        return (self, [])

    def compile_neg(self, index_counter, glueFormulaFactory):
        """
        From Iddo Lev's PhD Dissertation p108-109

        @param index_counter: C{Counter} for unique indices
        @param glueFormulaFactory: C{GlueFormula} for creating new glue formulas
        @return: (C{Expression},C{set}) for the compiled linear logic and any newly created glue formulas
        """
        self.dependencies = []
        return (self, [])

    def initialize_labels(self, fstruct):
        self.name = fstruct.initialize_label(self.name.lower())

    def __eq__(self, other):
        return self.__class__ == other.__class__ and self.name == other.name

    def __str__(self):
        accum = self.name
        if self.dependencies:
            accum += str(self.dependencies)
        return accum

    def __hash__(self):
        return hash(self.name)

class ConstantExpression(AtomicExpression):
    def unify(self, other, bindings):
        """
        If 'other' is a constant, then it must be equal to 'self'.  If 'other' is a variable,
        then it must not be bound to anything other than 'self'.

        @param other: C{Expression}
        @param bindings: C{BindingDict} A dictionary of all current bindings
        @return: C{BindingDict} A new combined dictionary of of 'bindings' and any new binding
        @raise UnificationException: If 'self' and 'other' cannot be unified in the context of 'bindings'
        """
        assert isinstance(other, Expression)
        if isinstance(other, VariableExpression):
            try:
                return bindings + BindingDict([(other, self)])
            except VariableBindingException:
                pass
        elif self == other:
                return bindings
        raise UnificationException(self, other, bindings)

class VariableExpression(AtomicExpression):
    def unify(self, other, bindings):
        """
        'self' must not be bound to anything other than 'other'.

        @param other: C{Expression}
        @param bindings: C{BindingDict} A dictionary of all current bindings
        @return: C{BindingDict} A new combined dictionary of of 'bindings' and the new binding
        @raise UnificationException: If 'self' and 'other' cannot be unified in the context of 'bindings'
        """
        assert isinstance(other, Expression)
        try:
            if self == other:
                return bindings
            else:
                return bindings + BindingDict([(self, other)])
        except VariableBindingException:
            raise UnificationException(self, other, bindings)

class ImpExpression(Expression):
    def __init__(self, antecedent, consequent):
        """
        @param antecedent: C{Expression} for the antecedent
        @param consequent: C{Expression} for the consequent
        """
        assert isinstance(antecedent, Expression)
        assert isinstance(consequent, Expression)
        self.antecedent = antecedent
        self.consequent = consequent

    def simplify(self, bindings=None):
        return self.__class__(self.antecedent.simplify(bindings), self.consequent.simplify(bindings))

    def unify(self, other, bindings):
        """
        Both the antecedent and consequent of 'self' and 'other' must unify.

        @param other: C{ImpExpression}
        @param bindings: C{BindingDict} A dictionary of all current bindings
        @return: C{BindingDict} A new combined dictionary of of 'bindings' and any new bindings
        @raise UnificationException: If 'self' and 'other' cannot be unified in the context of 'bindings'
        """
        assert isinstance(other, ImpExpression)
        try:
            return bindings + self.antecedent.unify(other.antecedent, bindings) + self.consequent.unify(other.consequent, bindings)
        except VariableBindingException:
            raise UnificationException(self, other, bindings)

    def compile_pos(self, index_counter, glueFormulaFactory):
        """
        From Iddo Lev's PhD Dissertation p108-109

        @param index_counter: C{Counter} for unique indices
        @param glueFormulaFactory: C{GlueFormula} for creating new glue formulas
        @return: (C{Expression},C{set}) for the compiled linear logic and any newly created glue formulas
        """
        (a, a_new) = self.antecedent.compile_neg(index_counter, glueFormulaFactory)
        (c, c_new) = self.consequent.compile_pos(index_counter, glueFormulaFactory)
        return (ImpExpression(a,c), a_new + c_new)

    def compile_neg(self, index_counter, glueFormulaFactory):
        """
        From Iddo Lev's PhD Dissertation p108-109

        @param index_counter: C{Counter} for unique indices
        @param glueFormulaFactory: C{GlueFormula} for creating new glue formulas
        @return: (C{Expression},C{list} of C{GlueFormula}) for the compiled linear logic and any newly created glue formulas
        """
        (a, a_new) = self.antecedent.compile_pos(index_counter, glueFormulaFactory)
        (c, c_new) = self.consequent.compile_neg(index_counter, glueFormulaFactory)
        fresh_index = index_counter.get()
        c.dependencies.append(fresh_index)
        new_v = glueFormulaFactory('v%s' % fresh_index, a, set([fresh_index]))
        return (c, a_new + c_new + [new_v])

    def initialize_labels(self, fstruct):
        self.antecedent.initialize_labels(fstruct)
        self.consequent.initialize_labels(fstruct)

    def __eq__(self, other):
        return self.__class__ == other.__class__ and \
                self.antecedent == other.antecedent and self.consequent == other.consequent

    def __str__(self):
        return Tokens.OPEN + str(self.antecedent) + ' ' + Tokens.IMP + \
               ' ' + str(self.consequent) + Tokens.CLOSE

    def __hash__(self):
        return hash('%s%s%s' % (hash(self.antecedent), Tokens.IMP, hash(self.consequent)))

class ApplicationExpression(Expression):
    def __init__(self, function, argument, argument_indices=None):
        """
        @param function: C{Expression} for the function
        @param argument: C{Expression} for the argument
        @param argument_indices: C{set} for the indices of the glue formula from which the argument came
        @raise LinearLogicApplicationException: If 'function' cannot be applied to 'argument' given 'argument_indices'.
        """
        function_simp = function.simplify()
        argument_simp = argument.simplify()

        assert isinstance(function_simp, ImpExpression)
        assert isinstance(argument_simp, Expression)

        bindings = BindingDict()

        try:
            if isinstance(function, ApplicationExpression):
                bindings += function.bindings
            if isinstance(argument, ApplicationExpression):
                bindings += argument.bindings
            bindings += function_simp.antecedent.unify(argument_simp, bindings)
        except UnificationException, e:
            raise LinearLogicApplicationException, 'Cannot apply %s to %s. %s' % (function_simp, argument_simp, e)

        # If you are running it on complied premises, more conditions apply
        if argument_indices:
            # A.dependencies of (A -o (B -o C)) must be a proper subset of argument_indices
            if not set(function_simp.antecedent.dependencies) < argument_indices:
                raise LinearLogicApplicationException, 'Dependencies unfulfilled when attempting to apply Linear Logic formula %s to %s' % (function_simp, argument_simp)
            if set(function_simp.antecedent.dependencies) == argument_indices:
                raise LinearLogicApplicationException, 'Dependencies not a proper subset of indices when attempting to apply Linear Logic formula %s to %s' % (function_simp, argument_simp)

        self.function = function
        self.argument = argument
        self.bindings = bindings

    def simplify(self, bindings=None):
        """
        Since function is an implication, return its consequent.  There should be
        no need to check that the application is valid since the checking is done
        by the constructor.

        @param bindings: C{BindingDict} A dictionary of bindings used to simplify
        @return: C{Expression}
        """
        if not bindings:
            bindings = self.bindings

        return self.function.simplify(bindings).consequent

    def __eq__(self, other):
        return self.__class__ == other.__class__ and \
                self.function == other.function and self.argument == other.argument

    def __str__(self):
        return str(self.function) + Tokens.OPEN + str(self.argument) + Tokens.CLOSE

    def __hash__(self):
        return hash('%s%s%s' % (hash(self.antecedent), Tokens.OPEN, hash(self.consequent)))

class BindingDict(object):
    def __init__(self, binding_list=None):
        """
        @param binding_list: C{list} of (C{VariableExpression}, C{AtomicExpression}) to initialize the dictionary
        """
        self.d = {}

        if binding_list:
            for (v, b) in binding_list:
                self[v] = b

    def __setitem__(self, variable, binding):
        """
        A binding is consistent with the dict if its variable is not already bound, OR if its
        variable is already bound to its argument.

        @param variable: C{VariableExpression} The variable bind
        @param binding: C{Expression} The expression to which 'variable' should be bound
        @raise VariableBindingException: If the variable cannot be bound in this dictionary
        """
        assert isinstance(variable, VariableExpression)
        assert isinstance(binding, Expression)

        assert variable != binding

        try:
            existing = self.d[variable]
        except KeyError:
            existing = None

        if not existing or binding == existing:
            self.d[variable] = binding
        else:
            raise VariableBindingException, 'Variable %s already bound to another value' % (variable)

    def __getitem__(self, variable):
        """
        Return the expression to which 'variable' is bound
        """
        assert isinstance(variable, VariableExpression)

        intermediate = self.d[variable]
        while intermediate:
            try:
                intermediate = self.d[intermediate]
            except KeyError:
                return intermediate

    def __contains__(self, item):
        return item in self.d

    def __add__(self, other):
        """
        @param other: C{BindingDict} The dict with which to combine self
        @return: C{BindingDict} A new dict containing all the elements of both parameters
        @raise VariableBindingException: If the parameter dictionaries are not consistent with each other
        """
        try:
            combined = BindingDict()
            for v in self.d:
                combined[v] = self.d[v]
            for v in other.d:
                combined[v] = other.d[v]
            return combined
        except VariableBindingException:
            raise VariableBindingException, 'Attempting to add two contradicting'\
                        ' VariableBindingsLists: %s, %s' % (self, other)

    def __str__(self):
        return '{' + ', '.join(['%s: %s' % (v, self.d[v]) for v in self.d]) + '}'

    def __repr__(self):
        return 'BindingDict: ' + str(self)

class VariableBindingException(Exception): pass

class UnificationException(Exception):
    def __init__(self, a, b, bindings):
        Exception.__init__(self, 'Cannot unify %s with %s given %s' % (a, b, bindings))

class LinearLogicApplicationException(Exception): pass

class Tokens(object):
    #Punctuation
    OPEN = '('
    CLOSE = ')'

    #Operations
    IMP = '-o'

    PUNCT = [OPEN, CLOSE]
    TOKENS = PUNCT + [IMP]

class LinearLogicParser(LogicParser):
    """A linear logic expression parser."""
    def __init__(self):
        LogicParser.__init__(self)

        self.operator_precedence = {'APP': 1, Tokens.IMP: 2, None: 3}
        self.right_associated_operations += [Tokens.IMP]

    def get_all_symbols(self):
        return Tokens.TOKENS

    def handle(self, tok, context):
        if tok not in Tokens.TOKENS:
            return self.handle_variable(tok, context)
        elif tok == Tokens.OPEN:
            return self.handle_open(tok, context)

    def get_BooleanExpression_factory(self, tok):
        if tok == Tokens.IMP:
            return ImpExpression
        else:
            return None

    def make_BooleanExpression(self, factory, first, second):
        return factory(first, second)

    def attempt_ApplicationExpression(self, expression, context):
        """Attempt to make an application expression.  If the next tokens
        are an argument in parens, then the argument expression is a
        function being applied to the arguments.  Otherwise, return the
        argument expression."""
        if self.has_priority('APP', context):
            if self.inRange(0) and self.token(0) == Tokens.OPEN:
                self.token() #swallow then open paren
                argument = self.parse_Expression('APP')
                self.assertNextToken(Tokens.CLOSE)
                expression = ApplicationExpression(expression, argument, None)
        return expression

    def make_VariableExpression(self, name):
        if name[0].isupper():
            return VariableExpression(name)
        else:
            return ConstantExpression(name)

def demo():
    llp = LinearLogicParser()

    print llp.parse(r'f')
    print llp.parse(r'(g -o f)')
    print llp.parse(r'((g -o G) -o G)')
    print llp.parse(r'g -o h -o f')
    print llp.parse(r'(g -o f)(g)').simplify()
    print llp.parse(r'(H -o f)(g)').simplify()
    print llp.parse(r'((g -o G) -o G)((g -o f))').simplify()
    print llp.parse(r'(H -o H)((g -o f))').simplify()

if __name__ == '__main__':
    demo()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""This Python script formats nginx configuration files in consistent way.

Originally published under https://github.com/1connect/nginx-config-formatter
"""

import argparse
import codecs

import re

__author__ = "Michał Słomkowski"
__license__ = "Apache 2.0"
__version__ = "1.0.2"

INDENTATION = ' ' * 4

TEMPLATE_VARIABLE_OPENING_TAG = '___TEMPLATE_VARIABLE_OPENING_TAG___'
TEMPLATE_VARIABLE_CLOSING_TAG = '___TEMPLATE_VARIABLE_CLOSING_TAG___'

def strip_line(single_line):
    """Strips the line and replaces neighbouring whitespaces with single space (except when within quotation marks)."""
    single_line = single_line.strip()
    if single_line.startswith('#'):
        return single_line

    within_quotes = False
    parts = []
    for part in re.split('"', single_line):
        if within_quotes:
            parts.append(part)
        else:
            parts.append(re.sub(r'[\s]+', ' ', part))
        within_quotes = not within_quotes
    return '"'.join(parts)

def apply_variable_template_tags(line: str) -> str:
    """Replaces variable indicators ${ and } with tags, so subsequent formatting is easier."""
    return re.sub(r'\${\s*(\w+)\s*}',
                  TEMPLATE_VARIABLE_OPENING_TAG + r"\1" + TEMPLATE_VARIABLE_CLOSING_TAG,
                  line,
                  flags=re.UNICODE)

def strip_variable_template_tags(line: str) -> str:
    """Replaces tags back with ${ and } respectively."""
    return re.sub(TEMPLATE_VARIABLE_OPENING_TAG + r'\s*(\w+)\s*' + TEMPLATE_VARIABLE_CLOSING_TAG,
                  r'${\1}',
                  line,
                  flags=re.UNICODE)

def clean_lines(orig_lines) -> list:
    """Strips the lines and splits them if they contain curly brackets."""
    cleaned_lines = []
    for line in orig_lines:
        line = strip_line(line)
        line = apply_variable_template_tags(line)
        if line == "":
            cleaned_lines.append("")
            continue
        else:
            if line.startswith("#"):
                cleaned_lines.append(strip_variable_template_tags(line))
            elif line.startswith("rewrite"):
                cleaned_lines.append(strip_variable_template_tags(line))
            else:
                cleaned_lines.extend(
                    [strip_variable_template_tags(l).strip() for l in re.split(r"([{}])", line) if l != ""])

    return cleaned_lines

def join_opening_bracket(lines):
    """When opening curly bracket is in it's own line (K&R convention), it's joined with precluding line (Java)."""
    modified_lines = []
    for i in range(len(lines)):
        if i > 0 and lines[i] == "{":
            modified_lines[-1] += " {"
        else:
            modified_lines.append(lines[i])
    return modified_lines

def perform_indentation(lines):
    """Indents the lines according to their nesting level determined by curly brackets."""
    indented_lines = []
    current_indent = 0
    for line in lines:
        if not line.startswith("#") and line.endswith('}') and current_indent > 0:
            current_indent -= 1

        if line != "":
            indented_lines.append(current_indent * INDENTATION + line)
        else:
            indented_lines.append("")

        if not line.startswith("#") and line.endswith('{'):
            current_indent += 1

    return indented_lines

def format_config_contents(contents):
    """Accepts the string containing nginx configuration and returns formatted one. Adds newline at the end."""
    lines = contents.splitlines()
    lines = clean_lines(lines)
    lines = join_opening_bracket(lines)
    lines = perform_indentation(lines)

    text = '\n'.join(lines)

    for pattern, substitute in ((r'\n{3,}', '\n\n\n'), (r'^\n', ''), (r'\n$', '')):
        text = re.sub(pattern, substitute, text, re.MULTILINE)

    return text + '\n'

def format_config_file(file_path, original_backup_file_path=None, verbose=True):
    """
    Performs the formatting on the given file. The function tries to detect file encoding first.
    :param file_path: path to original nginx configuration file. This file will be overridden.
    :param original_backup_file_path: optional path, where original file will be backed up.
    :param verbose: show messages
    """
    encodings = ('utf-8', 'latin1')

    encoding_failures = []
    chosen_encoding = None

    for enc in encodings:
        try:
            with codecs.open(file_path, 'r', encoding=enc) as rfp:
                original_file_content = rfp.read()
            chosen_encoding = enc
            break
        except ValueError as e:
            encoding_failures.append(e)

    if chosen_encoding is None:
        raise Exception('none of encodings %s are valid for file %s. Errors: %s'
                        % (encodings, file_path, [e.message for e in encoding_failures]))

    assert original_file_content is not None

    with codecs.open(file_path, 'w', encoding=chosen_encoding) as wfp:
        wfp.write(format_config_contents(original_file_content))

    if verbose:
        print("Formatted file '%s' (detected encoding %s)." % (file_path, chosen_encoding))

    if original_backup_file_path:
        with codecs.open(original_backup_file_path, 'w', encoding=chosen_encoding) as wfp:
            wfp.write(original_file_content)
        if verbose:
            print("Original saved to '%s'." % original_backup_file_path)

if __name__ == "__main__":
    arg_parser = argparse.ArgumentParser(description=__doc__)

    arg_parser.add_argument("-v", "--verbose", action="store_true", help="show formatted file names")
    arg_parser.add_argument("-b", "--backup-original", action="store_true", help="backup original config file")
    arg_parser.add_argument("config_files", nargs='+', help="configuration files to format")

    args = arg_parser.parse_args()

    for config_file_path in args.config_files:
        backup_file_path = config_file_path + '~' if args.backup_original else None
        format_config_file(config_file_path, backup_file_path, args.verbose)
<commit_msg>fix multiline `;` in one line case  and rewrite regex curly bracket case.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""This Python script formats nginx configuration files in consistent way.

Originally published under https://github.com/1connect/nginx-config-formatter
"""

import argparse
import codecs

import re

__author__ = "Michał Słomkowski"
__license__ = "Apache 2.0"
__version__ = "1.0.2"

INDENTATION = ' ' * 4

TEMPLATE_VARIABLE_OPENING_TAG = '___TEMPLATE_VARIABLE_OPENING_TAG___'
TEMPLATE_VARIABLE_CLOSING_TAG = '___TEMPLATE_VARIABLE_CLOSING_TAG___'

def strip_line(single_line):
    """Strips the line and replaces neighbouring whitespaces with single space (except when within quotation marks)."""
    single_line = single_line.strip()
    if single_line.startswith('#'):
        return single_line

    within_quotes = False
    parts = []
    for part in re.split('"', single_line):
        if within_quotes:
            parts.append(part)
        else:
            parts.append(re.sub(r'[\s]+', ' ', part))
        within_quotes = not within_quotes
    return '"'.join(parts)

def apply_variable_template_tags(line: str) -> str:
    """Replaces variable indicators ${ and } with tags, so subsequent formatting is easier."""
    return re.sub(r'\${\s*(\w+)\s*}',
                  TEMPLATE_VARIABLE_OPENING_TAG + r"\1" + TEMPLATE_VARIABLE_CLOSING_TAG,
                  line,
                  flags=re.UNICODE)

def strip_variable_template_tags(line: str) -> str:
    """Replaces tags back with ${ and } respectively."""
    return re.sub(TEMPLATE_VARIABLE_OPENING_TAG + r'\s*(\w+)\s*' + TEMPLATE_VARIABLE_CLOSING_TAG,
                  r'${\1}',
                  line,
                  flags=re.UNICODE)

def clean_lines(orig_lines) -> list:
    """Strips the lines and splits them if they contain curly brackets."""
    cleaned_lines = []
    for line in orig_lines:
        line = strip_line(line)
        line = apply_variable_template_tags(line)
        if line == "":
            cleaned_lines.append("")
            continue
        else:
            if line.startswith("#"):
                cleaned_lines.append(strip_variable_template_tags(line))
            else:
                if line.count(";") > 1:
                    newlines = line.split(";")
                    cleaned_lines.extend(clean_lines(["".join([ln, ";"]) for ln in newlines if ln != ""]))
                else:
                    if line.startswith("rewrite"):
                        cleaned_lines.append(strip_variable_template_tags(line))
                    else:
                        cleaned_lines.extend(
                            [strip_variable_template_tags(l).strip() for l in re.split(r"([{}])", line) if l != ""])
    return cleaned_lines

def join_opening_bracket(lines):
    """When opening curly bracket is in it's own line (K&R convention), it's joined with precluding line (Java)."""
    modified_lines = []
    for i in range(len(lines)):
        if i > 0 and lines[i] == "{":
            modified_lines[-1] += " {"
        else:
            modified_lines.append(lines[i])
    return modified_lines

def perform_indentation(lines):
    """Indents the lines according to their nesting level determined by curly brackets."""
    indented_lines = []
    current_indent = 0
    for line in lines:
        if not line.startswith("#") and line.endswith('}') and current_indent > 0:
            current_indent -= 1

        if line != "":
            indented_lines.append(current_indent * INDENTATION + line)
        else:
            indented_lines.append("")

        if not line.startswith("#") and line.endswith('{'):
            current_indent += 1

    return indented_lines

def format_config_contents(contents):
    """Accepts the string containing nginx configuration and returns formatted one. Adds newline at the end."""
    lines = contents.splitlines()
    lines = clean_lines(lines)
    lines = join_opening_bracket(lines)
    lines = perform_indentation(lines)

    text = '\n'.join(lines)

    for pattern, substitute in ((r'\n{3,}', '\n\n\n'), (r'^\n', ''), (r'\n$', '')):
        text = re.sub(pattern, substitute, text, re.MULTILINE)

    return text + '\n'

def format_config_file(file_path, original_backup_file_path=None, verbose=True):
    """
    Performs the formatting on the given file. The function tries to detect file encoding first.
    :param file_path: path to original nginx configuration file. This file will be overridden.
    :param original_backup_file_path: optional path, where original file will be backed up.
    :param verbose: show messages
    """
    encodings = ('utf-8', 'latin1')

    encoding_failures = []
    chosen_encoding = None

    for enc in encodings:
        try:
            with codecs.open(file_path, 'r', encoding=enc) as rfp:
                original_file_content = rfp.read()
            chosen_encoding = enc
            break
        except ValueError as e:
            encoding_failures.append(e)

    if chosen_encoding is None:
        raise Exception('none of encodings %s are valid for file %s. Errors: %s'
                        % (encodings, file_path, [e.message for e in encoding_failures]))

    assert original_file_content is not None

    with codecs.open(file_path, 'w', encoding=chosen_encoding) as wfp:
        wfp.write(format_config_contents(original_file_content))

    if verbose:
        print("Formatted file '%s' (detected encoding %s)." % (file_path, chosen_encoding))

    if original_backup_file_path:
        with codecs.open(original_backup_file_path, 'w', encoding=chosen_encoding) as wfp:
            wfp.write(original_file_content)
        if verbose:
            print("Original saved to '%s'." % original_backup_file_path)

if __name__ == "__main__":
    arg_parser = argparse.ArgumentParser(description=__doc__)

    arg_parser.add_argument("-v", "--verbose", action="store_true", help="show formatted file names")
    arg_parser.add_argument("-b", "--backup-original", action="store_true", help="backup original config file")
    arg_parser.add_argument("config_files", nargs='+', help="configuration files to format")

    args = arg_parser.parse_args()

    for config_file_path in args.config_files:
        backup_file_path = config_file_path + '~' if args.backup_original else None
        format_config_file(config_file_path, backup_file_path, args.verbose)
<|endoftext|>


<commit_before>"""
calculatepi.py
Author: Avery Wallis

This was an experiment for Math Modeling and Fractions

"""
import math
lists = []
a = 1/2
lists.append[a]

"""
n = int(input("I will estimate pi. How many terms should I use? "))
decimals = int(input("How many decimal places should I use in the result? "))
pi = float(4*(sum([((-1.0)**k)/(2*k+1) for k in range(0,n)])))
print("The approximate value of pi is {0}".format(round(pi, decimals)))
"""<commit_msg>Updated from Brython Server: 9/18/2016 12:06:31 PM<commit_after>"""
calculatepi.py
Author: Avery Wallis

This was an experiment for Math Modeling and Fractions

"""
import math
lists = []
a = 1/2
lists.append(a)

"""
n = int(input("I will estimate pi. How many terms should I use? "))
decimals = int(input("How many decimal places should I use in the result? "))
pi = float(4*(sum([((-1.0)**k)/(2*k+1) for k in range(0,n)])))
print("The approximate value of pi is {0}".format(round(pi, decimals)))
"""<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

## DEPENDENCIES: primer3_core, exonerate, blast (?)

def main():
    print("Yarr! I be Cap'n Primer.")
    # CROSS-SPECIES CDS SEQUENCE EXPLORATION -- "in-species" (e.g. Bdor) and "out-species" (e.g. Dmel)
    # blast CDS of genes of interest from out-species against in-species genome
        # get cds.fasta for out-species
        # tweak headers of fasta somehow, magically
        # prepare blast database(s) for each in-species genome or whatever you wanna use
        # run blast against your in-species fastas using cross-species sequence exploration parameters

    # CROSS-SPECIES PROTEIN SEQUENCE EXPLORATION
    # blast translations (proteins) of genes of interest from out-species against in-species genome
        # get translations.fasta for out-species
        # tweak headers
        # run blast again, using protein-to-genome parameters

    # OK, ALL THAT GOT YOU WAS A BUNCH OF BLAST RESULTS. SO.

    # PARSING BLAST RESULTS
    # should have max e-value and min alignment length for blast hits
    # apparently 'filter_blast_hits.py' does this bit
    # so filter each set of blast results

    # should have priorities for each of the above in-species fastas
    # so somehow go through all the results in order of priority and, for each gene of interest, pick the first match.

    # GETTING SEQUENCES FOR MATCHING TRANSCRIPTS/CDSs
    # look into 'fasta_from_blast_hits.py' -- basically given original fasta and blast hit (contains indices?) write fasta of matching seq
    # do this for all the results (coming from multiple in-species fastas, remember)

    # TRIMMING SCAFFOLDS FOR GENOME ALIGNMENTS (?)
    # so apparently the step above gave us the whole dern scaffolds. which is fine for the transcriptome, but too much for the genome. so.
    # use exonerate to map the out-species genes of interest to their corresponding scaffolds
        # get a list of the out-species genes that mapped to the genome
        # get a fasta of those PROTEIN sequences
        # run exonerate --model protein2genome ...etc.
        # TODO maybe skip blast and just align? this was in my notes from before. because one gene actually aligned better to a different spot
    # exonerate outputs subsequences in fasta format
    # if any sequence is too short, use its indices (?) to pull a longer subsequence from original in-species fasta (?)

    # COMBINE ALL SEQUENCES INTO ONE FILE
    # the matches from all in-species fastas get turned into one big fasta.

    # ADD IN ANY OTHER KNOWN-INTERESTING IN-SPECIES TRANSCRIPTS TO FASTA

    # PREPARE INPUT FOR PRIMER3_CORE
        # make .boulder-io file
        # make primer_settings.conf file
        # cat 'em together

    # RUN PRIMER3_CORE

    # BLAST PRIMER SEQUENCES AGAINST GENOME TO MAKE SURE THEY ONLY AMPLIFY ONE REGION
        # concatenate left primer with reverse complement of right primer
        # make fasta
        # blastall against in-species genome (i guess)
        # using e-value and alignment length cutoffs, discard matches you don't believe
        # each primer should have only one legit match. if it's got more than one, it's trasssh

if __name__ == '__main__':
    main()
<commit_msg>more comments<commit_after>#!/usr/bin/env python
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

## DEPENDENCIES: primer3_core, exonerate, blast (?)
## data: any number of .fasta files representing in-species sequences (genome, transcriptome, etc.)
##       out-species genes of interest in nucleotide and protein fasta format
##       blast results a-plenty
##       alignment results from exonerate
##       various config data

## maybe work backwards -- given fasta file of sequences to create primers for, run primer3_core and then blast against reference to verify.
## then can add in previous exploration steps.

def main():
    print("Yarr! I be Cap'n Primer.")
    # CROSS-SPECIES CDS SEQUENCE EXPLORATION -- "in-species" (e.g. Bdor) and "out-species" (e.g. Dmel)
    # blast CDS of genes of interest from out-species against in-species genome
        # get cds.fasta for out-species
        # tweak headers of fasta somehow, magically
        # prepare blast database(s) for each in-species genome or whatever you wanna use
        # run blast against your in-species fastas using cross-species sequence exploration parameters

    # CROSS-SPECIES PROTEIN SEQUENCE EXPLORATION
    # blast translations (proteins) of genes of interest from out-species against in-species genome
        # get translations.fasta for out-species
        # tweak headers
        # run blast again, using protein-to-genome parameters

    # OK, ALL THAT GOT YOU WAS A BUNCH OF BLAST RESULTS. SO.

    # PARSING BLAST RESULTS
    # should have max e-value and min alignment length for blast hits
    # apparently 'filter_blast_hits.py' does this bit
    # so filter each set of blast results

    # should have priorities for each of the above in-species fastas
    # so somehow go through all the results in order of priority and, for each gene of interest, pick the first match.

    # GETTING SEQUENCES FOR MATCHING TRANSCRIPTS/CDSs
    # look into 'fasta_from_blast_hits.py' -- basically given original fasta and blast hit (contains indices?) write fasta of matching seq
    # do this for all the results (coming from multiple in-species fastas, remember)

    # TRIMMING SCAFFOLDS FOR GENOME ALIGNMENTS (?)
    # so apparently the step above gave us the whole dern scaffolds. which is fine for the transcriptome, but too much for the genome. so.
    # use exonerate to map the out-species genes of interest to their corresponding scaffolds
        # get a list of the out-species genes that mapped to the genome
        # get a fasta of those PROTEIN sequences
        # run exonerate --model protein2genome ...etc.
        # TODO maybe skip blast and just align? this was in my notes from before. because one gene actually aligned better to a different spot
    # exonerate outputs subsequences in fasta format
    # if any sequence is too short, use its indices (?) to pull a longer subsequence from original in-species fasta (?)

    # COMBINE ALL SEQUENCES INTO ONE FILE
    # the matches from all in-species fastas get turned into one big fasta.

    # ADD IN ANY OTHER KNOWN-INTERESTING IN-SPECIES TRANSCRIPTS TO FASTA

    # PREPARE INPUT FOR PRIMER3_CORE
        # make .boulder-io file
        # make primer_settings.conf file
        # cat 'em together

    # RUN PRIMER3_CORE

    # BLAST PRIMER SEQUENCES AGAINST GENOME TO MAKE SURE THEY ONLY AMPLIFY ONE REGION
        # concatenate left primer with reverse complement of right primer
        # make fasta
        # blastall against in-species genome (i guess)
        # using e-value and alignment length cutoffs, discard matches you don't believe
        # each primer should have only one legit match. if it's got more than one, it's trasssh

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2011 credativ Ltd (<http://www.credativ.co.uk>).
#    All Rights Reserved
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    'name': 'HSBC Account Banking',
    'version': '0.2',
    'license': 'AGPL-3',
    'author': 'credativ Ltd',
    'website': 'http://www.credativ.co.uk',
    'category': 'Account Banking',
    'depends': ['account_banking'],
    'init_xml': [],
    'update_xml': [
        'account_banking_uk_hsbc.xml',
        'data/banking_export_hsbc.xml',
        'wizard/export_hsbc_view.xml',
    ],
    'demo_xml': [],
    'description': '''
    Module to import HSBC format transation files (S.W.I.F.T MT940) and to export payments for HSBC.net (PAYMUL).

    Currently it is targetting UK market, due to country variances of the MT940 and PAYMUL.

    It is possible to extend this module to work with HSBC.net in other countries and potentially other banks.

    This module adds above import/export filter to the account_banking module.
    All business logic is in account_banking module.
    ''',
    'active': False,
    'installable': True,
}
<commit_msg>Releasing 0.3<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2011 credativ Ltd (<http://www.credativ.co.uk>).
#    All Rights Reserved
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    'name': 'HSBC Account Banking',
    'version': '0.3',
    'license': 'AGPL-3',
    'author': 'credativ Ltd',
    'website': 'http://www.credativ.co.uk',
    'category': 'Account Banking',
    'depends': ['account_banking'],
    'init_xml': [],
    'update_xml': [
        'account_banking_uk_hsbc.xml',
        'data/banking_export_hsbc.xml',
        'wizard/export_hsbc_view.xml',
    ],
    'demo_xml': [],
    'description': '''
    Module to import HSBC format transation files (S.W.I.F.T MT940) and to export payments for HSBC.net (PAYMUL).

    Currently it is targetting UK market, due to country variances of the MT940 and PAYMUL.

    It is possible to extend this module to work with HSBC.net in other countries and potentially other banks.

    This module adds above import/export filter to the account_banking module.
    All business logic is in account_banking module.

    Initial release of this module was co-sponsored by canonical.
    ''',
    'active': False,
    'installable': True,
}
<|endoftext|>