<commit_before>import os.path
import re
import cx_Oracle

from django.db import connection, models
from django.db.backends.util import truncate_name
from django.core.management.color import no_style
from django.db.backends.oracle.base import get_sequence_name
from south.db import generic

print " ! WARNING: South's Oracle support is still alpha."
print " !          Be wary of posible bugs."

class DatabaseOperations(generic.DatabaseOperations):
    """
    Oracle implementation of database operations.
    """
    backend_name = 'oracle'

    alter_string_set_type =     'ALTER TABLE %(table_name)s MODIFY "%(column)s" %(type)s %(nullity)s;'
    alter_string_set_default =  'ALTER TABLE %(table_name)s MODIFY "%(column)s" DEFAULT %(default)s;'
    add_column_string =         'ALTER TABLE %s ADD %s;'
    delete_column_string =      'ALTER TABLE %s DROP COLUMN %s;'

    allows_combined_alters = False

    constraits_dict = {
        'PRIMARY KEY': 'P',
        'UNIQUE': 'U',
        'CHECK': 'C',
        'REFERENCES': 'R'
    }
    table_names_cache = set()

    def adj_column_sql(self, col):
        col = re.sub('(?P<constr>CHECK \(.*\))(?P<any>.*)(?P<default>DEFAULT [0|1])',
                     lambda mo: '%s %s%s'%(mo.group('default'), mo.group('constr'), mo.group('any')), col) #syntax fix for boolean field only
        col = re.sub('(?P<not_null>NOT NULL) (?P<default>DEFAULT.+)',
                     lambda mo: '%s %s'%(mo.group('default'), mo.group('not_null')), col) #fix order  of DEFAULT and NOT NULL
        return col

    def check_m2m(self, table_name):
        m2m_table_name = table_name
        existing_tables = []

        if not self.table_names_cache:
            self.check_meta(table_name)
            self.table_names_cache = set(connection.introspection.table_names())
        tn = table_name.rsplit('_', 1)

        while len(tn) == 2:
            tn2qn = self.quote_name(tn[0], upper = False, check_m2m = False)
            if tn2qn in self.table_names_cache:
                m2m_table_name = table_name.replace(tn[0], tn2qn)
                break
            else:
                if not existing_tables:
                    existing_tables = connection.introspection.table_names()
                if tn2qn in existing_tables:
                    m2m_table_name = table_name.replace(tn[0], tn2qn)
                    break
            tn = tn[0].rsplit('_', 1)

        self.table_names_cache.add(m2m_table_name)
        return m2m_table_name

    def check_meta(self, table_name):
        return table_name in [ m._meta.db_table for m in models.get_models() ] #caching provided by Django

    def quote_name(self, name, upper=True, column = False, check_m2m = True):
        if not column:
            if check_m2m:
                name = self.check_m2m(name)
            if self.check_meta(name): #replication of Django flow for models where Meta.db_table is set by user
                name = name.upper()
        tn = truncate_name(name, connection.ops.max_name_length())

        return upper and tn.upper() or tn.lower()

    def create_table(self, table_name, fields):
        qn = self.quote_name(table_name, upper = False)
        qn_upper = qn.upper()
        columns = []
        autoinc_sql = ''

        for field_name, field in fields:
            col = self.column_sql(qn_upper, field_name, field)
            if not col:
                continue
            col = self.adj_column_sql(col)

            columns.append(col)
            if isinstance(field, models.AutoField):
                autoinc_sql = connection.ops.autoinc_sql(self.check_meta(table_name) and table_name or qn, field_name)

        sql = 'CREATE TABLE %s (%s);' % (qn_upper, ', '.join([col for col in columns]))
        self.execute(sql)
        if autoinc_sql:
            self.execute(autoinc_sql[0])
            self.execute(autoinc_sql[1])

    def delete_table(self, table_name, cascade=True):
        qn = self.quote_name(table_name, upper = False)

        if cascade:
            self.execute('DROP TABLE %s CASCADE CONSTRAINTS PURGE;' % qn.upper())
        else:
            self.execute('DROP TABLE %s;' % qn.upper())
        self.execute('DROP SEQUENCE %s;'%get_sequence_name(qn))

    def alter_column(self, table_name, name, field, explicit_name=True):
        qn = self.quote_name(table_name)

        # hook for the field to do any resolution prior to it's attributes being queried
        if hasattr(field, 'south_init'):
            field.south_init()
        field = self._field_sanity(field)

        # Add _id or whatever if we need to
        field.set_attributes_from_name(name)
        if not explicit_name:
            name = field.column
        qn_col = self.quote_name(name, column = True)

        # First, change the type
        params = {
            'table_name':qn,
            'column': qn_col,
            'type': self._db_type_for_alter_column(field),
            'nullity': 'NOT NULL',
            'default': 'NULL'
        }
        if field.null:
            params['nullity'] = ''
        sqls = [self.alter_string_set_type % params]

        if not field.null and field.has_default():
            params['default'] = field.get_default()

        sqls.append(self.alter_string_set_default % params)

        #UNIQUE constraint
        unique_constraint = list(self._constraints_affecting_columns(qn, [qn_col]))

        if field.unique and not unique_constraint:
            self.create_unique(qn, [qn_col])
        elif not field.unique and unique_constraint:
            self.delete_unique(qn, [qn_col])

        #CHECK constraint is not handled

        for sql in sqls:
            try:
                self.execute(sql)
            except cx_Oracle.DatabaseError, exc:
                if str(exc).find('ORA-01442') == -1:
                    raise

    def add_column(self, table_name, name, field, keep_default=True):
        qn = self.quote_name(table_name, upper = False)
        sql = self.column_sql(qn, name, field)
        sql = self.adj_column_sql(sql)

        if sql:
            params = (
                qn.upper(),
                sql
            )
            sql = self.add_column_string % params
            self.execute(sql)

            # Now, drop the default if we need to
            if not keep_default and field.default is not None:
                field.default = NOT_PROVIDED
                self.alter_column(table_name, name, field, explicit_name=False)

    def delete_column(self, table_name, name):
        return super(DatabaseOperations, self).delete_column(self.quote_name(table_name), name)

    def _field_sanity(self, field):
        """
        This particular override stops us sending DEFAULTs for BooleanField.
        """
        if isinstance(field, models.BooleanField) and field.has_default():
            field.default = int(field.to_python(field.get_default()))
        return field

    def _constraints_affecting_columns(self, table_name, columns, type='UNIQUE'):
        """
        Gets the names of the constraints affecting the given columns.
        """
        qn = self.quote_name

        if self.dry_run:
            raise ValueError("Cannot get constraints for columns during a dry run.")
        columns = set(columns)
        rows = self.execute("""
            SELECT user_cons_columns.constraint_name, user_cons_columns.column_name
            FROM user_constraints
            JOIN user_cons_columns ON
                 user_constraints.table_name = user_cons_columns.table_name AND
                 user_constraints.constraint_name = user_cons_columns.constraint_name
            WHERE user_constraints.table_name = '%s' AND
                  user_constraints.constraint_type = '%s'
        """ % (qn(table_name), self.constraits_dict[type]))
        # Load into a dict
        mapping = {}
        for constraint, column in rows:
            mapping.setdefault(constraint, set())
            mapping[constraint].add(column)
        # Find ones affecting these columns
        for constraint, itscols in mapping.items():
            if itscols == columns:
                yield constraint<commit_msg>Fix to oracle backend<commit_after>import os.path
import re
import cx_Oracle

from django.db import connection, models
from django.db.backends.util import truncate_name
from django.core.management.color import no_style
from django.db.backends.oracle.base import get_sequence_name
from django.db.models.fields import NOT_PROVIDED
from south.db import generic

print " ! WARNING: South's Oracle support is still alpha."
print " !          Be wary of posible bugs."

class DatabaseOperations(generic.DatabaseOperations):
    """
    Oracle implementation of database operations.
    """
    backend_name = 'oracle'

    alter_string_set_type =     'ALTER TABLE %(table_name)s MODIFY "%(column)s" %(type)s %(nullity)s;'
    alter_string_set_default =  'ALTER TABLE %(table_name)s MODIFY "%(column)s" DEFAULT %(default)s;'
    add_column_string =         'ALTER TABLE %s ADD %s;'
    delete_column_string =      'ALTER TABLE %s DROP COLUMN %s;'

    allows_combined_alters = False

    constraits_dict = {
        'PRIMARY KEY': 'P',
        'UNIQUE': 'U',
        'CHECK': 'C',
        'REFERENCES': 'R'
    }
    table_names_cache = set()

    def adj_column_sql(self, col):
        col = re.sub('(?P<constr>CHECK \(.*\))(?P<any>.*)(?P<default>DEFAULT [0|1])',
                     lambda mo: '%s %s%s'%(mo.group('default'), mo.group('constr'), mo.group('any')), col) #syntax fix for boolean field only
        col = re.sub('(?P<not_null>NOT NULL) (?P<default>DEFAULT.+)',
                     lambda mo: '%s %s'%(mo.group('default'), mo.group('not_null')), col) #fix order  of DEFAULT and NOT NULL
        return col

    def check_m2m(self, table_name):
        m2m_table_name = table_name
        existing_tables = []

        if not self.table_names_cache:
            self.check_meta(table_name)
            self.table_names_cache = set(connection.introspection.table_names())
        tn = table_name.rsplit('_', 1)

        while len(tn) == 2:
            tn2qn = self.quote_name(tn[0], upper = False, check_m2m = False)
            if tn2qn in self.table_names_cache:
                m2m_table_name = table_name.replace(tn[0], tn2qn)
                break
            else:
                if not existing_tables:
                    existing_tables = connection.introspection.table_names()
                if tn2qn in existing_tables:
                    m2m_table_name = table_name.replace(tn[0], tn2qn)
                    break
            tn = tn[0].rsplit('_', 1)

        self.table_names_cache.add(m2m_table_name)
        return m2m_table_name

    def check_meta(self, table_name):
        return table_name in [ m._meta.db_table for m in models.get_models() ] #caching provided by Django

    def quote_name(self, name, upper=True, column = False, check_m2m = True):
        if not column:
            if check_m2m:
                name = self.check_m2m(name)
            if self.check_meta(name): #replication of Django flow for models where Meta.db_table is set by user
                name = name.upper()
        tn = truncate_name(name, connection.ops.max_name_length())

        return upper and tn.upper() or tn.lower()

    def create_table(self, table_name, fields):
        qn = self.quote_name(table_name, upper = False)
        qn_upper = qn.upper()
        columns = []
        autoinc_sql = ''

        for field_name, field in fields:
            col = self.column_sql(qn_upper, field_name, field)
            if not col:
                continue
            col = self.adj_column_sql(col)

            columns.append(col)
            if isinstance(field, models.AutoField):
                autoinc_sql = connection.ops.autoinc_sql(self.check_meta(table_name) and table_name or qn, field_name)

        sql = 'CREATE TABLE %s (%s);' % (qn_upper, ', '.join([col for col in columns]))
        self.execute(sql)
        if autoinc_sql:
            self.execute(autoinc_sql[0])
            self.execute(autoinc_sql[1])

    def delete_table(self, table_name, cascade=True):
        qn = self.quote_name(table_name, upper = False)

        if cascade:
            self.execute('DROP TABLE %s CASCADE CONSTRAINTS PURGE;' % qn.upper())
        else:
            self.execute('DROP TABLE %s;' % qn.upper())
        self.execute('DROP SEQUENCE %s;'%get_sequence_name(qn))

    def alter_column(self, table_name, name, field, explicit_name=True):
        qn = self.quote_name(table_name)

        # hook for the field to do any resolution prior to it's attributes being queried
        if hasattr(field, 'south_init'):
            field.south_init()
        field = self._field_sanity(field)

        # Add _id or whatever if we need to
        field.set_attributes_from_name(name)
        if not explicit_name:
            name = field.column
        qn_col = self.quote_name(name, column = True)

        # First, change the type
        params = {
            'table_name':qn,
            'column': qn_col,
            'type': self._db_type_for_alter_column(field),
            'nullity': 'NOT NULL',
            'default': 'NULL'
        }
        if field.null:
            params['nullity'] = ''
        sqls = [self.alter_string_set_type % params]

        if not field.null and field.has_default():
            params['default'] = field.get_default()

        sqls.append(self.alter_string_set_default % params)

        #UNIQUE constraint
        unique_constraint = list(self._constraints_affecting_columns(qn, [qn_col]))

        if field.unique and not unique_constraint:
            self.create_unique(qn, [qn_col])
        elif not field.unique and unique_constraint:
            self.delete_unique(qn, [qn_col])

        #CHECK constraint is not handled

        for sql in sqls:
            try:
                self.execute(sql)
            except cx_Oracle.DatabaseError, exc:
                if str(exc).find('ORA-01442') == -1:
                    raise

    def add_column(self, table_name, name, field, keep_default=True):
        qn = self.quote_name(table_name, upper = False)
        sql = self.column_sql(qn, name, field)
        sql = self.adj_column_sql(sql)

        if sql:
            params = (
                qn.upper(),
                sql
            )
            sql = self.add_column_string % params
            self.execute(sql)

            # Now, drop the default if we need to
            if not keep_default and field.default is not None:
                field.default = NOT_PROVIDED
                self.alter_column(table_name, name, field, explicit_name=False)

    def delete_column(self, table_name, name):
        return super(DatabaseOperations, self).delete_column(self.quote_name(table_name), name)

    def _field_sanity(self, field):
        """
        This particular override stops us sending DEFAULTs for BooleanField.
        """
        if isinstance(field, models.BooleanField) and field.has_default():
            field.default = int(field.to_python(field.get_default()))
        return field

    def _constraints_affecting_columns(self, table_name, columns, type='UNIQUE'):
        """
        Gets the names of the constraints affecting the given columns.
        """
        qn = self.quote_name

        if self.dry_run:
            raise ValueError("Cannot get constraints for columns during a dry run.")
        columns = set(columns)
        rows = self.execute("""
            SELECT user_cons_columns.constraint_name, user_cons_columns.column_name
            FROM user_constraints
            JOIN user_cons_columns ON
                 user_constraints.table_name = user_cons_columns.table_name AND
                 user_constraints.constraint_name = user_cons_columns.constraint_name
            WHERE user_constraints.table_name = '%s' AND
                  user_constraints.constraint_type = '%s'
        """ % (qn(table_name), self.constraits_dict[type]))
        # Load into a dict
        mapping = {}
        for constraint, column in rows:
            mapping.setdefault(constraint, set())
            mapping[constraint].add(column)
        # Find ones affecting these columns
        for constraint, itscols in mapping.items():
            if itscols == columns:
                yield constraint<|endoftext|>


<commit_before># coding: utf8
from __future__ import unicode_literals, division, print_function

import plac
import json
from collections import defaultdict
import cytoolz
from pathlib import Path
import dill
import tqdm
from thinc.neural._classes.model import Model
from thinc.neural.optimizers import linear_decay
from timeit import default_timer as timer
import random
import numpy.random

from ..tokens.doc import Doc
from ..scorer import Scorer
from ..gold import GoldParse, merge_sents
from ..gold import GoldCorpus, minibatch
from ..util import prints
from .. import util
from .. import about
from .. import displacy
from ..compat import json_dumps

random.seed(0)
numpy.random.seed(0)

@plac.annotations(
    lang=("model language", "positional", None, str),
    output_dir=("output directory to store model in", "positional", None, str),
    train_data=("location of JSON-formatted training data", "positional", None, str),
    dev_data=("location of JSON-formatted development data (optional)", "positional", None, str),
    n_iter=("number of iterations", "option", "n", int),
    n_sents=("number of sentences", "option", "ns", int),
    use_gpu=("Use GPU", "option", "g", int),
    vectors=("Model to load vectors from", "option", "v"),
    no_tagger=("Don't train tagger", "flag", "T", bool),
    no_parser=("Don't train parser", "flag", "P", bool),
    no_entities=("Don't train NER", "flag", "N", bool),
    gold_preproc=("Use gold preprocessing", "flag", "G", bool),
    version=("Model version", "option", "V", str),
    meta_path=("Optional path to meta.json. All relevant properties will be overwritten.", "option", "m", Path)
)
def train(cmd, lang, output_dir, train_data, dev_data, n_iter=10, n_sents=0,
          use_gpu=-1, vectors=None, no_tagger=False, no_parser=False, no_entities=False,
          gold_preproc=False, version="0.0.0", meta_path=None):
    """
    Train a model. Expects data in spaCy's JSON format.
    """
    util.set_env_log(True)
    n_sents = n_sents or None
    output_path = util.ensure_path(output_dir)
    train_path = util.ensure_path(train_data)
    dev_path = util.ensure_path(dev_data)
    meta_path = util.ensure_path(meta_path)
    if not output_path.exists():
        output_path.mkdir()
    if not train_path.exists():
        prints(train_path, title="Training data not found", exits=1)
    if dev_path and not dev_path.exists():
        prints(dev_path, title="Development data not found", exits=1)
    if meta_path is not None and not meta_path.exists():
        prints(meta_path, title="meta.json not found", exits=1)
    meta = util.read_json(meta_path) if meta_path else {}
    if not isinstance(meta, dict):
        prints("Expected dict but got: {}".format(type(meta)),
               title="Not a valid meta.json format", exits=1)

    pipeline = ['tagger', 'parser', 'ner']
    if no_tagger and 'tagger' in pipeline: pipeline.remove('tagger')
    if no_parser and 'parser' in pipeline: pipeline.remove('parser')
    if no_entities and 'ner' in pipeline: pipeline.remove('ner')

    # Take dropout and batch size as generators of values -- dropout
    # starts high and decays sharply, to force the optimizer to explore.
    # Batch size starts at 1 and grows, so that we make updates quickly
    # at the beginning of training.
    dropout_rates = util.decaying(util.env_opt('dropout_from', 0.6),
                                  util.env_opt('dropout_to', 0.1),
                                  util.env_opt('dropout_decay', 1e-5))
    batch_sizes = util.compounding(util.env_opt('batch_from', 1),
                                   util.env_opt('batch_to', 4),
                                   util.env_opt('batch_compound', 1.001))
    corpus = GoldCorpus(train_path, dev_path, limit=n_sents)
    n_train_words = corpus.count_train()

    lang_class = util.get_lang_class(lang)
    nlp = lang_class(pipeline=pipeline)
    if vectors:
        util.load_model(vectors, vocab=nlp.vocab)
    optimizer = nlp.begin_training(lambda: corpus.train_tuples, device=use_gpu)
    nlp._optimizer = None

    print("Itn.\tP.Loss\tN.Loss\tUAS\tNER P.\tNER R.\tNER F.\tTag %\tToken %")
    try:
        train_docs = corpus.train_docs(nlp, projectivize=True, noise_level=0.0,
                                       gold_preproc=gold_preproc, max_length=0)
        train_docs = list(train_docs)
        for i in range(n_iter):
            with tqdm.tqdm(total=n_train_words, leave=False) as pbar:
                losses = {}
                for batch in minibatch(train_docs, size=batch_sizes):
                    docs, golds = zip(*batch)
                    nlp.update(docs, golds, sgd=optimizer,
                               drop=next(dropout_rates), losses=losses)
                    pbar.update(sum(len(doc) for doc in docs))

            with nlp.use_params(optimizer.averages):
                util.set_env_log(False)
                epoch_model_path = output_path / ('model%d' % i)
                nlp.to_disk(epoch_model_path)
                nlp_loaded = lang_class(pipeline=pipeline)
                nlp_loaded = nlp_loaded.from_disk(epoch_model_path)
                scorer = nlp_loaded.evaluate(
                            list(corpus.dev_docs(
                                nlp_loaded,
                                gold_preproc=gold_preproc)))
                acc_loc =(output_path / ('model%d' % i) / 'accuracy.json')
                with acc_loc.open('w') as file_:
                    file_.write(json_dumps(scorer.scores))
                meta_loc = output_path / ('model%d' % i) / 'meta.json'
                meta['accuracy'] = scorer.scores
                meta['lang'] = nlp.lang
                meta['pipeline'] = pipeline
                meta['spacy_version'] = '>=%s' % about.__version__
                meta.setdefault('name', 'model%d' % i)
                meta.setdefault('version', version)

                with meta_loc.open('w') as file_:
                    file_.write(json_dumps(meta))
                util.set_env_log(True)
            print_progress(i, losses, scorer.scores)
    finally:
        print("Saving model...")
        try:
            with (output_path / 'model-final.pickle').open('wb') as file_:
                with nlp.use_params(optimizer.averages):
                    dill.dump(nlp, file_, -1)
        except:
            print("Error saving model")

def _render_parses(i, to_render):
    to_render[0].user_data['title'] = "Batch %d" % i
    with Path('/tmp/entities.html').open('w') as file_:
        html = displacy.render(to_render[:5], style='ent', page=True)
        file_.write(html)
    with Path('/tmp/parses.html').open('w') as file_:
        html = displacy.render(to_render[:5], style='dep', page=True)
        file_.write(html)

def print_progress(itn, losses, dev_scores, wps=0.0):
    scores = {}
    for col in ['dep_loss', 'tag_loss', 'uas', 'tags_acc', 'token_acc',
                'ents_p', 'ents_r', 'ents_f', 'wps']:
        scores[col] = 0.0
    scores['dep_loss'] = losses.get('parser', 0.0)
    scores['ner_loss'] = losses.get('ner', 0.0)
    scores['tag_loss'] = losses.get('tagger', 0.0)
    scores.update(dev_scores)
    scores['wps'] = wps
    tpl = '\t'.join((
        '{:d}',
        '{dep_loss:.3f}',
        '{ner_loss:.3f}',
        '{uas:.3f}',
        '{ents_p:.3f}',
        '{ents_r:.3f}',
        '{ents_f:.3f}',
        '{tags_acc:.3f}',
        '{token_acc:.3f}',
        '{wps:.1f}'))
    print(tpl.format(itn, **scores))

def print_results(scorer):
    results = {
        'TOK': '%.2f' % scorer.token_acc,
        'POS': '%.2f' % scorer.tags_acc,
        'UAS': '%.2f' % scorer.uas,
        'LAS': '%.2f' % scorer.las,
        'NER P': '%.2f' % scorer.ents_p,
        'NER R': '%.2f' % scorer.ents_r,
        'NER F': '%.2f' % scorer.ents_f}
    util.print_table(results, title="Results")
<commit_msg>Update defaults<commit_after># coding: utf8
from __future__ import unicode_literals, division, print_function

import plac
import json
from collections import defaultdict
import cytoolz
from pathlib import Path
import dill
import tqdm
from thinc.neural._classes.model import Model
from thinc.neural.optimizers import linear_decay
from timeit import default_timer as timer
import random
import numpy.random

from ..tokens.doc import Doc
from ..scorer import Scorer
from ..gold import GoldParse, merge_sents
from ..gold import GoldCorpus, minibatch
from ..util import prints
from .. import util
from .. import about
from .. import displacy
from ..compat import json_dumps

random.seed(0)
numpy.random.seed(0)

@plac.annotations(
    lang=("model language", "positional", None, str),
    output_dir=("output directory to store model in", "positional", None, str),
    train_data=("location of JSON-formatted training data", "positional", None, str),
    dev_data=("location of JSON-formatted development data (optional)", "positional", None, str),
    n_iter=("number of iterations", "option", "n", int),
    n_sents=("number of sentences", "option", "ns", int),
    use_gpu=("Use GPU", "option", "g", int),
    vectors=("Model to load vectors from", "option", "v"),
    no_tagger=("Don't train tagger", "flag", "T", bool),
    no_parser=("Don't train parser", "flag", "P", bool),
    no_entities=("Don't train NER", "flag", "N", bool),
    gold_preproc=("Use gold preprocessing", "flag", "G", bool),
    version=("Model version", "option", "V", str),
    meta_path=("Optional path to meta.json. All relevant properties will be overwritten.", "option", "m", Path)
)
def train(cmd, lang, output_dir, train_data, dev_data, n_iter=10, n_sents=0,
          use_gpu=-1, vectors=None, no_tagger=False, no_parser=False, no_entities=False,
          gold_preproc=False, version="0.0.0", meta_path=None):
    """
    Train a model. Expects data in spaCy's JSON format.
    """
    util.set_env_log(True)
    n_sents = n_sents or None
    output_path = util.ensure_path(output_dir)
    train_path = util.ensure_path(train_data)
    dev_path = util.ensure_path(dev_data)
    meta_path = util.ensure_path(meta_path)
    if not output_path.exists():
        output_path.mkdir()
    if not train_path.exists():
        prints(train_path, title="Training data not found", exits=1)
    if dev_path and not dev_path.exists():
        prints(dev_path, title="Development data not found", exits=1)
    if meta_path is not None and not meta_path.exists():
        prints(meta_path, title="meta.json not found", exits=1)
    meta = util.read_json(meta_path) if meta_path else {}
    if not isinstance(meta, dict):
        prints("Expected dict but got: {}".format(type(meta)),
               title="Not a valid meta.json format", exits=1)

    pipeline = ['tagger', 'parser', 'ner']
    if no_tagger and 'tagger' in pipeline: pipeline.remove('tagger')
    if no_parser and 'parser' in pipeline: pipeline.remove('parser')
    if no_entities and 'ner' in pipeline: pipeline.remove('ner')

    # Take dropout and batch size as generators of values -- dropout
    # starts high and decays sharply, to force the optimizer to explore.
    # Batch size starts at 1 and grows, so that we make updates quickly
    # at the beginning of training.
    dropout_rates = util.decaying(util.env_opt('dropout_from', 0.2),
                                  util.env_opt('dropout_to', 0.2),
                                  util.env_opt('dropout_decay', 0.0))
    batch_sizes = util.compounding(util.env_opt('batch_from', 1),
                                   util.env_opt('batch_to', 16),
                                   util.env_opt('batch_compound', 1.001))
    corpus = GoldCorpus(train_path, dev_path, limit=n_sents)
    n_train_words = corpus.count_train()

    lang_class = util.get_lang_class(lang)
    nlp = lang_class(pipeline=pipeline)
    if vectors:
        util.load_model(vectors, vocab=nlp.vocab)
    optimizer = nlp.begin_training(lambda: corpus.train_tuples, device=use_gpu)
    nlp._optimizer = None

    print("Itn.\tP.Loss\tN.Loss\tUAS\tNER P.\tNER R.\tNER F.\tTag %\tToken %")
    try:
        train_docs = corpus.train_docs(nlp, projectivize=True, noise_level=0.0,
                                       gold_preproc=gold_preproc, max_length=0)
        train_docs = list(train_docs)
        for i in range(n_iter):
            with tqdm.tqdm(total=n_train_words, leave=False) as pbar:
                losses = {}
                for batch in minibatch(train_docs, size=batch_sizes):
                    docs, golds = zip(*batch)
                    nlp.update(docs, golds, sgd=optimizer,
                               drop=next(dropout_rates), losses=losses)
                    pbar.update(sum(len(doc) for doc in docs))

            with nlp.use_params(optimizer.averages):
                util.set_env_log(False)
                epoch_model_path = output_path / ('model%d' % i)
                nlp.to_disk(epoch_model_path)
                nlp_loaded = lang_class(pipeline=pipeline)
                nlp_loaded = nlp_loaded.from_disk(epoch_model_path)
                scorer = nlp_loaded.evaluate(
                            list(corpus.dev_docs(
                                nlp_loaded,
                                gold_preproc=gold_preproc)))
                acc_loc =(output_path / ('model%d' % i) / 'accuracy.json')
                with acc_loc.open('w') as file_:
                    file_.write(json_dumps(scorer.scores))
                meta_loc = output_path / ('model%d' % i) / 'meta.json'
                meta['accuracy'] = scorer.scores
                meta['lang'] = nlp.lang
                meta['pipeline'] = pipeline
                meta['spacy_version'] = '>=%s' % about.__version__
                meta.setdefault('name', 'model%d' % i)
                meta.setdefault('version', version)

                with meta_loc.open('w') as file_:
                    file_.write(json_dumps(meta))
                util.set_env_log(True)
            print_progress(i, losses, scorer.scores)
    finally:
        print("Saving model...")
        try:
            with (output_path / 'model-final.pickle').open('wb') as file_:
                with nlp.use_params(optimizer.averages):
                    dill.dump(nlp, file_, -1)
        except:
            print("Error saving model")

def _render_parses(i, to_render):
    to_render[0].user_data['title'] = "Batch %d" % i
    with Path('/tmp/entities.html').open('w') as file_:
        html = displacy.render(to_render[:5], style='ent', page=True)
        file_.write(html)
    with Path('/tmp/parses.html').open('w') as file_:
        html = displacy.render(to_render[:5], style='dep', page=True)
        file_.write(html)

def print_progress(itn, losses, dev_scores, wps=0.0):
    scores = {}
    for col in ['dep_loss', 'tag_loss', 'uas', 'tags_acc', 'token_acc',
                'ents_p', 'ents_r', 'ents_f', 'wps']:
        scores[col] = 0.0
    scores['dep_loss'] = losses.get('parser', 0.0)
    scores['ner_loss'] = losses.get('ner', 0.0)
    scores['tag_loss'] = losses.get('tagger', 0.0)
    scores.update(dev_scores)
    scores['wps'] = wps
    tpl = '\t'.join((
        '{:d}',
        '{dep_loss:.3f}',
        '{ner_loss:.3f}',
        '{uas:.3f}',
        '{ents_p:.3f}',
        '{ents_r:.3f}',
        '{ents_f:.3f}',
        '{tags_acc:.3f}',
        '{token_acc:.3f}',
        '{wps:.1f}'))
    print(tpl.format(itn, **scores))

def print_results(scorer):
    results = {
        'TOK': '%.2f' % scorer.token_acc,
        'POS': '%.2f' % scorer.tags_acc,
        'UAS': '%.2f' % scorer.uas,
        'LAS': '%.2f' % scorer.las,
        'NER P': '%.2f' % scorer.ents_p,
        'NER R': '%.2f' % scorer.ents_r,
        'NER F': '%.2f' % scorer.ents_f}
    util.print_table(results, title="Results")
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Copyright (c) 2014 Anton Tsyganenko
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import os
import sys

################ FUNCTIONS

def bytesToString(bytes):
    return "".join(chr(x) for x in bytes)

def nextarg(arg): # just for better readability
    return sys.argv[sys.argv.index(arg) + 1]

def bxor(b1, b2): # use xor for bytes
    result = b""
    for b1, b2 in zip(b1, b2):
        result += bytes([b1 ^ b2])
    return result

def validate_key(key, text): # don't let user to use short key
    if len(key) < len(text):
        print("the key must have the text length or be longer")
        exit()
    return key

def binOut(data):
    if omode != "bin":
        return bytes(space.join('{:02x}'.format(x) for x in data), "utf-8")
    else:
        return data

def out(output):
    if fileout: # output to a file
        with open(nextarg("-o"), "bw") as outFile:
            outFile.write(output)
    else:
        print(bytesToString(output))

################ OPTIONS

fileout = False
filein = False
keyfromfile = False
imode = "auto"
omode = "auto"
genkey = False
deletekey = True
notenoughkeys = 5

if "-o" in sys.argv:
    fileout = True

if "-i" in sys.argv:
    filein = True

if "-ki" in sys.argv:
    keyfromfile = True

if "--imode" in sys.argv:
    imode = nextarg("--imode")

if "--omode" in sys.argv:
    omode = nextarg("--omode")

if "--gen-key" in sys.argv:
    genkey = True

if "--do-not-delete-key" in sys.argv:
    deletekey = False

if "--no-spaces" in sys.argv:
    space = ""
else:
    space = " "

decryption = False

################ KEY GENERATION

if genkey:
    number = int(input("number of keys > "))
    length = int(input("key length > "))

    if omode == "hex": # text with keys in hex format
        result = b""
        for i in range(number):
            result += binOut(os.urandom(length)) + b"\n"
        out(result)

    else: # folder with binary key files
        keyfolder = nextarg("-o")
        try:
            os.mkdir(keyfolder)
        except:
            pass
        for i in range(number):
            with open(keyfolder + os.sep + str(i), 'xb') as f:
                f.write(os.urandom(length))

    exit()

################ TEXT INPUT

if filein: # from a file
    with open(nextarg("-i"), "rb") as file:
        text = file.read()
    if omode == "auto":
        omode = "bin"
    if imode == "auto":
        imode = "bin"

else: # direct input
    print("enter the text, then press ENTER; CTRL+D")
    text = sys.stdin.read()[:-1]
    text = bytes(text, "utf-8")

if imode in ["auto", "hex"]:
    try: # try to decode hex
        text = bytes.fromhex(text.replace(b" ", b"").decode("utf-8"))
        if omode == "auto":
            omode = "bin"
    except:
        omode = "hex"

################ KEY INPUT

if keyfromfile: # use folder with keys
    keyfolder = nextarg("-ki")
    fileslist = os.listdir(keyfolder)

    if len(fileslist) == 0:
        print("================\nNO KEYS IN {kf}!".format(kf = keyfolder))
        exit()
    if len(fileslist) <= notenoughkeys:
        print("================\nWARNING! only {k} keys left, and one of them will be used now.".format(k = len(fileslist)))

    keyfile = keyfolder + os.sep + max(fileslist)
    with open(keyfile, 'br') as f:
        key = f.read()

else: # manually input the key
    key = bytes.fromhex(input("enter the key > ").replace(" ", ""))

################ ENCRYPTION/DECRYPTION

result = bxor(text, validate_key(key, text))

################ FINAL

if omode in ["hex", "auto"]: # encrypted result convert to hex format
    result = binOut(result)

if not fileout and not (filein and keyfromfile): # separator
    print("================\n")

out(result)

if deletekey and keyfromfile: # delete the key if evrything is ok
    os.remove(keyfile)
<commit_msg>stylized code a bit more<commit_after>#!/usr/bin/env python3

# Copyright (c) 2014 Anton Tsyganenko
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import os
import sys

################ FUNCTIONS

def bytesToString(bytes):
    return "".join(chr(x) for x in bytes)

def nextarg(arg): # just for better readability
    return sys.argv[sys.argv.index(arg) + 1]

def bxor(b1, b2): # use xor for bytes
    result = b""
    for b1, b2 in zip(b1, b2):
        result += bytes([b1 ^ b2])
    return result

def validate_key(key, text): # don't let user to use short key
    if len(key) < len(text):
        print("the key must have the text length or be longer")
        exit()
    return key

def binOut(data):
    if omode != "bin":
        return bytes(space.join('{:02x}'.format(x) for x in data), \
                     "utf-8")
    else:
        return data

def out(output):
    if fileout: # output to a file
        with open(nextarg("-o"), "bw") as outFile:
            outFile.write(output)
    else:
        print(bytesToString(output))

################ OPTIONS

fileout = False
filein = False
keyfromfile = False
imode = "auto"
omode = "auto"
genkey = False
deletekey = True
notenoughkeys = 5

if "-o" in sys.argv:
    fileout = True

if "-i" in sys.argv:
    filein = True

if "-ki" in sys.argv:
    keyfromfile = True

if "--imode" in sys.argv:
    imode = nextarg("--imode")

if "--omode" in sys.argv:
    omode = nextarg("--omode")

if "--gen-key" in sys.argv:
    genkey = True

if "--do-not-delete-key" in sys.argv:
    deletekey = False

if "--no-spaces" in sys.argv:
    space = ""
else:
    space = " "

decryption = False

################ KEY GENERATION

if genkey:
    number = int(input("number of keys > "))
    length = int(input("key length > "))

    if omode == "hex": # text with keys in hex format
        result = b""
        for i in range(number):
            result += binOut(os.urandom(length)) + b"\n"
        out(result)

    else: # folder with binary key files
        keyfolder = nextarg("-o")
        try:
            os.mkdir(keyfolder)
        except:
            pass
        for i in range(number):
            with open(keyfolder + os.sep + str(i), "xb") as f:
                f.write(os.urandom(length))

    exit()

################ TEXT INPUT

if filein: # from a file
    with open(nextarg("-i"), "rb") as file:
        text = file.read()
    if omode == "auto":
        omode = "bin"
    if imode == "auto":
        imode = "bin"

else: # direct input
    print("enter the text, then press ENTER; CTRL+D")
    text = sys.stdin.read()[:-1]
    text = bytes(text, "utf-8")

if imode in ["auto", "hex"]:
    try: # try to decode hex
        text = bytes.fromhex(text.replace(b" ", b"").decode("utf-8"))
        if omode == "auto":
            omode = "bin"
    except:
        omode = "hex"

################ KEY INPUT

if keyfromfile: # use folder with keys
    keyfolder = nextarg("-ki")
    fileslist = os.listdir(keyfolder)

    if len(fileslist) == 0:
        print("================\n" +
              "NO KEYS IN {kf}!".format(kf=keyfolder))
        exit()
    if len(fileslist) <= notenoughkeys:
        print("================\n" +
            "WARNING! only {k} keys left, ".format(k=len(fileslist)) +
            "and one of them will be used now.")

    keyfile = keyfolder + os.sep + max(fileslist)
    with open(keyfile, 'br') as f:
        key = f.read()

else: # manually input the key
    key = bytes.fromhex(input("enter the key > ").replace(" ", ""))

################ ENCRYPTION/DECRYPTION

result = bxor(text, validate_key(key, text))

################ FINAL

if omode in ["hex", "auto"]: # encrypted result convert to hex format
    result = binOut(result)

if not fileout and not (filein and keyfromfile): # separator
    print("================\n")

out(result)

if deletekey and keyfromfile: # delete the key if evrything is ok
    os.remove(keyfile)
<|endoftext|>


<commit_before>#/bin/python
import os,math
from sys import argv
import argparse
from shutil import copyfile,make_archive,rmtree
import xml.etree.ElementTree

def ensure_dir(s):
	if not os.path.exists(s):
		os.makedirs(s)

def ensure_no_dir(s):
	if os.path.exists(s):
		rmtree(s)

PACKAGE_DIR='poly/'
OUTPUT_PATH='.'
OUTPUT_DIR=OUTPUT_PATH+'/domjudge/'
EXTENSION_FOR_OUTPUT = '.a'
sample_tests = ['01']
PROBCODE = "PROB1"
PROBCOLOR = "#000000"
nodelete = False

#PARSING COMMAND LINE ARGUMENTS
parser = argparse.ArgumentParser(description='Process Polygon Package to Domjudge Package.')
parser.add_argument('package', type=str, help='path of the polygon package')
parser.add_argument('--code',  type=str, help='problem code for domjudge')
parser.add_argument('--sample',type=str, help='Specify the filename for sample test. Defaults to \'01\'')
parser.add_argument('--color', type=str, help='problem color for domjudge (in RRGGBB format)')
parser.add_argument('-o','--output', type=str, help='Output Package directory')
parser.add_argument('--no-delete', action='store_true', help='Don\'t delete the output directory')
parser.add_argument('--ext', type=str, help='Set extension for the OUTPUT files in testset')
args = parser.parse_args()
if args.code:
	PROBCODE = args.code

if args.color:
	PROBCOLOR = '#'+args.color

if args.sample:
	sample_tests = [args.sample]

if args.output:
	OUTPUT_PATH = args.output
	OUTPUT_DIR=OUTPUT_PATH+'/domjudge/'

if args.ext:
	EXTENSION_FOR_OUTPUT = args.ext

if args.no_delete:
	nodelete = True

#Extracting the package
import zipfile
if not os.path.isfile(args.package):
	print "[ERROR] PACKAGE ZIP NOT FOUND"
	exit(1)
zip_ref = zipfile.ZipFile(args.package, 'r')
zip_ref.extractall(PACKAGE_DIR)
zip_ref.close()

#Create the OUTPUT DIR
ensure_no_dir(OUTPUT_DIR)
ensure_dir(OUTPUT_DIR)

#Create Sub DIRS for tests
ensure_dir(OUTPUT_DIR+'/data')
ensure_dir(OUTPUT_DIR+'/data/sample')
ensure_dir(OUTPUT_DIR+'/data/secret')

#Parse XML for Problem Data
root = xml.etree.ElementTree.parse(PACKAGE_DIR+'problem.xml').getroot()
problem_name = root.find('names').find('name').attrib['value']
timelimit = int(math.ceil(float(root.find('judging').find('testset').find('time-limit').text)/1000.0))

desc = open(OUTPUT_DIR+'domjudge-problem.ini','w+')
desc.write("probid='"+PROBCODE+"'\n")
desc.write("name='"+problem_name+"'\n")
desc.write("timelimit='"+str(timelimit)+"'\n")
desc.write("color='"+PROBCOLOR+"'\n")
desc.close()

tests = filter(lambda x:not x.endswith(EXTENSION_FOR_OUTPUT),os.listdir(PACKAGE_DIR+'/tests'))
for test in tests:
	if test in sample_tests:
		copyfile(PACKAGE_DIR+'/tests/'+test,OUTPUT_DIR+'/data/sample/'+test+'.in')
		copyfile(PACKAGE_DIR+'/tests/'+test+EXTENSION_FOR_OUTPUT,OUTPUT_DIR+'/data/sample/'+test+'.ans')
	else:
		copyfile(PACKAGE_DIR+'/tests/'+test,OUTPUT_DIR+'/data/secret/'+test+'.in')
		copyfile(PACKAGE_DIR+'/tests/'+test+EXTENSION_FOR_OUTPUT,OUTPUT_DIR+'/data/secret/'+test+'.ans')

#ZIP the OUTPUT and DELETE Temp
make_archive(OUTPUT_PATH+'/domjudge', 'zip', OUTPUT_DIR)
rmtree(PACKAGE_DIR)
if nodelete == False:
	rmtree(OUTPUT_DIR)
<commit_msg>Added HTML Statement addition<commit_after>#/bin/python
import os,math
from sys import argv
import argparse
from shutil import copyfile,make_archive,rmtree
import xml.etree.ElementTree

def ensure_dir(s):
	if not os.path.exists(s):
		os.makedirs(s)

def ensure_no_dir(s):
	if os.path.exists(s):
		rmtree(s)

PACKAGE_DIR='poly/'
OUTPUT_PATH='.'
OUTPUT_DIR=OUTPUT_PATH+'/domjudge/'
EXTENSION_FOR_OUTPUT = '.a'
sample_tests = ['01']
PROBCODE = "PROB1"
PROBCOLOR = "#000000"
nodelete = False

#PARSING COMMAND LINE ARGUMENTS
parser = argparse.ArgumentParser(description='Process Polygon Package to Domjudge Package.')
parser.add_argument('package', type=str, help='path of the polygon package')
parser.add_argument('--code',  type=str, help='problem code for domjudge')
parser.add_argument('--sample',type=str, help='Specify the filename for sample test. Defaults to \'01\'')
parser.add_argument('--color', type=str, help='problem color for domjudge (in RRGGBB format)')
parser.add_argument('-o','--output', type=str, help='Output Package directory')
parser.add_argument('--no-delete', action='store_true', help='Don\'t delete the output directory')
parser.add_argument('--add-html', action='store_true', help='Add Problem statement in HTML form')
parser.add_argument('--ext', type=str, help='Set extension for the OUTPUT files in testset')
args = parser.parse_args()
if args.code:
	PROBCODE = args.code

if args.color:
	PROBCOLOR = '#'+args.color

if args.sample:
	sample_tests = [args.sample]

if args.output:
	OUTPUT_PATH = args.output
	OUTPUT_DIR=OUTPUT_PATH+'/domjudge/'

if args.ext:
	EXTENSION_FOR_OUTPUT = args.ext

if args.no_delete:
	nodelete = True

#Extracting the package
import zipfile
if not os.path.isfile(args.package):
	print "[ERROR] PACKAGE ZIP NOT FOUND"
	exit(1)
zip_ref = zipfile.ZipFile(args.package, 'r')
ensure_no_dir(PACKAGE_DIR)
zip_ref.extractall(PACKAGE_DIR)
zip_ref.close()

#Create the OUTPUT DIR
ensure_no_dir(OUTPUT_DIR)
ensure_dir(OUTPUT_DIR)

if args.add_html:
	copyfile(PACKAGE_DIR+'/statements/.html/english/problem.html',OUTPUT_DIR+'problem.html')
	copyfile(PACKAGE_DIR+'/statements/.html/english/problem-statement.css',OUTPUT_DIR+'problem-statement.css')

#Create Sub DIRS for tests
ensure_dir(OUTPUT_DIR+'/data')
ensure_dir(OUTPUT_DIR+'/data/sample')
ensure_dir(OUTPUT_DIR+'/data/secret')

#Parse XML for Problem Data
root = xml.etree.ElementTree.parse(PACKAGE_DIR+'problem.xml').getroot()
problem_name = root.find('names').find('name').attrib['value']
timelimit = int(math.ceil(float(root.find('judging').find('testset').find('time-limit').text)/1000.0))

desc = open(OUTPUT_DIR+'domjudge-problem.ini','w+')
desc.write("probid='"+PROBCODE+"'\n")
desc.write("name='"+problem_name+"'\n")
desc.write("timelimit='"+str(timelimit)+"'\n")
desc.write("color='"+PROBCOLOR+"'\n")
desc.close()

tests = filter(lambda x:not x.endswith(EXTENSION_FOR_OUTPUT),os.listdir(PACKAGE_DIR+'/tests'))
for test in tests:
	if test in sample_tests:
		copyfile(PACKAGE_DIR+'/tests/'+test,OUTPUT_DIR+'/data/sample/'+test+'.in')
		copyfile(PACKAGE_DIR+'/tests/'+test+EXTENSION_FOR_OUTPUT,OUTPUT_DIR+'/data/sample/'+test+'.ans')
	else:
		copyfile(PACKAGE_DIR+'/tests/'+test,OUTPUT_DIR+'/data/secret/'+test+'.in')
		copyfile(PACKAGE_DIR+'/tests/'+test+EXTENSION_FOR_OUTPUT,OUTPUT_DIR+'/data/secret/'+test+'.ans')

#ZIP the OUTPUT and DELETE Temp
make_archive(OUTPUT_PATH+'/domjudge', 'zip', OUTPUT_DIR)
rmtree(PACKAGE_DIR)
if nodelete == False:
	rmtree(OUTPUT_DIR)
<|endoftext|>


<commit_before># !/usr/local/bin/python3.4.2
# ----Copyright (c) 2017 Carnegie Hall | The MIT License (MIT)----
# ----For the full license terms, please visit https://github.com/CarnegieHall/linked-data/blob/master/LICENSE----

## Argument[0] is script to run
## Argument[1] is path to csv of Wikidata query results w/MIMO and MBZ IDs
## Argument[2] is path to csv file of CH instrument IDs, labels, MBZ links
## from CH-LOD

import csv
import json
import os
import sys
##from fuzzywuzzy import fuzz
##from fuzzywuzzy import process
from rdflib import Graph, Literal, Namespace, URIRef
from rdflib.namespace import FOAF, RDF, RDFS, SKOS, XSD
from rdflib.plugins.serializers.nt import NTSerializer

filePath_1 = sys.argv[1]
filePath_2 = sys.argv[2]

wikidataInstr_dict = {}
chInstr_dict = {}
ch_toMIMO_Dict = {}

g = Graph()

##def fuzzy_match(x, choices, scorer, cutoff):
##    return process.extractOne(
##        x, choices=choices, scorer=scorer, score_cutoff=cutoff
##        )

with open(filePath_1, 'rU') as f1:
    wikidataInstr = csv.reader(f1, dialect='excel', delimiter=',', quotechar='"')
    next(wikidataInstr, None)
    for row in wikidataInstr:
        wikidataID = row[0]
        wikidataLabel = row[1]
        mbzID = row[2]
        mimoID = row[3]

        wikidataInstr_dict[str(mbzID)] = {}
        wikidataInstr_dict[str(mbzID)]['label'] = wikidataLabel
        wikidataInstr_dict[str(mbzID)]['wikidataID'] = wikidataID
        wikidataInstr_dict[str(mbzID)]['mimo'] = mimoID

with open(filePath_2, 'rU') as f2:
    chInstruments = csv.reader(f2, dialect='excel', delimiter=',', quotechar='"')
    next(chInstruments, None)
    for row in chInstruments:
        chID = row[0]
        chLabel = row[1]
        mbzID = row[2]

        if mbzID in wikidataInstr_dict.keys():
            wikidataLabel = wikidataInstr_dict[str(mbzID)]['label']
            wikidataID = wikidataInstr_dict[str(mbzID)]['wikidataID']
            mimoID = wikidataInstr_dict[str(mbzID)]['mimo']

            ch_toMIMO_Dict[str(chID)] = {}
            ch_toMIMO_Dict[str(chID)]['chLabel'] = chLabel
            ch_toMIMO_Dict[str(chID)]['wikidataLabel'] = wikidataLabel
            ch_toMIMO_Dict[str(chID)]['wikidataID'] = wikidataID
            ch_toMIMO_Dict[str(chID)]['mimoID'] = mimoID

            g.add( (URIRef(chID), SKOS.exactMatch, URIRef(mimoID)) )
            g.add( (URIRef(chID), SKOS.exactMatch, URIRef(wikidataID)) )

ch_toMIMO_Dict_path = os.path.join(
    os.path.dirname(__file__), os.pardir, 'JSON_dicts', 'ch_toMIMO_Dict.json')

ch_toMIMO_Graph_path = os.path.join(
    os.path.dirname(__file__), os.pardir, 'graphs', 'ch_toMIMO_Graph.nt')

g.bind("skos", SKOS)
g = g.serialize(destination=ch_toMIMO_Graph_path, format='nt')

with open(ch_toMIMO_Dict_path, 'w') as f1:
    json.dump(ch_toMIMO_Dict, f1)

print(json.dumps(ch_toMIMO_Dict, indent=4))
print("Finished finding matches to MIMO instrument IDs")
<commit_msg>Added match count<commit_after># !/usr/local/bin/python3.4.2
# ----Copyright (c) 2017 Carnegie Hall | The MIT License (MIT)----
# ----For the full license terms, please visit https://github.com/CarnegieHall/linked-data/blob/master/LICENSE----

## Matches CH instrument IDs to Wikidata and MIMO IDs using existing
## MusicBrainz ID alignment
## Argument[0] is script to run
## Argument[1] is path to csv of Wikidata query results w/MIMO and MBZ IDs
## Argument[2] is path to csv file of CH instrument IDs, labels, MBZ links
## from CH-LOD

import csv
import json
import os
import sys
from rdflib import Graph, Literal, Namespace, URIRef
from rdflib.namespace import FOAF, RDF, RDFS, SKOS, XSD
from rdflib.plugins.serializers.nt import NTSerializer

filePath_1 = sys.argv[1]
filePath_2 = sys.argv[2]

wikidataInstr_dict = {}
chInstr_dict = {}
ch_toMIMO_Dict = {}

g = Graph()

with open(filePath_1, encoding='utf-8', errors='ignore') as f1:
    wikidataInstr = csv.reader(f1, dialect='excel', delimiter=',', quotechar='"')
    next(wikidataInstr, None)
    for row in wikidataInstr:
        wikidataID = row[0]
        wikidataLabel = row[1]
        mbzID = row[2]
        mimoID = row[3]

        wikidataInstr_dict[str(mbzID)] = {}
        wikidataInstr_dict[str(mbzID)]['label'] = wikidataLabel
        wikidataInstr_dict[str(mbzID)]['wikidataID'] = wikidataID
        wikidataInstr_dict[str(mbzID)]['mimo'] = mimoID

with open(filePath_2, encoding='utf-8', errors='ignore') as f2:
    chInstruments = csv.reader(f2, dialect='excel', delimiter=',', quotechar='"')
    next(chInstruments, None)
    for row in chInstruments:
        chID = row[0]
        chLabel = row[1]
        mbzID = row[2]

        if mbzID in wikidataInstr_dict.keys():
            wikidataLabel = wikidataInstr_dict[str(mbzID)]['label']
            wikidataID = wikidataInstr_dict[str(mbzID)]['wikidataID']
            mimoID = wikidataInstr_dict[str(mbzID)]['mimo']

            ch_toMIMO_Dict[str(chID)] = {}
            ch_toMIMO_Dict[str(chID)]['chLabel'] = chLabel
            ch_toMIMO_Dict[str(chID)]['wikidataLabel'] = wikidataLabel
            ch_toMIMO_Dict[str(chID)]['wikidataID'] = wikidataID
            ch_toMIMO_Dict[str(chID)]['mimoID'] = mimoID

            g.add( (URIRef(chID), SKOS.exactMatch, URIRef(mimoID)) )
            g.add( (URIRef(chID), SKOS.exactMatch, URIRef(wikidataID)) )

ch_toMIMO_Dict_path = os.path.join(
    os.path.dirname(__file__), os.pardir, 'JSON_dicts', 'ch_toMIMO_Dict.json')

ch_toMIMO_Graph_path = os.path.join(
    os.path.dirname(__file__), os.pardir, 'graphs', 'ch_toMIMO_Graph.nt')

g.bind("skos", SKOS)
g = g.serialize(destination=ch_toMIMO_Graph_path, format='nt')

with open(ch_toMIMO_Dict_path, 'w') as f1:
    json.dump(ch_toMIMO_Dict, f1)

print("Finished finding matches to MIMO instrument IDs")
print(len(ch_toMIMO_Dict), "instruments matched")
<|endoftext|>


<commit_before>"""Script for retracting pending retractions that are more than 48 hours old."""

import logging

import django
from django.db import transaction
from django.utils import timezone
from modularodm import Q
django.setup()

from framework.auth import Auth
from framework.celery_tasks import app as celery_app

from website.app import init_app
from website import models, settings
from osf.models import NodeLog

from scripts import utils as scripts_utils

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def main(dry_run=True):
    pending_retractions = models.Retraction.find(Q('state', 'eq', models.Retraction.UNAPPROVED))
    for retraction in pending_retractions:
        if should_be_retracted(retraction):
            if dry_run:
                logger.warn('Dry run mode')
            try:
                parent_registration = models.Node.find_one(Q('retraction', 'eq', retraction))
            except Exception as err:
                logger.error('Could not find registration associated with retraction {}'.format(retraction))
                logger.error('Skipping...'.format(retraction))
                continue

            logger.warn(
                'Retraction {0} approved. Retracting registration {1}'
                .format(retraction._id, parent_registration._id)
            )
            if not dry_run:
                with transaction.atomic():
                    retraction.state = models.Retraction.APPROVED
                    try:
                        parent_registration.registered_from.add_log(
                            action=NodeLog.RETRACTION_APPROVED,
                            params={
                                'node': parent_registration.registered_from._id,
                                'registration': parent_registration._id,
                                'retraction_id': parent_registration.retraction._id,
                            },
                            auth=Auth(parent_registration.retraction.initiated_by),
                        )
                        retraction.save()
                        parent_registration.update_search()
                        for node in parent_registration.get_descendants_recursive():
                            node.update_search()
                    except Exception as err:
                        logger.error(
                            'Unexpected error raised when retracting '
                            'registration {}. Continuing...'.format(parent_registration))
                        logger.exception(err)

def should_be_retracted(retraction):
    """Returns true if retraction was initiated more than 48 hours prior"""
    return (timezone.now() - retraction.initiation_date) >= settings.RETRACTION_PENDING_TIME

@celery_app.task(name='scripts.retract_registrations')
def run_main(dry_run=True):
    init_app(routes=False)
    if not dry_run:
        scripts_utils.add_file_logger(logger, __file__)
    main(dry_run=dry_run)

<commit_msg>Remove missed `website.models` import   -update script<commit_after>"""Script for retracting pending retractions that are more than 48 hours old."""

import logging

import django
from django.db import transaction
from django.utils import timezone
from modularodm import Q
django.setup()

from framework.auth import Auth
from framework.celery_tasks import app as celery_app

from website.app import init_app
from website import settings
from osf.models import Node, NodeLog, Retraction

from scripts import utils as scripts_utils

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def main(dry_run=True):
    pending_retractions = Retraction.find(Q('state', 'eq', Retraction.UNAPPROVED))
    for retraction in pending_retractions:
        if should_be_retracted(retraction):
            if dry_run:
                logger.warn('Dry run mode')
            try:
                parent_registration = Node.find_one(Q('retraction', 'eq', retraction))
            except Exception as err:
                logger.error('Could not find registration associated with retraction {}'.format(retraction))
                logger.error('Skipping...'.format(retraction))
                continue

            logger.warn(
                'Retraction {0} approved. Retracting registration {1}'
                .format(retraction._id, parent_registration._id)
            )
            if not dry_run:
                with transaction.atomic():
                    retraction.state = Retraction.APPROVED
                    try:
                        parent_registration.registered_from.add_log(
                            action=NodeLog.RETRACTION_APPROVED,
                            params={
                                'node': parent_registration.registered_from._id,
                                'registration': parent_registration._id,
                                'retraction_id': parent_registration.retraction._id,
                            },
                            auth=Auth(parent_registration.retraction.initiated_by),
                        )
                        retraction.save()
                        parent_registration.update_search()
                        for node in parent_registration.get_descendants_recursive():
                            node.update_search()
                    except Exception as err:
                        logger.error(
                            'Unexpected error raised when retracting '
                            'registration {}. Continuing...'.format(parent_registration))
                        logger.exception(err)

def should_be_retracted(retraction):
    """Returns true if retraction was initiated more than 48 hours prior"""
    return (timezone.now() - retraction.initiation_date) >= settings.RETRACTION_PENDING_TIME

@celery_app.task(name='scripts.retract_registrations')
def run_main(dry_run=True):
    init_app(routes=False)
    if not dry_run:
        scripts_utils.add_file_logger(logger, __file__)
    main(dry_run=dry_run)

<|endoftext|>


<commit_before>#!/usr/bin/env python
#########################################################################################
#
# Check the installation and environment variables of the toolbox and its dependences.
#
# ---------------------------------------------------------------------------------------
# Copyright (c) 2013 Polytechnique Montreal <www.neuro.polymtl.ca>
# Author: Julien Cohen-Adad
# Modified: 2014-06-05
#
# About the license: see the file LICENSE.TXT
#########################################################################################

# TODO: also check the SCT
# TODO: find another way to create log file. E.g. sct.print(). For color as well.
# TODO: manage .cshrc files

# DEFAULT PARAMETERS
class param:
    ## The constructor
    def __init__(self):
        self.log                = 0

import os
import sys
#import getopt
import commands
import time
#import logging

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

# MAIN
# ==========================================================================================
def main():

    # initialization
    fsl_is_installed = 1
    ants_is_installed = 1
    c3d_is_installed = 1
    install_software = 0
    restart_terminal = 0
    os_running = ''
    print

    # check if user is root (should not be!)
    if os.geteuid() == 0:
       print 'Looks like you are root. Please run this script without sudo. Exit program\n'
       sys.exit(2)

    # check OS
    print 'Check which OS is running ... '
    if (sys.platform == 'darwin'):
        os_running = 'osx'
    print '  '+os_running+' ('+sys.platform+')'

    # check installation packages
    print 'Check which Python is running ... '
    print '  '+sys.executable

    # check numpy
    print_line('Check if numpy is installed ................... ')
    try:
        import numpy
        print_ok()
    except ImportError:
        print_fail()
        print '  numpy is not installed! Please install it via miniconda (https://sourceforge.net/p/spinalcordtoolbox/wiki/install_python/)'
        install_software = 1

    # check scipy
    print_line('Check if scipy is installed ................... ')
    try:
        import scipy
        print_ok()
    except ImportError:
        print_fail()
        print '  scipy is not installed! Please install it via miniconda (https://sourceforge.net/p/spinalcordtoolbox/wiki/install_python/)'
        install_software = 1

    # check nibabel
    print_line('Check if nibabel is installed ................. ')
    try:
        import nibabel
        print_ok()
    except ImportError:
        print_fail()
        print '  nibabel is not installed! See instructions (https://sourceforge.net/p/spinalcordtoolbox/wiki/install_python/)'
        install_software = 1

    # check if FSL is installed
    print_line('Check if FSL is installed ..................... ')
    (status, output) = commands.getstatusoutput('find /usr -name "fsl" -type d -print -quit 2>/dev/null')
    if output:
        print_ok()
        path_fsl = output
        print '  '+path_fsl
    else:
        print_fail()
        print '  FSL is not installed! Install it from: http://fsl.fmrib.ox.ac.uk/'
        fsl_is_installed = 0
        install_software = 1

    # check if FSL is declared
    if fsl_is_installed:
        print_line('Check if FSL is declared ...................... ')
        (status, output) = commands.getstatusoutput('which fsl')
        if output:
            print_ok()
        else:
            print_warning()
            print '  FSL is not declared! Modifying .bash_profile ...'
            add_bash_profile('#FSL (added on '+time.strftime("%Y-%m-%d")+')\n' \
                'FSLDIR='+path_fsl+'\n' \
                '. ${FSLDIR}/etc/fslconf/fsl.sh\n' \
                'PATH=${FSLDIR}/bin:${PATH}\n' \
                'export FSLDIR PATH')
            restart_terminal = 1

    # # check if ANTS is installed
    # print_line('Check if ANTs is installed .................... ')
    # (status, output) = commands.getstatusoutput('find /usr -name "antsRegistration" -type f -print -quit 2>/dev/null')
    # if output:
    #     print_ok()
    #     path_ants = os.path.dirname(output)
    #     print '  '+path_ants
    # else:
    #     print_fail()
    #     print '  ANTs is not installed! Follow instructions here: https://sourceforge.net/p/spinalcordtoolbox/wiki/ants_installation/'
    #     ants_is_installed = 0
    #     install_software = 1
    #
    # # check if ANTS is declared
    # if ants_is_installed:
    #     print_line('Check if ANTs is declared ..................... ')
    #     (status, output) = commands.getstatusoutput('which antsRegistration')
    #     if output:
    #         print_ok()
    #     else:
    #         print_warning()
    #         print '  ANTs is not declared! Modifying .bash_profile ...'
    #         add_bash_profile('#ANTS (added on '+time.strftime("%Y-%m-%d")+')\n' \
    #             'PATH=${PATH}:'+path_ants)
    #         restart_terminal = 1

    # # check if C3D is installed
    # print_line('Check if c3d is installed ..................... ')
    # output = ''
    # if os_running == 'osx':
    #     # in OSX, c3d is typically installed under /Applications
    #     (status, output) = commands.getstatusoutput('find /Applications -name "c3d" -type f -print -quit 2>/dev/null')
    # if not output:
    #     # check the typical /usr folder
    #     (status, output) = commands.getstatusoutput('find /usr -name "c3d" -type f -print -quit 2>/dev/null')
    # if not output:
    #     # if still not found, check everywhere (takes a while)
    #     (status, output) = commands.getstatusoutput('find / -name "c3d" -type f -print -quit 2>/dev/null')
    # if output:
    #     print_ok()
    #     path_c3d = os.path.dirname(output)
    #     print '  '+path_c3d
    # else:
    #     print_fail()
    #     print '  Please install it from there: http://www.itksnap.org/pmwiki/pmwiki.php?n=Downloads.C3D'
    #     c3d_is_installed = 0
    #     install_software = 1
    #
    # # check if C3D is declared
    # if c3d_is_installed:
    #     print_line('Check if c3d is declared ...................... ')
    #     (status, output) = commands.getstatusoutput('which c3d')
    #     if output:
    #         print_ok()
    #     else:
    #         print_warning()
    #         print '  c3d is not declared! Modifying .bash_profile ...'
    #         add_bash_profile('#C3D (added on '+time.strftime("%Y-%m-%d")+')\n' \
    #             'PATH=${PATH}:'+path_c3d)
    #         restart_terminal = 1
    #
    # if install_software:
    #     print '\nDone! Please install the required software, then run this script again.'
    # elif restart_terminal:
    #     print '\nDone! Please restart your Terminal for changes to take effect.'
    # else:
    #     print '\nDone! Everything is in order :-)'
    # print

# Print without new carriage return
# ==========================================================================================
def print_line(string):
    import sys
    sys.stdout.write(string)
    sys.stdout.flush()

def print_ok():
    print "[" + bcolors.OKGREEN + "OK" + bcolors.ENDC + "]"

def print_warning():
    print "[" + bcolors.WARNING + "WARNING" + bcolors.ENDC + "]"

def print_fail():
    print "[" + bcolors.FAIL + "FAIL" + bcolors.ENDC + "]"

def add_bash_profile(string):
    from os.path import expanduser
    home = expanduser("~")
    with open(home+"/.bash_profile", "a") as file_bash:
    # with open("test.txt", "a") as file_bash:
        file_bash.write("\n"+string)

# Print usage
# ==========================================================================================
def usage():
    print '\n' \
        ''+os.path.basename(__file__)+'\n' \
        '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' \
        'Part of the Spinal Cord Toolbox <https://sourceforge.net/projects/spinalcordtoolbox>\n' \
        '\n'\
        'DESCRIPTION\n' \
        '  Check the installation and environment variables of the toolbox and its dependences.\n' \
        '\n' \
        'USAGE\n' \
        '  '+os.path.basename(__file__)+'\n' \
        '\n' \
        'OPTIONAL ARGUMENTS\n' \
        '  -h            print this help.\n'

    # exit program
    sys.exit(2)

# START PROGRAM
# ==========================================================================================
if __name__ == "__main__":
    # initialize parameters
    param = param()
    # call main function
    main()<commit_msg>INST: now checking for sympy and OS compatibility for ANTS and c3d<commit_after>#!/usr/bin/env python
#########################################################################################
#
# Check the installation and environment variables of the toolbox and its dependences.
#
# ---------------------------------------------------------------------------------------
# Copyright (c) 2013 Polytechnique Montreal <www.neuro.polymtl.ca>
# Author: Julien Cohen-Adad
# Modified: 2014-06-05
#
# About the license: see the file LICENSE.TXT
#########################################################################################

# TODO: also check the SCT
# TODO: find another way to create log file. E.g. sct.print(). For color as well.
# TODO: manage .cshrc files

# DEFAULT PARAMETERS
class param:
    ## The constructor
    def __init__(self):
        self.log                = 0

import os
import sys
#import getopt
import commands
import time
#import logging

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

# MAIN
# ==========================================================================================
def main():

    # initialization
    fsl_is_installed = 1
    ants_is_installed = 1
    c3d_is_installed = 1
    install_software = 0
    restart_terminal = 0
    os_running = ''
    print

    # check if user is root (should not be!)
    if os.geteuid() == 0:
       print 'Looks like you are root. Please run this script without sudo. Exit program\n'
       sys.exit(2)

    # check OS
    print 'Check which OS is running ... '
    if (sys.platform == 'darwin'):
        os_running = 'osx'
    print '  '+os_running+' ('+sys.platform+')'

    # check installation packages
    print 'Check which Python is running ... '
    print '  '+sys.executable

    # check numpy
    print_line('Check if numpy is installed ................... ')
    try:
        import numpy
        print_ok()
    except ImportError:
        print_fail()
        print '  numpy is not installed! Please install it via miniconda (https://sourceforge.net/p/spinalcordtoolbox/wiki/install_python/)'
        install_software = 1

    # check scipy
    print_line('Check if scipy is installed ................... ')
    try:
        import scipy
        print_ok()
    except ImportError:
        print_fail()
        print '  scipy is not installed! Please install it via miniconda (https://sourceforge.net/p/spinalcordtoolbox/wiki/install_python/)'
        install_software = 1

    # check sympy
    print_line('Check if sympy is installed ................... ')
    try:
        import sympy
        print_ok()
    except ImportError:
        print_fail()
        print '  sympy is not installed! Please install it via miniconda (https://sourceforge.net/p/spinalcordtoolbox/wiki/install_python/)'
        install_software = 1

    # check nibabel
    print_line('Check if nibabel is installed ................. ')
    try:
        import nibabel
        print_ok()
    except ImportError:
        print_fail()
        print '  nibabel is not installed! See instructions (https://sourceforge.net/p/spinalcordtoolbox/wiki/install_python/)'
        install_software = 1

    # check if FSL is installed
    print_line('Check if FSL is installed ..................... ')
    (status, output) = commands.getstatusoutput('find /usr -name "flirt" -type f -print -quit 2>/dev/null')
    if output:
        print_ok()
        path_fsl = output[:-10]
        print '  '+path_fsl
    else:
        print_fail()
        print '  FSL is not installed! Install it from: http://fsl.fmrib.ox.ac.uk/'
        fsl_is_installed = 0
        install_software = 1

    # check if FSL is declared
    if fsl_is_installed:
        print_line('Check if FSL is declared ...................... ')
        (status, output) = commands.getstatusoutput('which fsl')
        if output:
            print_ok()
        else:
            print_warning()
            print '  FSL is not declared! Modifying .bash_profile ...'
            add_bash_profile('#FSL (added on '+time.strftime("%Y-%m-%d")+')\n' \
                'FSLDIR='+path_fsl+'\n' \
                '. ${FSLDIR}/etc/fslconf/fsl.sh\n' \
                'PATH=${FSLDIR}/bin:${PATH}\n' \
                'export FSLDIR PATH')
            restart_terminal = 1

    # check ANTs
    print_line('Check if ANTs is declared ..................... ')
    (status, output) = commands.getstatusoutput('command -v antsRegistration >/dev/null 2>&1 || { echo >&2 "nope";}')
    if not output:
        print_ok()
    else:
        print_warning()
        print '  ANTs is not declared.'

    # check if ANTs is compatible with OS
    print_line('Check ANTs compatibility with your OS ......... ')
    (status, output) = commands.getstatusoutput('antsRegistration')
    if status in [0, 256]:
        print_ok()
    else:
        print_fail()

    # check c3d
    print_line('Check if c3d is working ....................... ')
    (status, output) = commands.getstatusoutput('command -v c3d >/dev/null 2>&1 || { echo >&2 "nope";}')
    if not output:
        print_ok()
    else:
        print_warning()
        print '  c3d is not installed or not declared.'

    # check c3d compatibility with OS
    print_line('Check c3d compatibility with your OS .......... ')
    (status, output) = commands.getstatusoutput('c3d -h')
    if status in [0, 256]:
        print_ok()
    else:
        print_fail()

    # check PropSeg compatibility with OS
    print_line('Check PropSeg compatibility with your OS ...... ')
    (status, output) = commands.getstatusoutput('sct_segmentation_propagation')
    if status in [0, 256]:
        print_ok()
    else:
        print_fail()

    # # check if ANTS is installed
    # print_line('Check if ANTs is installed .................... ')
    # (status, output) = commands.getstatusoutput('find /usr -name "antsRegistration" -type f -print -quit 2>/dev/null')
    # if output:
    #     print_ok()
    #     path_ants = os.path.dirname(output)
    #     print '  '+path_ants
    # else:
    #     print_fail()
    #     print '  ANTs is not installed! Follow instructions here: https://sourceforge.net/p/spinalcordtoolbox/wiki/ants_installation/'
    #     ants_is_installed = 0
    #     install_software = 1
    #
    # # check if ANTS is declared
    # if ants_is_installed:
    #     print_line('Check if ANTs is declared ..................... ')
    #     (status, output) = commands.getstatusoutput('which antsRegistration')
    #     if output:
    #         print_ok()
    #     else:
    #         print_warning()
    #         print '  ANTs is not declared! Modifying .bash_profile ...'
    #         add_bash_profile('#ANTS (added on '+time.strftime("%Y-%m-%d")+')\n' \
    #             'PATH=${PATH}:'+path_ants)
    #         restart_terminal = 1

    # # check if C3D is installed
    # print_line('Check if c3d is installed ..................... ')
    # output = ''
    # if os_running == 'osx':
    #     # in OSX, c3d is typically installed under /Applications
    #     (status, output) = commands.getstatusoutput('find /Applications -name "c3d" -type f -print -quit 2>/dev/null')
    # if not output:
    #     # check the typical /usr folder
    #     (status, output) = commands.getstatusoutput('find /usr -name "c3d" -type f -print -quit 2>/dev/null')
    # if not output:
    #     # if still not found, check everywhere (takes a while)
    #     (status, output) = commands.getstatusoutput('find / -name "c3d" -type f -print -quit 2>/dev/null')
    # if output:
    #     print_ok()
    #     path_c3d = os.path.dirname(output)
    #     print '  '+path_c3d
    # else:
    #     print_fail()
    #     print '  Please install it from there: http://www.itksnap.org/pmwiki/pmwiki.php?n=Downloads.C3D'
    #     c3d_is_installed = 0
    #     install_software = 1
    #
    # # check if C3D is declared
    # if c3d_is_installed:
    #     print_line('Check if c3d is declared ...................... ')
    #     (status, output) = commands.getstatusoutput('which c3d')
    #     if output:
    #         print_ok()
    #     else:
    #         print_warning()
    #         print '  c3d is not declared! Modifying .bash_profile ...'
    #         add_bash_profile('#C3D (added on '+time.strftime("%Y-%m-%d")+')\n' \
    #             'PATH=${PATH}:'+path_c3d)
    #         restart_terminal = 1
    #
    if install_software:
        print '\nDone! Please install the required software, then run this script again.'
    elif restart_terminal:
        print '\nDone! Please restart your Terminal for changes to take effect.'
    else:
        print '\nDone! Everything is in order :-)'
    print

# Print without new carriage return
# ==========================================================================================
def print_line(string):
    import sys
    sys.stdout.write(string)
    sys.stdout.flush()

def print_ok():
    print "[" + bcolors.OKGREEN + "OK" + bcolors.ENDC + "]"

def print_warning():
    print "[" + bcolors.WARNING + "WARNING" + bcolors.ENDC + "]"

def print_fail():
    print "[" + bcolors.FAIL + "FAIL" + bcolors.ENDC + "]"

def add_bash_profile(string):
    from os.path import expanduser
    home = expanduser("~")
    with open(home+"/.bash_profile", "a") as file_bash:
    # with open("test.txt", "a") as file_bash:
        file_bash.write("\n"+string)

# Print usage
# ==========================================================================================
def usage():
    print '\n' \
        ''+os.path.basename(__file__)+'\n' \
        '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' \
        'Part of the Spinal Cord Toolbox <https://sourceforge.net/projects/spinalcordtoolbox>\n' \
        '\n'\
        'DESCRIPTION\n' \
        '  Check the installation and environment variables of the toolbox and its dependences.\n' \
        '\n' \
        'USAGE\n' \
        '  '+os.path.basename(__file__)+'\n' \
        '\n' \
        'OPTIONAL ARGUMENTS\n' \
        '  -h            print this help.\n'

    # exit program
    sys.exit(2)

# START PROGRAM
# ==========================================================================================
if __name__ == "__main__":
    # initialize parameters
    param = param()
    # call main function
    main()<|endoftext|>


<commit_before>"""
Probability density functions compliant with the lmfit package
Author: Cyril Grima <cyril.grima@gmail.com>
"""

import math
import numpy as np
from scipy import stats, integrate
from scipy.special import jv, kv, j0, i0, digamma

def gamma(params, x, data=None, eps=None):
    """Gamma PDF from scipy with adequate variables
    """
    # Initialisation
    mu = params['mu']
    # Debug inputs
    if hasattr(mu, 'value'): mu = mu.value #idem
    x = np.array([x]).flatten(0) # debug for iteration over 0-d element
    # Model function
    model = stats.gamma.pdf(x, mu, scale = 1)
    model = np.nan_to_num(model)

    if data is None:
        return model
    if eps is None:
        return (model - data) #residual
    return (model - data)/eps

def rayleigh(params, x, data=None, eps=None):
    """Rayleigh PDF from scipy with adequate variables
    """
    # Initialisation
    s = params['s']
    # Debug inputs
    if hasattr(s, 'value'): s = s.value #idem
    x = np.array([x]).flatten(0) # debug for iteration over 0-d element
    # Model function
    model = stats.rayleigh.pdf(x, scale = s)
    model = np.nan_to_num(model)

    if data is None:
        return model
    if eps is None:
        return (model - data) #residual
    return (model - data)/eps

def rice(params, x, data=None, eps=None, method = None):
    """Rice PDF from scipy with adequate variables
    """
    # Initialisation
    a = params['a']
    s = params['s']
    # Debug inputs
    if hasattr(a, 'value'): a = a.value #debug due to lmfit.minimize
    if hasattr(s, 'value'): s = s.value #idem
    x = np.array([x]).flatten(0) # debug for iteration over 0-d element
    # Model function
    model = stats.rice.pdf(x, a/s, scale = s)
    #model = (x/s**2) * np.exp(-(x**2+a**2)/(2*s**2)) * np.i0(a*x/s**2)
    #model = np.nan_to_num(model)

    if data is None:
        return model
    if eps is None:
        return (model - data) #residual
    return (model - data)/eps

def k(params, x, data=None, eps=None):
    """K PDF
    """
    # Initialisation
    s = params['s']
    mu = params['mu']
    # Debug inputs
    if hasattr(s, 'value'): s = s.value #idem
    if hasattr(mu, 'value'): mu = mu.value #idem
    x = np.array([x]).flatten(0) # debug for iteration over 0-d element

    # Model function
    b = np.sqrt(2*mu)/s
    model = 2*(x/2.)**mu *b**(mu+1.) /math.gamma(mu) *kv(mu-1,b*x)
    model = np.nan_to_num(model)

    if data is None:
        return model
    if eps is None:
        return (model - data) #residual
    return (model - data)/eps

def hk(params, x, data=None, eps=None, method = 'analytic'):
    """Homodyne K-distribution from various methods

    Arguments
    ---------
    params : dict
        params for the hk function {'a', 's', 'mu'}
    x : sequence
        x coordinates

    Keywords
    --------
    data : sequence
        data to compare the result with (for minimization)
    eps : sequence
        error on the data
    method : string
        'analytic' = from the common analytic form
        'compound' = from the compound representation [Destrempes and Cloutier,
                     2010, Ultrasound in Med. and Biol. 36(7): 1037-51, Eq. 16]
        NB: 'compound' is less unstable than 'analytic' but is ~10x longer!
    verbose : bool
        print fitting report
    """
    # Initialisation
    a = params['a']
    s = params['s']
    mu = params['mu']
    # Debug inputs
    if hasattr(a, 'value'): a = a.value #debug due to lmfit.minimize
    if hasattr(s, 'value'): s = s.value #idem
    if hasattr(mu, 'value'): mu = mu.value #idem
    x = np.array([x]).flatten('C') # debug for iteration over 0-d element

    def integrand_analytic(w, x, a, s, mu):
        return x*w*j0(w*a)*j0(w*x)*(1. +w*w*s*s/(2.*mu))**-mu
    def integrand_compound(w, x, a, s, mu):
        return rice({'a':a,'s':s*np.sqrt(w/mu)}, x) * gamma({'mu':mu}, w)

    integrands = {
        'analytic': integrand_analytic,
        'compound': integrand_compound,
    }
    integrand = integrands[method]

    model = [integrate.quad(integrand, 0., np.inf, args=(i, a, s, mu), full_output=1)[0]
            for i in x] # Integration
    model = np.array(model)

    #def integrand(w, x, a, s, mu, method=method):
    #    if method == 'analytic':
    #        return x*w*j0(w*a)*j0(w*x)*(1. +w*w*s*s/(2.*mu))**-mu
    #    if method == 'compound':
    #        return rice({'a':a,'s':s*np.sqrt(w/mu)}, x) * gamma({'mu':mu}, w)

    #model = [integrate.quad(integrand, 0., np.inf, args=(i, a, s, mu, method), full_output=1)[0]
    #         for i in x] # Integration
    #model = np.array(model)

    if data is None:
        return model
    if eps is None:
        return (model - data) #residual
    return (model - data)/eps
<commit_msg>compatibility with numpy > 1.4.0<commit_after>"""
Probability density functions compliant with the lmfit package
Author: Cyril Grima <cyril.grima@gmail.com>
"""

import math
import numpy as np
from scipy import stats, integrate
from scipy.special import jv, kv, j0, i0, digamma

def gamma(params, x, data=None, eps=None):
    """Gamma PDF from scipy with adequate variables
    """
    # Initialisation
    mu = params['mu']
    # Debug inputs
    if hasattr(mu, 'value'): mu = mu.value #idem
    x = np.array([x]).flatten('C') # debug for iteration over 0-d element
    # Model function
    model = stats.gamma.pdf(x, mu, scale = 1)
    model = np.nan_to_num(model)

    if data is None:
        return model
    if eps is None:
        return (model - data) #residual
    return (model - data)/eps

def rayleigh(params, x, data=None, eps=None):
    """Rayleigh PDF from scipy with adequate variables
    """
    # Initialisation
    s = params['s']
    # Debug inputs
    if hasattr(s, 'value'): s = s.value #idem
    x = np.array([x]).flatten('C') # debug for iteration over 0-d element
    # Model function
    model = stats.rayleigh.pdf(x, scale = s)
    model = np.nan_to_num(model)

    if data is None:
        return model
    if eps is None:
        return (model - data) #residual
    return (model - data)/eps

def rice(params, x, data=None, eps=None, method = None):
    """Rice PDF from scipy with adequate variables
    """
    # Initialisation
    a = params['a']
    s = params['s']
    # Debug inputs
    if hasattr(a, 'value'): a = a.value #debug due to lmfit.minimize
    if hasattr(s, 'value'): s = s.value #idem
    x = np.array([x]).flatten('C') # debug for iteration over 0-d element
    # Model function
    model = stats.rice.pdf(x, a/s, scale = s)
    #model = (x/s**2) * np.exp(-(x**2+a**2)/(2*s**2)) * np.i0(a*x/s**2)
    #model = np.nan_to_num(model)

    if data is None:
        return model
    if eps is None:
        return (model - data) #residual
    return (model - data)/eps

def k(params, x, data=None, eps=None):
    """K PDF
    """
    # Initialisation
    s = params['s']
    mu = params['mu']
    # Debug inputs
    if hasattr(s, 'value'): s = s.value #idem
    if hasattr(mu, 'value'): mu = mu.value #idem
    x = np.array([x]).flatten('C') # debug for iteration over 0-d element

    # Model function
    b = np.sqrt(2*mu)/s
    model = 2*(x/2.)**mu *b**(mu+1.) /math.gamma(mu) *kv(mu-1,b*x)
    model = np.nan_to_num(model)

    if data is None:
        return model
    if eps is None:
        return (model - data) #residual
    return (model - data)/eps

def hk(params, x, data=None, eps=None, method = 'analytic'):
    """Homodyne K-distribution from various methods

    Arguments
    ---------
    params : dict
        params for the hk function {'a', 's', 'mu'}
    x : sequence
        x coordinates

    Keywords
    --------
    data : sequence
        data to compare the result with (for minimization)
    eps : sequence
        error on the data
    method : string
        'analytic' = from the common analytic form
        'compound' = from the compound representation [Destrempes and Cloutier,
                     2010, Ultrasound in Med. and Biol. 36(7): 1037-51, Eq. 16]
        NB: 'compound' is less unstable than 'analytic' but is ~10x longer!
    verbose : bool
        print fitting report
    """
    # Initialisation
    a = params['a']
    s = params['s']
    mu = params['mu']
    # Debug inputs
    if hasattr(a, 'value'): a = a.value #debug due to lmfit.minimize
    if hasattr(s, 'value'): s = s.value #idem
    if hasattr(mu, 'value'): mu = mu.value #idem
    x = np.array([x]).flatten('C') # debug for iteration over 0-d element

    def integrand_analytic(w, x, a, s, mu):
        return x*w*j0(w*a)*j0(w*x)*(1. +w*w*s*s/(2.*mu))**-mu
    def integrand_compound(w, x, a, s, mu):
        return rice({'a':a,'s':s*np.sqrt(w/mu)}, x) * gamma({'mu':mu}, w)

    integrands = {
        'analytic': integrand_analytic,
        'compound': integrand_compound,
    }
    integrand = integrands[method]

    model = [integrate.quad(integrand, 0., np.inf, args=(i, a, s, mu), full_output=1)[0]
            for i in x] # Integration
    model = np.array(model)

    #def integrand(w, x, a, s, mu, method=method):
    #    if method == 'analytic':
    #        return x*w*j0(w*a)*j0(w*x)*(1. +w*w*s*s/(2.*mu))**-mu
    #    if method == 'compound':
    #        return rice({'a':a,'s':s*np.sqrt(w/mu)}, x) * gamma({'mu':mu}, w)

    #model = [integrate.quad(integrand, 0., np.inf, args=(i, a, s, mu, method), full_output=1)[0]
    #         for i in x] # Integration
    #model = np.array(model)

    if data is None:
        return model
    if eps is None:
        return (model - data) #residual
    return (model - data)/eps
<|endoftext|>


<commit_before>#!/usr/bin/env python
#########################################################################################
#
# Concatenate bval files in time.
#
# ---------------------------------------------------------------------------------------
# Copyright (c) 2015 Polytechnique Montreal <www.neuro.polymtl.ca>
# Author: Simon LEVY
#
# About the license: see the file LICENSE.TXT
#########################################################################################

import sys
from msct_parser import Parser
from sct_utils import extract_fname

# PARSER
# ==========================================================================================
def get_parser():

    # Initialize the parser
    parser = Parser(__file__)
    parser.usage.set_description('Concatenate bval files in time.')
    parser.add_option(name="-i",
                      type_value=[[','], 'file'],
                      description="List of the bval files to concatenate.",
                      mandatory=True,
                      example="dmri_b700.bval,dmri_b2000.bval")
    parser.add_option(name="-o",
                      type_value="file_output",
                      description='Output file with bvals merged.',
                      mandatory=False,
                      example='dmri_b700_b2000_concat.bval')
    return parser

# MAIN
# ==========================================================================================
def main():

    # Get parser info
    parser = get_parser()
    arguments = parser.parse(sys.argv[1:])
    fname_bval_list = arguments["-i"]
    # Build fname_out
    if "-o" in arguments:
        fname_out = arguments["-o"]
    else:
        path_in, file_in, ext_in = extract_fname(fname_bval_list[0])
        fname_out = path_in+'bvals_concat'+ext_in

    # Open bval files and concatenate
    bvals_concat = ''
    for i_file in range(0,len(fname_bval_list)):
        f = open(fname_bval_list[i_file],'r')
        for line in f:
            bvals_concat += line
        f.close()

    # Write new bval
    new_f = open(fname_out,'w')
    new_f.write(bvals_concat)
    new_f.close()

# START PROGRAM
# ==========================================================================================
if __name__ == "__main__":
    # call main function
    main()

<commit_msg>OPT: use dopy function to read bvals<commit_after>#!/usr/bin/env python
#########################################################################################
#
# Concatenate bval files in time.
#
# ---------------------------------------------------------------------------------------
# Copyright (c) 2015 Polytechnique Montreal <www.neuro.polymtl.ca>
# Author: Simon LEVY
#
# About the license: see the file LICENSE.TXT
#########################################################################################

import sys
from msct_parser import Parser
from sct_utils import extract_fname
from dipy.data.fetcher import read_bvals_bvecs

# PARSER
# ==========================================================================================
def get_parser():

    # Initialize the parser
    parser = Parser(__file__)
    parser.usage.set_description('Concatenate bval files in time.')
    parser.add_option(name="-i",
                      type_value=[[','], 'file'],
                      description="List of the bval files to concatenate.",
                      mandatory=True,
                      example="dmri_b700.bval,dmri_b2000.bval")
    parser.add_option(name="-o",
                      type_value="file_output",
                      description='Output file with bvals merged.',
                      mandatory=False,
                      example='dmri_b700_b2000_concat.bval')
    return parser

# MAIN
# ==========================================================================================
def main():

    # Get parser info
    parser = get_parser()
    arguments = parser.parse(sys.argv[1:])
    fname_bval_list = arguments["-i"]
    # Build fname_out
    if "-o" in arguments:
        fname_out = arguments["-o"]
    else:
        path_in, file_in, ext_in = extract_fname(fname_bval_list[0])
        fname_out = path_in+'bvals_concat'+ext_in

    # Open bval files and concatenate
    bvals_concat = ''
    # for file_i in fname_bval_list:
    #     f = open(file_i, 'r')
    #     for line in f:
    #         bvals_concat += line
    #     f.close()
    for i_fname in fname_bval_list:
        bval_i, bvec_i = read_bvals_bvecs(i_fname, None)
        bvals_concat += ' '.join(str(v) for v in bval_i)
        bvals_concat += ' '

    # Write new bval
    new_f = open(fname_out, 'w')
    new_f.write(bvals_concat)
    new_f.close()

# START PROGRAM
# ==========================================================================================
if __name__ == "__main__":
    # call main function
    main()

<|endoftext|>


<commit_before>import sys, screed.fasta, os
import khmer

K = 32
HASHTABLE_SIZE=int(8e9)
N_HT = 4

###

RADIUS=2
MAX_CIRCUM=4                            # 4 seems to eliminate lump in 1m.fa
MAX_VOLUME=200

infile = sys.argv[1]
outprefix = sys.argv[2]

lowfile = outprefix + '.low'
highfile = outprefix + '.high'

print 'saving low-density to:', lowfile
print 'saving high-density to:', highfile

print 'making hashtable'
ht = khmer.new_hashbits(K, HASHTABLE_SIZE, N_HT)

lowfp = open(lowfile, 'w')
highfp = open(highfile, 'w')

print 'eating', infile
ht.consume_fasta(infile)

start = RADIUS
incr = 2*RADIUS

for n, record in enumerate(screed.fasta.fasta_iter(open(infile),
                                                   parse_description=False)):
    if n % 10000 == 0:
        print '... saving', n

    seq = record['sequence']

    # between [RADIUS:-RADIUS] kmers, calculate circumference every 2*RADIUS.
    # chop sequence from first high circumference, onwards.

    end = len(seq) - K + 1 - incr/2
    is_high = False

    for pos in range(start, end, incr):
        circum = ht.count_kmers_on_radius(seq[pos:pos+K], RADIUS, MAX_VOLUME)

        if circum >= MAX_CIRCUM:
            is_high = True
#            if pos == start:    # entire sequence is crud => high file
#                is_high = True
#            else:
#                chop = pos - incr
#                seq = seq[:chop + K]    # may be salvageable

            break

    # sort "high circumference" and "low" circumerence sequences separately.
    if is_high:
        fp = highfp
    else:
        fp = lowfp

    print >>fp, '>%s\n%s' % (record['name'], seq)
<commit_msg>updated to include start<commit_after>import sys, screed.fasta, os
import khmer

K = 32
HASHTABLE_SIZE=int(8e9)
N_HT = 4

###

RADIUS=2
MAX_CIRCUM=4                            # 4 seems to eliminate lump in 1m.fa
MAX_VOLUME=200

infile = sys.argv[1]
outprefix = sys.argv[2]

lowfile = outprefix + '.low'
highfile = outprefix + '.high'

print 'saving low-density to:', lowfile
print 'saving high-density to:', highfile

print 'making hashtable'
ht = khmer.new_hashbits(K, HASHTABLE_SIZE, N_HT)

lowfp = open(lowfile, 'w')
highfp = open(highfile, 'w')

print 'eating', infile
ht.consume_fasta(infile)

start = RADIUS
incr = 2*RADIUS

for n, record in enumerate(screed.fasta.fasta_iter(open(infile),
                                                   parse_description=False)):
    if n % 10000 == 0:
        print '... saving', n

    seq = record['sequence']

    # between [RADIUS:-RADIUS] kmers, calculate circumference every 2*RADIUS.
    # chop sequence from first high circumference, onwards.

    end = len(seq) - K + 1 - incr/2
    is_high = False

    for pos in range(end, start, -incr):
        circum = ht.count_kmers_on_radius(seq[pos:pos+K], RADIUS, MAX_VOLUME)

        if circum >= MAX_CIRCUM:
            is_high = True
#            if pos == start:    # entire sequence is crud => high file
#                is_high = True
#            else:
#                chop = pos - incr
#                seq = seq[:chop + K]    # may be salvageable

            break

    if pos != start:
        pos = start
        circum = ht.count_kmers_on_radius(seq[pos:pos+K], RADIUS, MAX_VOLUME)

        if circum >= MAX_CIRCUM:
            is_high = True

    # sort "high circumference" and "low" circumerence sequences separately.
    if is_high:
        fp = highfp
    else:
        fp = lowfp

    print >>fp, '>%s\n%s' % (record['name'], seq)
<|endoftext|>