<commit_before>#!/usr/bin/env python
#Dan Blankenberg
# adapted from Dan's BWA one for rna star
# ross lazarus sept 2014
#
import sys
import os
import tempfile
import optparse
import subprocess

from json import loads, dumps

DEFAULT_DATA_TABLE_NAME = "rnastar_index"
CHUNK_SIZE = 2**20

def get_id_name( params, dbkey, fasta_description=None):
    #TODO: ensure sequence_id is unique and does not already appear in location file
    sequence_id = params['param_dict']['sequence_id']
    if not sequence_id:
        sequence_id = dbkey

    sequence_name = params['param_dict']['sequence_name']
    if not sequence_name:
        sequence_name = fasta_description
        if not sequence_name:
            sequence_name = dbkey
    return sequence_id, sequence_name

def _run_command( command, target_directory ):
    tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-rnastar_index_builder-stderr" )
    return_code = subprocess.call( command, shell=False, stdout=sys.stdout, stderr=tmp_stderr.fileno() )
    if return_code:
        tmp_stderr.flush()
        tmp_stderr.seek( 0 )
        sys.stderr.write( "### star reports an error:\n" )
        while True:
            chunk = tmp_stderr.read( CHUNK_SIZE )
            if not chunk:
                break
            sys.stderr.write( chunk )
        sys.exit( return_code )
    tmp_stderr.close()
    return return_code

def build_rnastar_index( data_manager_dict, fasta_filename,  target_directory,
    dbkey, sequence_id, sequence_name, data_table_name,
    sjdbOverhang,sjdbGTFfile, sjdbFileChrStartEnd,sjdbGTFtagExonParentTranscript,sjdbGTFfeatureExon,sjdbGTFchrPrefix,n_threads):
    #TODO: allow multiple FASTA input files
    fasta_base_name = os.path.basename( fasta_filename )
    #sym_linked_fasta_filename = os.path.join( tmp_dir, fasta_base_name )
    #os.symlink( fasta_filename, sym_linked_fasta_filename )
    #print >> sys.stdout,'made',sym_linked_fasta_filename
    cl = ['STAR','--runMode', 'genomeGenerate', '--genomeFastaFiles', fasta_filename, '--genomeDir', target_directory, '--runThreadN',n_threads ]

    if sjdbGTFfile:
         cl += [ '--sjdbGTFfeatureExon', sjdbGTFfeatureExon,'--sjdbGTFtagExonParentTranscript',sjdbGTFtagExonParentTranscript]
         if (sjdbGTFchrPrefix > ''):
             cl += ['--sjdbGTFchrPrefix', sjdbGTFchrPrefix]
         cl += ['--sjdbOverhang', sjdbOverhang, '--sjdbGTFfile', sjdbGTFfile]
    elif sjdbFileChrStartEnd:
        cl += ['--sjdbFileChrStartEnd', sjdbFileChrStartEnd, '--sjdbOverhang', sjdbOverhang]
    return_code = _run_command(command=cl,target_directory=target_directory)
    data_table_entry = dict( value=sequence_id, dbkey=dbkey, name=sequence_name, path=fasta_base_name )
    data_manager_dict.setdefault('data_tables',{})
    data_manager_dict['data_tables'].setdefault('data_table_name',[])
    data_manager_dict['data_tables'][ 'data_table_name' ].append( data_table_entry )
    return data_manager_dict

def main():
    #Parse Command Line
    parser = optparse.OptionParser()
    parser.add_option( '--fasta_filename', dest='fasta_filename', action='store', type="string", default=None, help='fasta_filename' )
    parser.add_option( '--fasta_dbkey', dest='fasta_dbkey', action='store', type="string", default=None, help='fasta_dbkey' )
    parser.add_option( '--fasta_description', dest='fasta_description', action='store', type="string", default=None, help='fasta_description' )
    parser.add_option( '--data_table_name', dest='data_table_name', action='store', type="string", default=None, help='data_table_name' )
    parser.add_option( '--sjdbGTFfile', type="string", default=None )
    parser.add_option( '--sjdbGTFchrPrefix', type="string", default=None )
    parser.add_option( '--sjdbGTFfeatureExon', type="string", default=None )
    parser.add_option( '--sjdbGTFtagExonParentTranscript', type="string", default=None )
    parser.add_option( '--sjdbFileChrStartEnd', type="string", default=None )
    parser.add_option( '--sjdbOverhang', type="string", default='100' )
    parser.add_option( '--runThreadN', type="string", default='4' )
    (options, args) = parser.parse_args()
    filename = args[0]
    # this is passed as an positional parameter in the original wrapper with the extra_files_path from the params
    params = loads( open( filename ).read() )
    target_directory = params[ 'output_data' ][0]['extra_files_path'].encode('ascii','replace')
    try:
         os.mkdir( target_directory )
    except OSError:
        pass

    data_manager_dict = {}

    dbkey = options.fasta_dbkey

    if dbkey in [ None, '', '?' ]:
        raise Exception( 'dbkey "%s" does not specify any locally known genome to be indexed.' % ( dbkey ) )

    sequence_id, sequence_name = get_id_name( params, dbkey=dbkey, fasta_description=options.fasta_description )

    #build the index
    data_manager_dict = build_rnastar_index( data_manager_dict = data_manager_dict, fasta_filename = options.fasta_filename, target_directory = target_directory,
      dbkey = dbkey, sequence_id = sequence_id, sequence_name = sequence_name, data_table_name=options.data_table_name,
      sjdbOverhang=options.sjdbOverhang,sjdbGTFfile=options.sjdbGTFfile,
      sjdbFileChrStartEnd=options.sjdbFileChrStartEnd,sjdbGTFtagExonParentTranscript=options.sjdbGTFtagExonParentTranscript,
      sjdbGTFfeatureExon=options.sjdbGTFfeatureExon,sjdbGTFchrPrefix=options.sjdbGTFchrPrefix,
      n_threads=options.runThreadN )

    #save info to json file
    open( filename, 'wb' ).write( dumps( data_manager_dict ) )

if __name__ == "__main__": main()
<commit_msg>eeesh. tracking down failure to update tables after star datamanager run. data_table_name =/= 'data_table_name'<commit_after>#!/usr/bin/env python
#Dan Blankenberg
# adapted from Dan's BWA one for rna star
# ross lazarus sept 2014
#
import sys
import os
import tempfile
import optparse
import subprocess

from json import loads, dumps

DEFAULT_DATA_TABLE_NAME = "rnastar_index"
CHUNK_SIZE = 2**20

def get_id_name( params, dbkey, fasta_description=None):
    #TODO: ensure sequence_id is unique and does not already appear in location file
    sequence_id = params['param_dict']['sequence_id']
    if not sequence_id:
        sequence_id = dbkey

    sequence_name = params['param_dict']['sequence_name']
    if not sequence_name:
        sequence_name = fasta_description
        if not sequence_name:
            sequence_name = dbkey
    return sequence_id, sequence_name

def _run_command( command, target_directory ):
    tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-rnastar_index_builder-stderr" )
    return_code = subprocess.call( command, shell=False, stdout=sys.stdout, stderr=tmp_stderr.fileno() )
    if return_code:
        tmp_stderr.flush()
        tmp_stderr.seek( 0 )
        sys.stderr.write( "### star reports an error:\n" )
        while True:
            chunk = tmp_stderr.read( CHUNK_SIZE )
            if not chunk:
                break
            sys.stderr.write( chunk )
        sys.exit( return_code )
    tmp_stderr.close()
    return return_code

def build_rnastar_index( data_manager_dict, fasta_filename,  target_directory,
    dbkey, sequence_id, sequence_name, data_table_name,
    sjdbOverhang,sjdbGTFfile, sjdbFileChrStartEnd,sjdbGTFtagExonParentTranscript,sjdbGTFfeatureExon,sjdbGTFchrPrefix,n_threads):
    #TODO: allow multiple FASTA input files
    fasta_base_name = os.path.basename( fasta_filename )
    #sym_linked_fasta_filename = os.path.join( tmp_dir, fasta_base_name )
    #os.symlink( fasta_filename, sym_linked_fasta_filename )
    #print >> sys.stdout,'made',sym_linked_fasta_filename
    cl = ['STAR','--runMode', 'genomeGenerate', '--genomeFastaFiles', fasta_filename, '--genomeDir', target_directory, '--runThreadN',n_threads ]

    if sjdbGTFfile:
         cl += [ '--sjdbGTFfeatureExon', sjdbGTFfeatureExon,'--sjdbGTFtagExonParentTranscript',sjdbGTFtagExonParentTranscript]
         if (sjdbGTFchrPrefix > ''):
             cl += ['--sjdbGTFchrPrefix', sjdbGTFchrPrefix]
         cl += ['--sjdbOverhang', sjdbOverhang, '--sjdbGTFfile', sjdbGTFfile]
    elif sjdbFileChrStartEnd:
        cl += ['--sjdbFileChrStartEnd', sjdbFileChrStartEnd, '--sjdbOverhang', sjdbOverhang]
    return_code = _run_command(command=cl,target_directory=target_directory)
    data_table_entry = dict( value=sequence_id, dbkey=dbkey, name=sequence_name, path=fasta_base_name )
    data_manager_dict.setdefault('data_tables',{})
    data_manager_dict['data_tables'].setdefault(data_table_name,[])
    data_manager_dict['data_tables'][ data_table_name ].append( data_table_entry )
    return data_manager_dict

def main():
    #Parse Command Line
    parser = optparse.OptionParser()
    parser.add_option( '--fasta_filename', dest='fasta_filename', action='store', type="string", default=None, help='fasta_filename' )
    parser.add_option( '--fasta_dbkey', dest='fasta_dbkey', action='store', type="string", default=None, help='fasta_dbkey' )
    parser.add_option( '--fasta_description', dest='fasta_description', action='store', type="string", default=None, help='fasta_description' )
    parser.add_option( '--data_table_name', dest='data_table_name', action='store', type="string", default=None, help='data_table_name' )
    parser.add_option( '--sjdbGTFfile', type="string", default=None )
    parser.add_option( '--sjdbGTFchrPrefix', type="string", default=None )
    parser.add_option( '--sjdbGTFfeatureExon', type="string", default=None )
    parser.add_option( '--sjdbGTFtagExonParentTranscript', type="string", default=None )
    parser.add_option( '--sjdbFileChrStartEnd', type="string", default=None )
    parser.add_option( '--sjdbOverhang', type="string", default='100' )
    parser.add_option( '--runThreadN', type="string", default='4' )
    (options, args) = parser.parse_args()
    filename = args[0]
    # this is passed as an positional parameter in the original wrapper with the extra_files_path from the params
    params = loads( open( filename ).read() )
    target_directory = params[ 'output_data' ][0]['extra_files_path'].encode('ascii','replace')
    try:
         os.mkdir( target_directory )
    except OSError:
        pass

    data_manager_dict = {}

    dbkey = options.fasta_dbkey

    if dbkey in [ None, '', '?' ]:
        raise Exception( 'dbkey "%s" does not specify any locally known genome to be indexed.' % ( dbkey ) )

    sequence_id, sequence_name = get_id_name( params, dbkey=dbkey, fasta_description=options.fasta_description )

    #build the index
    data_manager_dict = build_rnastar_index( data_manager_dict = data_manager_dict, fasta_filename = options.fasta_filename, target_directory = target_directory,
      dbkey = dbkey, sequence_id = sequence_id, sequence_name = sequence_name, data_table_name=options.data_table_name,
      sjdbOverhang=options.sjdbOverhang,sjdbGTFfile=options.sjdbGTFfile,
      sjdbFileChrStartEnd=options.sjdbFileChrStartEnd,sjdbGTFtagExonParentTranscript=options.sjdbGTFtagExonParentTranscript,
      sjdbGTFfeatureExon=options.sjdbGTFfeatureExon,sjdbGTFchrPrefix=options.sjdbGTFchrPrefix,
      n_threads=options.runThreadN )

    #save info to json file
    open( filename, 'wb' ).write( dumps( data_manager_dict ) )

if __name__ == "__main__": main()
<|endoftext|>


<commit_before>from __future__ import absolute_import

import cgi
import cStringIO as StringIO
from itertools import islice
import urllib, urlparse
import warnings

import httplib2

from .schema import SolrSchema, SolrError
from .search import SolrSearch, LuceneQuery

class SolrConnection(object):
    def __init__(self, url, http_connection=None):
        if not http_connection:
            http_connection = httplib2.Http()
        self.url = url.rstrip("/") + "/"
        self.update_url = self.url + "update/"
        self.select_url = self.url + "select/"
        self.request = http_connection.request

    def commit(self, wait_flush=True, wait_searcher=True):
        response = self.commit_or_optimize("commit",
                                           wait_flush, wait_searcher)

    def optimize(self, wait_flush=True, wait_searcher=True):
        response = self.commit_or_optimize("optimize",
                                           wait_flush, wait_searcher)

    def commit_or_optimize(self, verb, wait_flush, wait_searcher):
        wait_flush = "true" if wait_flush else "false"
        wait_searcher = "true" if wait_searcher else "false"
        response = self.update('<%s waitFlush="%s" waitSearcher="%s"/>' %
                               (verb, wait_flush, wait_searcher))

    def rollback(self):
        response = self.update("<rollback/>")

    def update(self, update_doc):
        body = update_doc
        headers = {"Content-Type":"text/xml; charset=utf-8"}
        r, c = self.request(self.update_url, method="POST", body=body,
                            headers=headers)
        if r.status != 200:
            raise SolrError(r, c)

    def select(self, params):
        qs = utf8_urlencode(params)
        url = "%s?%s" % (self.select_url, qs)
        r, c = self.request(url)
        if r.status != 200:
            raise SolrError(r, c)
        return c

class SolrInterface(object):
    readable = True
    writeable = True
    remote_schema_file = "admin/file/?file=schema.xml"
    def __init__(self, url, schemadoc=None, http_connection=None, mode=''):
        if not http_connection:
            http_connection = httplib2.Http()
        self.conn = SolrConnection(url, http_connection)
        if not schemadoc:
            r, c = http_connection.request(
                urlparse.urljoin(url, self.remote_schema_file))
            if r.status != 200:
                raise EnvironmentError("Couldn't retrieve schema document from server - received status code %s\n%s" % (r.status, c))
            schemadoc = StringIO.StringIO(c)
        self.schema = SolrSchema(schemadoc)
        if mode == 'r':
            self.writeable = False
        elif mode == 'w':
            self.readable = False

    def add(self, docs, chunk=100):
        if not self.writeable:
            raise TypeError("This Solr instance is only for reading")
        if hasattr(docs, "items") or not hasattr(docs, "__iter__"):
            docs = [docs]
        # to avoid making messages too large, we break the message every
        # chunk docs.
        for doc_chunk in grouper(docs, chunk):
            update_message = self.schema.make_update(doc_chunk)
            self.conn.update(str(update_message))

    def delete(self, docs=None, queries=None):
        if not self.writeable:
            raise TypeError("This Solr instance is only for reading")
        if not docs and not queries:
            raise SolrError("No docs or query specified for deletion")
        elif docs is not None and (hasattr(docs, "items") or not hasattr(docs, "__iter__")):
            docs = [docs]
        delete_message = self.schema.make_delete(docs, queries)
        self.conn.update(str(delete_message))

    def commit(self, *args, **kwargs):
        if not self.writeable:
            raise TypeError("This Solr instance is only for reading")
        self.conn.commit(*args, **kwargs)

    def optimize(self, *args, **kwargs):
        if not self.writeable:
            raise TypeError("This Solr instance is only for reading")
        self.conn.optimize(*args, **kwargs)

    def rollback(self):
        if not self.writeable:
            raise TypeError("This Solr instance is only for reading")
        self.conn.rollback()

    def clear_all(self):
        if not self.writeable:
            raise TypeError("This Solr instance is only for reading")
        # When deletion is fixed to escape query strings, this will need fixed.
        self.delete(queries="*:*")

    def search(self, **kwargs):
        if not self.readable:
            raise TypeError("This Solr instance is only for writing")
        params = self.params(**kwargs)
        return self.schema.parse_results(self.conn.select(params))

    def query(self, *args, **kwargs):
        if not self.readable:
            raise TypeError("This Solr instance is only for writing")
        q = SolrSearch(self)
        if len(args) + len(kwargs) > 0:
            return q.query(*args, **kwargs)
        else:
            return q

    def params(self, **kwargs):
        params = kwargs.copy()
        for k, v in kwargs.items():
            if hasattr(v, "items"):
                del params[k]
                params.update(v)
        if 'q' not in params:
            if 'fq' in params:
                params['q'] = params['fq']
                warnings.warn("No query parameter in search, re-using filter query")
            else:
                raise SolrError("No query parameter in search")
        return params

    def Q(self, *args, **kwargs):
        q = LuceneQuery(self.schema)
        q.add(args, kwargs)
        return q

def utf8_urlencode(params):
    utf8_params = []
    for k, vs in params.items():
        if isinstance(k, unicode):
            k = k.encode('utf-8')
        # We allow for multivalued options with lists.
        if not hasattr(vs, "__iter__"):
            vs = [vs]
        for v in vs:
            if isinstance(v, bool):
                v = u"true" if v else u"false"
            else:
                v = unicode(v)
            v = v.encode('utf-8')
            utf8_params.append((k, v))
    return urllib.urlencode(utf8_params)

def grouper(iterable, n):
    "grouper('ABCDEFG', 3) --> [['ABC'], ['DEF'], ['G']]"
    i = iter(iterable)
    g = list(islice(i, 0, n))
    while g:
        yield g
        g = list(islice(i, 0, n))
<commit_msg>Remove useless dictionary-transformation code<commit_after>from __future__ import absolute_import

import cgi
import cStringIO as StringIO
from itertools import islice
import urllib, urlparse
import warnings

import httplib2

from .schema import SolrSchema, SolrError
from .search import SolrSearch, LuceneQuery

class SolrConnection(object):
    def __init__(self, url, http_connection=None):
        if not http_connection:
            http_connection = httplib2.Http()
        self.url = url.rstrip("/") + "/"
        self.update_url = self.url + "update/"
        self.select_url = self.url + "select/"
        self.request = http_connection.request

    def commit(self, wait_flush=True, wait_searcher=True):
        response = self.commit_or_optimize("commit",
                                           wait_flush, wait_searcher)

    def optimize(self, wait_flush=True, wait_searcher=True):
        response = self.commit_or_optimize("optimize",
                                           wait_flush, wait_searcher)

    def commit_or_optimize(self, verb, wait_flush, wait_searcher):
        wait_flush = "true" if wait_flush else "false"
        wait_searcher = "true" if wait_searcher else "false"
        response = self.update('<%s waitFlush="%s" waitSearcher="%s"/>' %
                               (verb, wait_flush, wait_searcher))

    def rollback(self):
        response = self.update("<rollback/>")

    def update(self, update_doc):
        body = update_doc
        headers = {"Content-Type":"text/xml; charset=utf-8"}
        r, c = self.request(self.update_url, method="POST", body=body,
                            headers=headers)
        if r.status != 200:
            raise SolrError(r, c)

    def select(self, params):
        qs = utf8_urlencode(params)
        url = "%s?%s" % (self.select_url, qs)
        r, c = self.request(url)
        if r.status != 200:
            raise SolrError(r, c)
        return c

class SolrInterface(object):
    readable = True
    writeable = True
    remote_schema_file = "admin/file/?file=schema.xml"
    def __init__(self, url, schemadoc=None, http_connection=None, mode=''):
        if not http_connection:
            http_connection = httplib2.Http()
        self.conn = SolrConnection(url, http_connection)
        if not schemadoc:
            r, c = http_connection.request(
                urlparse.urljoin(url, self.remote_schema_file))
            if r.status != 200:
                raise EnvironmentError("Couldn't retrieve schema document from server - received status code %s\n%s" % (r.status, c))
            schemadoc = StringIO.StringIO(c)
        self.schema = SolrSchema(schemadoc)
        if mode == 'r':
            self.writeable = False
        elif mode == 'w':
            self.readable = False

    def add(self, docs, chunk=100):
        if not self.writeable:
            raise TypeError("This Solr instance is only for reading")
        if hasattr(docs, "items") or not hasattr(docs, "__iter__"):
            docs = [docs]
        # to avoid making messages too large, we break the message every
        # chunk docs.
        for doc_chunk in grouper(docs, chunk):
            update_message = self.schema.make_update(doc_chunk)
            self.conn.update(str(update_message))

    def delete(self, docs=None, queries=None):
        if not self.writeable:
            raise TypeError("This Solr instance is only for reading")
        if not docs and not queries:
            raise SolrError("No docs or query specified for deletion")
        elif docs is not None and (hasattr(docs, "items") or not hasattr(docs, "__iter__")):
            docs = [docs]
        delete_message = self.schema.make_delete(docs, queries)
        self.conn.update(str(delete_message))

    def commit(self, *args, **kwargs):
        if not self.writeable:
            raise TypeError("This Solr instance is only for reading")
        self.conn.commit(*args, **kwargs)

    def optimize(self, *args, **kwargs):
        if not self.writeable:
            raise TypeError("This Solr instance is only for reading")
        self.conn.optimize(*args, **kwargs)

    def rollback(self):
        if not self.writeable:
            raise TypeError("This Solr instance is only for reading")
        self.conn.rollback()

    def clear_all(self):
        if not self.writeable:
            raise TypeError("This Solr instance is only for reading")
        # When deletion is fixed to escape query strings, this will need fixed.
        self.delete(queries="*:*")

    def search(self, **kwargs):
        if not self.readable:
            raise TypeError("This Solr instance is only for writing")
        params = self.params(**kwargs)
        return self.schema.parse_results(self.conn.select(params))

    def query(self, *args, **kwargs):
        if not self.readable:
            raise TypeError("This Solr instance is only for writing")
        q = SolrSearch(self)
        if len(args) + len(kwargs) > 0:
            return q.query(*args, **kwargs)
        else:
            return q

    def params(self, **kwargs):
        params = kwargs.copy()
        if 'q' not in params:
            if 'fq' in params:
                params['q'] = params['fq']
                warnings.warn("No query parameter in search, re-using filter query")
            else:
                raise SolrError("No query parameter in search")
        return params

    def Q(self, *args, **kwargs):
        q = LuceneQuery(self.schema)
        q.add(args, kwargs)
        return q

def utf8_urlencode(params):
    utf8_params = []
    for k, vs in params.items():
        if isinstance(k, unicode):
            k = k.encode('utf-8')
        # We allow for multivalued options with lists.
        if not hasattr(vs, "__iter__"):
            vs = [vs]
        for v in vs:
            if isinstance(v, bool):
                v = u"true" if v else u"false"
            else:
                v = unicode(v)
            v = v.encode('utf-8')
            utf8_params.append((k, v))
    return urllib.urlencode(utf8_params)

def grouper(iterable, n):
    "grouper('ABCDEFG', 3) --> [['ABC'], ['DEF'], ['G']]"
    i = iter(iterable)
    g = list(islice(i, 0, n))
    while g:
        yield g
        g = list(islice(i, 0, n))
<|endoftext|>


<commit_before>from teafacto.util import ticktock, argprun
import os, pickle, random
from teafacto.procutil import *
from IPython import embed
from scipy import sparse

from teafacto.blocks.lang.wordvec import Glove, WordEmb
from teafacto.blocks.lang.sentenc import TwoLevelEncoder
from teafacto.blocks.seq.rnn import RNNSeqEncoder, MaskMode
from teafacto.blocks.seq.enc import SimpleSeq2Vec
from teafacto.blocks.cnn import CNNSeqEncoder
from teafacto.blocks.basic import VectorEmbed
from teafacto.blocks.memory import MemVec
from teafacto.blocks.match import SeqMatchScore, CosineDistance, MatchScore

from teafacto.core.base import Block, tensorops as T, Val

def readdata(p="../../../../data/simplequestions/clean/datamat.word.fb2m.pkl",
             entinfp="../../../../data/simplequestions/clean/subjs-counts-labels-types.fb2m.tsv",
             cachep=None, #"subjpredcharns.readdata.cache.pkl",
             maskid=-1,
             ):
    tt = ticktock("dataloader")
    if cachep is not None and os.path.isfile(cachep):      # load
        tt.tick("loading from cache")
        ret = pickle.load(open(cachep))
        tt.tock("loaded from cache")
    else:
        tt.tick("loading datamat")
        x = pickle.load(open(p))
        tt.tock("datamat loaded")
        worddic = x["worddic"]
        entdic = x["entdic"]
        entmat = x["entmat"]
        numents = x["numents"]
        traindata, traingold = x["train"]
        validdata, validgold = x["valid"]
        testdata, testgold = x["test"]
        traingold[:, 1] -= numents
        validgold[:, 1] -= numents
        testgold[:, 1] -= numents

        rwd = {v: k for k, v in worddic.items()}

        subjdic = {k: v for k, v in entdic.items() if v < numents}
        reldic = {k: v - numents for k, v in entdic.items() if v >= numents}

        subjmat = entmat[:numents]
        relmat = entmat[numents:]

        traindata = wordmat2wordchartensor(traindata, rwd=rwd, maskid=maskid)
        validdata = wordmat2wordchartensor(validdata, rwd=rwd, maskid=maskid)
        testdata = wordmat2wordchartensor(testdata, rwd=rwd, maskid=maskid)

        subjmat = wordmat2charmat(subjmat, rwd=rwd, maskid=maskid, maxlen=75)

        ret = ((traindata, traingold), (validdata, validgold),
               (testdata, testgold), (subjmat, relmat), (subjdic, reldic),
               worddic)
        if cachep is not None:
            tt.tick("dumping to cache")
            pickle.dump(ret, open(cachep, "w"))
            tt.tock("dumped to cache")

    subjinfo = loadsubjinfo(entinfp, subjdic)
    testsubjcans = loadsubjtestcans()
    testrelcans = loadreltestcans(testgold,subjdic, reldic)
    return ret + (subjinfo, (testsubjcans, testrelcans))

def loadreltestcans(testgold, subjdic, reldic, relsperentp="../../../../data/simplequestions/allrelsperent.dmp"):
    tt = ticktock("test rel can loader")
    testsubjs = testgold[:, 0]
    relsoftestsubjs = {k: ([], []) for k in set(list(testsubjs))}
    tt.tick("loading rel test cans")
    for line in open(relsperentp):
        subj, relsout, relsin = line[:-1].split("\t")
        if subj in subjdic and subjdic[subj] in relsoftestsubjs:
            relsoftestsubjs[subjdic[subj]] = (
                [reldic[x] for x in relsout.split(" ")] if relsout != "" else [],
                [reldic[x] for x in relsin.split(" ")] if relsin != "" else []
            )
    tt.tock("test cans loaded")
    relsoftestexamples = [(relsoftestsubjs[x][0], relsoftestsubjs[x][1])
                          for x in testsubjs]
    return relsoftestexamples

def loadsubjtestcans(p="../../../../data/simplequestions/clean/testcans.pkl"):
    tt = ticktock("test subjects candidate loader")
    tt.tick("loading candidates")
    ret = pickle.load(open(p))
    tt.tock("canddiates loaded")
    return ret

def loadsubjinfo(entinfp, entdic, cachep=None):#"subjinfo.cache.pkl"):
    tt = ticktock("subjinfoloader")
    def make():
        tt.tick("making subject info from file")
        subjinf = {}
        c = 0
        for line in open(entinfp):
            subjuri, subjc, objc, subjname, typuri, typname = line[:-1].split("\t")
            subjinf[entdic[subjuri]] = (subjname, typname.lower().split(), typuri, subjc, objc)
            if c % 1000 == 0:
                tt.live(str(c))
            c += 1
        tt.tock("made subject info from file")
        return subjinf
    if cachep is not None:
        if os.path.isfile(cachep):      # load
            tt.tick("loading cached subject info")
            subjinfo = pickle.load(open(cachep))
            tt.tock("loaded cached subject info")
        else:                           # make  and dump
            subjinfo = make()
            tt.tick("dumping subject info in cache")
            pickle.dump(subjinfo, open(cachep, "w"))
            tt.tock("dumped subject info in cache")
    else:       # just make
        subjinfo = make()
    return subjinfo

def buildrelsamplespace(entmat, wd, maskid=-1):
    tt = ticktock("samplespace")
    tt.tick("making sample space")
    #rwd = {v: k for k, v in wd.items()}
    entmatm = sparse.dok_matrix((entmat.shape[0], np.max(entmat) + 1))
    posblacklist = {0: {wd["base"], wd["user"]}}
    blacklist = set([wd[x] for x in "default domain of by the in at s this for with type".split()])
    #revin = {k: set() for k in np.unique(entmat)}
    #revinm = sparse.dok_matrix((np.max(entmat), entmat.shape[0]))
    samdic = {k: set() for k in range(entmat.shape[0])}     # from ent ids to sets of ent ids
    #samdic = np.zeros((entmat.shape[0], entmat.shape[0]))
    for i in range(entmat.shape[0]):
        for j in range(entmat.shape[1]):
            w = entmat[i, j]
            if w == -1:     # beginning of padding
                break
            if j in posblacklist:
                if w in posblacklist[j]:
                    continue
            if w in blacklist:
                continue
            entmatm[i, w] = 1
            #for oe in revin[w]:     # other entities already in revind
            #    samdic[oe].add(i)
            #    samdic[i].add(oe)
            #revin[w].add(i)
            #revinm[w, i] = 1
    samdicm = entmatm.dot(entmatm.T)
    for i in range(samdicm.shape[0]):
        samdic[i] = list(np.argwhere(samdicm[i, :])[:, 1])
    tt.tock("made sample space")
    return samdic, entmatm.T

class SeqLeftBlock(Block):
    def __init__(self, inner, **kw):
        super(SeqLeftBlock, self).__init__(**kw)
        self.inner = inner

    def apply(self, x):
        # idxs^(batsize, seqlen, ...) --> (batsize, seqlen, 2, encdim)
        res = self.inner(x).dimshuffle(0, "x", 1)
        ret = T.concatenate([res, res], axis=1)
        return ret

class ConcatLeftBlock(Block):
    def __init__(self, inner, **kw):
        super(ConcatLeftBlock, self).__init__(**kw)
        self.inner = inner

    def apply(self, x):
        res = self.inner(x).dimshuffle(0, "x", 1) # (batsize, q_enc_dim)
        mid = res.shape[2]/2
        ret = T.concatenate([res[:, :, mid], res[:, :, mid:]], axis=1)
        return ret

class RightBlock(Block):
    def __init__(self, a, b, **kw):
        super(RightBlock, self).__init__(**kw)
        self.subjenc = a
        self.predenc = b

    def apply(self, subjslice, relslice):  # idxs^(batsize, len)
        aret = self.subjenc(subjslice).dimshuffle(0, "x", 1)
        bret = self.predenc(relslice).dimshuffle(0, "x", 1)
        ret = T.concatenate([aret, bret], axis=1)
        return ret

def run(closenegsam=False,
        checkdata=False,
        glove=True,
        embdim=100,
        charencdim=100,
        charembdim=50,
        encdim=400,
        bidir=False,
        charenc="cnn",  # "cnn" or "rnn"
        mode="seq",      # "seq" or "concat"
        margin=0.5,
        lr=0.1,
        numbats=700,
        epochs=15,
        ):
    tt = ticktock("script")
    tt.tick("loading data")
    (traindata, traingold), (validdata, validgold), (testdata, testgold), \
    (subjmat, relmat), (subjdic, reldic), worddic, \
    subjinfo, (testsubjcans, testrelcans) = readdata()

    revsamplespace = None
    if closenegsam:
        relsamplespace, revind = buildrelsamplespace(relmat, worddic)
    tt.tock("data loaded")

    if checkdata:
        embed()

    numwords = max(worddic.values()) + 1
    numsubjs = max(subjdic.values()) + 1
    numrels = max(reldic.values()) + 1
    maskid = -1
    numchars = 256

    if mode == "seq":
        decdim = encdim
    elif mode == "concat":
        decdim = encdim / 2
    else:
        raise Exception("unrecognized mode")

    # defining model
    if glove:
        wordemb = Glove(embdim).adapt(worddic)
    else:
        wordemb = WordEmb(dim=embdim, indim=numwords)
    charemb = VectorEmbed(indim=numchars, dim=charembdim)
    if charenc == "cnn":
        print "using CNN char encoder"
        charenc = CNNSeqEncoder(inpemb=charemb,
                                innerdim=[charencdim]*2, maskid=maskid,
                                stride=1)
    elif charenc == "rnn":
        print "using RNN char encoder"
        charenc = RNNSeqEncoder(inpemb=charemb, innerdim=charencdim)\
            .maskoptions(maskid, MaskMode.AUTO)
    else:
        raise Exception("no other modes available")
    wordenc = RNNSeqEncoder(inpemb=False, inpembdim=wordemb.outdim + charencdim,
                            innerdim=encdim, bidir=bidir).maskoptions(MaskMode.NONE)
    question_encoder = TwoLevelEncoder(l1enc=charenc, l2emb=wordemb,
                                       l2enc=wordenc, maskid=maskid)

    # encode predicate on word level
    predemb = SimpleSeq2Vec(inpemb=wordemb,
                           innerdim=decdim,
                           maskid=maskid,
                           bidir=bidir,
                           layers=1)
    #predemb.load(relmat)

    # encode subject on character level
    subjemb = SimpleSeq2Vec(inpemb=charemb,
                           innerdim=decdim,
                           maskid=maskid,
                           bidir=bidir,
                           layers=2)
    #subjemb.load(subjmat)

    # package
    if mode == "seq":
        lb = SeqLeftBlock(question_encoder)
        rb = RightBlock(subjemb, predemb)
    elif mode == "concat":
        lb = ConcatLeftBlock(question_encoder)
        rb = RightBlock(subjemb, predemb)
    else:
        raise Exception("unrecognized mode")
    scorer = SeqMatchScore(lb, rb, scorer=CosineDistance(),
                           aggregator=lambda x: x, argproc=lambda x, y, z: ((x,), (y, z)))

    obj = lambda p, n: T.sum((n - p + margin).clip(0, np.infty), axis=1)

    # negative sampling
    class NegIdxGen(object):
        def __init__(self, maxentid, maxrelid, relclose=None):
            self.maxentid = maxentid
            self.maxrelid = maxrelid
            self.relclose = relclose

        def __call__(self, datas, gold):
            entrand = np.random.randint(0, self.maxentid+1, (gold.shape[0], 1))
            relrand = self.samplerels(gold[:, 1:2])
            ret = np.concatenate([entrand, relrand], axis=1)
            #embed()
            return datas, ret.astype("int32")

        def samplerels(self, gold):
            assert(gold.ndim == 2 and gold.shape[1] == 1)
            if self.relclose is None:
                return np.random.randint(0, self.maxrelid+1, (gold.shape[0], 1))
            else:
                ret = np.zeros_like(gold)
                for i in range(gold.shape[0]):
                    sampleset = self.relclose[gold[i]]
                    if len(sampleset) > 5:
                        ret[i] = random.sample(sampleset, 1)[0]
                    else:
                        ret[i] = np.random.randint(0, self.maxrelid+1)
                return ret.astype("int32")

    class PreProc(object):
        def __init__(self, subjmat, relmat):
            self.f = PreProcE(subjmat, relmat)

        def __call__(self, data, gold):     # gold: idxs-(batsize, 2)
            ret = self.f(gold)[0]
            return (data,) + ret, {}

    class PreProcE(object):
        def __init__(self, subjmat, relmat):
            self.subjmat = Val(subjmat)
            self.relmat = Val(relmat)

        def __call__(self, x):
            subjslice = self.subjmat[x[:, 0]]
            relslice = self.relmat[x[:, 1]]
            return (subjslice, relslice), {}

    transf = PreProc(subjmat, relmat)

    #embed()
    tt.tick("training")
    nscorer = scorer.nstrain([traindata, traingold]).transform(transf)\
        .negsamplegen(NegIdxGen(numsubjs-1, numrels-1, relclose=revsamplespace)) \
        .objective(obj).adagrad(lr=lr).grad_total_norm(1.0)\
        .validate_on([validdata, validgold])\
        .train(numbats=numbats, epochs=epochs)
    tt.tock("trained").tick()

    # saving
    saveid = "".join([str(np.random.randint(0, 10)) for i in range(4)])
    scorer.save("fullrank{}.model".format(saveid))
    tt.tock("saved: {}".format(saveid))

    # evaluation
    # TODO

if __name__ == "__main__":
    argprun(run)<commit_msg>fullrank fixes<commit_after>from teafacto.util import ticktock, argprun
import os, pickle, random
from teafacto.procutil import *
from IPython import embed
from scipy import sparse

from teafacto.blocks.lang.wordvec import Glove, WordEmb
from teafacto.blocks.lang.sentenc import TwoLevelEncoder
from teafacto.blocks.seq.rnn import RNNSeqEncoder, MaskMode
from teafacto.blocks.seq.enc import SimpleSeq2Vec
from teafacto.blocks.cnn import CNNSeqEncoder
from teafacto.blocks.basic import VectorEmbed
from teafacto.blocks.memory import MemVec
from teafacto.blocks.match import SeqMatchScore, CosineDistance, MatchScore

from teafacto.core.base import Block, tensorops as T, Val

def readdata(p="../../../../data/simplequestions/clean/datamat.word.fb2m.pkl",
             entinfp="../../../../data/simplequestions/clean/subjs-counts-labels-types.fb2m.tsv",
             cachep=None, #"subjpredcharns.readdata.cache.pkl",
             maskid=-1,
             ):
    tt = ticktock("dataloader")
    if cachep is not None and os.path.isfile(cachep):      # load
        tt.tick("loading from cache")
        ret = pickle.load(open(cachep))
        tt.tock("loaded from cache")
    else:
        tt.tick("loading datamat")
        x = pickle.load(open(p))
        tt.tock("datamat loaded")
        worddic = x["worddic"]
        entdic = x["entdic"]
        entmat = x["entmat"]
        numents = x["numents"]
        traindata, traingold = x["train"]
        validdata, validgold = x["valid"]
        testdata, testgold = x["test"]
        traingold[:, 1] -= numents
        validgold[:, 1] -= numents
        testgold[:, 1] -= numents

        rwd = {v: k for k, v in worddic.items()}

        subjdic = {k: v for k, v in entdic.items() if v < numents}
        reldic = {k: v - numents for k, v in entdic.items() if v >= numents}

        subjmat = entmat[:numents]
        relmat = entmat[numents:]

        traindata = wordmat2wordchartensor(traindata, rwd=rwd, maskid=maskid)
        validdata = wordmat2wordchartensor(validdata, rwd=rwd, maskid=maskid)
        testdata = wordmat2wordchartensor(testdata, rwd=rwd, maskid=maskid)

        subjmat = wordmat2charmat(subjmat, rwd=rwd, maskid=maskid, maxlen=75)

        ret = ((traindata, traingold), (validdata, validgold),
               (testdata, testgold), (subjmat, relmat), (subjdic, reldic),
               worddic)
        if cachep is not None:
            tt.tick("dumping to cache")
            pickle.dump(ret, open(cachep, "w"))
            tt.tock("dumped to cache")

    subjinfo = loadsubjinfo(entinfp, subjdic)
    testsubjcans = loadsubjtestcans()
    testrelcans = loadreltestcans(testgold,subjdic, reldic)
    return ret + (subjinfo, (testsubjcans, testrelcans))

def loadreltestcans(testgold, subjdic, reldic, relsperentp="../../../../data/simplequestions/allrelsperent.dmp"):
    tt = ticktock("test rel can loader")
    testsubjs = testgold[:, 0]
    relsoftestsubjs = {k: ([], []) for k in set(list(testsubjs))}
    tt.tick("loading rel test cans")
    for line in open(relsperentp):
        subj, relsout, relsin = line[:-1].split("\t")
        if subj in subjdic and subjdic[subj] in relsoftestsubjs:
            relsoftestsubjs[subjdic[subj]] = (
                [reldic[x] for x in relsout.split(" ")] if relsout != "" else [],
                [reldic[x] for x in relsin.split(" ")] if relsin != "" else []
            )
    tt.tock("test cans loaded")
    relsoftestexamples = [(relsoftestsubjs[x][0], relsoftestsubjs[x][1])
                          for x in testsubjs]
    return relsoftestexamples

def loadsubjtestcans(p="../../../../data/simplequestions/clean/testcans.pkl"):
    tt = ticktock("test subjects candidate loader")
    tt.tick("loading candidates")
    ret = pickle.load(open(p))
    tt.tock("canddiates loaded")
    return ret

def loadsubjinfo(entinfp, entdic, cachep=None):#"subjinfo.cache.pkl"):
    tt = ticktock("subjinfoloader")
    def make():
        tt.tick("making subject info from file")
        subjinf = {}
        c = 0
        for line in open(entinfp):
            subjuri, subjc, objc, subjname, typuri, typname = line[:-1].split("\t")
            subjinf[entdic[subjuri]] = (subjname, typname.lower().split(), typuri, subjc, objc)
            if c % 1000 == 0:
                tt.live(str(c))
            c += 1
        tt.tock("made subject info from file")
        return subjinf
    if cachep is not None:
        if os.path.isfile(cachep):      # load
            tt.tick("loading cached subject info")
            subjinfo = pickle.load(open(cachep))
            tt.tock("loaded cached subject info")
        else:                           # make  and dump
            subjinfo = make()
            tt.tick("dumping subject info in cache")
            pickle.dump(subjinfo, open(cachep, "w"))
            tt.tock("dumped subject info in cache")
    else:       # just make
        subjinfo = make()
    return subjinfo

def buildrelsamplespace(entmat, wd, maskid=-1):
    tt = ticktock("samplespace")
    tt.tick("making sample space")
    #rwd = {v: k for k, v in wd.items()}
    entmatm = sparse.dok_matrix((entmat.shape[0], np.max(entmat) + 1))
    posblacklist = {0: {wd["base"], wd["user"]}}
    blacklist = set([wd[x] for x in "default domain of by the in at s this for with type".split()])
    #revin = {k: set() for k in np.unique(entmat)}
    #revinm = sparse.dok_matrix((np.max(entmat), entmat.shape[0]))
    samdic = {k: set() for k in range(entmat.shape[0])}     # from ent ids to sets of ent ids
    #samdic = np.zeros((entmat.shape[0], entmat.shape[0]))
    for i in range(entmat.shape[0]):
        for j in range(entmat.shape[1]):
            w = entmat[i, j]
            if w == -1:     # beginning of padding
                break
            if j in posblacklist:
                if w in posblacklist[j]:
                    continue
            if w in blacklist:
                continue
            entmatm[i, w] = 1
            #for oe in revin[w]:     # other entities already in revind
            #    samdic[oe].add(i)
            #    samdic[i].add(oe)
            #revin[w].add(i)
            #revinm[w, i] = 1
    samdicm = entmatm.dot(entmatm.T)
    for i in range(samdicm.shape[0]):
        samdic[i] = list(np.argwhere(samdicm[i, :])[:, 1])
    tt.tock("made sample space")
    return samdic, entmatm.T

class SeqLeftBlock(Block):
    def __init__(self, inner, **kw):
        super(SeqLeftBlock, self).__init__(**kw)
        self.inner = inner

    def apply(self, x):
        # idxs^(batsize, seqlen, ...) --> (batsize, seqlen, 2, encdim)
        res = self.inner(x).dimshuffle(0, "x", 1)
        ret = T.concatenate([res, res], axis=1)
        return ret

class ConcatLeftBlock(Block):
    def __init__(self, inner, **kw):
        super(ConcatLeftBlock, self).__init__(**kw)
        self.inner = inner

    def apply(self, x):
        res = self.inner(x).dimshuffle(0, "x", 1) # (batsize, q_enc_dim)
        mid = res.shape[2]/2
        ret = T.concatenate([res[:, :, mid], res[:, :, mid:]], axis=1)
        return ret

class RightBlock(Block):
    def __init__(self, a, b, **kw):
        super(RightBlock, self).__init__(**kw)
        self.subjenc = a
        self.predenc = b

    def apply(self, subjslice, relslice):  # idxs^(batsize, len)
        aret = self.subjenc(subjslice).dimshuffle(0, "x", 1)
        bret = self.predenc(relslice).dimshuffle(0, "x", 1)
        ret = T.concatenate([aret, bret], axis=1)
        return ret

def run(closenegsam=False,
        checkdata=False,
        glove=True,
        embdim=100,
        charencdim=100,
        charembdim=50,
        encdim=400,
        bidir=False,
        charenc="cnn",  # "cnn" or "rnn"
        mode="seq",      # "seq" or "concat"
        margin=0.5,
        lr=0.1,
        numbats=700,
        epochs=15,
        ):
    tt = ticktock("script")
    tt.tick("loading data")
    (traindata, traingold), (validdata, validgold), (testdata, testgold), \
    (subjmat, relmat), (subjdic, reldic), worddic, \
    subjinfo, (testsubjcans, testrelcans) = readdata()

    revsamplespace = None
    if closenegsam:
        relsamplespace, revind = buildrelsamplespace(relmat, worddic)
    tt.tock("data loaded")

    if checkdata:
        embed()

    numwords = max(worddic.values()) + 1
    numsubjs = max(subjdic.values()) + 1
    numrels = max(reldic.values()) + 1
    maskid = -1
    numchars = 256

    if mode == "seq":
        decdim = encdim
        print "seq mode: {} decdim".format(decdim)
    elif mode == "concat":
        decdim = encdim / 2
        print "concat mode: {} decdim".format(decdim)
    else:
        raise Exception("unrecognized mode")

    # defining model
    if glove:
        wordemb = Glove(embdim).adapt(worddic)
    else:
        wordemb = WordEmb(dim=embdim, indim=numwords)
    charemb = VectorEmbed(indim=numchars, dim=charembdim)
    if charenc == "cnn":
        print "using CNN char encoder"
        charenc = CNNSeqEncoder(inpemb=charemb,
                                innerdim=[charencdim]*2, maskid=maskid,
                                stride=1)
    elif charenc == "rnn":
        print "using RNN char encoder"
        charenc = RNNSeqEncoder(inpemb=charemb, innerdim=charencdim)\
            .maskoptions(maskid, MaskMode.AUTO)
    else:
        raise Exception("no other modes available")
    wordenc = RNNSeqEncoder(inpemb=False, inpembdim=wordemb.outdim + charencdim,
                            innerdim=encdim, bidir=bidir).maskoptions(MaskMode.NONE)
    question_encoder = TwoLevelEncoder(l1enc=charenc, l2emb=wordemb,
                                       l2enc=wordenc, maskid=maskid)

    # encode predicate on word level
    predemb = SimpleSeq2Vec(inpemb=wordemb,
                           innerdim=decdim,
                           maskid=maskid,
                           bidir=bidir,
                           layers=1)
    #predemb.load(relmat)

    # encode subject on character level
    subjemb = SimpleSeq2Vec(inpemb=charemb,
                           innerdim=decdim,
                           maskid=maskid,
                           bidir=bidir,
                           layers=2)
    #subjemb.load(subjmat)

    # package
    if mode == "seq":
        lb = SeqLeftBlock(question_encoder)
        rb = RightBlock(subjemb, predemb)
    elif mode == "concat":
        lb = ConcatLeftBlock(question_encoder)
        rb = RightBlock(subjemb, predemb)
    else:
        raise Exception("unrecognized mode")
    scorer = SeqMatchScore(lb, rb, scorer=CosineDistance(),
                           aggregator=lambda x: x, argproc=lambda x, y, z: ((x,), (y, z)))

    obj = lambda p, n: T.sum((n - p + margin).clip(0, np.infty), axis=1)

    # negative sampling
    class NegIdxGen(object):
        def __init__(self, maxentid, maxrelid, relclose=None):
            self.maxentid = maxentid
            self.maxrelid = maxrelid
            self.relclose = relclose

        def __call__(self, datas, gold):
            entrand = np.random.randint(0, self.maxentid+1, (gold.shape[0], 1))
            relrand = self.samplerels(gold[:, 1:2])
            ret = np.concatenate([entrand, relrand], axis=1)
            #embed()
            return datas, ret.astype("int32")

        def samplerels(self, gold):
            assert(gold.ndim == 2 and gold.shape[1] == 1)
            if self.relclose is None:
                return np.random.randint(0, self.maxrelid+1, (gold.shape[0], 1))
            else:
                ret = np.zeros_like(gold)
                for i in range(gold.shape[0]):
                    sampleset = self.relclose[gold[i]]
                    if len(sampleset) > 5:
                        ret[i] = random.sample(sampleset, 1)[0]
                    else:
                        ret[i] = np.random.randint(0, self.maxrelid+1)
                return ret.astype("int32")

    class PreProc(object):
        def __init__(self, subjmat, relmat):
            self.f = PreProcE(subjmat, relmat)

        def __call__(self, data, gold):     # gold: idxs-(batsize, 2)
            ret = self.f(gold)[0]
            return (data,) + ret, {}

    class PreProcE(object):
        def __init__(self, subjmat, relmat):
            self.subjmat = Val(subjmat)
            self.relmat = Val(relmat)

        def __call__(self, x):
            subjslice = self.subjmat[x[:, 0]]
            relslice = self.relmat[x[:, 1]]
            return (subjslice, relslice), {}

    transf = PreProc(subjmat, relmat)

    #embed()
    tt.tick("training")
    nscorer = scorer.nstrain([traindata, traingold]).transform(transf)\
        .negsamplegen(NegIdxGen(numsubjs-1, numrels-1, relclose=revsamplespace)) \
        .objective(obj).adagrad(lr=lr).grad_total_norm(1.0)\
        .validate_on([validdata, validgold])\
        .train(numbats=numbats, epochs=epochs)
    tt.tock("trained").tick()

    # saving
    saveid = "".join([str(np.random.randint(0, 10)) for i in range(4)])
    scorer.save("fullrank{}.model".format(saveid))
    tt.tock("saved: {}".format(saveid))

    # evaluation
    # TODO

if __name__ == "__main__":
    argprun(run)<|endoftext|>


<commit_before># Copyright 2013 IBM Corp.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import json
import time

from tempest.common.rest_client import RestClient
from tempest import config
from tempest import exceptions

CONF = config.CONF

class InterfacesV3ClientJSON(RestClient):

    def __init__(self, username, password, auth_url, tenant_name=None):
        super(InterfacesV3ClientJSON, self).__init__(username,
                                                     password, auth_url,
                                                     tenant_name)
        self.service = CONF.compute.catalog_v3_type

    def list_interfaces(self, server):
        resp, body = self.get('servers/%s/os-attach-interfaces' % server)
        body = json.loads(body)
        return resp, body['interface_attachments']

    def create_interface(self, server, port_id=None, network_id=None,
                         fixed_ip=None):
        post_body = dict(interface_attachment=dict())
        if port_id:
            post_body['port_id'] = port_id
        if network_id:
            post_body['net_id'] = network_id
        if fixed_ip:
            post_body['fixed_ips'] = [dict(ip_address=fixed_ip)]
        post_body = json.dumps(post_body)
        resp, body = self.post('servers/%s/os-attach-interfaces' % server,
                               headers=self.headers,
                               body=post_body)
        body = json.loads(body)
        return resp, body['interface_attachment']

    def show_interface(self, server, port_id):
        resp, body =\
            self.get('servers/%s/os-attach-interfaces/%s' % (server, port_id))
        body = json.loads(body)
        return resp, body['interface_attachment']

    def delete_interface(self, server, port_id):
        resp, body =\
            self.delete('servers/%s/os-attach-interfaces/%s' % (server,
                                                                port_id))
        return resp, body

    def wait_for_interface_status(self, server, port_id, status):
        """Waits for a interface to reach a given status."""
        resp, body = self.show_interface(server, port_id)
        interface_status = body['port_state']
        start = int(time.time())

        while(interface_status != status):
            time.sleep(self.build_interval)
            resp, body = self.show_interface(server, port_id)
            interface_status = body['port_state']

            timed_out = int(time.time()) - start >= self.build_timeout

            if interface_status != status and timed_out:
                message = ('Interface %s failed to reach %s status within '
                           'the required time (%s s).' %
                           (port_id, status, self.build_timeout))
                raise exceptions.TimeoutException(message)

        return resp, body
<commit_msg>Fix attach_interfaces tests of Nova v3<commit_after># Copyright 2013 IBM Corp.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import json
import time

from tempest.common.rest_client import RestClient
from tempest import config
from tempest import exceptions

CONF = config.CONF

class InterfacesV3ClientJSON(RestClient):

    def __init__(self, username, password, auth_url, tenant_name=None):
        super(InterfacesV3ClientJSON, self).__init__(username,
                                                     password, auth_url,
                                                     tenant_name)
        self.service = CONF.compute.catalog_v3_type

    def list_interfaces(self, server):
        resp, body = self.get('servers/%s/os-attach-interfaces' % server)
        body = json.loads(body)
        return resp, body['interface_attachments']

    def create_interface(self, server, port_id=None, network_id=None,
                         fixed_ip=None):
        post_body = dict()
        if port_id:
            post_body['port_id'] = port_id
        if network_id:
            post_body['net_id'] = network_id
        if fixed_ip:
            post_body['fixed_ips'] = [dict(ip_address=fixed_ip)]
        post_body = json.dumps({'interface_attachment': post_body})
        resp, body = self.post('servers/%s/os-attach-interfaces' % server,
                               headers=self.headers,
                               body=post_body)
        body = json.loads(body)
        return resp, body['interface_attachment']

    def show_interface(self, server, port_id):
        resp, body =\
            self.get('servers/%s/os-attach-interfaces/%s' % (server, port_id))
        body = json.loads(body)
        return resp, body['interface_attachment']

    def delete_interface(self, server, port_id):
        resp, body =\
            self.delete('servers/%s/os-attach-interfaces/%s' % (server,
                                                                port_id))
        return resp, body

    def wait_for_interface_status(self, server, port_id, status):
        """Waits for a interface to reach a given status."""
        resp, body = self.show_interface(server, port_id)
        interface_status = body['port_state']
        start = int(time.time())

        while(interface_status != status):
            time.sleep(self.build_interval)
            resp, body = self.show_interface(server, port_id)
            interface_status = body['port_state']

            timed_out = int(time.time()) - start >= self.build_timeout

            if interface_status != status and timed_out:
                message = ('Interface %s failed to reach %s status within '
                           'the required time (%s s).' %
                           (port_id, status, self.build_timeout))
                raise exceptions.TimeoutException(message)

        return resp, body
<|endoftext|>


<commit_before>import os
import time
import re

import unittest

from selenium                       import webdriver
from selenium.webdriver             import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions     import NoSuchElementException
from selenium.webdriver.support.ui  import Select

import track_selectors

class JBrowseTest (object):

    data_dir = None
    base_url = None

    tracksel_type = 'Simple'

    ## TestCase override - use instead of constructor
    def setUp( self ):

        self.track_selector = getattr( track_selectors, '%sTrackSelector' % self.tracksel_type )( self )

        fp = webdriver.FirefoxProfile()

        fp.set_preference("browser.download.folderList",2)
        fp.set_preference("browser.download.manager.showWhenStarting",False)
        fp.set_preference("browser.download.dir", os.getcwd())
        fp.set_preference("browser.helperApps.neverAsk.saveToDisk","application/x-bedgraph,application/x-wiggle,application/x-bed")

        self.browser = webdriver.Firefox( firefox_profile = fp )

        base = self.baseURL()
        self.browser.get(
            base + ( '&' if base.find('?') >= 0 else '?' )
            + ( "data="+self.data_dir if self.data_dir else "" )
        )
        time.sleep(0.5)  # give selenium some time to get its head on straight

    def baseURL( self ):
        if not self.base_url:
            self.base_url = os.environ['JBROWSE_URL'] if 'JBROWSE_URL' in os.environ else "file://"+os.getcwd()+"/index.html"
        return self.base_url

    ## convenience methods for us

    def maybe_find_element_by_xpath( self, xpathExpression ):
        try:
            el = self.browser.find_element_by_xpath( xpathExpression )
        except NoSuchElementException:
            return None
        return el

    def assert_element( self, expression ):
        try:
            if expression.find('/') >= 0:
                el = self.browser.find_element_by_xpath( expression )
            else:
                el = self.browser.find_element_by_css_selector( expression )
        except NoSuchElementException:
            assert 0, ( "can't find %s" % expression )
        return el

    def assert_elements( self, expression ):
        try:
            if expression.find('/') >= 0:
                el = self.browser.find_elements_by_xpath( expression )
            else:
                el = self.browser.find_elements_by_css_selector( expression )
        except NoSuchElementException:
            assert 0, ( "can't find %s" % expression )
        return el

    def assert_no_element( self, expression ):
        try:
            if expression.find('/') >= 0:
                el = self.browser.find_element_by_xpath( expression )
            else:
                el = self.browser.find_element_by_css_selector( expression )
            assert 0, ( "not supposed to find %s" % expression )
        except NoSuchElementException:
            pass

    def assert_no_js_errors( self ):
        assert self.browser.find_element_by_xpath('/html/body') \
                      .get_attribute('JSError') == None

    def do_typed_query( self, text ):
        # Find the query box and put f15 into it and hit enter
        qbox = self.browser.find_element_by_id("location")
        qbox.send_keys( Keys.BACK_SPACE * 40 )
        time.sleep( 0.05 )
        for i in range( len(text) ):
            qbox.send_keys( text[i] )
        qbox.send_keys( Keys.RETURN );
        time.sleep( 0.3 )

    def _rubberband( self, el_xpath, start_pct, end_pct, modkey = None ):
        el = self.assert_element( el_xpath )
        start_offset = el.size['width'] * start_pct - el.size['width']/2;
        c = self.actionchains() \
            .move_to_element( el ) \
            .move_by_offset( start_offset, 0 )

        if( modkey ):
            c = c.key_down( modkey )

        c =  c \
            .click_and_hold( None ) \
            .move_by_offset( el.size['width']*(end_pct-start_pct), 0 ) \
            .release( None )

        if( modkey ):
            c = c.key_up( modkey )

        c \
            .release( None ) \
            .perform()

        self.assert_no_js_errors()

    def export_track( self, track_name, region, format, button ):
        time.sleep(2);

        track_name = re.sub( '\W', '_', track_name.lower() )

        self.track_menu_click( track_name, 'Save')

        # test view export
        self.assert_element("//label[contains(.,'%s')]" % region ).click()
        self.assert_element("//label[contains(.,'%s')]" % format ).click()
        self.assert_element("//*[contains(@class,'dijitButton')]//*[contains(@class,'dijitButtonText')][contains(.,'%s')]" % button ).click()
        self.assert_no_js_errors()

    def close_dialog( self, title ):
        self.assert_element("//div[@class='dijitDialogTitleBar'][contains(@title,'%s')]/span[contains(@class,'dijitDialogCloseIcon')]" % title ).click()
        self.assert_no_js_errors()

    def track_menu_click( self, track_name, item_name ):
        self.assert_element( "//div[contains(@class,'track_%s')]//div[contains(@class,'track-label')]//div[contains(@class,'track-menu-button')]" % re.sub( '\W', '_', track_name ) ) \
            .click()

        time.sleep(2)

        self.menu_item_click( item_name )

    def menu_item_click( self, text ):
        self.assert_element( "//div[contains(@class,'dijitMenuPopup')][not(contains(@style,'display: none'))]//td[contains(@class,'dijitMenuItemLabel')][contains(.,'%s')]" % text ) \
            .click()

    def overview_rubberband( self, start_pct, end_pct ):
        """Executes a rubberband gesture from start_pct to end_pct on the overview bar"""
        self._rubberband( "//div[@id='overview']", start_pct, end_pct )

    # I can't get a mainscale_rubberband() working, can't find an
    # element to tell selenium to move to that will hit it.  can't
    # move to the scale itself because it's so wide.

    def trackpane_rubberband( self, start_pct, end_pct ):
        """Executes a rubberband gesture from start_pct to end_pct in the main track pane"""
        self._rubberband( "//div[contains(@class,'dragWindow')]", start_pct, end_pct, Keys.SHIFT )

    def is_track_on( self, tracktext ):
        # find the track label in the track pane
        return not self.maybe_find_element_by_xpath( "//div[contains(@class,'dragWindow')]//div[@class='track-label'][contains(.,'%s')]" % tracktext )

    def turn_on_track( self, tracktext ):
        return self.track_selector.turn_on_track( tracktext )

    def turn_off_track( self, tracktext ):
        return self.track_selector.turn_off_track( tracktext )

    def actionchains( self ):
        return ActionChains( self.browser )

    def get_track_labels_containing( self, string ):
        return self.assert_elements( "//div[contains(@class,'track-label')][contains(.,'%s')]" % string )

    def select_refseq( self, name ):
        self.do_typed_query( name );

    def scroll( self ):
        move_right_button = self.browser.find_element_by_id('moveRight')
        move_right_button.click()
        time.sleep(0.5)
        move_left_button = self.browser.find_element_by_id('moveLeft')
        move_left_button.click()
        # TODO: check the outcome of this
        time.sleep(0.5)

        self.assert_no_js_errors()

        # scroll back and forth with the mouse
        self.actionchains() \
           .move_to_element( move_right_button ) \
           .move_by_offset( 0, 300 ) \
           .click_and_hold( None ) \
           .move_by_offset( 300, 0 ) \
           .release( None ) \
           .move_by_offset( -100,100 ) \
           .click_and_hold( None ) \
           .move_by_offset( -300, 0 ) \
           .release( None ) \
           .perform()

        self.assert_no_js_errors()

<commit_msg>try some more to get track-menu clicking in selenium tests less brittle<commit_after>import os
import time
import re

import unittest

from selenium                       import webdriver
from selenium.webdriver             import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions     import NoSuchElementException
from selenium.webdriver.support.ui  import Select

import track_selectors

class JBrowseTest (object):

    data_dir = None
    base_url = None

    tracksel_type = 'Simple'

    ## TestCase override - use instead of constructor
    def setUp( self ):

        self.track_selector = getattr( track_selectors, '%sTrackSelector' % self.tracksel_type )( self )

        fp = webdriver.FirefoxProfile()

        fp.set_preference("browser.download.folderList",2)
        fp.set_preference("browser.download.manager.showWhenStarting",False)
        fp.set_preference("browser.download.dir", os.getcwd())
        fp.set_preference("browser.helperApps.neverAsk.saveToDisk","application/x-bedgraph,application/x-wiggle,application/x-bed")

        self.browser = webdriver.Firefox( firefox_profile = fp )

        base = self.baseURL()
        self.browser.get(
            base + ( '&' if base.find('?') >= 0 else '?' )
            + ( "data="+self.data_dir if self.data_dir else "" )
        )
        time.sleep(0.5)  # give selenium some time to get its head on straight

    def baseURL( self ):
        if not self.base_url:
            self.base_url = os.environ['JBROWSE_URL'] if 'JBROWSE_URL' in os.environ else "file://"+os.getcwd()+"/index.html"
        return self.base_url

    ## convenience methods for us

    def maybe_find_element_by_xpath( self, xpathExpression ):
        try:
            el = self.browser.find_element_by_xpath( xpathExpression )
        except NoSuchElementException:
            return None
        return el

    def assert_element( self, expression ):
        try:
            if expression.find('/') >= 0:
                el = self.browser.find_element_by_xpath( expression )
            else:
                el = self.browser.find_element_by_css_selector( expression )
        except NoSuchElementException:
            assert 0, ( "can't find %s" % expression )
        return el

    def assert_elements( self, expression ):
        try:
            if expression.find('/') >= 0:
                el = self.browser.find_elements_by_xpath( expression )
            else:
                el = self.browser.find_elements_by_css_selector( expression )
        except NoSuchElementException:
            assert 0, ( "can't find %s" % expression )
        return el

    def assert_no_element( self, expression ):
        try:
            if expression.find('/') >= 0:
                el = self.browser.find_element_by_xpath( expression )
            else:
                el = self.browser.find_element_by_css_selector( expression )
            assert 0, ( "not supposed to find %s" % expression )
        except NoSuchElementException:
            pass

    def assert_no_js_errors( self ):
        assert self.browser.find_element_by_xpath('/html/body') \
                      .get_attribute('JSError') == None

    def do_typed_query( self, text ):
        # Find the query box and put f15 into it and hit enter
        qbox = self.browser.find_element_by_id("location")
        qbox.send_keys( Keys.BACK_SPACE * 40 )
        time.sleep( 0.05 )
        for i in range( len(text) ):
            qbox.send_keys( text[i] )
        qbox.send_keys( Keys.RETURN );
        time.sleep( 0.3 )

    def _rubberband( self, el_xpath, start_pct, end_pct, modkey = None ):
        el = self.assert_element( el_xpath )
        start_offset = el.size['width'] * start_pct - el.size['width']/2;
        c = self.actionchains() \
            .move_to_element( el ) \
            .move_by_offset( start_offset, 0 )

        if( modkey ):
            c = c.key_down( modkey )

        c =  c \
            .click_and_hold( None ) \
            .move_by_offset( el.size['width']*(end_pct-start_pct), 0 ) \
            .release( None )

        if( modkey ):
            c = c.key_up( modkey )

        c \
            .release( None ) \
            .perform()

        self.assert_no_js_errors()

    def export_track( self, track_name, region, format, button ):
        time.sleep(2);

        track_name = re.sub( '\W', '_', track_name.lower() )

        self.track_menu_click( track_name, 'Save')

        # test view export
        self.assert_element("//label[contains(.,'%s')]" % region ).click()
        self.assert_element("//label[contains(.,'%s')]" % format ).click()
        self.assert_element("//*[contains(@class,'dijitButton')]//*[contains(@class,'dijitButtonText')][contains(.,'%s')]" % button ).click()
        self.assert_no_js_errors()

    def close_dialog( self, title ):
        self.assert_element("//div[@class='dijitDialogTitleBar'][contains(@title,'%s')]/span[contains(@class,'dijitDialogCloseIcon')]" % title ).click()
        self.assert_no_js_errors()

    def track_menu_click( self, track_name, item_name ):
        self.assert_element( "//div[contains(@class,'track_%s')]//div[contains(@class,'track-label')]//div[contains(@class,'track-menu-button')]" % re.sub( '\W', '_', track_name ) ) \
            .click()

        time.sleep(1)

        self.assert_element( "//div[contains(@class,'track_%s')]//div[contains(@class,'track-label')]//div[contains(@class,'track-menu-button')]" % re.sub( '\W', '_', track_name ) ) \
            .click()

        time.sleep(1)

        self.menu_item_click( item_name )

    def menu_item_click( self, text ):
        self.assert_element( "//div[contains(@class,'dijitMenuPopup')][not(contains(@style,'display: none'))]//td[contains(@class,'dijitMenuItemLabel')][contains(.,'%s')]" % text ) \
            .click()

    def overview_rubberband( self, start_pct, end_pct ):
        """Executes a rubberband gesture from start_pct to end_pct on the overview bar"""
        self._rubberband( "//div[@id='overview']", start_pct, end_pct )

    # I can't get a mainscale_rubberband() working, can't find an
    # element to tell selenium to move to that will hit it.  can't
    # move to the scale itself because it's so wide.

    def trackpane_rubberband( self, start_pct, end_pct ):
        """Executes a rubberband gesture from start_pct to end_pct in the main track pane"""
        self._rubberband( "//div[contains(@class,'dragWindow')]", start_pct, end_pct, Keys.SHIFT )

    def is_track_on( self, tracktext ):
        # find the track label in the track pane
        return not self.maybe_find_element_by_xpath( "//div[contains(@class,'dragWindow')]//div[@class='track-label'][contains(.,'%s')]" % tracktext )

    def turn_on_track( self, tracktext ):
        return self.track_selector.turn_on_track( tracktext )

    def turn_off_track( self, tracktext ):
        return self.track_selector.turn_off_track( tracktext )

    def actionchains( self ):
        return ActionChains( self.browser )

    def get_track_labels_containing( self, string ):
        return self.assert_elements( "//div[contains(@class,'track-label')][contains(.,'%s')]" % string )

    def select_refseq( self, name ):
        self.do_typed_query( name );

    def scroll( self ):
        move_right_button = self.browser.find_element_by_id('moveRight')
        move_right_button.click()
        time.sleep(0.5)
        move_left_button = self.browser.find_element_by_id('moveLeft')
        move_left_button.click()
        # TODO: check the outcome of this
        time.sleep(0.5)

        self.assert_no_js_errors()

        # scroll back and forth with the mouse
        self.actionchains() \
           .move_to_element( move_right_button ) \
           .move_by_offset( 0, 300 ) \
           .click_and_hold( None ) \
           .move_by_offset( 300, 0 ) \
           .release( None ) \
           .move_by_offset( -100,100 ) \
           .click_and_hold( None ) \
           .move_by_offset( -300, 0 ) \
           .release( None ) \
           .perform()

        self.assert_no_js_errors()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from openerp.osv import fields, orm
import sys

class DummyModel(orm.Model):
    # expect "no access rules" error
    _name = 'another.dummy.model'
    _columns = {
        'name': fields.char('Dummy', size=100),
    }
<commit_msg>fix flake8 error in new test module<commit_after># -*- coding: utf-8 -*-

from openerp.osv import fields, orm

class DummyModel(orm.Model):
    # expect "no access rules" error
    _name = 'another.dummy.model'
    _columns = {
        'name': fields.char('Dummy', size=100),
    }
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from ...tests.helper import pytest

import numpy as np
from numpy import testing as npt

from ... import units as u

def test_angle_arrays():
    """
    Test arrays values with Angle objects.
    """
    from .. import Angle

    # Tests incomplete
    a1 = Angle([0, 45, 90, 180, 270, 360, 720.], unit=u.degree)
    npt.assert_almost_equal([0., 45., 90., 180., 270., 360., 720.], a1.value)

    a2 = Angle(np.array([-90, -45, 0, 45, 90, 180, 270, 360]), unit=u.degree)
    npt.assert_almost_equal([-90, -45, 0, 45, 90, 180, 270, 360],
                            a2.value)

    a3 = Angle(["12 degrees", "3 hours", "5 deg", "4rad"])
    npt.assert_almost_equal([12., 45., 5., 229.18311805],
                            a3.value)
    assert a3.unit == u.degree

    a4 = Angle(["12 degrees", "3 hours", "5 deg", "4rad"], u.radian)
    npt.assert_almost_equal(a4.degree, a3.value)
    assert a4.unit == u.radian

    a5 = Angle([0, 45, 90, 180, 270, 360], unit=u.degree)
    a6 = a5.sum()
    npt.assert_almost_equal(a6.value, 945.0)
    assert a6.unit is u.degree

    with pytest.raises(TypeError):
        # Arrays of Angle objects are not supported -- that's really
        # tricky to do correctly, if at all, due to the possibility of
        # nesting.
        a7 = Angle([a1, a2, a3], unit=u.degree)

    a8 = Angle(["04:02:02", "03:02:01", "06:02:01"], unit=u.degree)
    npt.assert_almost_equal(a8.value, [4.03388889, 3.03361111, 6.03361111])

    a9 = Angle(np.array(["04:02:02", "03:02:01", "06:02:01"]), unit=u.degree)
    npt.assert_almost_equal(a9.value, a8.value)

    with pytest.raises(u.UnitsError):
        a10 = Angle(["04:02:02", "03:02:01", "06:02:01"])

def test_dms():
    from .. import Angle
    from ..angle_utilities import dms_to_degrees

    a1 = Angle([0, 45.5, -45.5], unit=u.degree)
    d, m, s = a1.dms
    npt.assert_almost_equal(d, [0, 45, -45])
    npt.assert_almost_equal(m, [0, 30, 30])
    npt.assert_almost_equal(s, [0, 0, 0])

    dms = a1.dms
    degrees = dms_to_degrees(*dms)
    npt.assert_almost_equal(a1.degree, degrees)

    a2 = Angle(dms, unit=u.degree)

    npt.assert_almost_equal(a2.radian, a1.radian)

def test_hms():
    from .. import Angle
    from ..angle_utilities import hms_to_hours

    a1 = Angle([0, 11.5, -11.5], unit=u.hour)
    h, m, s = a1.hms
    npt.assert_almost_equal(h, [0, 11, -11])
    npt.assert_almost_equal(m, [0, 30, 30])
    npt.assert_almost_equal(s, [0, 0, 0])

    hms = a1.hms
    hours = hms_to_hours(*hms)
    npt.assert_almost_equal(a1.hour, hours)

    a2 = Angle(hms, unit=u.hour)

    npt.assert_almost_equal(a2.radian, a1.radian)

def test_array_coordinates_creation():
    """
    Test creating coordinates from arrays.
    """
    from .. import ICRSCoordinates

    c = ICRSCoordinates(np.array([1, 2]), np.array([3, 4]), unit=(u.deg, u.deg))
    assert not c.isscalar

    with pytest.raises(ValueError):
        c = ICRSCoordinates(np.array([1, 2]), np.array([3, 4, 5]), unit=(u.deg, u.deg))
    with pytest.raises(ValueError):
        c = ICRSCoordinates(np.array([1, 2]), np.array([[3, 4], [5, 6]]), unit=(u.deg, u.deg))

    #make sure cartesian initialization also works
    c = ICRSCoordinates(x=np.array([1, 2]), y=np.array([3, 4]), z=np.array([5, 6]), unit=u.kpc)

    #also ensure strings can be arrays
    c = ICRSCoordinates(np.array(['1d0m0s', '2h02m00.3s']), np.array(['3d', '4d']), unit=(u.deg, u.deg))

    #but invalid strings cannot
    with pytest.raises(ValueError):
        c = ICRSCoordinates(np.array(['10m0s', '2h02m00.3s']), np.array(['3d', '4d']), unit=(u.deg, u.deg))
    with pytest.raises(ValueError):
        c = ICRSCoordinates(np.array(['1d0m0s', '2h02m00.3s']), np.array(['3x', '4d']), unit=(u.deg, u.deg))

def test_array_coordinates_distances():
    """
    Test creating coordinates from arrays and distances.
    """
    from .. import ICRSCoordinates

    #correct way
    ICRSCoordinates(np.array([1, 2]), np.array([3, 4]), unit=(u.deg, u.deg), distance= [.1, .2] * u.kpc)

    with pytest.raises(ValueError):
        #scalar distance and array coordinates
        ICRSCoordinates(np.array([1, 2]), np.array([[3, 4], [5, 6]]), unit=(u.deg, u.deg), distance= 2. * u.kpc)
    with pytest.raises(ValueError):
        #scalar coordinates and array distance
        ICRSCoordinates(1., 2., unit=(u.deg, u.deg), distance= [.1, .2, 3.] * u.kpc)
    with pytest.raises(ValueError):
        #more distance values than coordinates
        ICRSCoordinates(np.array([1, 2]), np.array([[3, 4], [5, 6]]), unit=(u.deg, u.deg), distance= [.1, .2, 3.] * u.kpc)

@pytest.mark.parametrize(('arrshape', 'distance'), [((2, ), None), ((4, 2, 5), None), ((4, 2, 5), 2 * u.kpc)])
def test_array_coordinates_transformations(arrshape, distance):
    """
    Test transformation on coordinates with array content (first length-2 1D, then a 3D array)
    """
    from .. import ICRSCoordinates, GalacticCoordinates

    #M31 coordinates from test_transformations
    raarr = np.ones(arrshape) * 10.6847929
    decarr = np.ones(arrshape) * 41.2690650
    if distance is not None:
        distance = np.ones(arrshape) * distance

    c = ICRSCoordinates(raarr, decarr, unit=(u.deg, u.deg), distance=distance)
    g = c.transform_to(GalacticCoordinates)

    assert g.l.shape == arrshape

    npt.assert_array_almost_equal(g.l.degree, 121.17447049007306)
    npt.assert_array_almost_equal(g.b.degree, -21.57291080408368)

    if distance is not None:
        assert g.distance.unit == c.distance.unit

    #now make sure round-tripping works through FK5 - this exercises both static and dynamic transform matricies
    c2 = c.fk5.icrs
    npt.assert_array_almost_equal(c.ra, c2.ra)
    npt.assert_array_almost_equal(c.dec, c2.dec)

    assert c2.ra.shape == arrshape

    if distance is not None:
        assert c2.distance.unit == c.distance.unit

    #also make sure it's possible to get to FK4, which uses a direct transform function.
    fk4 = c.fk4

    npt.assert_array_almost_equal(fk4.ra.degree, 10.0004, decimal=4)
    npt.assert_array_almost_equal(fk4.dec.degree, 40.9953, decimal=4)

    assert fk4.ra.shape == arrshape
    if distance is not None:
        assert fk4.distance.unit == c.distance.unit

    #now check the reverse transforms run
    cfk4 = fk4.icrs
    assert cfk4.ra.shape == arrshape

def test_array_coordinates_string():
    """
    tests for string representations of aarray coordinates
    """
    from .. import ICRSCoordinates

    c = ICRSCoordinates(np.array([1, 2]), np.array([3, 4]), unit=(u.deg, u.deg))
    str(c)
    unicode(c)
    repr(c)

    assert repr(c) == '<ICRSCoordinates RA=[1 2] deg, Dec=[3 4] deg>'

    #also check with distance

    c = ICRSCoordinates(np.array([1, 2]), np.array([3, 4]), unit=(u.deg, u.deg), distance= u.kpc * [0.5, 1.5])
    str(c)
    unicode(c)
    repr(c)

    print(repr(c))

    assert repr(c) == '<ICRSCoordinates RA=[1 2] deg, Dec=[3 4] deg, Distance=[ 0.5  1.5] kpc>'

def test_array_precession():
    """
    Ensures that FK5 coordinates as arrays precess their equinoxes
    """
    from ...time import Time
    from .. import FK5Coordinates

    j2000 = Time('J2000', scale='utc')
    j1975 = Time('J1975', scale='utc')

    fk5 = FK5Coordinates([1, 1.1], [0.5, 0.6], unit=(u.radian, u.radian))
    assert fk5.equinox.jyear == j2000.jyear
    fk5_2 = fk5.precess_to(j1975)
    assert fk5_2.equinox.jyear == j1975.jyear

    npt.assert_array_less(0.05, np.abs(fk5.ra.degree - fk5_2.ra.degree))
    npt.assert_array_less(0.05, np.abs(fk5.dec.degree - fk5_2.dec.degree))
<commit_msg>make tests py 3.x compliant via `unicode`->`six.text_type`<commit_after># -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from ..extern import six
from ...tests.helper import pytest

import numpy as np
from numpy import testing as npt

from ... import units as u

def test_angle_arrays():
    """
    Test arrays values with Angle objects.
    """
    from .. import Angle

    # Tests incomplete
    a1 = Angle([0, 45, 90, 180, 270, 360, 720.], unit=u.degree)
    npt.assert_almost_equal([0., 45., 90., 180., 270., 360., 720.], a1.value)

    a2 = Angle(np.array([-90, -45, 0, 45, 90, 180, 270, 360]), unit=u.degree)
    npt.assert_almost_equal([-90, -45, 0, 45, 90, 180, 270, 360],
                            a2.value)

    a3 = Angle(["12 degrees", "3 hours", "5 deg", "4rad"])
    npt.assert_almost_equal([12., 45., 5., 229.18311805],
                            a3.value)
    assert a3.unit == u.degree

    a4 = Angle(["12 degrees", "3 hours", "5 deg", "4rad"], u.radian)
    npt.assert_almost_equal(a4.degree, a3.value)
    assert a4.unit == u.radian

    a5 = Angle([0, 45, 90, 180, 270, 360], unit=u.degree)
    a6 = a5.sum()
    npt.assert_almost_equal(a6.value, 945.0)
    assert a6.unit is u.degree

    with pytest.raises(TypeError):
        # Arrays of Angle objects are not supported -- that's really
        # tricky to do correctly, if at all, due to the possibility of
        # nesting.
        a7 = Angle([a1, a2, a3], unit=u.degree)

    a8 = Angle(["04:02:02", "03:02:01", "06:02:01"], unit=u.degree)
    npt.assert_almost_equal(a8.value, [4.03388889, 3.03361111, 6.03361111])

    a9 = Angle(np.array(["04:02:02", "03:02:01", "06:02:01"]), unit=u.degree)
    npt.assert_almost_equal(a9.value, a8.value)

    with pytest.raises(u.UnitsError):
        a10 = Angle(["04:02:02", "03:02:01", "06:02:01"])

def test_dms():
    from .. import Angle
    from ..angle_utilities import dms_to_degrees

    a1 = Angle([0, 45.5, -45.5], unit=u.degree)
    d, m, s = a1.dms
    npt.assert_almost_equal(d, [0, 45, -45])
    npt.assert_almost_equal(m, [0, 30, 30])
    npt.assert_almost_equal(s, [0, 0, 0])

    dms = a1.dms
    degrees = dms_to_degrees(*dms)
    npt.assert_almost_equal(a1.degree, degrees)

    a2 = Angle(dms, unit=u.degree)

    npt.assert_almost_equal(a2.radian, a1.radian)

def test_hms():
    from .. import Angle
    from ..angle_utilities import hms_to_hours

    a1 = Angle([0, 11.5, -11.5], unit=u.hour)
    h, m, s = a1.hms
    npt.assert_almost_equal(h, [0, 11, -11])
    npt.assert_almost_equal(m, [0, 30, 30])
    npt.assert_almost_equal(s, [0, 0, 0])

    hms = a1.hms
    hours = hms_to_hours(*hms)
    npt.assert_almost_equal(a1.hour, hours)

    a2 = Angle(hms, unit=u.hour)

    npt.assert_almost_equal(a2.radian, a1.radian)

def test_array_coordinates_creation():
    """
    Test creating coordinates from arrays.
    """
    from .. import ICRSCoordinates

    c = ICRSCoordinates(np.array([1, 2]), np.array([3, 4]), unit=(u.deg, u.deg))
    assert not c.isscalar

    with pytest.raises(ValueError):
        c = ICRSCoordinates(np.array([1, 2]), np.array([3, 4, 5]), unit=(u.deg, u.deg))
    with pytest.raises(ValueError):
        c = ICRSCoordinates(np.array([1, 2]), np.array([[3, 4], [5, 6]]), unit=(u.deg, u.deg))

    #make sure cartesian initialization also works
    c = ICRSCoordinates(x=np.array([1, 2]), y=np.array([3, 4]), z=np.array([5, 6]), unit=u.kpc)

    #also ensure strings can be arrays
    c = ICRSCoordinates(np.array(['1d0m0s', '2h02m00.3s']), np.array(['3d', '4d']), unit=(u.deg, u.deg))

    #but invalid strings cannot
    with pytest.raises(ValueError):
        c = ICRSCoordinates(np.array(['10m0s', '2h02m00.3s']), np.array(['3d', '4d']), unit=(u.deg, u.deg))
    with pytest.raises(ValueError):
        c = ICRSCoordinates(np.array(['1d0m0s', '2h02m00.3s']), np.array(['3x', '4d']), unit=(u.deg, u.deg))

def test_array_coordinates_distances():
    """
    Test creating coordinates from arrays and distances.
    """
    from .. import ICRSCoordinates

    #correct way
    ICRSCoordinates(np.array([1, 2]), np.array([3, 4]), unit=(u.deg, u.deg), distance= [.1, .2] * u.kpc)

    with pytest.raises(ValueError):
        #scalar distance and array coordinates
        ICRSCoordinates(np.array([1, 2]), np.array([[3, 4], [5, 6]]), unit=(u.deg, u.deg), distance= 2. * u.kpc)
    with pytest.raises(ValueError):
        #scalar coordinates and array distance
        ICRSCoordinates(1., 2., unit=(u.deg, u.deg), distance= [.1, .2, 3.] * u.kpc)
    with pytest.raises(ValueError):
        #more distance values than coordinates
        ICRSCoordinates(np.array([1, 2]), np.array([[3, 4], [5, 6]]), unit=(u.deg, u.deg), distance= [.1, .2, 3.] * u.kpc)

@pytest.mark.parametrize(('arrshape', 'distance'), [((2, ), None), ((4, 2, 5), None), ((4, 2, 5), 2 * u.kpc)])
def test_array_coordinates_transformations(arrshape, distance):
    """
    Test transformation on coordinates with array content (first length-2 1D, then a 3D array)
    """
    from .. import ICRSCoordinates, GalacticCoordinates

    #M31 coordinates from test_transformations
    raarr = np.ones(arrshape) * 10.6847929
    decarr = np.ones(arrshape) * 41.2690650
    if distance is not None:
        distance = np.ones(arrshape) * distance

    c = ICRSCoordinates(raarr, decarr, unit=(u.deg, u.deg), distance=distance)
    g = c.transform_to(GalacticCoordinates)

    assert g.l.shape == arrshape

    npt.assert_array_almost_equal(g.l.degree, 121.17447049007306)
    npt.assert_array_almost_equal(g.b.degree, -21.57291080408368)

    if distance is not None:
        assert g.distance.unit == c.distance.unit

    #now make sure round-tripping works through FK5 - this exercises both static and dynamic transform matricies
    c2 = c.fk5.icrs
    npt.assert_array_almost_equal(c.ra, c2.ra)
    npt.assert_array_almost_equal(c.dec, c2.dec)

    assert c2.ra.shape == arrshape

    if distance is not None:
        assert c2.distance.unit == c.distance.unit

    #also make sure it's possible to get to FK4, which uses a direct transform function.
    fk4 = c.fk4

    npt.assert_array_almost_equal(fk4.ra.degree, 10.0004, decimal=4)
    npt.assert_array_almost_equal(fk4.dec.degree, 40.9953, decimal=4)

    assert fk4.ra.shape == arrshape
    if distance is not None:
        assert fk4.distance.unit == c.distance.unit

    #now check the reverse transforms run
    cfk4 = fk4.icrs
    assert cfk4.ra.shape == arrshape

def test_array_coordinates_string():
    """
    tests for string representations of aarray coordinates
    """
    from .. import ICRSCoordinates

    c = ICRSCoordinates(np.array([1, 2]), np.array([3, 4]), unit=(u.deg, u.deg))
    str(c)
    six.text_type(c)
    repr(c)

    assert repr(c) == '<ICRSCoordinates RA=[1 2] deg, Dec=[3 4] deg>'

    #also check with distance

    c = ICRSCoordinates(np.array([1, 2]), np.array([3, 4]), unit=(u.deg, u.deg), distance= u.kpc * [0.5, 1.5])
    str(c)
    six.text_type(c)
    repr(c)

    print(repr(c))

    assert repr(c) == '<ICRSCoordinates RA=[1 2] deg, Dec=[3 4] deg, Distance=[ 0.5  1.5] kpc>'

def test_array_precession():
    """
    Ensures that FK5 coordinates as arrays precess their equinoxes
    """
    from ...time import Time
    from .. import FK5Coordinates

    j2000 = Time('J2000', scale='utc')
    j1975 = Time('J1975', scale='utc')

    fk5 = FK5Coordinates([1, 1.1], [0.5, 0.6], unit=(u.radian, u.radian))
    assert fk5.equinox.jyear == j2000.jyear
    fk5_2 = fk5.precess_to(j1975)
    assert fk5_2.equinox.jyear == j1975.jyear

    npt.assert_array_less(0.05, np.abs(fk5.ra.degree - fk5_2.ra.degree))
    npt.assert_array_less(0.05, np.abs(fk5.dec.degree - fk5_2.dec.degree))
<|endoftext|>


<commit_before>from model_utils import Choices

LOCATION_STATUS = Choices('active', 'review')
OBSERVATION_STATUS = Choices('active', 'review', 'deleted')
COMMENT_STATUS = Choices('active', 'deleted')
<commit_msg>Added additional status options for observations<commit_after>from model_utils import Choices

LOCATION_STATUS = Choices('active', 'review')
OBSERVATION_STATUS = Choices(
    'active', 'draft', 'review', 'pending', 'suspended', 'deleted'
)
COMMENT_STATUS = Choices('active', 'deleted')
<|endoftext|>