<commit_before>#!/usr/bin/env python
from optparse import OptionParser

from collections import OrderedDict
import json
import os
import pdb
import sys

import numpy as np
import pickle
import pybedtools
import pysam
import tensorflow as tf

import pygene
from basenji import dna_io
from basenji import rnann
from basenji import vcf as bvcf
from basenji_sad_ref import make_alt_1hot

'''
saluki_ssd.py

Notes:
-trimming the RNA can result in removing the variant and producing a zero.
-consider removing NMD, splice sites, etc, from eQTL task.
'''

################################################################################
# main
################################################################################
def main():
  usage = 'usage: %prog [options] <params> <model> <vcf>'
  parser = OptionParser(usage)
  parser.add_option('-f', dest='genome_fasta',
    default='%s/data/hg38.fa' % os.environ['BASENJIDIR'],
    help='Genome FASTA for sequences [Default: %default]')
  parser.add_option('-g', dest='genes_gtf',
    default='/home/drk/rnaml/data/genes/gencode36_saluki.gtf',
    help='Genes GTF [Default: %default]')
  parser.add_option('-o', dest='out_dir',
    default='ssd',
    help='Output directory for tables and plots [Default: %default]')
  parser.add_option('--shifts', dest='shifts',
    default='0', type='str',
    help='Ensemble prediction shifts [Default: %default]')
  (options,args) = parser.parse_args()

  if len(args) == 3:
    # single worker
    params_file = args[0]
    model_file = args[1]
    vcf_file = args[2]

  elif len(args) == 4:
    # multi separate
    options_pkl_file = args[0]
    params_file = args[1]
    model_file = args[2]
    vcf_file = args[3]

    # save out dir
    out_dir = options.out_dir

    # load options
    options_pkl = open(options_pkl_file, 'rb')
    options = pickle.load(options_pkl)
    options_pkl.close()

    # update output directory
    options.out_dir = out_dir

  else:
    parser.error('Must provide parameters, model, and VCF')

  options.shifts = [int(shift) for shift in options.shifts.split(',')]

  os.makedirs(options.out_dir, exist_ok=True)

  #################################################################
  # variants to transcripts

  # hash variant identifier to a list of (saluki_score, transcript_id, gene_id)
  variant_transcripts = {}
  variants_bedt = pybedtools.BedTool(vcf_file)
  transcripts_bedt = pybedtools.BedTool(options.genes_gtf)
  for overlap in variants_bedt.intersect(transcripts_bedt, wo=True):
    if overlap[-8] == 'exon':
      vid = overlap[2]
      gene_kv = pygene.gtf_kv(overlap[-2])
      gene_tup = (gene_kv['saluki_score'], gene_kv['transcript_id'], gene_kv['gene_id'])
      variant_transcripts.setdefault(vid,[]).append(gene_tup)

  # assign variants to transcripts
  transcript_variants = OrderedDict()
  for vid in variant_transcripts:
    tid = sorted(variant_transcripts[vid])[-1][1]
    transcript_variants.setdefault(tid,[]).append(vid)

  print('Variants overlapping transcripts: %d' % len(variant_transcripts))
  print('Transcripts overlapping variants: %d' % len(transcript_variants))

  #################################################################
  # parameters and model

  # read model parameters
  with open(params_file) as params_open:
    params = json.load(params_open)
  params_model = params['model']
  params_train = params['train']

  # setup model
  seqnn_model = rnann.RnaNN(params_model)
  seqnn_model.restore(model_file)
  # seqnn_model.build_ensemble(options.shifts)

  #################################################################
  # generator

  # read variants
  variants_list = bvcf.vcf_snps(vcf_file)
  variants_dict = dict([(v.rsid,v) for v in variants_list])

  # read genes
  genes_gtf = pygene.GTF(options.genes_gtf)

  # note: VCF and GTF use 1-based indexing

  # open genome FASTA
  genome_open = pysam.Fastafile(options.genome_fasta)

  def seq_gen():
    for tx_id, tx_variants in transcript_variants.items():
      transcript = genes_gtf.transcripts[tx_id]

      # skip features
      tx_features = np.array([len(transcript.exons)], dtype='float32')

      # encode RNA input
      tx_dna, tx_seq1 = rna_1hot_cds(transcript, genome_open)

      # correct strand
      if transcript.strand == '-':
        tx_seq1_yield = dna_io.hot1_rc(tx_seq1)
      else:
        tx_seq1_yield = tx_seq1

      # pad/trim to length
      tx_seq1_yield = set_rna_length(tx_seq1_yield, params_model['seq_length'])

      # yield
      yield tx_seq1_yield, tx_features

      # for each variant
      for vid in tx_variants:
        variant = variants_dict[vid]

        # map genome position to transcript position
        variant.pos_tx = 0
        for exon in transcript.exons:
          if exon.start <= variant.pos <= exon.end:
            variant.pos_tx += variant.pos - exon.start
            break
          else:
            variant.pos_tx += exon.end - exon.start + 1

        if variant.ref_allele != tx_dna[variant.pos_tx]:
          print('%s ref allele %s does not match transcriptome %s' % \
            (variant.rsid, variant.ref_allele, tx_dna[variant.pos_tx]), file=sys.stderr)
          exit(1)

          # cannot flip alleles using this approach
          # if variant.alt_allele == tx_dna[variant.pos_tx]:
          #   variant.flip_alleles()
          # else:
          #   # bail
          #   print('%s ref allele %s and alt allele %s do not match transcriptome %s' % \
          #   (variant.rsid, variant.ref_allele, variant.alt_allele, tx_dna[variant.pos_tx]), file=sys.stderr)

        # substitue alternative allele
        vtx_seq1 = make_alt_1hot(tx_seq1, variant.pos_tx,
          variant.ref_allele, variant.alt_allele)

        # correct strand
        if transcript.strand == '-':
          vtx_seq1_yield = dna_io.hot1_rc(vtx_seq1)
        else:
          vtx_seq1_yield = vtx_seq1

        # pad/trim to length
        vtx_seq1_yield = set_rna_length(vtx_seq1_yield, params_model['seq_length'])

        # yield
        yield vtx_seq1_yield, tx_features

  ################################################################
  # predict SNP scores, write output

  # initialize output table
  ssd_out = open('%s/ssd.tsv' % options.out_dir, 'w')
  headers = ['variant', 'transcript', 'SSD']
  print('\t'.join(headers), file=ssd_out)

  # initialize predictions stream
  preds_stream = PredStreamGen(seqnn_model, seq_gen(), params['train']['batch_size'])

  # predictions index
  pi = 0

  for tx_id, tx_variants in transcript_variants.items():
    # get reference prediction
    ref_preds = preds_stream[pi]
    pi += 1

    # for each variant
    for vid in tx_variants:
      # get alternative prediction
      alt_preds = preds_stream[pi]
      pi += 1

      # compute SNP stability difference
      ssd = alt_preds - ref_preds
      print(ssd.shape)

      # print
      cols = [vid, tx_id, '%.4f'%ssd]
      print('\t'.join(cols), file=ssd_out)

  # close genome
  genome_open.close()

  # close SNP output
  ssd_out.close()

def rna_1hot_cds(transcript, genome_open):
  """add cds channel"""

  # exons RNA
  tx_exons = transcript.fasta_exons(genome_open)
  seq_len = len(tx_exons)

  # CDS RNA
  tx_cds = transcript.fasta_cds(genome_open)
  cds_start = tx_exons.find(tx_cds)
  assert(cds_start != -1)
  cds_end = cds_start + len(tx_cds)

  # add CDS channel
  cds_channel = np.zeros((seq_len,1))
  cds_channel[cds_start:cds_end] = 1

  # 1 hot encode
  tx_seq1 = dna_io.dna_1hot(tx_exons, n_uniform=True)
  tx_seq1 = np.concatenate([tx_seq1, cds_channel], axis=-1)

  return tx_exons, tx_seq1.astype('float32')

def set_rna_length(tx_seq1, length):
  """Set RNA length to specific value by padding or trimming."""
  seq_len, seq_depth = tx_seq1.shape

  if seq_len < length:
    # pad
    pad_len = length - seq_len
    pad_seq1 = np.zeros((pad_len,seq_depth))
    tx_seq1 = np.concatenate([tx_seq1,pad_seq1], axis=0)

  elif seq_len > length:
    # trim 5'
    tx_seq1 = tx_seq1[-length:]

  return tx_seq1

class PredStreamGen:
  """ Interface to acquire predictions via a buffered stream mechanism
        rather than getting them all at once and using excessive memory.
        Accepts generator and constructs stream batches from it. """
  def __init__(self, model, seqs_gen, batch_size, stream_seqs=256, verbose=False):
    self.model = model
    self.seqs_gen = seqs_gen
    self.stream_seqs = stream_seqs
    self.batch_size = batch_size
    self.verbose = verbose

    self.stream_start = 0
    self.stream_end = 0

  def __getitem__(self, i):
    # acquire predictions, if needed
    if i >= self.stream_end:
      # update start
      self.stream_start = self.stream_end

      if self.verbose:
        print('Predicting from %d' % self.stream_start, flush=True)

      # predict
      self.stream_preds = self.model.predict(self.make_dataset())

      # update end
      self.stream_end = self.stream_start + self.stream_preds.shape[0]

    return self.stream_preds[i - self.stream_start]

  def make_dataset(self):
    """ Construct Dataset object for this stream chunk. """
    seqs_1hot = []
    seqs_features = []
    stream_end = self.stream_start+self.stream_seqs
    for si in range(self.stream_start, stream_end):
      try:
        seq_1hot, seq_features = self.seqs_gen.__next__()
        seqs_1hot.append(seq_1hot)
        seqs_features.append(seq_features)
      except StopIteration:
        continue

    seqs_1hot = np.array(seqs_1hot)
    seqs_features = np.array(seqs_features)

    dataset = tf.data.Dataset.from_tensor_slices(((seqs_1hot,seqs_features),))
    dataset = dataset.batch(self.batch_size)

    return dataset

################################################################################
# __main__
################################################################################
if __name__ == '__main__':
    main()
<commit_msg>coding/splice tracks<commit_after>#!/usr/bin/env python
from optparse import OptionParser

from collections import OrderedDict
import json
import os
import pdb
import sys

import numpy as np
import pickle
import pybedtools
import pysam
import tensorflow as tf

import pygene
from basenji import dna_io
from basenji import rnann
from basenji import vcf as bvcf
from basenji_sad_ref import make_alt_1hot

'''
saluki_ssd.py

Notes:
-trimming the RNA can result in removing the variant and producing a zero.
-consider removing NMD, splice sites, etc, from eQTL task.
'''

################################################################################
# main
################################################################################
def main():
  usage = 'usage: %prog [options] <params> <model> <vcf>'
  parser = OptionParser(usage)
  parser.add_option('-f', dest='genome_fasta',
    default='%s/data/hg38.fa' % os.environ['BASENJIDIR'],
    help='Genome FASTA for sequences [Default: %default]')
  parser.add_option('-g', dest='genes_gtf',
    default='/home/drk/rnaml/data/genes/gencode36_saluki.gtf',
    help='Genes GTF [Default: %default]')
  parser.add_option('-o', dest='out_dir',
    default='ssd',
    help='Output directory for tables and plots [Default: %default]')
  parser.add_option('--shifts', dest='shifts',
    default='0', type='str',
    help='Ensemble prediction shifts [Default: %default]')
  (options,args) = parser.parse_args()

  if len(args) == 3:
    # single worker
    params_file = args[0]
    model_file = args[1]
    vcf_file = args[2]

  elif len(args) == 4:
    # multi separate
    options_pkl_file = args[0]
    params_file = args[1]
    model_file = args[2]
    vcf_file = args[3]

    # save out dir
    out_dir = options.out_dir

    # load options
    options_pkl = open(options_pkl_file, 'rb')
    options = pickle.load(options_pkl)
    options_pkl.close()

    # update output directory
    options.out_dir = out_dir

  else:
    parser.error('Must provide parameters, model, and VCF')

  options.shifts = [int(shift) for shift in options.shifts.split(',')]

  os.makedirs(options.out_dir, exist_ok=True)

  #################################################################
  # variants to transcripts

  # hash variant identifier to a list of (saluki_score, transcript_id, gene_id)
  variant_transcripts = {}
  variants_bedt = pybedtools.BedTool(vcf_file)
  transcripts_bedt = pybedtools.BedTool(options.genes_gtf)
  for overlap in variants_bedt.intersect(transcripts_bedt, wo=True):
    if overlap[-8] == 'exon':
      vid = overlap[2]
      gene_kv = pygene.gtf_kv(overlap[-2])
      gene_tup = (gene_kv['saluki_score'], gene_kv['transcript_id'], gene_kv['gene_id'])
      variant_transcripts.setdefault(vid,[]).append(gene_tup)

  # assign variants to transcripts
  transcript_variants = OrderedDict()
  for vid in variant_transcripts:
    tid = sorted(variant_transcripts[vid])[-1][1]
    transcript_variants.setdefault(tid,[]).append(vid)

  print('Variants overlapping transcripts: %d' % len(variant_transcripts))
  print('Transcripts overlapping variants: %d' % len(transcript_variants))

  #################################################################
  # parameters and model

  # read model parameters
  with open(params_file) as params_open:
    params = json.load(params_open)
  params_model = params['model']
  params_train = params['train']

  # setup model
  seqnn_model = rnann.RnaNN(params_model)
  seqnn_model.restore(model_file)
  # seqnn_model.build_ensemble(False, options.shifts)

  #################################################################
  # generator

  # read variants
  variants_list = bvcf.vcf_snps(vcf_file)
  variants_dict = dict([(v.rsid,v) for v in variants_list])

  # read genes
  genes_gtf = pygene.GTF(options.genes_gtf)

  # note: VCF and GTF use 1-based indexing

  # open genome FASTA
  genome_open = pysam.Fastafile(options.genome_fasta)

  def seq_gen():
    for tx_id, tx_variants in transcript_variants.items():
      transcript = genes_gtf.transcripts[tx_id]

      # skip features
      # tx_features = np.array([len(transcript.exons)], dtype='float32')

      # encode RNA input
      tx_dna, tx_seq1 = rna_1hot_splice(transcript, genome_open)

      # correct strand
      if transcript.strand == '-':
        tx_seq1_yield = dna_io.hot1_rc(tx_seq1)
      else:
        tx_seq1_yield = tx_seq1

      # pad/trim to length
      tx_seq1_yield = set_rna_length(tx_seq1_yield, params_model['seq_length'])

      # yield
      yield tx_seq1_yield #, tx_features

      # for each variant
      for vid in tx_variants:
        variant = variants_dict[vid]

        # map genome position to transcript position
        variant.pos_tx = 0
        for exon in transcript.exons:
          if exon.start <= variant.pos <= exon.end:
            variant.pos_tx += variant.pos - exon.start
            break
          else:
            variant.pos_tx += exon.end - exon.start + 1

        if variant.ref_allele != tx_dna[variant.pos_tx]:
          print('%s ref allele %s does not match transcriptome %s' % \
            (variant.rsid, variant.ref_allele, tx_dna[variant.pos_tx]), file=sys.stderr)
          exit(1)

          # cannot flip alleles using this approach
          # if variant.alt_allele == tx_dna[variant.pos_tx]:
          #   variant.flip_alleles()
          # else:
          #   # bail
          #   print('%s ref allele %s and alt allele %s do not match transcriptome %s' % \
          #   (variant.rsid, variant.ref_allele, variant.alt_allele, tx_dna[variant.pos_tx]), file=sys.stderr)

        # substitute alternative allele
        vtx_seq1 = make_alt_1hot(tx_seq1, variant.pos_tx,
          variant.ref_allele, variant.alt_allele)

        # correct strand
        if transcript.strand == '-':
          vtx_seq1_yield = dna_io.hot1_rc(vtx_seq1)
        else:
          vtx_seq1_yield = vtx_seq1

        # pad/trim to length
        vtx_seq1_yield = set_rna_length(vtx_seq1_yield, params_model['seq_length'])

        # yield
        yield vtx_seq1_yield #, tx_features

  ################################################################
  # predict SNP scores, write output

  # initialize output table
  ssd_out = open('%s/ssd.tsv' % options.out_dir, 'w')
  headers = ['variant', 'transcript', 'SSD']
  print('\t'.join(headers), file=ssd_out)

  # initialize predictions stream
  preds_stream = PredStreamGen(seqnn_model, seq_gen(), params['train']['batch_size'])

  # predictions index
  pi = 0

  for tx_id, tx_variants in transcript_variants.items():
    # get reference prediction
    ref_preds = preds_stream[pi]
    pi += 1

    # for each variant
    for vid in tx_variants:
      # get alternative prediction
      alt_preds = preds_stream[pi]
      pi += 1

      # compute SNP stability difference
      ssd = alt_preds - ref_preds
      print(ssd.shape)

      # print
      cols = [vid, tx_id, '%.6f'%ssd]
      print('\t'.join(cols), file=ssd_out)

  # close genome
  genome_open.close()

  # close SNP output
  ssd_out.close()

def rna_1hot_splice(transcript, genome_open):
  """Extract RNA nucleotides and add splice and coding frame channels.
      Consider strand, but do not yet reverse complement, because its
      easier to handle variant substitution."""

   # exons RNA
  tx_exons = transcript.fasta_exons(genome_open, stranded=False)
  seq_len = len(tx_exons)

  # CDS RNA
  tx_cds = transcript.fasta_cds(genome_open, stranded=False)
  if len(tx_cds) % 3 == 0:
    valid_coding = True
  else:
    # print('WARNING: %d mod 3 == %d length:' % (len(tx_cds),len(tx_cds)%3), transcript)
    valid_coding = False
  cds_start = tx_exons.find(tx_cds)
  assert(cds_start != -1)
  cds_end = cds_start + len(tx_cds)

  # add frame channel
  frame_channel = np.zeros((seq_len,1))
  if valid_coding:
    aa_len = (cds_end - cds_start) // 3
    if transcript.strand == '+':
      frame_channel[cds_start:cds_end,0] = np.tile([1,0,0], aa_len)
    else:
      frame_channel[cds_start:cds_end,0] = np.tile([0,0,1], aa_len)

  # splice track
  splice_channel = np.zeros((seq_len,1))
  ti = 0
  exon_lens = [ex.end-ex.start for ex in transcript.exons]
  for el in exon_lens:
    if transcript.strand == '+':
      ti += el
      splice_channel[ti-1,0] = 1
    else:
      splice_channel[ti,0] = 1
      ti += el

  # 1 hot encode
  tx_seq1 = dna_io.dna_1hot(tx_exons, n_uniform=True)
  tx_seq1 = np.concatenate([tx_seq1, frame_channel, splice_channel], axis=-1)

  return tx_exons, tx_seq1.astype('float32')

def rna_1hot_splice_vikram(transcript, genome_open):
  """Extract RNA nucleotides and add splice and coding frame channels.
      Consider strand, but do not yet reverse complement, because its
      easier to handle variant substitution."""

  # exons RNA
  tx_exons = transcript.fasta_exons(genome_open, stranded=False)
  seq_len = len(tx_exons)

  # CDS RNA
  tx_cds = transcript.fasta_cds(genome_open, stranded=False)
  if len(tx_cds) % 3 != 0:
    pdb.set_trace()
  assert(len(tx_cds) % 3 == 0)
  cds_start = tx_exons.find(tx_cds)
  assert(cds_start != -1)
  cds_end = cds_start + len(tx_cds)

  # add frame channel
  frame_channel = np.zeros((seq_len,1))
  aa_len = (cds_end - cds_start) // 3
  if transcript.strand == '+':
    frame_channel[cds_start:cds_end,0] = np.tile([1,0,0], aa_len)
  else:
    frame_channel[cds_start:cds_end,0] = np.tile([0,0,1], aa_len)

  # splice tracks
  splice5_channel = np.zeros((seq_len,1))
  splice3_channel = np.zeros((seq_len,1))
  ti = 0
  exon_lens = [ex.end-ex.start for ex in transcript.exons]
  for el in exon_lens:
    if transcript.strand == '+':
      splice5_channel[ti,0] = 1
      ti += el
      splice3_channel[ti-1,0] = 1
    else:
      splice3_channel[ti,0] = 1
      ti += el
      splice5_channel[ti-1,0] = 1

  # 1 hot encode
  tx_seq1 = dna_io.dna_1hot(tx_exons, n_uniform=True)
  tx_seq1 = np.concatenate([tx_seq1, frame_channel, splice5_channel, splice3_channel], axis=-1)

  return tx_exons, tx_seq1.astype('float32')

def rna_1hot_cds(transcript, genome_open):
  """add cds channel"""

  # exons RNA
  tx_exons = transcript.fasta_exons(genome_open)
  seq_len = len(tx_exons)

  # CDS RNA
  tx_cds = transcript.fasta_cds(genome_open)
  cds_start = tx_exons.find(tx_cds)
  assert(cds_start != -1)
  cds_end = cds_start + len(tx_cds)

  # add CDS channel
  cds_channel = np.zeros((seq_len,1))
  cds_channel[cds_start:cds_end] = 1

  # 1 hot encode
  tx_seq1 = dna_io.dna_1hot(tx_exons, n_uniform=True)
  tx_seq1 = np.concatenate([tx_seq1, cds_channel], axis=-1)

  return tx_exons, tx_seq1.astype('float32')

def set_rna_length(tx_seq1, length):
  """Set RNA length to specific value by padding or trimming."""
  seq_len, seq_depth = tx_seq1.shape

  if seq_len < length:
    # pad
    pad_len = length - seq_len
    pad_seq1 = np.zeros((pad_len,seq_depth))
    tx_seq1 = np.concatenate([tx_seq1,pad_seq1], axis=0)

  elif seq_len > length:
    # trim 5'
    tx_seq1 = tx_seq1[-length:]

  return tx_seq1

class PredStreamGen:
  """ Interface to acquire predictions via a buffered stream mechanism
        rather than getting them all at once and using excessive memory.
        Accepts generator and constructs stream batches from it. """
  def __init__(self, model, seqs_gen, batch_size, stream_seqs=256, verbose=False):
    self.model = model
    self.seqs_gen = seqs_gen
    self.stream_seqs = stream_seqs
    self.batch_size = batch_size
    self.verbose = verbose

    self.stream_start = 0
    self.stream_end = 0

  def __getitem__(self, i):
    # acquire predictions, if needed
    if i >= self.stream_end:
      # update start
      self.stream_start = self.stream_end

      if self.verbose:
        print('Predicting from %d' % self.stream_start, flush=True)

      # predict
      self.stream_preds = self.model.predict(self.make_dataset())

      # update end
      self.stream_end = self.stream_start + self.stream_preds.shape[0]

    return self.stream_preds[i - self.stream_start]

  def make_dataset(self):
    """ Construct Dataset object for this stream chunk. """
    seqs_1hot = []
    stream_end = self.stream_start+self.stream_seqs
    for si in range(self.stream_start, stream_end):
      try:
        seq_1hot = self.seqs_gen.__next__()
        seqs_1hot.append(seq_1hot)
      except StopIteration:
        continue

    seqs_1hot = np.array(seqs_1hot)

    dataset = tf.data.Dataset.from_tensor_slices((seqs_1hot,))
    dataset = dataset.batch(self.batch_size)

    return dataset

################################################################################
# __main__
################################################################################
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Process a list of Python and/or XML files containing SCons documentation.
#
# This script creates formatted lists of the Builders, Tools or
# construction variables documented in the specified XML files.
#
# Dependening on the options, the lists are output in either
# DocBook-formatted generated XML files containing the summary text
# and/or .mod files contining the ENTITY definitions for each item,
# or in man-page-formatted output.
#
import getopt
import os.path
import re
import string
import StringIO
import sys
import xml.sax

import SConsDoc

base_sys_path = [os.getcwd() + '/build/test-tar-gz/lib/scons'] + sys.path

helpstr = """\
Usage: scons-proc.py [--man|--xml]
                     [-b file(s)] [-t file(s)] [-v file(s)] [infile ...]
Options:
  -b file(s)        dump builder information to the specified file(s)
  -t file(s)        dump tool information to the specified file(s)
  -v file(s)        dump variable information to the specified file(s)
  --man             print info in man page format, each -[btv] argument
                    is a single file name
  --xml             (default) print info in SML format, each -[btv] argument
                    is a pair of comma-separated .gen,.mod file names
"""

opts, args = getopt.getopt(sys.argv[1:],
                           "b:ht:v:",
                           ['builders=', 'help',
                            'man', 'xml', 'tools=', 'variables='])

buildersfiles = None
output_type = '--xml'
toolsfiles = None
variablesfiles = None

for o, a in opts:
    if o in ['-b', '--builders']:
        buildersfiles = a
    elif o in ['-h', '--help']:
        sys.stdout.write(helpstr)
        sys.exit(0)
    elif o in ['--man', '--xml']:
        output_type = o
    elif o in ['-t', '--tools']:
        toolsfiles = a
    elif o in ['-v', '--variables']:
        variablesfiles = a

h = SConsDoc.SConsDocHandler()
saxparser = xml.sax.make_parser()
saxparser.setContentHandler(h)
saxparser.setErrorHandler(h)

xml_preamble = """\
<?xml version="1.0"?>
<scons_doc>
"""

xml_postamble = """\
</scons_doc>
"""

for f in args:
    _, ext = os.path.splitext(f)
    if ext == '.py':
        dir, _ = os.path.split(f)
        if dir:
            sys.path = [dir] + base_sys_path
        module = SConsDoc.importfile(f)
        h.set_file_info(f, len(xml_preamble.split('\n')))
        try:
            content = module.__scons_doc__
        except AttributeError:
            content = None
        else:
            del module.__scons_doc__
    else:
        h.set_file_info(f, len(xml_preamble.split('\n')))
        content = open(f).read()
    if content:
        content = content.replace('&', '&')
        input = xml_preamble + content + xml_postamble
        try:
            saxparser.parse(StringIO.StringIO(input))
        except:
            sys.stderr.write("error in %s\n" % f)
            raise

Warning = """\
<!--
THIS IS AN AUTOMATICALLY-GENERATED FILE.  DO NOT EDIT.
-->
"""

Regular_Entities_Header = """\
<!--

  Regular %s entities.

-->
"""

Link_Entities_Header = """\
<!--

  Entities that are links to the %s entries in the appendix.

-->
"""

class SCons_XML:
    def __init__(self, entries, **kw):
        values = entries.values()
        values.sort()
        self.values = values
        for k, v in kw.items():
            setattr(self, k, v)
    def fopen(self, name):
        if name == '-':
            return sys.stdout
        return open(name, 'w')

class SCons_XML_to_XML(SCons_XML):
    def write(self, files):
        gen, mod = string.split(files, ',')
        g.write_gen(gen)
        g.write_mod(mod)
    def write_gen(self, filename):
        if not filename:
            return
        f = self.fopen(filename)
        for v in self.values:
            f.write('\n<varlistentry id="%s%s">\n' %
                        (self.prefix, self.idfunc(v.name)))
            for term in self.termfunc(v.name):
                f.write('<term><%s>%s</%s></term>\n' %
                        (self.tag, term, self.tag))
            f.write('<listitem>\n')
            for chunk in v.summary.body:
                f.write(str(chunk))
            if v.sets:
                s = map(lambda x: '&cv-link-%s;' % x, v.sets)
                f.write('<para>\n')
                f.write('Sets:  ' + ', '.join(s) + '.\n')
                f.write('</para>\n')
            if v.uses:
                u = map(lambda x: '&cv-link-%s;' % x, v.uses)
                f.write('<para>\n')
                f.write('Uses:  ' + ', '.join(u) + '.\n')
                f.write('</para>\n')
            f.write('</listitem>\n')
            f.write('</varlistentry>\n')
    def write_mod(self, filename):
        if not filename:
            return
        f = self.fopen(filename)
        f.write(Warning)
        f.write('\n')
        f.write(Regular_Entities_Header % self.description)
        f.write('\n')
        for v in self.values:
            f.write('<!ENTITY %s%s "<%s>%s</%s>">\n' %
                        (self.prefix, self.idfunc(v.name),
                         self.tag, self.entityfunc(v.name), self.tag))
        f.write('\n')
        f.write(Warning)
        f.write('\n')
        f.write(Link_Entities_Header % self.description)
        f.write('\n')
        for v in self.values:
            f.write('<!ENTITY %slink-%s \'<link linkend="%s%s"><%s>%s</%s></link>\'>\n' %
                        (self.prefix, self.idfunc(v.name),
                         self.prefix, self.idfunc(v.name),
                         self.tag, self.entityfunc(v.name), self.tag))
        f.write('\n')
        f.write(Warning)

class SCons_XML_to_man(SCons_XML):
    def mansep(self):
        return ['\n']
    def initial_chunks(self, name):
        return [name]
    def write(self, filename):
        if not filename:
            return
        f = self.fopen(filename)
        chunks = []
        for v in self.values:
            chunks.extend(self.mansep())
            for n in self.initial_chunks(v.name):
                chunks.append('.IP %s\n' % n)
            chunks.extend(map(str, v.summary.body))

        body = ''.join(chunks)
        body = string.replace(body, '<programlisting>', '.ES')
        body = string.replace(body, '</programlisting>', '.EE')
        body = string.replace(body, '\n</para>\n<para>\n', '\n\n')
        body = string.replace(body, '<para>\n', '')
        body = string.replace(body, '<para>', '\n')
        body = string.replace(body, '</para>\n', '')
        body = re.sub('\.EE\n\n+(?!\.IP)', '.EE\n.IP\n', body)
        body = re.sub('&(scons|SConstruct|SConscript|jar);', r'\\fB\1\\fP', body)
        body = string.replace(body, '&Dir;', r'\fBDir\fP')
        body = string.replace(body, '⌖', r'\fItarget\fP')
        body = string.replace(body, '&source;', r'\fIsource\fP')
        body = re.sub('&b(-link)?-([^;]*);', r'\\fB\2\\fP()', body)
        body = re.sub('&cv(-link)?-([^;]*);', r'$\2', body)
        body = re.sub(r'<(command|envar|filename|literal|option)>([^<]*)</\1>',
                      r'\\fB\2\\fP', body)
        body = re.sub(r'<(classname|emphasis|varname)>([^<]*)</\1>',
                      r'\\fI\2\\fP', body)
        body = re.compile(r'^\\f([BI])(.*)\\fP\s*$', re.M).sub(r'.\1 \2', body)
        body = re.compile(r'^\\f([BI])(.*)\\fP(\S+)', re.M).sub(r'.\1R \2 \3', body)
        body = string.replace(body, '<', '<')
        body = string.replace(body, '>', '>')
        body = re.sub(r'\\([^f])', r'\\\\\1', body)
        body = re.compile("^'\\\\\\\\", re.M).sub("'\\\\", body)
        body = re.compile(r'^\.([BI]R?) -', re.M).sub(r'.\1 \-', body)
        body = re.compile(r'^\.([BI]R?) (\S+)\\\\(\S+)', re.M).sub(r'.\1 "\2\\\\\\\\\2"', body)
        body = re.compile(r'\\f([BI])-', re.M).sub(r'\\f\1\-', body)
        f.write(body)

if output_type == '--man':
    processor_class = SCons_XML_to_man
elif output_type == '--xml':
    processor_class = SCons_XML_to_XML
else:
    sys.stderr.write("Unknown output type '%s'\n" % output_type)
    sys.exit(1)

if buildersfiles:
    g = processor_class(h.builders,
            description = 'builder',
            prefix = 'b-',
            tag = 'function',
            idfunc = lambda x: x,
            termfunc = lambda x: [x+'()', 'env.'+x+'()'],
            entityfunc = lambda x: x)

    g.mansep = lambda: ['\n', "'\\" + '"'*69 + '\n']
    g.initial_chunks = lambda n: [n+'()', 'env.'+n+'()']

    g.write(buildersfiles)

if toolsfiles:
    g = processor_class(h.tools,
            description = 'tool',
            prefix = 't-',
            tag = 'literal',
            idfunc = lambda x: string.replace(x, '+', 'X'),
            termfunc = lambda x: [x],
            entityfunc = lambda x: x)

    g.write(toolsfiles)

if variablesfiles:
    g = processor_class(h.cvars,
            description = 'construction variable',
            prefix = 'cv-',
            tag = 'envar',
            idfunc = lambda x: x,
            termfunc = lambda x: [x],
            entityfunc = lambda x: '$'+x)

    g.write(variablesfiles)

# Local Variables:
# tab-width:4
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=4 shiftwidth=4:
<commit_msg>Refactor the XMl processing of various entity types to make the code more OO and extensible.<commit_after>#!/usr/bin/env python
#
# Process a list of Python and/or XML files containing SCons documentation.
#
# This script creates formatted lists of the Builders, Tools or
# construction variables documented in the specified XML files.
#
# Dependening on the options, the lists are output in either
# DocBook-formatted generated XML files containing the summary text
# and/or .mod files contining the ENTITY definitions for each item,
# or in man-page-formatted output.
#
import getopt
import os.path
import re
import string
import StringIO
import sys
import xml.sax

import SConsDoc

base_sys_path = [os.getcwd() + '/build/test-tar-gz/lib/scons'] + sys.path

helpstr = """\
Usage: scons-proc.py [--man|--xml]
                     [-b file(s)] [-t file(s)] [-v file(s)] [infile ...]
Options:
  -b file(s)        dump builder information to the specified file(s)
  -t file(s)        dump tool information to the specified file(s)
  -v file(s)        dump variable information to the specified file(s)
  --man             print info in man page format, each -[btv] argument
                    is a single file name
  --xml             (default) print info in SML format, each -[btv] argument
                    is a pair of comma-separated .gen,.mod file names
"""

opts, args = getopt.getopt(sys.argv[1:],
                           "b:ht:v:",
                           ['builders=', 'help',
                            'man', 'xml', 'tools=', 'variables='])

buildersfiles = None
output_type = '--xml'
toolsfiles = None
variablesfiles = None

for o, a in opts:
    if o in ['-b', '--builders']:
        buildersfiles = a
    elif o in ['-h', '--help']:
        sys.stdout.write(helpstr)
        sys.exit(0)
    elif o in ['--man', '--xml']:
        output_type = o
    elif o in ['-t', '--tools']:
        toolsfiles = a
    elif o in ['-v', '--variables']:
        variablesfiles = a

h = SConsDoc.SConsDocHandler()
saxparser = xml.sax.make_parser()
saxparser.setContentHandler(h)
saxparser.setErrorHandler(h)

xml_preamble = """\
<?xml version="1.0"?>
<scons_doc>
"""

xml_postamble = """\
</scons_doc>
"""

for f in args:
    _, ext = os.path.splitext(f)
    if ext == '.py':
        dir, _ = os.path.split(f)
        if dir:
            sys.path = [dir] + base_sys_path
        module = SConsDoc.importfile(f)
        h.set_file_info(f, len(xml_preamble.split('\n')))
        try:
            content = module.__scons_doc__
        except AttributeError:
            content = None
        else:
            del module.__scons_doc__
    else:
        h.set_file_info(f, len(xml_preamble.split('\n')))
        content = open(f).read()
    if content:
        content = content.replace('&', '&')
        input = xml_preamble + content + xml_postamble
        try:
            saxparser.parse(StringIO.StringIO(input))
        except:
            sys.stderr.write("error in %s\n" % f)
            raise

Warning = """\
<!--
THIS IS AN AUTOMATICALLY-GENERATED FILE.  DO NOT EDIT.
-->
"""

Regular_Entities_Header = """\
<!--

  Regular %s entities.

-->
"""

Link_Entities_Header = """\
<!--

  Entities that are links to the %s entries in the appendix.

-->
"""

class SCons_XML:
    def __init__(self, entries, **kw):
        self.values = entries
        for k, v in kw.items():
            setattr(self, k, v)
    def fopen(self, name):
        if name == '-':
            return sys.stdout
        return open(name, 'w')

class SCons_XML_to_XML(SCons_XML):
    def write(self, files):
        gen, mod = string.split(files, ',')
        g.write_gen(gen)
        g.write_mod(mod)
    def write_gen(self, filename):
        if not filename:
            return
        f = self.fopen(filename)
        for v in self.values:
            f.write('\n<varlistentry id="%s%s">\n' %
                        (v.prefix, v.idfunc()))
            for term in v.termfunc():
                f.write('<term><%s>%s</%s></term>\n' %
                        (v.tag, term, v.tag))
            f.write('<listitem>\n')
            for chunk in v.summary.body:
                f.write(str(chunk))
            if v.sets:
                s = map(lambda x: '&cv-link-%s;' % x, v.sets)
                f.write('<para>\n')
                f.write('Sets:  ' + ', '.join(s) + '.\n')
                f.write('</para>\n')
            if v.uses:
                u = map(lambda x: '&cv-link-%s;' % x, v.uses)
                f.write('<para>\n')
                f.write('Uses:  ' + ', '.join(u) + '.\n')
                f.write('</para>\n')
            f.write('</listitem>\n')
            f.write('</varlistentry>\n')
    def write_mod(self, filename):
        description = self.values[0].description
        if not filename:
            return
        f = self.fopen(filename)
        f.write(Warning)
        f.write('\n')
        f.write(Regular_Entities_Header % description)
        f.write('\n')
        for v in self.values:
            f.write('<!ENTITY %s%s "<%s>%s</%s>">\n' %
                        (v.prefix, v.idfunc(),
                         v.tag, v.entityfunc(), v.tag))
        f.write('\n')
        f.write(Warning)
        f.write('\n')
        f.write(Link_Entities_Header % description)
        f.write('\n')
        for v in self.values:
            f.write('<!ENTITY %slink-%s \'<link linkend="%s%s"><%s>%s</%s></link>\'>\n' %
                        (v.prefix, v.idfunc(),
                         v.prefix, v.idfunc(),
                         v.tag, v.entityfunc(), v.tag))
        f.write('\n')
        f.write(Warning)

class SCons_XML_to_man(SCons_XML):
    def write(self, filename):
        if not filename:
            return
        f = self.fopen(filename)
        chunks = []
        for v in self.values:
            chunks.extend(v.mansep())
            for n in v.initial_chunks():
                chunks.append('.IP %s\n' % n)
            chunks.extend(map(str, v.summary.body))

        body = ''.join(chunks)
        body = string.replace(body, '<programlisting>', '.ES')
        body = string.replace(body, '</programlisting>', '.EE')
        body = string.replace(body, '\n</para>\n<para>\n', '\n\n')
        body = string.replace(body, '<para>\n', '')
        body = string.replace(body, '<para>', '\n')
        body = string.replace(body, '</para>\n', '')
        body = re.sub('\.EE\n\n+(?!\.IP)', '.EE\n.IP\n', body)
        body = re.sub('&(scons|SConstruct|SConscript|jar);', r'\\fB\1\\fP', body)
        body = string.replace(body, '&Dir;', r'\fBDir\fP')
        body = string.replace(body, '⌖', r'\fItarget\fP')
        body = string.replace(body, '&source;', r'\fIsource\fP')
        body = re.sub('&b(-link)?-([^;]*);', r'\\fB\2\\fP()', body)
        body = re.sub('&cv(-link)?-([^;]*);', r'$\2', body)
        body = re.sub(r'<(command|envar|filename|literal|option)>([^<]*)</\1>',
                      r'\\fB\2\\fP', body)
        body = re.sub(r'<(classname|emphasis|varname)>([^<]*)</\1>',
                      r'\\fI\2\\fP', body)
        body = re.compile(r'^\\f([BI])(.*)\\fP\s*$', re.M).sub(r'.\1 \2', body)
        body = re.compile(r'^\\f([BI])(.*)\\fP(\S+)', re.M).sub(r'.\1R \2 \3', body)
        body = string.replace(body, '<', '<')
        body = string.replace(body, '>', '>')
        body = re.sub(r'\\([^f])', r'\\\\\1', body)
        body = re.compile("^'\\\\\\\\", re.M).sub("'\\\\", body)
        body = re.compile(r'^\.([BI]R?) -', re.M).sub(r'.\1 \-', body)
        body = re.compile(r'^\.([BI]R?) (\S+)\\\\(\S+)', re.M).sub(r'.\1 "\2\\\\\\\\\2"', body)
        body = re.compile(r'\\f([BI])-', re.M).sub(r'\\f\1\-', body)
        f.write(body)

class Proxy:
    def __init__(self, subject):
        """Wrap an object as a Proxy object"""
        self.__subject = subject

    def __getattr__(self, name):
        """Retrieve an attribute from the wrapped object.  If the named
           attribute doesn't exist, AttributeError is raised"""
        return getattr(self.__subject, name)

    def get(self):
        """Retrieve the entire wrapped object"""
        return self.__subject

    def __cmp__(self, other):
        if issubclass(other.__class__, self.__subject.__class__):
            return cmp(self.__subject, other)
        return cmp(self.__dict__, other.__dict__)

class Builder(Proxy):
    description = 'builder'
    prefix = 'b-'
    tag = 'function'
    def idfunc(self):
        return self.name
    def termfunc(self):
        return ['%s()' % self.name, 'env.%s()' % self.name]
    def entityfunc(self):
        return self.name
    def mansep(self):
        return ['\n', "'\\" + '"'*69 + '\n']
    def initial_chunks(self):
        return self.termfunc()

class Tool(Proxy):
    description = 'tool'
    prefix = 't-'
    tag = 'literal'
    def idfunc(self):
        return string.replace(self.name, '+', 'X')
    def termfunc(self):
        return [self.name]
    def entityfunc(self):
        return self.name
    def mansep(self):
        return ['\n']
    def initial_chunks(self):
        return [self.name]

class Variable(Proxy):
    description = 'construction variable'
    prefix = 'cv-'
    tag = 'envar'
    def idfunc(self):
        return self.name
    def termfunc(self):
        return [self.name]
    def entityfunc(self):
        return '$' + self.name
    def mansep(self):
        return ['\n']
    def initial_chunks(self):
        return [self.name]

if output_type == '--man':
    processor_class = SCons_XML_to_man
elif output_type == '--xml':
    processor_class = SCons_XML_to_XML
else:
    sys.stderr.write("Unknown output type '%s'\n" % output_type)
    sys.exit(1)

if buildersfiles:
    g = processor_class([ Builder(b) for b in sorted(h.builders.values()) ])
    g.write(buildersfiles)

if toolsfiles:
    g = processor_class([ Tool(t) for t in sorted(h.tools.values()) ])
    g.write(toolsfiles)

if variablesfiles:
    g = processor_class([ Variable(v) for v in sorted(h.cvars.values()) ])
    g.write(variablesfiles)

# Local Variables:
# tab-width:4
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=4 shiftwidth=4:
<|endoftext|>


<commit_before>#!/usr/bin/env python

#Copyright (C) 2013 by Glenn Hickey
#
#Released under the MIT license, see LICENSE.txt
import unittest
import sys
import os
import argparse
import logging

from teHmm.track import TrackData
from teHmm.trackIO import readBedIntervals, getMergedBedIntervals
from teHmm.hmm import MultitrackHmm
from teHmm.emission import IndependentMultinomialEmissionModel
from teHmm.track import CategoryMap

def main(argv=None):
    if argv is None:
        argv = sys.argv
    logging.basicConfig(level=logging.INFO)

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="Create a teHMM")

    parser.add_argument("tracksInfo", help="Path of Tracks Info file "
                        "containing paths to genome annotation tracks")
    parser.add_argument("trainingBed", help="Path of BED file containing"
                        " elements to train  model on")
    parser.add_argument("outputModel", help="Path of output hmm")
    parser.add_argument("--numStates", help="Number of states in model",
                        type = int, default=2)
    parser.add_argument("--iter", help="Number of EM iterations",
                        type = int, default=10)
    parser.add_argument("--supervised", help="Use name (4th) column of "
                        "<traingingBed> for the true hidden states of the"
                        " model.  Transition parameters will be estimated"
                        " directly from this information rather than EM."
                        " NOTE: The number of states will be determined "
                        "from the bed.  States must be labled 0,1,2 etc.",
                        action = "store_true", default = False)
    parser.add_argument("--verbose", help="Print out detailed logging messages",
                        action = "store_true", default = False)

    args = parser.parse_args()
    if args.verbose is True:
        logging.basicConfig(level=logging.DEBUG)

    # read training intervals from the bed file
    logging.info("loading training intervals from %s" % args.trainingBed)
    mergedIntervals = getMergedBedIntervals(args.trainingBed, ncol=4)
    if mergedIntervals is None or len(mergedIntervals) < 1:
        raise RuntimeError("Could not read any intervals from %s" %
                           args.trainingBed)

    # read the tracks, while intersecting them with the training intervals
    logging.info("loading tracks %s" % args.tracksInfo)
    trackData = TrackData()
    trackData.loadTrackData(args.tracksInfo, mergedIntervals)

    catMap = None
    truthIntervals = None
    # state number is overrided by the input bed file in supervised mode
    if args.supervised is True:
        logging.info("processing supervised state names")
        # we reload because we don't want to be merging them here
        truthIntervals = readBedIntervals(args.trainingBed, ncol=4)
        catMap = mapStateNames(truthIntervals)
        args.numStates = len(catMap)

    # create the independent emission model
    logging.info("creating model")
    numSymbolsPerTrack = trackData.getNumSymbolsPerTrack()
    emissionModel = IndependentMultinomialEmissionModel(args.numStates,
                                                        numSymbolsPerTrack)

    # create the hmm
    hmm = MultitrackHmm(emissionModel, n_iter=args.iter, state_name_map=catMap)

    # do the training
    if args.supervised is False:
        logging.info("training via EM")
        hmm.train(trackData)
    else:
        logging.info("training from input bed states")
        hmm.supervisedTrain(trackData, truthIntervals)

    # write the model to a pickle
    logging.info("saving trained model to %s" % args.outputModel)
    hmm.save(args.outputModel)

def mapStateNames(bedIntervals):
    """ sanitize the states (column 4) of each bed interval, mapping to unique
    integer in place.  return the map"""
    catMap = CategoryMap()
    for idx, interval in enumerate(bedIntervals):
        if len(interval) < 4 or interval[3] is None:
            raise RuntimeError("Could not read state from 4th column" %
                               str(interval))
        bedIntervals[idx] = (interval[0], interval[1], interval[2],
                             catMap.getMap(interval[3], update=True))
    return catMap

if __name__ == "__main__":
    sys.exit(main())

<commit_msg>fix loggign option<commit_after>#!/usr/bin/env python

#Copyright (C) 2013 by Glenn Hickey
#
#Released under the MIT license, see LICENSE.txt
import unittest
import sys
import os
import argparse
import logging

from teHmm.track import TrackData
from teHmm.trackIO import readBedIntervals, getMergedBedIntervals
from teHmm.hmm import MultitrackHmm
from teHmm.emission import IndependentMultinomialEmissionModel
from teHmm.track import CategoryMap

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="Create a teHMM")

    parser.add_argument("tracksInfo", help="Path of Tracks Info file "
                        "containing paths to genome annotation tracks")
    parser.add_argument("trainingBed", help="Path of BED file containing"
                        " elements to train  model on")
    parser.add_argument("outputModel", help="Path of output hmm")
    parser.add_argument("--numStates", help="Number of states in model",
                        type = int, default=2)
    parser.add_argument("--iter", help="Number of EM iterations",
                        type = int, default=10)
    parser.add_argument("--supervised", help="Use name (4th) column of "
                        "<traingingBed> for the true hidden states of the"
                        " model.  Transition parameters will be estimated"
                        " directly from this information rather than EM."
                        " NOTE: The number of states will be determined "
                        "from the bed.  States must be labled 0,1,2 etc.",
                        action = "store_true", default = False)
    parser.add_argument("--verbose", help="Print out detailed logging messages",
                        action = "store_true", default = False)

    args = parser.parse_args()
    if args.verbose is True:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    # read training intervals from the bed file
    logging.info("loading training intervals from %s" % args.trainingBed)
    mergedIntervals = getMergedBedIntervals(args.trainingBed, ncol=4)
    if mergedIntervals is None or len(mergedIntervals) < 1:
        raise RuntimeError("Could not read any intervals from %s" %
                           args.trainingBed)

    # read the tracks, while intersecting them with the training intervals
    logging.info("loading tracks %s" % args.tracksInfo)
    trackData = TrackData()
    trackData.loadTrackData(args.tracksInfo, mergedIntervals)

    catMap = None
    truthIntervals = None
    # state number is overrided by the input bed file in supervised mode
    if args.supervised is True:
        logging.info("processing supervised state names")
        # we reload because we don't want to be merging them here
        truthIntervals = readBedIntervals(args.trainingBed, ncol=4)
        catMap = mapStateNames(truthIntervals)
        args.numStates = len(catMap)

    # create the independent emission model
    logging.info("creating model")
    numSymbolsPerTrack = trackData.getNumSymbolsPerTrack()
    emissionModel = IndependentMultinomialEmissionModel(args.numStates,
                                                        numSymbolsPerTrack)

    # create the hmm
    hmm = MultitrackHmm(emissionModel, n_iter=args.iter, state_name_map=catMap)

    # do the training
    if args.supervised is False:
        logging.info("training via EM")
        hmm.train(trackData)
    else:
        logging.info("training from input bed states")
        hmm.supervisedTrain(trackData, truthIntervals)

    # write the model to a pickle
    logging.info("saving trained model to %s" % args.outputModel)
    hmm.save(args.outputModel)

def mapStateNames(bedIntervals):
    """ sanitize the states (column 4) of each bed interval, mapping to unique
    integer in place.  return the map"""
    catMap = CategoryMap()
    for idx, interval in enumerate(bedIntervals):
        if len(interval) < 4 or interval[3] is None:
            raise RuntimeError("Could not read state from 4th column" %
                               str(interval))
        bedIntervals[idx] = (interval[0], interval[1], interval[2],
                             catMap.getMap(interval[3], update=True))
    return catMap

if __name__ == "__main__":
    sys.exit(main())

<|endoftext|>


<commit_before>from __future__ import absolute_import

import unittest

import os
import os.path

def isTestFileName(fn):
	return len(fn) >= 9 and fn[:5] == "test_" and fn[-3:] == '.py'

def moduleName(base, root, fn):
	fullPath = os.path.join(root, fn)
	path, ext = os.path.splitext(fullPath)
	path = removePrefix(base, path)
	path = path.replace(os.path.sep, ".")
	return path

def removePrefix(base, path):
	assert path[:len(base)] == base
	return path[len(base)+len(os.path.sep):]

def wantFile(filename, only, exclude):
	old = filename

	base, ext = os.path.splitext(old)

	while old != base:
		if base in only:
			return True

		if base in exclude:
			return False

		old = base
		base, part = os.path.split(old)

	return not only

def findTests(base, only, exclude):
	names = []

	for root, dirs, files in os.walk(base):
		for fn in files:
			if isTestFileName(fn) and wantFile(os.path.join(root, fn), only, exclude):
				name = moduleName(base, root, fn)
				names.append(name)
	return names

def runTests(path, only=set(), exclude=set()):
	# If no tests are specified, find them.
	if not os.path.isdir(path):
		raise TypeError, '"%s" is not a directory.' % path

	testList = findTests(path, only, exclude)

	print "/========== Tests ==========\\"
	for test in testList:
		print "| %s" % test
	print "\\===========================/"
	print

	suite = unittest.defaultTestLoader.loadTestsFromNames(testList)

	#unittest.TextTestRunner(verbosity=2).run(suite)
	unittest.TextTestRunner().run(suite)
<commit_msg>Test runner no longer masks import errors.<commit_after>from __future__ import absolute_import

import unittest

import os
import os.path

def isTestFileName(fn):
	return len(fn) >= 9 and fn[:5] == "test_" and fn[-3:] == '.py'

def moduleName(base, root, fn):
	fullPath = os.path.join(root, fn)
	path, ext = os.path.splitext(fullPath)
	path = removePrefix(base, path)
	path = path.replace(os.path.sep, ".")
	return path

def removePrefix(base, path):
	assert path[:len(base)] == base
	return path[len(base)+len(os.path.sep):]

def wantFile(filename, only, exclude):
	old = filename

	base, ext = os.path.splitext(old)

	while old != base:
		if base in only:
			return True

		if base in exclude:
			return False

		old = base
		base, part = os.path.split(old)

	return not only

def findTests(base, only, exclude):
	names = []

	for root, dirs, files in os.walk(base):
		for fn in files:
			if isTestFileName(fn) and wantFile(os.path.join(root, fn), only, exclude):
				name = moduleName(base, root, fn)
				names.append(name)
	return names

def getModule(name):
	parts  = name.split('.')
	module = __import__(name)

	for part in parts[1:]:
		module = getattr(module, part)

	return module

# NOTE unlike the builtin version, this does NOT catch import errors.  This helps debugging.
# TODO turn import errors into test errors?
def loadTestsFromNames(self, names):
	suites = [self.loadTestsFromModule(getModule(name)) for name in names]
	return self.suiteClass(suites)

def runTests(path, only=set(), exclude=set()):
	# If no tests are specified, find them.
	if not os.path.isdir(path):
		raise TypeError, '"%s" is not a directory.' % path

	testList = findTests(path, only, exclude)

	print "/========== Tests ==========\\"
	for test in testList:
		print "| %s" % test
	print "\\===========================/"
	print

	suite = loadTestsFromNames(unittest.defaultTestLoader, testList)

	#unittest.TextTestRunner(verbosity=2).run(suite)
	unittest.TextTestRunner().run(suite)
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

try:
	from functools import wraps
except ImportError:
	# this module doesn't exist in python < 2.5
	# we define a identity decorator
	def wraps(f):
		def identity(x):
			return x
		return identity
<commit_msg>[FIX] fix the decorator "@wraps" to work on python < 2.5<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

__all__ = ['partial', 'wraps', 'update_wrapper']

try:
    from functools import partial, wraps, update_wrapper
except ImportError:
    # The functools module doesn't exist in python < 2.5
    # Code taken from python 2.5
    # http://svn.python.org/view/python/tags/r254/Lib/functools.py?view=markup

    def partial(fun, *args, **kwargs):
        def _partial(*args2, **kwargs2):
            return fun(*(args+args2), **dict(kwargs, **kwargs2))
        return _partial

    ### --- code from python 2.5

    # update_wrapper() and wraps() are tools to help write
    # wrapper functions that can handle naive introspection

    WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__doc__')
    WRAPPER_UPDATES = ('__dict__',)
    def update_wrapper(wrapper,
                       wrapped,
                       assigned = WRAPPER_ASSIGNMENTS,
                       updated = WRAPPER_UPDATES):
        """Update a wrapper function to look like the wrapped function

           wrapper is the function to be updated
           wrapped is the original function
           assigned is a tuple naming the attributes assigned directly
           from the wrapped function to the wrapper function (defaults to
           functools.WRAPPER_ASSIGNMENTS)
           updated is a tuple naming the attributes off the wrapper that
           are updated with the corresponding attribute from the wrapped
           function (defaults to functools.WRAPPER_UPDATES)
        """
        for attr in assigned:
            setattr(wrapper, attr, getattr(wrapped, attr))
        for attr in updated:
            getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
        # Return the wrapper so this can be used as a decorator via partial()
        return wrapper

    def wraps(wrapped,
              assigned = WRAPPER_ASSIGNMENTS,
              updated = WRAPPER_UPDATES):
        """Decorator factory to apply update_wrapper() to a wrapper function

           Returns a decorator that invokes update_wrapper() with the decorated
           function as the wrapper argument and the arguments to wraps() as the
           remaining arguments. Default arguments are as for update_wrapper().
           This is a convenience function to simplify applying partial() to
           update_wrapper().
        """
        return partial(update_wrapper, wrapped=wrapped,
                       assigned=assigned, updated=updated)

<|endoftext|>


<commit_before>import os
import json

class Struct:
    def __init__(self, **args):
        self.__dict__.update(args)

try:
    nar_path = os.environ["NARRATIVEDIR"]
    config_json = open(os.path.join(nar_path, "config.json")).read()
    config = json.loads(config_json)
    url_config = config[config['config']]  #fun, right?

    URLS = Struct(**url_config)
except:
    url_dict = {
        "workspace" : "https://kbase.us/services/ws/",
        "invocation" : "https://kbase.us/services/invocation",
        "fba" : "https://kbase.us/services/KBaseFBAModeling",
        "genomeCmp" : "https://kbase.us/services/genome_comparison/jsonrpc",
        "trees" : "https://kbase.us/services/trees"
    }
    URLS = Struct(**url_dict)
<commit_msg>Temporarily hard coded host and port for log proxy<commit_after>import os
import json

class Struct:
    def __init__(self, **args):
        self.__dict__.update(args)

try:
    nar_path = os.environ["NARRATIVEDIR"]
    config_json = open(os.path.join(nar_path, "config.json")).read()
    config = json.loads(config_json)
    url_config = config[config['config']]  #fun, right?

    URLS = Struct(**url_config)
except:
    url_dict = {
        "workspace" : "https://kbase.us/services/ws/",
        "invocation" : "https://kbase.us/services/invocation",
        "fba" : "https://kbase.us/services/KBaseFBAModeling",
        "genomeCmp" : "https://kbase.us/services/genome_comparison/jsonrpc",
        "trees" : "https://kbase.us/services/trees",
        "log_proxy_port" : 32001,
        "log_proxy_host" : "172.17.42.1"
    }
    URLS = Struct(**url_dict)
<|endoftext|>


<commit_before>from __future__ import division
from __future__ import print_function

import numpy as np

class FitFunction:
    def __init__(self, function, num_fit_params, force_zero=None, name=None):
        self.fn = function
        self.num_fit_params = num_fit_params
        self.fz = force_zero
        if name is not None:
            self.__name__ = name
        else:
            self.__name__ = self.fn.__name__

    def eval(self, x, params, const_list, const_dict):
        if self.fz is not None:
            f = self.fn
            return (f(x, params, const_list, const_dict) -
                    f(self.fz, params, const_list, const_dict))
        else:
            return self.fn(x, params, const_list, const_dict)

def combine(list_of_ffn, force_zero=None, name_pref='', name_sep=', '):
    """Combines multiple fit functions (and/or dependencies) into one fit
    function.

    :param list_of_ffn: A list of FitFunctions to combine into one
    :param force_zero: (optional) force the zero of the overall result to a
    specified point
    :param name_pref: prefix for the name of the combined fit function
    :param name_sep: separator to go between fit functions
    :return: A combined fit function object, which may be used to optimize with
    respect to all of the degrees of freedom of its sub-functions
    """
    params_lengths = [ffn.num_fit_params for ffn in list_of_ffn]
    params_breaks = [0]
    for pl, i in zip(params_lengths, range(len(params_lengths))):
        params_breaks.append(pl + params_breaks[i])
    total_params_length = params_breaks[-1]
    combined_name = name_pref
    for ffn in list_of_ffn:
        combined_name += ffn.__name__ + name_sep
    combined_name = combined_name[:combined_name.rfind(name_sep)]

    def combined_ffns(x, params, const_list, const_dict):
        result = 0
        for ffn, i, j in zip(list_of_ffn, params_breaks, params_breaks[1:]):
            result += ffn.eval(x, params[i:j], const_list, const_dict)
        return result

    return FitFunction(combined_ffns, total_params_length,
                       force_zero=force_zero, name=combined_name)

# INDEPENDENT
def scalar():
    def sf(x, params, const_list, const_dict):
        a = params[0]
        return a
    return FitFunction(sf, 1, name='scalar')

def x1(force_zero=None):
    def x1f(x, params, const_list, const_dict):
        a = params[0]
        return a * x
    return FitFunction(x1f, 1, force_zero, name='x^1')

def linear(force_zero=None):
    if force_zero is None:
        def lf(x, params, const_list, const_dict):
            a, b = params[0:2]
            return a * x + b
        return FitFunction(lf, 2, force_zero, name='linear')
    else:
        def lf(x, params, const_list, const_dict):
            a = params[0]
            return a * x
        return FitFunction(lf, 1, force_zero, name='linear')

def x2(force_zero=None):
    def x2f(x, params, const_list, const_dict):
        a = params[0]
        return a * x ** 2
    return FitFunction(x2f, 1, force_zero, name='x^2')

def quadratic(force_zero=None):
    if force_zero is None:
        def qf(x, params, const_list, const_dict):
            a, b, c = params[0:3]
            return np.polyval([a, b, c], x)
        return FitFunction(qf, 3, force_zero, name='quadratic')
    else:
        def qf(x, params, const_list, const_dict):
            a, b = params[0:2]
            return np.polyval([a, b, 0], x)
        return FitFunction(qf, 2, force_zero, name='quadratic')

def x_power(n, force_zero=None):
    def xnf(x, params, const_list, const_dict):
        a = params[0]
        return a * x ** n
    return FitFunction(xnf, 1, force_zero, name='x^{}'.format(n))

def poly(n, force_zero=None):
    if force_zero is None:
        def pf(x, params, const_list, const_dict):
            return np.polyval(params, x)
        return FitFunction(pf, n+1, force_zero, name='poly{}'.format(n))
    else:
        def pf(x, params, const_list, const_dict):
            return np.polyval(np.concatenate((params, np.zeros(1))), x)
        return FitFunction(pf, n, force_zero, name='poly{}'.format(n))

def asymptote(n, force_zero=None):
    def af(x, params, const_list, const_dict):
        a = params[0]
        return - a / x**n
    return FitFunction(af, 1, force_zero, name='asymptote{}'.format(n))

def asymptote_n(force_zero=None):
    def anf(x, params, const_list, const_dict):
        a, n = params[0:2]
        return - a / x**n
    return FitFunction(anf, 2, force_zero, name='asymptote_n')

# DEPENDENTS
def scalar_dependence(dep_keys, ctfs=list()):
    return _dependence(f=lambda p, x: p[0],
                       n_params=1,
                       dep_keys=dep_keys,
                       ctfs=ctfs,
                       name='scalar dependence')

def x1_dependence(dep_keys, ctfs=list(), force_zero=None):
    return _dependence(lambda p, x: p[0] * x,
                       n_params=1,
                       dep_keys=dep_keys,
                       ctfs=ctfs,
                       force_zero=force_zero,
                       name='x dependence')

def linear_dependence(dep_keys, ctfs=list(), force_zero=None):
    if force_zero is None:
        return _dependence(np.polyval,
                           n_params=2,
                           dep_keys=dep_keys,
                           ctfs=ctfs,
                           force_zero=force_zero,
                           name='linear dependence')
    else:
        return _dependence(lambda p, x:
                           np.polyval(np.concatenate((p, np.zeros(1))), x),
                           n_params=1,
                           dep_keys=dep_keys,
                           ctfs=ctfs,
                           force_zero=force_zero,
                           name='linear dependence')

def x2_dependence(dep_keys, ctfs=list(), force_zero=None):
    return _dependence(lambda p, x: p[0] * x ** 2,
                       n_params=1,
                       dep_keys=dep_keys,
                       ctfs=ctfs,
                       force_zero=force_zero,
                       name='x^2 dependence')

def quadratic_dependence(dep_keys, ctfs=list(), force_zero=None):
    if force_zero is None:
        return _dependence(np.polyval,
                           n_params=3,
                           dep_keys=dep_keys,
                           ctfs=ctfs,
                           force_zero=force_zero,
                           name='quadratic dependence')
    else:
        return _dependence(lambda p, x:
                           np.polyval(np.concatenate((p, np.zeros(1))), x),
                           n_params=2,
                           dep_keys=dep_keys,
                           ctfs=ctfs,
                           force_zero=force_zero,
                           name='quadratic dependence')

def x_power_dependence(n, dep_keys, ctfs=list(), force_zero=None):
    return _dependence(lambda p, x: p[0] * x ** n,
                       n_params=1,
                       dep_keys=dep_keys,
                       ctfs=ctfs,
                       force_zero=force_zero,
                       name='x^{} dependence'.format(n))

def poly_dependence(n, dep_keys, ctfs=list(), force_zero=None):
    if force_zero is None:
        return _dependence(np.polyval,
                           n_params=n + 1,
                           dep_keys=dep_keys,
                           ctfs=ctfs,
                           force_zero=force_zero,
                           name='poly{n} dependence'.format(n=n))
    else:
        return _dependence(lambda p, x:
                           np.polyval(np.concatenate((p, np.zeros(1))), x),
                           n_params=n,
                           dep_keys=dep_keys,
                           ctfs=ctfs,
                           force_zero=force_zero,
                           name='poly{n} dependence'.format(n=n))

def asymptotic_dependence(n, dep_keys, ctfs=list(), force_zero=None):
    return _dependence(lambda p, x: - p[0] / x**n,
                       n_params=1,
                       dep_keys=dep_keys,
                       ctfs=ctfs,
                       force_zero=force_zero,
                       name='asymptotic{} dependence'.format(n))

def _dependence(f, n_params, dep_keys, name, ctfs=list(), force_zero=None):
    """An abstract function to determine f-dependence on constants given by
    dep_keys and ctfs

    :param f: f(p, x) -> y, a function that maps an array of parameters and an
    x value to a y value. Example: If one wants linear dependence on x,
    f(p, x) = p[0] * x + p[1], would be the correct function to use
    :param n_params: the number of parameters that f requires
    :param dep_keys: the keys to use with a constants dictionary to determine
    constants values. Example: If one wants dependence on the values of n and
    j, dep_keys=['n', 'j']
    :param name: the name of the dependence function
    :param ctfs: (Optional) constants transform functions are functions of the
    constants dictionary that return special combinations of the constants.
    Example: If one wants dependence on j^2, one would add the following
    function to ctfs: lambda cd: cd['j]^2
    :param force_zero: (Optional) an x value at which to force the dependence
    function to be 0
    :return: The dependence fit function
    """
    l1 = len(dep_keys) * n_params
    l2 = len(ctfs) * n_params

    def d(x, params, const_list, const_dict):
        more_constants = _do_transforms(ctfs, const_dict)
        p = np.zeros(n_params)
        dep_psubs = [params[i:i + n_params] for i in range(0, l1, n_params)]
        ctf_psubs = [params[i:i + n_params] for i in range(l1, l1+l2, n_params)]
        for dep in zip(dep_keys, *dep_psubs):
            k, p0 = dep[0], dep[1:]
            v = const_dict[k]
            for j, p0j in zip(range(n_params), p0):
                p[j] = p[j] + p0j * v
        for ctf in zip(more_constants, *ctf_psubs):
            c, p0 = ctf[0], ctf[1:]
            for j, p0j in zip(range(n_params), p0):
                p[j] = p[j] + p0j * c
        return f(p, x)
    return FitFunction(d,
                       num_fit_params=(len(dep_keys)+len(ctfs)) * n_params,
                       force_zero=force_zero,
                       name=name + ' on {}'.format(_dep_str(dep_keys, ctfs)))

def _dep_str(dep_keys, ctfs):
    return ('(' +
            ', '.join((dep_keys + list(map(lambda c: c.__name__, ctfs)))) +
            ')')

# FITTERS WITH DEPENDENCIES
def linear_with_linear_dependence(dep_keys, ctfs=list(), force_zero=None):
    return combine([linear(force_zero=force_zero),
                    linear_dependence(dep_keys, ctfs, force_zero=force_zero)],
                   force_zero=force_zero)

def poly_with_linear_dependence(n, dep_keys, ctfs=list(), force_zero=None):
    return combine([poly(n), linear_dependence(dep_keys, ctfs)],
                   force_zero=force_zero)

def asymptote_with_linear_dependence(n, dep_keys, ctfs=list(),
                                     force_zero=None):
    return combine([asymptote(n), linear_dependence(dep_keys, ctfs)],
                   force_zero=force_zero)

def asymptote_with_asymptotic_dependence(n, dep_keys, ctfs=list(),
                                         force_zero=None):
    return combine([asymptote(n), asymptotic_dependence(n, dep_keys, ctfs)],
                   force_zero=force_zero)

# CONSTANT TRANSFORMS
def _do_transforms(ctfs, const_dict):
    r = list()
    for ctf in ctfs:
        r.append(ctf(const_dict))
    return r

def joff2(const_dict):
    j = const_dict['j']
    return (j-1) * abs(j-1)

def jjoff(const_dict):
    j = const_dict['j']
    return j * (j - 1)

def ephw(const_dict):
    e = const_dict['e']
    hw = const_dict['hw']
    return e + hw

def y0pzbt0(const_dict):
    y0 = const_dict['y0']
    zbt0 = const_dict['zbt0']
    return y0 + zbt0
<commit_msg>Allowed use of a force_zero_func, which returns a zero based on a plot's specific constants.<commit_after>from __future__ import division
from __future__ import print_function

import numpy as np

class FitFunction:
    def __init__(self, function, num_fit_params, force_zero=None, name=None,
                 force_zero_func=None):
        self.fn = function
        self.num_fit_params = num_fit_params
        self.fz = force_zero
        self.fzfn = force_zero_func
        if name is not None:
            self.__name__ = name
        else:
            self.__name__ = self.fn.__name__

    def eval(self, x, params, const_list, const_dict):
        if self.fz is not None:
            f = self.fn
            return (f(x, params, const_list, const_dict) -
                    f(self.fz, params, const_list, const_dict))
        elif self.fzfn is not None:
            f = self.fn
            zero = self.fzfn(const_dict)
            return (f(x, params, const_list, const_dict) -
                    f(zero, params, const_list, const_dict))
        else:
            return self.fn(x, params, const_list, const_dict)

def combine(list_of_ffn, force_zero=None, name_pref='', name_sep=', ',
            **kwargs):
    """Combines multiple fit functions (and/or dependencies) into one fit
    function.

    :param list_of_ffn: A list of FitFunctions to combine into one
    :param force_zero: (optional) force the zero of the overall result to a
    specified point
    :param name_pref: prefix for the name of the combined fit function
    :param name_sep: separator to go between fit functions
    :return: A combined fit function object, which may be used to optimize with
    respect to all of the degrees of freedom of its sub-functions
    """
    params_lengths = [ffn.num_fit_params for ffn in list_of_ffn]
    params_breaks = [0]
    for pl, i in zip(params_lengths, range(len(params_lengths))):
        params_breaks.append(pl + params_breaks[i])
    total_params_length = params_breaks[-1]
    combined_name = name_pref
    for ffn in list_of_ffn:
        combined_name += ffn.__name__ + name_sep
    combined_name = combined_name[:combined_name.rfind(name_sep)]

    def combined_ffns(x, params, const_list, const_dict):
        result = 0
        for ffn, i, j in zip(list_of_ffn, params_breaks, params_breaks[1:]):
            result += ffn.eval(x, params[i:j], const_list, const_dict)
        return result

    return FitFunction(combined_ffns, total_params_length,
                       force_zero=force_zero, name=combined_name, **kwargs)

# INDEPENDENT
def scalar():
    def sf(x, params, const_list, const_dict):
        a = params[0]
        return a
    return FitFunction(sf, 1, name='scalar')

def x1(force_zero=None, **kwargs):
    def x1f(x, params, const_list, const_dict):
        a = params[0]
        return a * x
    return FitFunction(x1f, 1, force_zero, name='x^1', **kwargs)

def linear(force_zero=None, **kwargs):
    if force_zero is None:
        def lf(x, params, const_list, const_dict):
            a, b = params[0:2]
            return a * x + b
        return FitFunction(lf, 2, force_zero, name='linear', **kwargs)
    else:
        def lf(x, params, const_list, const_dict):
            a = params[0]
            return a * x
        return FitFunction(lf, 1, force_zero, name='linear', **kwargs)

def x2(force_zero=None, **kwargs):
    def x2f(x, params, const_list, const_dict):
        a = params[0]
        return a * x ** 2
    return FitFunction(x2f, 1, force_zero, name='x^2', **kwargs)

def quadratic(force_zero=None, **kwargs):
    if force_zero is None:
        def qf(x, params, const_list, const_dict):
            a, b, c = params[0:3]
            return np.polyval([a, b, c], x)
        return FitFunction(qf, 3, force_zero, name='quadratic', **kwargs)
    else:
        def qf(x, params, const_list, const_dict):
            a, b = params[0:2]
            return np.polyval([a, b, 0], x)
        return FitFunction(qf, 2, force_zero, name='quadratic', **kwargs)

def x_power(n, force_zero=None, **kwargs):
    def xnf(x, params, const_list, const_dict):
        a = params[0]
        return a * x ** n
    return FitFunction(xnf, 1, force_zero, name='x^{}'.format(n), **kwargs)

def poly(n, force_zero=None, **kwargs):
    if force_zero is None:
        def pf(x, params, const_list, const_dict):
            return np.polyval(params, x)
        return FitFunction(pf, n+1, force_zero, name='poly{}'.format(n),
                           **kwargs)
    else:
        def pf(x, params, const_list, const_dict):
            return np.polyval(np.concatenate((params, np.zeros(1))), x)
        return FitFunction(pf, n, force_zero, name='poly{}'.format(n), **kwargs)

def asymptote(n, force_zero=None, **kwargs):
    def af(x, params, const_list, const_dict):
        a = params[0]
        return - a / x**n
    return FitFunction(af, 1, force_zero, name='asymptote{}'.format(n),
                       **kwargs)

def asymptote_n(force_zero=None, **kwargs):
    def anf(x, params, const_list, const_dict):
        a, n = params[0:2]
        return - a / x**n
    return FitFunction(anf, 2, force_zero, name='asymptote_n', **kwargs)

# DEPENDENTS
def scalar_dependence(dep_keys, ctfs=list()):
    return _dependence(f=lambda p, x: p[0],
                       n_params=1,
                       dep_keys=dep_keys,
                       ctfs=ctfs,
                       name='scalar dependence')

def x1_dependence(dep_keys, ctfs=list(), force_zero=None, **kwargs):
    return _dependence(lambda p, x: p[0] * x,
                       n_params=1,
                       dep_keys=dep_keys,
                       ctfs=ctfs,
                       force_zero=force_zero,
                       name='x dependence',
                       **kwargs)

def linear_dependence(dep_keys, ctfs=list(), force_zero=None, **kwargs):
    if force_zero is None:
        return _dependence(np.polyval,
                           n_params=2,
                           dep_keys=dep_keys,
                           ctfs=ctfs,
                           force_zero=force_zero,
                           name='linear dependence',
                           **kwargs)
    else:
        return _dependence(lambda p, x:
                           np.polyval(np.concatenate((p, np.zeros(1))), x),
                           n_params=1,
                           dep_keys=dep_keys,
                           ctfs=ctfs,
                           force_zero=force_zero,
                           name='linear dependence',
                           **kwargs)

def x2_dependence(dep_keys, ctfs=list(), force_zero=None, **kwargs):
    return _dependence(lambda p, x: p[0] * x ** 2,
                       n_params=1,
                       dep_keys=dep_keys,
                       ctfs=ctfs,
                       force_zero=force_zero,
                       name='x^2 dependence',
                       **kwargs)

def quadratic_dependence(dep_keys, ctfs=list(), force_zero=None, **kwargs):
    if force_zero is None:
        return _dependence(np.polyval,
                           n_params=3,
                           dep_keys=dep_keys,
                           ctfs=ctfs,
                           force_zero=force_zero,
                           name='quadratic dependence',
                           **kwargs)
    else:
        return _dependence(lambda p, x:
                           np.polyval(np.concatenate((p, np.zeros(1))), x),
                           n_params=2,
                           dep_keys=dep_keys,
                           ctfs=ctfs,
                           force_zero=force_zero,
                           name='quadratic dependence',
                           **kwargs)

def x_power_dependence(n, dep_keys, ctfs=list(), force_zero=None, **kwargs):
    return _dependence(lambda p, x: p[0] * x ** n,
                       n_params=1,
                       dep_keys=dep_keys,
                       ctfs=ctfs,
                       force_zero=force_zero,
                       name='x^{} dependence'.format(n),
                       **kwargs)

def poly_dependence(n, dep_keys, ctfs=list(), force_zero=None, **kwargs):
    if force_zero is None:
        return _dependence(np.polyval,
                           n_params=n + 1,
                           dep_keys=dep_keys,
                           ctfs=ctfs,
                           force_zero=force_zero,
                           name='poly{n} dependence'.format(n=n),
                           **kwargs)
    else:
        return _dependence(lambda p, x:
                           np.polyval(np.concatenate((p, np.zeros(1))), x),
                           n_params=n,
                           dep_keys=dep_keys,
                           ctfs=ctfs,
                           force_zero=force_zero,
                           name='poly{n} dependence'.format(n=n),
                           **kwargs)

def asymptotic_dependence(n, dep_keys, ctfs=list(), force_zero=None, **kwargs):
    return _dependence(lambda p, x: - p[0] / x**n,
                       n_params=1,
                       dep_keys=dep_keys,
                       ctfs=ctfs,
                       force_zero=force_zero,
                       name='asymptotic{} dependence'.format(n),
                       **kwargs)

def _dependence(f, n_params, dep_keys, name, ctfs=list(), force_zero=None,
                **kwargs):
    """An abstract function to determine f-dependence on constants given by
    dep_keys and ctfs

    :param f: f(p, x) -> y, a function that maps an array of parameters and an
    x value to a y value. Example: If one wants linear dependence on x,
    f(p, x) = p[0] * x + p[1], would be the correct function to use
    :param n_params: the number of parameters that f requires
    :param dep_keys: the keys to use with a constants dictionary to determine
    constants values. Example: If one wants dependence on the values of n and
    j, dep_keys=['n', 'j']
    :param name: the name of the dependence function
    :param ctfs: (Optional) constants transform functions are functions of the
    constants dictionary that return special combinations of the constants.
    Example: If one wants dependence on j^2, one would add the following
    function to ctfs: lambda cd: cd['j]^2
    :param force_zero: (Optional) an x value at which to force the dependence
    function to be 0
    :return: The dependence fit function
    """
    l1 = len(dep_keys) * n_params
    l2 = len(ctfs) * n_params

    def d(x, params, const_list, const_dict):
        more_constants = _do_transforms(ctfs, const_dict)
        p = np.zeros(n_params)
        dep_psubs = [params[i:i + n_params] for i in range(0, l1, n_params)]
        ctf_psubs = [params[i:i + n_params] for i in range(l1, l1+l2, n_params)]
        for dep in zip(dep_keys, *dep_psubs):
            k, p0 = dep[0], dep[1:]
            v = const_dict[k]
            for j, p0j in zip(range(n_params), p0):
                p[j] = p[j] + p0j * v
        for ctf in zip(more_constants, *ctf_psubs):
            c, p0 = ctf[0], ctf[1:]
            for j, p0j in zip(range(n_params), p0):
                p[j] = p[j] + p0j * c
        return f(p, x)
    return FitFunction(d,
                       num_fit_params=(len(dep_keys)+len(ctfs)) * n_params,
                       force_zero=force_zero,
                       name=name + ' on {}'.format(_dep_str(dep_keys, ctfs)),
                       **kwargs)

def _dep_str(dep_keys, ctfs):
    return ('(' +
            ', '.join((dep_keys + list(map(lambda c: c.__name__, ctfs)))) +
            ')')

# FITTERS WITH DEPENDENCIES
def linear_with_linear_dependence(dep_keys, ctfs=list(), force_zero=None,
                                  **kwargs):
    return combine([linear(force_zero=force_zero),
                    linear_dependence(dep_keys, ctfs, force_zero=force_zero)],
                   force_zero=force_zero,
                   **kwargs)

def poly_with_linear_dependence(n, dep_keys, ctfs=list(), force_zero=None,
                                **kwargs):
    return combine([poly(n), linear_dependence(dep_keys, ctfs)],
                   force_zero=force_zero,
                   **kwargs)

def asymptote_with_linear_dependence(n, dep_keys, ctfs=list(),
                                     force_zero=None,
                                     **kwargs):
    return combine([asymptote(n), linear_dependence(dep_keys, ctfs)],
                   force_zero=force_zero,
                   **kwargs)

def asymptote_with_asymptotic_dependence(n, dep_keys, ctfs=list(),
                                         force_zero=None, **kwargs):
    return combine([asymptote(n), asymptotic_dependence(n, dep_keys, ctfs)],
                   force_zero=force_zero, **kwargs)

# CONSTANT TRANSFORMS
def _do_transforms(ctfs, const_dict):
    r = list()
    for ctf in ctfs:
        r.append(ctf(const_dict))
    return r

def joff2(const_dict):
    j = const_dict['j']
    return (j-1) * abs(j-1)

def jjoff(const_dict):
    j = const_dict['j']
    return j * (j - 1)

def ephw(const_dict):
    e = const_dict['e']
    hw = const_dict['hw']
    return e + hw

def y0pzbt0(const_dict):
    y0 = const_dict['y0']
    zbt0 = const_dict['zbt0']
    return y0 + zbt0

# FORCE ZERO FUNCTIONS
def fz_to_x0(const_dict):
    return const_dict['x0']
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Xcode打包工具类

Author: liyan
File: xcode_builder.py

Features:
1. read_local_version() 获取项目版本号
2. build_xcode_project() 给xcode 项目打 ipa 包
3. make_ota_link() 生成OTA下载链接
"""
from util import Command

def read_local_version():
	"""read get version command from a file,
	excute command to get local project's version
	"""
	try:
		# grep MyEditionId ../MyServiceDefine.h |cut -d "\\"" -f 2
		version_cmd = open('version_cmd').readline().strip()
		(cost, out, err) = Command.excute(version_cmd)
		if len(err) > 0:
			print version_cmd, '\nerr: ', err
		return out.strip('\n')
	except:
		return 'unknown'

def build_xcode_project(args):
	"""调用方法: python ~/apk-builder/xcode_builder.py -a ipa --ipa SDK_V5 -p FLGamePlatformDemo -s FLGamePlatformDemoRelease --clean
工作步骤：
	1. 获取项目版本号
	2. 获取当前svn版本号
	3. 打包
	4. 重命名ipa为%(export_path)s/%(scheme_name)s_V%(version)s(%(build)s).ipa
	"""
	version = read_local_version()
	print 'ipa file =', args.ipa, 'version =', version
	# xcodebuild -list
	Command.xcodebuild_ipa(project=args.project+'.xcodeproj', scheme=args.scheme, export=args.export, is_clean=args.clean, dry_run=args.dry_run)
	rename_cmd = 'mv %(export_path)s/%(scheme_name)s.ipa %(export_path)s/%(ipa_file)s_V%(version)s\\(%(build)s\\).ipa' % {
			'scheme_name': args.scheme,
			'export_path': args.export,
			'ipa_file': args.ipa,
			'version': version,
			'build': Command.svn_ver('..')
		}
	(cost, out, err) = Command.excute(rename_cmd, args.dry_run)
	print err
def make_ota_link():
	"""生成OTA下载链接
	"""

def main():
	import argparse
	parser = argparse.ArgumentParser(description='Xcode Builder.')
	parser.add_argument('-a', dest='action', type=str, default='ipa', choices=('test', 'lib', 'ipa'),
											help='Action perform')
	parser.add_argument('--ipa', dest='ipa', type=str, default='',
											help='the IPA file name')
	parser.add_argument('-p', dest='project', type=str, default='Demo.xcodeproj',
											help='Xcode Project Folder Name')
	parser.add_argument('-s', dest='scheme', type=str, default='DemoRelease',
											help='Xcode Project Scheme')
	parser.add_argument('-e', dest='export', type=str, default='./build',
											help='Build IPA to this directory')
	parser.add_argument('--clean', dest='clean', action='store_false',
											help='Remove files in CONFIGURATION_BUILD_DIR & CONFIGURATION_TEMP_DIR')
	parser.set_defaults(clean=True)
	parser.add_argument('--dry-run', dest='dry_run', action='store_true',
											help='Dry Run Mode: do not excute time-consuming operation')
	parser.set_defaults(dry_run=False)

	args = parser.parse_args()

	if args.dry_run: print 'in dry-run mode'

	if len(args.ipa) ==0: args.ipa = args.scheme

	if args.action == 'test':
		Command.xcodebuild_test(project=args.project+'.xcodeproj', scheme=args.scheme)
	elif args.action == 'lib':
		Command.xcodebuild_lib(project=args.project+'.xcodeproj', scheme=args.scheme, is_clean=True)
	elif args.action == 'ipa':
		build_xcode_project(args)

if __name__ == '__main__':
	import os, sys, logging
	config_file = os.path.join(sys.path[0],'logging.conf')
	try:
		logging.config.fileConfig( config_file )
	except Exception, ex:
		print 'Failed to load logging config file: ', config_file ,'\nCatch Exception: ', ex
		exit(-1)
	main()<commit_msg>增加生成ota plist的能力 python xcode_builder.py -a ipa -p DemoProject -s DemoRelease --ipa SDK_V5 --root /srv/www/html/ota --url https://123/456/ --clean --dry-run<commit_after># -*- coding: utf-8 -*-
"""
Xcode打包工具类

Author: liyan
File: xcode_builder.py

Features:
1. read_version_from_project() 获取项目版本号
2. build_xcode_project() 给xcode 项目打 ipa 包
3. make_ota_link() 生成OTA下载链接
"""
from util import Command, PlistBuddy

def read_version_from_project():
	"""read get version command from a file,
	excute command to get local project's version
	"""
	try:
		# grep MyEditionId ../MyServiceDefine.h |cut -d "\\"" -f 2
		version_cmd = open('version_cmd').readline().strip()
		(cost, out, err) = Command.excute(version_cmd)
		if len(err) > 0:
			print version_cmd, '\nerr: ', err
		return out.strip('\n')
	except:
		return 'unknown'

def build_xcode_project(args):
	"""调用方法: python ~/apk-builder/xcode_builder.py -a ipa --ipa SDK_V5 -p FLGamePlatformDemo -s FLGamePlatformDemoRelease --clean
工作步骤：
	1. XCode项目打包
	2. 重命名ipa为 %(ipa_name)s_v%(code_ver)s_r%(repo_ver)s_b%(build_num)s.ipa
	例如： SDK_V5_v5.1.56_r168988_b62.jar
	3. create ota plist
	"""
	# 1. build xcode project and export ipa
	Command.xcodebuild_ipa(project=args.project+'.xcodeproj',
							scheme=args.scheme,
	 						export=args.export,
	 						is_clean=args.clean, dry_run=args.dry_run)
	print 'ipa file name:', args.ipa_name

	# 2. rename
	ipa_name = '%(ipa_name)s_v%(code_ver)s_r%(repo_ver)s_b%(build_num)s.ipa' % {
			'ipa_name': args.ipa_name,
			'code_ver': read_version_from_project(),
			'repo_ver': Command.svn_ver('..'),
			'build_num': os.environ.get('BUILD_NUMBER')
		}
	rename_cmd = 'mv %(export_path)s/%(scheme_name)s.ipa %(export_path)s/%(ipa_name)s' % {
			'scheme_name': args.scheme,
			'export_path': args.export,
			'ipa_name': ipa_name
		}
	(cost, out, err) = Command.excute(rename_cmd, args.dry_run)
	if len(err) == 0:
		logging.info('mission done for building [%s]'%ipa_name)
	else:
		logging.error('mission failed for building [%s]'%ipa_name)
	# 3. create ota plist
	# cp ipa to nginx root path
	rename_cmd = 'cp %(export_path)s/%(ipa_name)s %(root_path)s' % {
			'export_path': args.export,
			'ipa_name': ipa_name,
			'root_path': args.root_path
		}
	(cost, out, err) = Command.excute(rename_cmd, args.dry_run)

	# create plist
	plistBuddy = PlistBuddy(args.root_path, ipa_name)
	plistBuddy.create_ota_plist( os.path.join(args.ipa_url, ipa_name), os.path.join(args.ipa_url, 'icon.png'))

def make_ota_link():
	"""生成OTA下载链接
	"""

def main():
	import argparse
	parser = argparse.ArgumentParser(description='Xcode Builder.')
	parser.add_argument('-a', dest='action', type=str, default='ipa', choices=('test', 'lib', 'ipa'),
											help='Action perform')
	parser.add_argument('--ipa', dest='ipa_name', type=str, default='',
											help='the IPA file name')
	parser.add_argument('--root', dest='root_path', type=str, default='.',
											help='put IPA here to make nginx find it')
	parser.add_argument('--url', dest='ipa_url', type=str, default='',
											help='the IPA download links, must be HTTPS!')
	parser.add_argument('-p', dest='project', type=str, default='Demo.xcodeproj',
											help='Xcode Project Folder Name')
	parser.add_argument('-s', dest='scheme', type=str, default='DemoRelease',
											help='Xcode Project Scheme')
	parser.add_argument('-e', dest='export', type=str, default='./build',
											help='Build IPA to this directory')
	parser.add_argument('--clean', dest='clean', action='store_false',
											help='Remove files in CONFIGURATION_BUILD_DIR & CONFIGURATION_TEMP_DIR')
	parser.set_defaults(clean=True)
	parser.add_argument('--dry-run', dest='dry_run', action='store_true',
											help='Dry Run Mode: do not excute time-consuming operation')
	parser.set_defaults(dry_run=False)

	args = parser.parse_args()

	if args.dry_run: print 'in dry-run mode'

	if len(args.ipa_name) ==0: args.ipa_name = args.scheme

	if args.action == 'test':
		Command.xcodebuild_test(project=args.project+'.xcodeproj', scheme=args.scheme)
	elif args.action == 'lib':
		Command.xcodebuild_lib(project=args.project+'.xcodeproj', scheme=args.scheme, is_clean=True)
	elif args.action == 'ipa':
		build_xcode_project(args)

if __name__ == '__main__':
	import os, sys, logging
	config_file = os.path.join(sys.path[0],'logging.conf')
	try:
		logging.config.fileConfig( config_file )
	except Exception, ex:
		print 'Failed to load logging config file: ', config_file ,'\nCatch Exception: ', ex
		exit(-1)
	main()<|endoftext|>