<commit_before>#!/usr/bin/python

import argparse
import os
import struct
import sys

CP_STRUCT_SIZES = { 0xa:3 }

###############
### CLASSES ###
###############
class MyParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

###################
### SUBROUTINES ###
###################
def lenCpStruct(tag):
  if tag in CP_STRUCT_SIZES:
    return CP_STRUCT_SIZES[tag]
  else:
    return -1

############
### MAIN ###
############
parser = MyParser('Run bytecode in jjvm')
parser.add_argument('path', help='path to class')
args = parser.parse_args()

with open(args.path, "rb") as c:
  c.seek(8)
  cpCount = struct.unpack(">H", c.read(2))[0] - 1

  print "Constant pool count: %d" % cpCount;

  while cpCount >= 0:
    cpTag = ord(c.read(1))

    print "Got tag: %d" % cpTag
    cpStructSize = lenCpStruct(cpTag)

    if cpStructSize < 0:
      print "ERROR: cpStructSize %d for tag %d" % (cpStructSize, cpTag)
      sys.exit(1)

    print "Size: %d" % cpStructSize

    cpCount -= 1
    c.seek(cpStructSize, os.SEEK_CUR)
<commit_msg>correct calculation for offset<commit_after>#!/usr/bin/python

import argparse
import os
import struct
import sys

CP_STRUCT_SIZES = { 0xa:5 }

###############
### CLASSES ###
###############
class MyParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

###################
### SUBROUTINES ###
###################
def lenCpStruct(tag):
  if tag in CP_STRUCT_SIZES:
    return CP_STRUCT_SIZES[tag]
  else:
    return -1

############
### MAIN ###
############
parser = MyParser('Run bytecode in jjvm')
parser.add_argument('path', help='path to class')
args = parser.parse_args()

with open(args.path, "rb") as c:
  c.seek(8)
  cpCount = struct.unpack(">H", c.read(2))[0] - 1

  print "Constant pool count: %d" % cpCount;

  while cpCount >= 0:
    cpTag = ord(c.read(1))

    print "Got tag: %d" % cpTag
    cpStructSize = lenCpStruct(cpTag)

    if cpStructSize < 0:
      print "ERROR: cpStructSize %d for tag %d" % (cpStructSize, cpTag)
      sys.exit(1)

    print "Size: %d" % cpStructSize

    cpCount -= 1
    c.seek(cpStructSize - 1, os.SEEK_CUR)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2011 Nigel Small
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Cypher utility module
"""

import argparse
try:
	import json
except ImportError:
	import simplejson as json
import neo4j
import string
import sys

__author__    = "Nigel Small <py2neo@nigelsmall.org>"
__copyright__ = "Copyright 2011 Nigel Small"
__license__   = "Apache License, Version 2.0"

def execute(query, database_uri=None):
	database_uri = database_uri or "http://localhost:7474/db/data/"
	graphdb = neo4j.GraphDatabaseService(database_uri)
	response = graphdb.execute_cypher_query(query)
	data, columns = response['data'], response['columns']
	return data, columns

def output(value):
	if isinstance(value, dict):
		if 'type' in value:
			# relationship
			out = "\"(%s)-[%d:%s]->(%s)\"" % (
				value['start'].rpartition("/")[2],
				int(value['self'].rpartition("/")[2]),
				value['type'],
				value['end'].rpartition("/")[2]
			)
		else:
			# node
			out = "\"(%d)\"" % (
				int(value['self'].rpartition("/")[2])
			)
	else:
		# property
		out = json.dumps(value)
	return out

def output_text(value):
	if isinstance(value, dict):
		if 'type' in value:
			# relationship
			out = "(%s)-[%d:%s]->(%s)" % (
				value['start'].rpartition("/")[2],
				int(value['self'].rpartition("/")[2]),
				value['type'],
				value['end'].rpartition("/")[2]
			)
		else:
			# node
			out = "(%d)" % (
				int(value['self'].rpartition("/")[2])
			)
	else:
		# property
		out = str(value)
	return out

def execute_and_output_as_delimited(query, database_uri=None, field_delimiter="\t"):
	data, columns = execute(query, database_uri)
	print string.join([
		json.dumps(column)
		for column in columns
	], field_delimiter)
	for row in data:
		print string.join([
			output(value)
			for value in row
		], field_delimiter)

def execute_and_output_as_json(query, database_uri=None):
	data, columns = execute(query, database_uri)
	columns = [json.dumps(column) for column in columns]
	row_count = 0
	sys.stdout.write("[\n")
	for row in data:
		row_count += 1
		if row_count > 1:
			sys.stdout.write(",\n")
		sys.stdout.write("\t{" + string.join([
			columns[i] + ": " + output(row[i])
			for i in range(len(row))
		], ", ") + "}")
	sys.stdout.write("\n]\n")

def execute_and_output_as_geoff(query, database_uri=None):
	nodes = {}
	rels = {}
	def update_descriptors(value, column):
		if isinstance(value, dict):
			if 'type' in value:
				# relationship
				rels["(%s)-[%d:%s]->(%s)" % (
					value['start'].rpartition("/")[2],
					int(value['self'].rpartition("/")[2]),
					value['type'],
					value['end'].rpartition("/")[2]
				)] = value['data']
			else:
				# node
				nodes["(%d)" % (
					int(value['self'].rpartition("/")[2])
				)] = value['data']
		else:
			# property - not supported in GEOFF format
			pass
	data, columns = execute(query, database_uri)
	for row in data:
		for i in range(len(row)):
			update_descriptors(row[i], columns[i])
	for key, value in nodes.items():
		print "%s\t%s" % (
			key,
			json.dumps(value)
		)
	for key, value in rels.items():
		print "%s\t%s" % (
			key,
			json.dumps(value)
		)

def execute_and_output_as_text(query, database_uri=None, field_delimiter="\t"):
	data, columns = execute(query, database_uri)
	column_widths = [len(column) for column in columns]
	for row in data:
		column_widths = [
			max(column_widths[i], None if row[i] is None else len(output_text(row[i])))
			for i in range(len(row))
		]
	print "+-" + string.join([
		"".ljust(column_widths[i], "-")
		for i in range(len(columns))
	], "---") + "-+"
	print "| " + string.join([
		columns[i].ljust(column_widths[i])
		for i in range(len(columns))
	], " | ") + " |"
	print "+-" + string.join([
		"".ljust(column_widths[i], "-")
		for i in range(len(columns))
	], "---") + "-+"
	for row in data:
		print "| " + string.join([
			output_text(row[i]).ljust(column_widths[i])
			for i in range(len(row))
		], " | ") + " |"
	print "+-" + string.join([
		"".ljust(column_widths[i], "-")
		for i in range(len(columns))
	], "---") + "-+"

if __name__ == "__main__":
	parser = argparse.ArgumentParser(description="Execute Cypher queries against a Neo4j database server and output the results.")
	parser.add_argument("-u", metavar="DATABASE_URI", default=None, help="the URI of the source Neo4j database server")
	parser.add_argument("-d", action="store_true", default=True, help="output all values in delimited format (default)")
	parser.add_argument("-j", action="store_true", default=False, help="output all values as a single JSON array")
	parser.add_argument("-g", action="store_true", default=False, help="output nodes and relationships in GEOFF format")
	parser.add_argument("-t", action="store_true", default=False, help="output all results in a plain text table")
	parser.add_argument("query", help="the Cypher query to execute")
	args = parser.parse_args()
	try:
		if args.g:
			execute_and_output_as_geoff(args.query, args.u)
		elif args.j:
			execute_and_output_as_json(args.query, args.u)
		elif args.t:
			execute_and_output_as_text(args.query, args.u)
		else:
			execute_and_output_as_delimited(args.query, args.u)
	except SystemError as err:
		content = err.args[0]['content']
		sys.stderr.write("%s\n" % (content['exception']))
		stacktrace = content['stacktrace']
		for frame in stacktrace:
			sys.stderr.write("\tat %s\n" % (frame))<commit_msg>updated text output to include properties<commit_after>#!/usr/bin/env python

# Copyright 2011 Nigel Small
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Cypher utility module
"""

import argparse
try:
	import json
except ImportError:
	import simplejson as json
import neo4j
import string
import sys

__author__    = "Nigel Small <py2neo@nigelsmall.org>"
__copyright__ = "Copyright 2011 Nigel Small"
__license__   = "Apache License, Version 2.0"

def execute(query, database_uri=None):
	database_uri = database_uri or "http://localhost:7474/db/data/"
	graphdb = neo4j.GraphDatabaseService(database_uri)
	response = graphdb.execute_cypher_query(query)
	data, columns = response['data'], response['columns']
	return data, columns

def output(value):
	if isinstance(value, dict):
		if 'type' in value:
			# relationship
			out = "\"(%s)-[%d:%s]->(%s)\"" % (
				value['start'].rpartition("/")[2],
				int(value['self'].rpartition("/")[2]),
				value['type'],
				value['end'].rpartition("/")[2]
			)
		else:
			# node
			out = "\"(%d)\"" % (
				int(value['self'].rpartition("/")[2])
			)
	else:
		# property
		out = json.dumps(value)
	return out

def output_text(value):
	if isinstance(value, dict):
		if 'type' in value:
			# relationship
			out = "(%s)-[%d:%s]->(%s) %s" % (
				value['start'].rpartition("/")[2],
				int(value['self'].rpartition("/")[2]),
				value['type'],
				value['end'].rpartition("/")[2],
				json.dumps(value['data'], separators=(',',':'))
			)
		else:
			# node
			out = "(%d) %s" % (
				int(value['self'].rpartition("/")[2]),
				json.dumps(value['data'], separators=(',',':'))
			)
	else:
		# property
		out = str(value)
	return out

def execute_and_output_as_delimited(query, database_uri=None, field_delimiter="\t"):
	data, columns = execute(query, database_uri)
	print string.join([
		json.dumps(column)
		for column in columns
	], field_delimiter)
	for row in data:
		print string.join([
			output(value)
			for value in row
		], field_delimiter)

def execute_and_output_as_json(query, database_uri=None):
	data, columns = execute(query, database_uri)
	columns = [json.dumps(column) for column in columns]
	row_count = 0
	sys.stdout.write("[\n")
	for row in data:
		row_count += 1
		if row_count > 1:
			sys.stdout.write(",\n")
		sys.stdout.write("\t{" + string.join([
			columns[i] + ": " + output(row[i])
			for i in range(len(row))
		], ", ") + "}")
	sys.stdout.write("\n]\n")

def execute_and_output_as_geoff(query, database_uri=None):
	nodes = {}
	rels = {}
	def update_descriptors(value, column):
		if isinstance(value, dict):
			if 'type' in value:
				# relationship
				rels["(%s)-[%d:%s]->(%s)" % (
					value['start'].rpartition("/")[2],
					int(value['self'].rpartition("/")[2]),
					value['type'],
					value['end'].rpartition("/")[2]
				)] = value['data']
			else:
				# node
				nodes["(%d)" % (
					int(value['self'].rpartition("/")[2])
				)] = value['data']
		else:
			# property - not supported in GEOFF format
			pass
	data, columns = execute(query, database_uri)
	for row in data:
		for i in range(len(row)):
			update_descriptors(row[i], columns[i])
	for key, value in nodes.items():
		print "%s\t%s" % (
			key,
			json.dumps(value)
		)
	for key, value in rels.items():
		print "%s\t%s" % (
			key,
			json.dumps(value)
		)

def execute_and_output_as_text(query, database_uri=None, field_delimiter="\t"):
	data, columns = execute(query, database_uri)
	column_widths = [len(column) for column in columns]
	for row in data:
		column_widths = [
			max(column_widths[i], None if row[i] is None else len(output_text(row[i])))
			for i in range(len(row))
		]
	print "+-" + string.join([
		"".ljust(column_widths[i], "-")
		for i in range(len(columns))
	], "---") + "-+"
	print "| " + string.join([
		columns[i].ljust(column_widths[i])
		for i in range(len(columns))
	], " | ") + " |"
	print "+-" + string.join([
		"".ljust(column_widths[i], "-")
		for i in range(len(columns))
	], "---") + "-+"
	for row in data:
		print "| " + string.join([
			output_text(row[i]).ljust(column_widths[i])
			for i in range(len(row))
		], " | ") + " |"
	print "+-" + string.join([
		"".ljust(column_widths[i], "-")
		for i in range(len(columns))
	], "---") + "-+"

if __name__ == "__main__":
	parser = argparse.ArgumentParser(description="Execute Cypher queries against a Neo4j database server and output the results.")
	parser.add_argument("-u", metavar="DATABASE_URI", default=None, help="the URI of the source Neo4j database server")
	parser.add_argument("-d", action="store_true", default=True, help="output all values in delimited format (default)")
	parser.add_argument("-j", action="store_true", default=False, help="output all values as a single JSON array")
	parser.add_argument("-g", action="store_true", default=False, help="output nodes and relationships in GEOFF format")
	parser.add_argument("-t", action="store_true", default=False, help="output all results in a plain text table")
	parser.add_argument("query", help="the Cypher query to execute")
	args = parser.parse_args()
	try:
		if args.g:
			execute_and_output_as_geoff(args.query, args.u)
		elif args.j:
			execute_and_output_as_json(args.query, args.u)
		elif args.t:
			execute_and_output_as_text(args.query, args.u)
		else:
			execute_and_output_as_delimited(args.query, args.u)
	except SystemError as err:
		content = err.args[0]['content']
		if 'exception' in content and 'stacktrace' in content:
			sys.stderr.write("%s\n" % (content['exception']))
			stacktrace = content['stacktrace']
			for frame in stacktrace:
				sys.stderr.write("\tat %s\n" % (frame))
		else:
			sys.stderr.write("%s\n" % (content))

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""

JSON Graph Interchange Format
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The JSON Graph Interchange Format (JGIF) is `specified <http://jsongraphformat.info/>`_ similarly to the Node-Link
JSON. Interchange with this format provides compatibilty with other software and repositories, such as the
`Causal Biological Network Database <http://causalbionet.com/>`_.

"""

import logging
from collections import defaultdict

from ..canonicalize import decanonicalize_node, decanonicalize_edge
from ..constants import *
from ..parser import BelParser
from ..struct import BELGraph

__all__ = [
    'from_cbn_jgif',
    'from_jgif',
    'to_jgif',
]

log = logging.getLogger(__name__)

annotation_map = {
    'tissue': 'Tissue',
    'disease': 'Disease',
    'species_common_name': 'Species',
    'cell': 'Cell',
}

species_map = {
    'human': '9606',
    'rat': '10116',
    'mouse': '10090',
}

placeholder_evidence = "This Network edge has no supporting evidence.  Please add real evidence to this edge prior to deleting."

EXPERIMENT_CONTEXT = 'experiment_context'

def get_citation(evidence):
    citation = {
        CITATION_TYPE: evidence['citation']['type'].strip(),
        CITATION_REFERENCE: evidence['citation']['id'].strip()
    }

    if 'name' in evidence['citation']:
        citation[CITATION_NAME] = evidence['citation']['name'].strip()

    return citation

def map_cbn(d):
    """Pre-processes the JSON from the CBN

    - removes statements without evidence, or with placeholder evidence

    :param dict d: Raw JGIF from the CBN
    :return: Preprocessed JGIF
    :rtype: dict
    """
    for i, edge in enumerate(d['graph']['edges']):
        if 'metadata' not in d['graph']['edges'][i]:
            continue

        if 'evidences' not in d['graph']['edges'][i]['metadata']:
            continue

        for j, evidence in enumerate(d['graph']['edges'][i]['metadata']['evidences']):
            if EXPERIMENT_CONTEXT not in evidence:
                continue

            # ctx = {k.strip().lower(): v.strip() for k, v in evidence[EXPERIMENT_CONTEXT].items() if v.strip()}

            new_context = {}

            for key, value in evidence[EXPERIMENT_CONTEXT].items():
                value = value.strip()

                if not value:
                    continue

                key = key.strip().lower()

                if key == 'species_common_name':
                    new_context['Species'] = species_map[value.lower()]
                elif key in annotation_map:
                    new_context[annotation_map[key]] = value
                else:
                    new_context[key] = value

            '''
            for k, v in annotation_map.items():
                if k not in ctx:
                    continue

                d['graph']['edges'][i]['metadata']['evidences'][j][EXPERIMENT_CONTEXT][v] = ctx[k]
                del d['graph']['edges'][i]['metadata']['evidences'][j][EXPERIMENT_CONTEXT][k]

            if 'species_common_name' in ctx:
                species_name = ctx['species_common_name'].strip().lower()
                d['graph']['edges'][i]['metadata']['evidences'][j][EXPERIMENT_CONTEXT]['Species'] = species_map[
                    species_name]
                del d['graph']['edges'][i]['metadata']['evidences'][j][EXPERIMENT_CONTEXT][
                    'species_common_name']
            '''

            d['graph']['edges'][i]['metadata']['evidences'][j][EXPERIMENT_CONTEXT] = new_context
    return d

def from_cbn_jgif(graph_jgif_dict):
    """Maps the JGIF used by the Causal Biological Network Database to standard namespace and annotations, then
    builds a BEL graph using :func:`pybel.from_jgif`.

    :param dict graph_jgif_dict: The JSON object representing the graph in JGIF format
    :return: A BEL graph
    :rtype: BELGraph

    Example:

    >>> import requests
    >>> from pybel import from_cbn_jgif
    >>> apoptosis_url = 'http://causalbionet.com/Networks/GetJSONGraphFile?networkId=810385422'
    >>> graph_jgif_dict = requests.get(apoptosis_url).json()
    >>> graph = from_cbn_jgif(graph_jgif_dict)
    """
    graph_jgif_dict = map_cbn(graph_jgif_dict)

    graph_jgif_dict['graph']['metadata'].update({
        METADATA_AUTHORS: 'Causal Biological Networks Database',
        METADATA_LICENSES: """
        Please cite:

        - www.causalbionet.com
        - https://bionet.sbvimprover.com

        as well as any relevant publications.

        The sbv IMPROVER project, the website and the Symposia are part of a collaborative project
        designed to enable scientists to learn about and contribute to the development of a new crowd
        sourcing method for verification of scientific data and results. The current challenges, website
        and biological network models were developed and are maintained as part of a collaboration among
        Selventa, OrangeBus and ADS. The project is led and funded by Philip Morris International. For more
        information on the focus of Philip Morris International’s research, please visit www.pmi.com.
        """.replace('\n', '\t')
    })

    graph = from_jgif(graph_jgif_dict)

    graph.namespace_url.update({
        'HGNC': 'https://arty.scai.fraunhofer.de/artifactory/bel/namespace/hgnc-human-genes/hgnc-human-genes-20150601.belns',
        'GOBP': 'https://arty.scai.fraunhofer.de/artifactory/bel/namespace/go-biological-process/go-biological-process-20150601.belns',
        'SFAM': 'https://arty.scai.fraunhofer.de/artifactory/bel/namespace/selventa-protein-families/selventa-protein-families-20150601.belns',
    })

    graph.annotation_url.update({
        'Cell': 'https://arty.scai.fraunhofer.de/artifactory/bel/annotation/cell/cell-line-20150601.belanno',
        'Disease': 'https://arty.scai.fraunhofer.de/artifactory/bel/annotation/disease/disease-20150601.belanno',
        'Species': 'https://arty.scai.fraunhofer.de/artifactory/bel/annotation/species-taxonomy-id/species-taxonomy-id-20170511.belanno',
        'Tissue': 'https://arty.scai.fraunhofer.de/artifactory/bel/annotation/mesh-anatomy/mesh-anatomy-20150601.belanno',
    })

    return graph

def from_jgif(graph_jgif_dict):
    """Builds a BEL graph from a JGIF JSON object.

    :param dict graph_jgif_dict: The JSON object representing the graph in JGIF format
    :return: A BEL graph
    :rtype: BELGraph
    """
    graph = BELGraph()

    root = graph_jgif_dict['graph']

    if 'label' in root:
        graph.name = root['label']

    if 'metadata' in root:
        metadata = root['metadata']

        for key in METADATA_INSERT_KEYS:
            if key in metadata:
                graph.document[key] = metadata[key]

    parser = BelParser(graph)
    parser.bel_term.addParseAction(parser.handle_term)

    for node in root['nodes']:
        if 'label' not in node:
            log.warning('node missing label: %s', node)
        node_label = node['label']
        parser.bel_term.parseString(node_label)

    for i, edge in enumerate(root['edges']):

        if edge['relation'] in {'actsIn'}:
            continue  # don't need legacy BEL format

        if edge['relation'] in UNQUALIFIED_EDGES:
            pass

        if not edge['relation'] in UNQUALIFIED_EDGES and (
                        'evidences' not in edge['metadata'] or not edge['metadata']['evidences']):
            log.debug('No evidence for edge %s', edge['label'])
            continue

        for evidence in edge['metadata']['evidences']:
            if 'citation' not in evidence or not evidence['citation']:
                continue

            if 'type' not in evidence['citation'] and 'id' not in evidence['citation']:
                continue

            summary_text = evidence['summary_text'].strip().strip('"')

            if not summary_text or summary_text == placeholder_evidence:
                continue

            parser.control_parser.clear()
            parser.control_parser.citation = get_citation(evidence)
            parser.control_parser.evidence = summary_text
            parser.control_parser.annotations.update(evidence[EXPERIMENT_CONTEXT])

            bel_statement = edge['label']

            try:
                parser.parseString(bel_statement, i)
            except Exception as e:
                log.warning('Error %s for %s', e, bel_statement)

    return graph

def to_jgif(graph):
    """Builds a JGIF dictionary from a BEL graph.

    :param BELGraph graph: A BEL graph
    :return: A JGIF dictionary
    :rtype: dict

    .. warning::

        Untested! This format is not general purpose and is therefore time is not heavily invested. If you want to
        use Cytoscape.js, we suggest using :func:`pybel.to_cx` instead.

    Example:

    >>> import pybel, os, json
    >>> graph_url = 'https://arty.scai.fraunhofer.de/artifactory/bel/knowledge/selventa-small-corpus/selventa-small-corpus-20150611.bel'
    >>> graph = pybel.from_url(graph_url)
    >>> graph_jgif_json = pybel.to_jgif(graph)
    >>> with open(os.path.expanduser('~/Desktop/small_corpus.json'), 'w') as f:
    ...     json.dump(graph_jgif_json, f)
    """
    node_to_bel = {}
    u_v_r_bel = {}

    nodes_entry = []
    edges_entry = []

    for i, node in enumerate(graph.nodes_iter()):
        bel = decanonicalize_node(graph, node)
        node_to_bel[node] = bel

        nodes_entry.append({
            'id': bel,
            'label': bel,
            'nodeId': i,
            'bel_function_type': graph.node[node][FUNCTION],
            'metadata': {}
        })

    for u, v in graph.edges_iter():
        relation_evidences = defaultdict(list)

        for k, d in graph.edge[u][v].items():

            if (u, v, d[RELATION]) not in u_v_r_bel:
                u_v_r_bel[u, v, d[RELATION]] = decanonicalize_edge(graph, u, v, k)

            bel = u_v_r_bel[u, v, d[RELATION]]

            evidence_dict = {
                'bel_statement': bel,
                'experiment_context': d[ANNOTATIONS],
            }

            if EVIDENCE in d:
                evidence_dict['summary_text'] = d[EVIDENCE]

            if CITATION in d:
                evidence_dict['citation'] = d[CITATION]

            relation_evidences[d[RELATION]].append(evidence_dict)

        for relation, evidences in relation_evidences.items():
            edges_entry.append({
                'source': node_to_bel[u],
                'target': node_to_bel[v],
                'relation': relation,
                'label': u_v_r_bel[u, v, relation],
                'metadata': {
                    'evidences': evidences
                }
            })

    return {
        'graph': {
            'metadata': graph.document,
            'nodes': nodes_entry,
            'edges': edges_entry
        }
    }
<commit_msg>Update URL<commit_after># -*- coding: utf-8 -*-

"""

JSON Graph Interchange Format
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The JSON Graph Interchange Format (JGIF) is `specified <http://jsongraphformat.info/>`_ similarly to the Node-Link
JSON. Interchange with this format provides compatibilty with other software and repositories, such as the
`Causal Biological Network Database <http://causalbionet.com/>`_.

"""

import logging
from collections import defaultdict

from ..canonicalize import decanonicalize_node, decanonicalize_edge
from ..constants import *
from ..parser import BelParser
from ..struct import BELGraph

__all__ = [
    'from_cbn_jgif',
    'from_jgif',
    'to_jgif',
]

log = logging.getLogger(__name__)

annotation_map = {
    'tissue': 'Tissue',
    'disease': 'Disease',
    'species_common_name': 'Species',
    'cell': 'Cell',
}

species_map = {
    'human': '9606',
    'rat': '10116',
    'mouse': '10090',
}

placeholder_evidence = "This Network edge has no supporting evidence.  Please add real evidence to this edge prior to deleting."

EXPERIMENT_CONTEXT = 'experiment_context'

def get_citation(evidence):
    citation = {
        CITATION_TYPE: evidence['citation']['type'].strip(),
        CITATION_REFERENCE: evidence['citation']['id'].strip()
    }

    if 'name' in evidence['citation']:
        citation[CITATION_NAME] = evidence['citation']['name'].strip()

    return citation

def map_cbn(d):
    """Pre-processes the JSON from the CBN

    - removes statements without evidence, or with placeholder evidence

    :param dict d: Raw JGIF from the CBN
    :return: Preprocessed JGIF
    :rtype: dict
    """
    for i, edge in enumerate(d['graph']['edges']):
        if 'metadata' not in d['graph']['edges'][i]:
            continue

        if 'evidences' not in d['graph']['edges'][i]['metadata']:
            continue

        for j, evidence in enumerate(d['graph']['edges'][i]['metadata']['evidences']):
            if EXPERIMENT_CONTEXT not in evidence:
                continue

            # ctx = {k.strip().lower(): v.strip() for k, v in evidence[EXPERIMENT_CONTEXT].items() if v.strip()}

            new_context = {}

            for key, value in evidence[EXPERIMENT_CONTEXT].items():
                value = value.strip()

                if not value:
                    continue

                key = key.strip().lower()

                if key == 'species_common_name':
                    new_context['Species'] = species_map[value.lower()]
                elif key in annotation_map:
                    new_context[annotation_map[key]] = value
                else:
                    new_context[key] = value

            '''
            for k, v in annotation_map.items():
                if k not in ctx:
                    continue

                d['graph']['edges'][i]['metadata']['evidences'][j][EXPERIMENT_CONTEXT][v] = ctx[k]
                del d['graph']['edges'][i]['metadata']['evidences'][j][EXPERIMENT_CONTEXT][k]

            if 'species_common_name' in ctx:
                species_name = ctx['species_common_name'].strip().lower()
                d['graph']['edges'][i]['metadata']['evidences'][j][EXPERIMENT_CONTEXT]['Species'] = species_map[
                    species_name]
                del d['graph']['edges'][i]['metadata']['evidences'][j][EXPERIMENT_CONTEXT][
                    'species_common_name']
            '''

            d['graph']['edges'][i]['metadata']['evidences'][j][EXPERIMENT_CONTEXT] = new_context
    return d

def from_cbn_jgif(graph_jgif_dict):
    """Maps the JGIF used by the Causal Biological Network Database to standard namespace and annotations, then
    builds a BEL graph using :func:`pybel.from_jgif`.

    :param dict graph_jgif_dict: The JSON object representing the graph in JGIF format
    :return: A BEL graph
    :rtype: BELGraph

    Example:

    >>> import requests
    >>> from pybel import from_cbn_jgif
    >>> apoptosis_url = 'http://causalbionet.com/Networks/GetJSONGraphFile?networkId=810385422'
    >>> graph_jgif_dict = requests.get(apoptosis_url).json()
    >>> graph = from_cbn_jgif(graph_jgif_dict)
    """
    graph_jgif_dict = map_cbn(graph_jgif_dict)

    graph_jgif_dict['graph']['metadata'].update({
        METADATA_AUTHORS: 'Causal Biological Networks Database',
        METADATA_LICENSES: """
        Please cite:

        - www.causalbionet.com
        - https://bionet.sbvimprover.com

        as well as any relevant publications.

        The sbv IMPROVER project, the website and the Symposia are part of a collaborative project
        designed to enable scientists to learn about and contribute to the development of a new crowd
        sourcing method for verification of scientific data and results. The current challenges, website
        and biological network models were developed and are maintained as part of a collaboration among
        Selventa, OrangeBus and ADS. The project is led and funded by Philip Morris International. For more
        information on the focus of Philip Morris International’s research, please visit www.pmi.com.
        """.replace('\n', '\t')
    })

    graph = from_jgif(graph_jgif_dict)

    graph.namespace_url.update({
        'HGNC': 'https://arty.scai.fraunhofer.de/artifactory/bel/namespace/hgnc-human-genes/hgnc-human-genes-20150601.belns',
        'GOBP': 'https://arty.scai.fraunhofer.de/artifactory/bel/namespace/go-biological-process/go-biological-process-20150601.belns',
        'SFAM': 'https://arty.scai.fraunhofer.de/artifactory/bel/namespace/selventa-protein-families/selventa-protein-families-20150601.belns',
    })

    graph.annotation_url.update({
        'Cell': 'https://arty.scai.fraunhofer.de/artifactory/bel/annotation/cell-line/cell-line-20150601.belanno',
        'Disease': 'https://arty.scai.fraunhofer.de/artifactory/bel/annotation/disease/disease-20150601.belanno',
        'Species': 'https://arty.scai.fraunhofer.de/artifactory/bel/annotation/species-taxonomy-id/species-taxonomy-id-20170511.belanno',
        'Tissue': 'https://arty.scai.fraunhofer.de/artifactory/bel/annotation/mesh-anatomy/mesh-anatomy-20150601.belanno',
    })

    return graph

def from_jgif(graph_jgif_dict):
    """Builds a BEL graph from a JGIF JSON object.

    :param dict graph_jgif_dict: The JSON object representing the graph in JGIF format
    :return: A BEL graph
    :rtype: BELGraph
    """
    graph = BELGraph()

    root = graph_jgif_dict['graph']

    if 'label' in root:
        graph.name = root['label']

    if 'metadata' in root:
        metadata = root['metadata']

        for key in METADATA_INSERT_KEYS:
            if key in metadata:
                graph.document[key] = metadata[key]

    parser = BelParser(graph)
    parser.bel_term.addParseAction(parser.handle_term)

    for node in root['nodes']:
        if 'label' not in node:
            log.warning('node missing label: %s', node)
        node_label = node['label']
        parser.bel_term.parseString(node_label)

    for i, edge in enumerate(root['edges']):

        if edge['relation'] in {'actsIn'}:
            continue  # don't need legacy BEL format

        if edge['relation'] in UNQUALIFIED_EDGES:
            pass

        if not edge['relation'] in UNQUALIFIED_EDGES and (
                        'evidences' not in edge['metadata'] or not edge['metadata']['evidences']):
            log.debug('No evidence for edge %s', edge['label'])
            continue

        for evidence in edge['metadata']['evidences']:
            if 'citation' not in evidence or not evidence['citation']:
                continue

            if 'type' not in evidence['citation'] and 'id' not in evidence['citation']:
                continue

            summary_text = evidence['summary_text'].strip().strip('"')

            if not summary_text or summary_text == placeholder_evidence:
                continue

            parser.control_parser.clear()
            parser.control_parser.citation = get_citation(evidence)
            parser.control_parser.evidence = summary_text
            parser.control_parser.annotations.update(evidence[EXPERIMENT_CONTEXT])

            bel_statement = edge['label']

            try:
                parser.parseString(bel_statement, i)
            except Exception as e:
                log.warning('Error %s for %s', e, bel_statement)

    return graph

def to_jgif(graph):
    """Builds a JGIF dictionary from a BEL graph.

    :param BELGraph graph: A BEL graph
    :return: A JGIF dictionary
    :rtype: dict

    .. warning::

        Untested! This format is not general purpose and is therefore time is not heavily invested. If you want to
        use Cytoscape.js, we suggest using :func:`pybel.to_cx` instead.

    Example:

    >>> import pybel, os, json
    >>> graph_url = 'https://arty.scai.fraunhofer.de/artifactory/bel/knowledge/selventa-small-corpus/selventa-small-corpus-20150611.bel'
    >>> graph = pybel.from_url(graph_url)
    >>> graph_jgif_json = pybel.to_jgif(graph)
    >>> with open(os.path.expanduser('~/Desktop/small_corpus.json'), 'w') as f:
    ...     json.dump(graph_jgif_json, f)
    """
    node_to_bel = {}
    u_v_r_bel = {}

    nodes_entry = []
    edges_entry = []

    for i, node in enumerate(graph.nodes_iter()):
        bel = decanonicalize_node(graph, node)
        node_to_bel[node] = bel

        nodes_entry.append({
            'id': bel,
            'label': bel,
            'nodeId': i,
            'bel_function_type': graph.node[node][FUNCTION],
            'metadata': {}
        })

    for u, v in graph.edges_iter():
        relation_evidences = defaultdict(list)

        for k, d in graph.edge[u][v].items():

            if (u, v, d[RELATION]) not in u_v_r_bel:
                u_v_r_bel[u, v, d[RELATION]] = decanonicalize_edge(graph, u, v, k)

            bel = u_v_r_bel[u, v, d[RELATION]]

            evidence_dict = {
                'bel_statement': bel,
                'experiment_context': d[ANNOTATIONS],
            }

            if EVIDENCE in d:
                evidence_dict['summary_text'] = d[EVIDENCE]

            if CITATION in d:
                evidence_dict['citation'] = d[CITATION]

            relation_evidences[d[RELATION]].append(evidence_dict)

        for relation, evidences in relation_evidences.items():
            edges_entry.append({
                'source': node_to_bel[u],
                'target': node_to_bel[v],
                'relation': relation,
                'label': u_v_r_bel[u, v, relation],
                'metadata': {
                    'evidences': evidences
                }
            })

    return {
        'graph': {
            'metadata': graph.document,
            'nodes': nodes_entry,
            'edges': edges_entry
        }
    }
<|endoftext|>


<commit_before>from collections import deque
from itertools import repeat, izip, count, product

__author__ = 'nhc'

def round_robin(*iterables):
    """Splices the given iterables fairly, see
       http://bugs.python.org/issue1757395
    """
    pending = deque(iter(i).next for i in reversed(iterables))
    rotate, pop, _StopIteration = pending.rotate, pending.pop, StopIteration
    while pending:
        try:
            while 1:
                yield pending[-1]()
                rotate()
        except _StopIteration:
            pop()

def _enumerated_fair_iterator(iterators):
    """
    Combines iterators into one. Example: with input [['a','b'], ['x','y']]
    the returned iterator is equivalent to
    [(1, (0, 'a')), (2, (1, 'x')), (3, (0, 'b')), (4, (1, 'y'))]
    Each original value is paired with the index of the iterator it came from.
    Each pair is annotated with a serial number independent which generator it came from.
    The iterators are polled using round_robin above.
    :param iterators: Any number of iterators, finite or infinite.
    :return: The combined iterator.
    """
    indexed_iterators = [izip(repeat(index), g) for index, g in enumerate(iterators)]
    fair_iterators = round_robin(*indexed_iterators)
    enumerated_fair_iterators = izip(count(1), fair_iterators)
    return enumerated_fair_iterators

def lazy_product(*iterators):
    """
    Akin to itertools.product except
        - works with infinite iterators
        - does not order the generated tuples in the same way
    For finite iterators, the following should hold:
        set(itertools.product(*iterators)) == set(lazy_product(*iterators))
    :param iterators: Any number of iterators
    :return: One generator that yields tuples. Each tuple contains
    one value from each of the input iterators.
    """
    tuple_size = len(iterators)
    if tuple_size == 0:  # Special case: Just yield the empty tuple
        yield ()
        return
    # General case: Keep a cache of every seen value, per iterator
    cache = [list() for _ in range(tuple_size)]
    #   TODO The above lists should be sets.
    #   To make our dicts hashable, see
    #   http://stackoverflow.com/questions/9997176/immutable-dictionary-only-use-as-a-key-for-another-dictionary
    # For each new value from one of the iterators...
    for counter, (index, value) in _enumerated_fair_iterator(iterators):
        # Add it to the cache
        cache[index].append(value)
        # If every iterator has been polled once and one was empty: Return
        if counter == tuple_size:
            if any([len(l) == 0 for l in cache]):
                return  # Some generator did not deliver a value
        # If every iterator has provided at least one value:
        if counter >= tuple_size:  # TODO: Only if value not seen before
            # Generate every tuple possible using the new value
            x = list(cache)
            x[index] = [value]
            for t in product(*x):
                yield t
<commit_msg>Fix TODO: Keep products as sparse as possible<commit_after>from collections import deque
from itertools import repeat, izip, count, product

__author__ = 'nhc'

def round_robin(*iterables):
    """Splices the given iterables fairly, see
       http://bugs.python.org/issue1757395
    """
    pending = deque(iter(i).next for i in reversed(iterables))
    rotate, pop, _StopIteration = pending.rotate, pending.pop, StopIteration
    while pending:
        try:
            while 1:
                yield pending[-1]()
                rotate()
        except _StopIteration:
            pop()

def _enumerated_fair_iterator(iterators):
    """
    Combines iterators into one. Example: with input [['a','b'], ['x','y']]
    the returned iterator is equivalent to
    [(1, (0, 'a')), (2, (1, 'x')), (3, (0, 'b')), (4, (1, 'y'))]
    Each original value is paired with the index of the iterator it came from.
    Each pair is annotated with a serial number independent which generator it came from.
    The iterators are polled using round_robin above.
    :param iterators: Any number of iterators, finite or infinite.
    :return: The combined iterator.
    """
    indexed_iterators = [izip(repeat(index), g) for index, g in enumerate(iterators)]
    fair_iterators = round_robin(*indexed_iterators)
    enumerated_fair_iterators = izip(count(1), fair_iterators)
    return enumerated_fair_iterators

def lazy_product(*iterators):
    """
    Akin to itertools.product except
        - works with infinite iterators
        - does not order the generated tuples in the same way
        - requires every value to be hashable
    For finite iterators, the following should hold:
        set(itertools.product(*iterators)) == set(lazy_product(*iterators))
    :param iterators: Any number of iterators
    :return: One generator that yields tuples. Each tuple contains
    one value from each of the input iterators.
    """
    tuple_size = len(iterators)
    if tuple_size == 0:  # Special case: Just yield the empty tuple
        yield ()
        return
    # General case: Keep a cache of every seen value, per iterator
    cache = [set() for _ in range(tuple_size)]
    # For each new value from one of the iterators...
    for counter, (index, value) in _enumerated_fair_iterator(iterators):
        # Add it to the cache
        cache[index].add(value)
        # If every iterator has been polled once and one was empty: Return
        if counter == tuple_size:
            if any([len(l) == 0 for l in cache]):
                return  # Some generator did not deliver a value
        # If every iterator has provided at least one value:
        if counter >= tuple_size:  # TODO: Only if value not seen before
            # Generate every tuple possible using the new value
            x = list(cache)
            x[index] = [value]
            for t in product(*x):
                yield t
<|endoftext|>


<commit_before>
import sys

class Edge:
    def __init__(self, trans, state, voided):
        self.trans = trans
        self.state = state
        self.voided = voided

class State:
    def __init__(self, edges, match):
        self.edges = edges
        self.match = match

class Automaton:
    def __init__(self, type, states, mode):
        self.type = type
        self.states = states
        self.mode = mode

class ParseError:
    def __init__(self, pos, line, col, nt):
        self.pos = pos
        self.line = line
        self.col = col
        self.nt = nt

    def display(self):
        print "parse error: failed to match '%s' at line=%s, col=%s, pos=%s" % (self.nt, self.line, self.col, self.pos)

class AST:
    def __init__(self, type, children, pos):
        self.type = type
        self.children = children
        self.pos = pos

    def display_iter(self, ast, indent):
        for i in range(0, indent[0] - 1):
            sys.stdout.write('    ')
        if indent[0] > 0:
            sys.stdout.write('->  ')
        print ast.type
        indent[0] += 1
        for a in ast.children:
            if isinstance(a, AST):
                self.display_iter(a, indent)
            else:
                for i in range(0, indent[0] - 1):
                    sys.stdout.write('    ')
                if indent[0] > 0:
                    sys.stdout.write('|   ')
                print a
        indent[0] -= 1

    def display(self):
        self.display_iter(self, [0])

#c = AST("num", ['1', '2'], [0, 2])
#a = AST("Hello Mama!", ['a', c, 'c'], [0, 4])

#a.display()

#err = ParseError(21, 8, 5, 'num')
#err.display()

class WaxeyeParser:
    def __init__(self, start, eof_check, automata):
        self.start = start
        self.eof_check = eof_check
        self.automata = automata

    def parse(self, input):
        WaxeyeParser.InnerParser(self.start, self.eof_check, self.automata, input).parse()

    class InnerParser:
        def __init__(self, start, eof_check, automata, input):
            self.start = start
            self.eof_check = eof_check
            self.automata = automata
            self.input = input

        def parse(self):
            print self.input

p = WaxeyeParser(0, True, [])
p.parse('1+2-3*4')
<commit_msg>More python runtime.<commit_after>
import sys

class Edge:
    def __init__(self, trans, state, voided):
        self.trans = trans
        self.state = state
        self.voided = voided

class State:
    def __init__(self, edges, match):
        self.edges = edges
        self.match = match

class FA:
    def __init__(self, type, states, mode):
        self.type = type
        self.states = states
        self.mode = mode

class ParseError:
    def __init__(self, pos, line, col, nt):
        self.pos = pos
        self.line = line
        self.col = col
        self.nt = nt

    def display(self):
        print "parse error: failed to match '%s' at line=%s, col=%s, pos=%s" % (self.nt, self.line, self.col, self.pos)

class AST:
    def __init__(self, type, children, pos):
        self.type = type
        self.children = children
        self.pos = pos

    def display_iter(self, ast, indent):
        for i in range(0, indent[0] - 1):
            sys.stdout.write('    ')
        if indent[0] > 0:
            sys.stdout.write('->  ')
        print ast.type
        indent[0] += 1
        for a in ast.children:
            if isinstance(a, AST):
                self.display_iter(a, indent)
            else:
                for i in range(0, indent[0] - 1):
                    sys.stdout.write('    ')
                if indent[0] > 0:
                    sys.stdout.write('|   ')
                print a
        indent[0] -= 1

    def display(self):
        self.display_iter(self, [0])

class WaxeyeParser:
    def __init__(self, start, eof_check, automata):
        self.start = start
        self.eof_check = eof_check
        self.automata = automata

    def parse(self, input):
        WaxeyeParser.InnerParser(self.start, self.eof_check, self.automata, input).parse()

    class InnerParser:
        def __init__(self, start, eof_check, automata, input):
            self.start = start
            self.eof_check = eof_check
            self.automata = automata
            self.input = input
            self.input_len = len(input)
            self.input_pos = 0
            self.line = 1
            self.column = 0
            self.last_cr = False
            self.error_pos = 0
            self.error_line = 1
            self.error_col = 0
            self.error_nt = automata[start].type
            self.fa_stack = []
            self.cache = {}

        def parse(self):
            print self.input

p = WaxeyeParser(0, True, [FA(0, [], 0)])
p.parse('1+2-3*4')
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os
import pickle
import re
from collections import Counter
from datetime import datetime

DATFILE = os.getenv('HOME') + '/.jump.dat'
LOGFILE = os.getenv('HOME') + '/.jump.log'
SPECIALDIRS = ['.', '..']

def log(msg):
    f = open(LOGFILE, 'a')
    f.write(str(datetime.today()) + ': ' + str(msg))
    f.write('\n')

def write_history(obj):
    f = open(DATFILE, 'wb')
    pickle.dump(obj, f)
    f.close()

def read_history():
    if not os.path.exists(DATFILE):
        return Counter()
    f = open(DATFILE, 'rb')
    obj = pickle.load(f)
    f.close()
    return obj

def sort_history(history):
    return sorted(history.items(), key=lambda e: e[1], reverse=True)

def delete_entry(entry):
    print('deleting "' + entry + '"')
    history = read_history()
    print(history)
    del history[entry]
    write_history(history)

def find_dir(pattern, history):
    p = re.compile(pattern)
    for d, cnt in sort_history(history):
        if p.search(d) and os.path.exists(d):
            yield d

def strip_common_prefix(l):
    if len(l) <= 1:
        return l
    prefix = os.path.commonprefix(l)
    # don't strip beginning of directory names
    prefix = prefix[:prefix.rfind('/')+1]
    prefixlen = len(prefix)
    return list(map(lambda s:
                    s[prefixlen:]
                    # only strip prefix if the result wouldn't be empty
                    if len(s[prefixlen:]) > 0
                    else s,
                    l))

def strip_leading_slash(l):
    if len(l) <= 1:
        return l
    return list(map(lambda s: s.lstrip('/'), l))

def only_special_dirs(name):
    return re.compile('^[./]+$').match(name) is not None

def find_candidates(name, history, canonical):
    try:
        # a directory that is reachable via 'name' has precedence
        os.chdir(name)
        return [os.getcwd()]
    except OSError:
        # a directory 'name' doesn't exist so carry on
        pass
    if not only_special_dirs(name):
        candidates = []
        for found in find_dir(name, history):
            candidates.append(found)
        if not canonical:
            candidates = strip_leading_slash(strip_common_prefix(candidates))
        return sorted(candidates, key=lambda e: e.count('/'))
    return [name]

def find_best_dir(d):
    history = read_history()
    candidates = find_candidates(d, history, True)
    if not candidates:
        return d
    best_dir = candidates[0]
    history[best_dir] += 1
    write_history(history)
    return best_dir

def complete(d):
    res = find_candidates(d, read_history(), False)
    width = len(str(len(res)-1))
    if len(res) > 1:
        res = list(map(
            lambda e:
            "{:0{width}} {}".format(res.index(e), e, width=width), res))
    print("\n".join(res))

def dump():
    history = read_history()
    width = len(str(history.most_common(1)[0][1]))
    for directory, count in sort_history(history):
        print("{:{width}} {}".format(count, directory, width=width))
<commit_msg>Fixes #2 Symlinks are ignored<commit_after>#!/usr/bin/env python3

import os
import pickle
import re
from collections import Counter
from datetime import datetime

DATFILE = os.getenv('HOME') + '/.jump.dat'
LOGFILE = os.getenv('HOME') + '/.jump.log'
SPECIALDIRS = ['.', '..']

def log(msg):
    f = open(LOGFILE, 'a')
    f.write(str(datetime.today()) + ': ' + str(msg))
    f.write('\n')

def write_history(obj):
    f = open(DATFILE, 'wb')
    pickle.dump(obj, f)
    f.close()

def read_history():
    if not os.path.exists(DATFILE):
        return Counter()
    f = open(DATFILE, 'rb')
    obj = pickle.load(f)
    f.close()
    return obj

def sort_history(history):
    return sorted(history.items(), key=lambda e: e[1], reverse=True)

def delete_entry(entry):
    print('deleting "' + entry + '"')
    history = read_history()
    del history[entry]
    write_history(history)

def find_dir(pattern, history):
    p = re.compile(pattern)
    for d, cnt in sort_history(history):
        if p.search(d) and os.path.exists(d):
            yield d

def strip_common_prefix(l):
    if len(l) <= 1:
        return l
    prefix = os.path.commonprefix(l)
    # don't strip beginning of directory names
    prefix = prefix[:prefix.rfind('/')+1]
    prefixlen = len(prefix)
    return list(map(lambda s:
                    s[prefixlen:]
                    # only strip prefix if the result wouldn't be empty
                    if len(s[prefixlen:]) > 0
                    else s,
                    l))

def strip_leading_slash(l):
    if len(l) <= 1:
        return l
    return list(map(lambda s: s.lstrip('/'), l))

def only_special_dirs(name):
    return re.compile('^[./]+$').match(name) is not None

def find_candidates(name, history, canonical):
    try:
        # a directory that is reachable via 'name' has precedence
        islink = os.path.islink(name)
        oldcwd = os.getcwd()
        os.chdir(name)
        cwd = os.getcwd()
        # return to the previous directory so completion works upon continuous
        # calls to this function
        os.chdir(oldcwd)
        # if we followed a symlink, return that, else return the CWD
        return [cwd if not islink else
                os.path.join(os.path.split(cwd)[0], name)]
    except OSError:
        # a directory 'name' doesn't exist so carry on
        pass
    if not only_special_dirs(name):
        candidates = []
        for found in find_dir(name, history):
            candidates.append(found)
        if not canonical:
            candidates = strip_leading_slash(strip_common_prefix(candidates))
        return sorted(candidates, key=lambda e: e.count('/'))
    return [name]

def find_best_dir(d):
    history = read_history()
    candidates = find_candidates(d, history, True)
    if not candidates:
        return d
    best_dir = candidates[0]
    history[best_dir] += 1
    write_history(history)
    return best_dir

def complete(d):
    res = find_candidates(d, read_history(), False)
    width = len(str(len(res)-1))
    if len(res) > 1:
        res = list(map(
            lambda e:
            "{:0{width}} {}".format(res.index(e), e, width=width), res))
    print("\n".join(res))

def dump():
    history = read_history()
    width = len(str(history.most_common(1)[0][1]))
    for directory, count in sort_history(history):
        print("{:{width}} {}".format(count, directory, width=width))
<|endoftext|>


<commit_before>"""Module containing class `CreateRandomClipsCommand`."""

from datetime import timedelta as TimeDelta
import logging

import numpy as np

from vesper.command.command import Command
from vesper.django.app.models import Clip, Job, Processor, RecordingChannel
from vesper.singleton.archive import archive
from vesper.singleton.preset_manager import preset_manager
from vesper.util.schedule import Interval, Schedule
import vesper.command.command_utils as command_utils
import vesper.util.signal_utils as signal_utils
import vesper.util.time_utils as time_utils

_PROCESSOR_NAME = 'Vesper Random Clip Creator 1.0'
_PROCESSOR_TYPE = 'Clip Creator'
_PROCESSOR_DESCRIPTION = '''
Virtual processor created by Vesper's Create Random Clips command.
The Create Random Clips command assigns this processor as the
creating processor of each clip that it creates.
'''.lstrip()

_logger = logging.getLogger()

# TODO: Consider avoiding collisions across multiple runs of this command.

# TODO: Consider optionally avoiding creating clips that intersect
# specified existing ones, e.g. Call* clips. The command might do this
# by excluding the intervals of the specified clips from consideration
# for random clip creation.

class CreateRandomClipsCommand(Command):

    extension_name = 'create_random_clips'

    def __init__(self, args):

        super().__init__(args)

        get = command_utils.get_required_arg
        self._sm_pair_ui_names = get('station_mics', args)
        self._start_date = get('start_date', args)
        self._end_date = get('end_date', args)
        self._schedule_name = get('schedule', args)
        self._clip_duration = get('clip_duration', args)
        self._clip_count = get('clip_count', args)

        self._schedule_preset = _get_schedule_preset(self._schedule_name)
        if self._schedule_preset is None:
            self._schedule_cache = None
        else:
            self._schedule_cache = ScheduleCache(self._schedule_preset)

    def execute(self, job_info):

        self._job_info = job_info

        _logger.info(
            'Gathering information about recording channel intervals in '
            'which to create clips...')

        # Get recording channel intervals in which to create clips,
        # as specified by command arguments.
        sm_pairs = command_utils.get_station_mic_output_pairs(
            self._sm_pair_ui_names)
        intervals = _get_processing_intervals(
            sm_pairs, self._start_date, self._end_date, self._schedule_cache)

        # Get subset of intervals that are *usable* in the sense that
        # they are long enough to contain a clip of duration
        # `self._clip_duration`. Also get time bounds of usable portions
        # of usable intervals in the concatenation of the usable portions.
        # The *usable portion* of a usable interval is the portion of
        # the interval in which a clip of duration `self._clip_duration`
        # can start and still be contained in the interval, i.e. the
        # initial portion ending `self._clip_duration` seconds before
        # the end of the interval. The length of the time bounds array
        # is one more than the number of usable intervals. Element i
        # of the array is the start time of interval i and the end
        # time of interval i - 1, in seconds from the start of the
        # concatenation.
        usable_intervals, time_bounds = \
            _get_concatenated_time_bounds(intervals, self._clip_duration)

        _logger.info('Generating random clip start times...')

        # Get start offsets of random clips in concatenation of usable
        # interval portions, in seconds.
        start_offsets = np.random.uniform(
            time_bounds[0], time_bounds[-1], self._clip_count)

        # Sort start offsets so they are in ascending order. We don't
        # have to do this for the call to `np.searchsorted` below to
        # work, but it seems like a good idea so we can create the
        # clips in a sensible order.
        start_offsets.sort()

        # Get index of usable interval of each clip.
        interval_indices = \
            np.searchsorted(time_bounds, start_offsets, 'right') - 1

        # Create clips.

        creation_time = time_utils.get_utc_now()
        creating_processor = self._get_creating_processor()
        creating_job = Job.objects.get(pk=self._job_info.job_id)

        _logger.info('Creating clips ...')

        # Bookkeeping for start index collision detection and resolution.
        last_interval_num = -1
        used_start_indices = set()

        for i, start_offset in zip(interval_indices, start_offsets):

            station, channel, interval_start_time, interval_end_time = \
                usable_intervals[i]
            recording = channel.recording
            sample_rate = recording.sample_rate

            # Get clip start index in recording.
            start_offset = TimeDelta(seconds=start_offset - time_bounds[i])
            start_time = interval_start_time + start_offset
            start_index = signal_utils.time_to_index(
                start_time, recording.start_time, sample_rate)

            # Get clip length in samples.
            length = signal_utils.seconds_to_frames(
                self._clip_duration, sample_rate)

            if i != last_interval_num:
                # new interval

                last_interval_num = i
                used_start_indices.clear()

            elif start_index in used_start_indices:
                # start index collision

                interval_end_index = signal_utils.time_to_index(
                    interval_end_time, interval_start_time, sample_rate)

                start_index = self._resolve_start_index_collision(
                    start_index, length, used_start_indices,
                    interval_end_index)

                if start_index is None:

                    _logger.warning(
                        'Could not resolve clip start index collision '
                        'while creating random clips in recording channel '
                        f'{str(channel)}. Clip for which collision '
                        'could not be resolved will not be created.')

                    # Move on to the next clip.
                    continue

            # Get time of first clip sample.
            start_time = signal_utils.index_to_time(
                start_index, recording.start_time, sample_rate)

            # Get time of last clip sample.
            end_time = \
                signal_utils.get_end_time(start_time, length, sample_rate)

            date = station.get_night(start_time)

            # TODO: Use Django's `bulk_create` to create clips.
            # Consider making a class, an instance of which you can push
            # clips (and perhaps accompanying annotations and tags) to
            # that will write them in batches to the database. The class
            # will include a `flush` method that writes all accumulated
            # data to the database, regardless of whether or not it is
            # a full batch. The class could be used by this command and
            # the detect command to create new clips. It might also
            # make sense to have analogous classes for writing tags
            # and/or annotations for existing clips. Pay careful
            # attention to what writes should happen in the same
            # transaction, e.g. all those pertaining to a particular
            # clip.

            clip = Clip.objects.create(
                station=station,
                mic_output=channel.mic_output,
                recording_channel=channel,
                start_index=start_index,
                length=length,
                sample_rate=sample_rate,
                start_time=start_time,
                end_time=end_time,
                date=date,
                creation_time=creation_time,
                creating_processor=creating_processor,
                creating_job=creating_job)

            used_start_indices.add(start_index)

        return True

    def _get_creating_processor(self):

        try:
            processor = Processor.objects.get(
                name=_PROCESSOR_NAME, type=_PROCESSOR_TYPE)

        except Processor.DoesNotExist:

            _logger.info(f'Creating processor "{_PROCESSOR_NAME}"...')

            processor = Processor.objects.create(
                name=_PROCESSOR_NAME,
                type=_PROCESSOR_TYPE,
                description=_PROCESSOR_DESCRIPTION
            )

        return processor

def _get_schedule_preset(schedule_name):

    if schedule_name == archive.NULL_CHOICE:
        # no schedule specified

        return None

    else:
        # schedule specified

        preset_path = ('Detection Schedule', schedule_name)
        preset = preset_manager.get_preset(preset_path)
        return preset.data

def _get_processing_intervals(sm_pairs, start_date, end_date, schedule_cache):

    processing_intervals = []

    for station, mic_output in sm_pairs:

        start_time, end_time = \
            station.get_night_interval_utc(start_date, end_date)

        channels = RecordingChannel.objects.filter(
            recording__station=station,
            mic_output=mic_output,
            recording__start_time__lt=end_time,
            recording__end_time__gt=start_time
        ).select_related('recording')

        if schedule_cache is None:
            schedule = None
        else:
            schedule = schedule_cache.get_schedule(station)

        for channel in channels:

            # Get time intervals to process for channel.
            intervals = _get_channel_intervals(channel, schedule)

            # Prepend other information to intervals.
            intervals = [(station, channel) + i for i in intervals]

            processing_intervals.extend(intervals)

    processing_intervals.sort(key=_create_interval_key)

    return processing_intervals

def _get_channel_intervals(channel, schedule):

    recording = channel.recording
    start_time = recording.start_time
    end_time = recording.end_time

    if schedule is None:
        return [(start_time, end_time)]

    else:
        # have schedule

        schedule_intervals = schedule.get_intervals(start_time, end_time)
        recording_interval = Interval(start=start_time, end=end_time)

        # Here we know from the contract of `Schedule.get_intervals`
        # that each interval of `schedule_intervals` intersects the
        # recording interval, so we don't have to worry about
        # `_get_time_intervals_intersection` returning `None`.
        return [
            _get_time_intervals_intersection(i, recording_interval)
            for i in schedule_intervals]

def _get_time_intervals_intersection(a, b):

    if a.end < b.start or b.end < a.start:
        # intervals do not intersect

        return None

    else:
        # intervals intersect

        start = a.start if b.start < a.start else b.start
        end = a.end if a.end < b.end else b.end
        return (start, end)

def _create_interval_key(interval):
    station, channel, start_time, _ = interval
    return (
        station.name, channel.recording.start_time, channel.channel_num,
        start_time)

def _get_concatenated_time_bounds(intervals, clip_duration):

   # The work of this function is divided as it is to make it easier
    # to unit test the `_get_concatenated_time_bounds_aux` function.

    # Prepare intervals for `_get_concatenated_time_bounds_aux`.
    diddled_intervals = _diddle_intervals(intervals)

    # Figure out which intervals are long enough to create clips in,
    # and get the time bounds of the portions of ths intervals in
    # which clips can start in their concatenation.
    usable_interval_indices, time_bounds = \
        _get_concatenated_time_bounds_aux(diddled_intervals, clip_duration)

    # Get usable intervals from indices.
    usable_intervals = [intervals[i] for i in usable_interval_indices]

    return usable_intervals, time_bounds

def _diddle_intervals(intervals):
    return [_diddle_interval(i) for i in intervals]

def _diddle_interval(interval):
    _, channel, start_time, end_time = interval
    recording = channel.recording
    sample_rate = recording.sample_rate
    recording_start_time = recording.start_time
    return recording_start_time, sample_rate, start_time, end_time

def _get_concatenated_time_bounds_aux(intervals, clip_duration):

    s2f = signal_utils.seconds_to_frames
    f2s = signal_utils.get_duration

    usable_interval_indices = []
    time_bounds = []
    last_end_time = 0.

    for i, interval in enumerate(intervals):

        recording_start_time, sample_rate, start_time, end_time = interval

        # Get usable subinterval length in samples. Work in samples
        # instead of in seconds to ensure that a clip of the desired
        # length will be entirely contained in the interval if it
        # starts in what we decide is the usable subinterval.

        offset = (start_time - recording_start_time).total_seconds()
        start_index = s2f(offset, sample_rate)

        offset = (end_time - recording_start_time).total_seconds()
        end_index = s2f(offset, sample_rate)

        interval_length = end_index - start_index

        clip_length = s2f(clip_duration, sample_rate)

        usable_length = interval_length - clip_length

        if usable_length > 0:
            # interval is longer than clip length

            usable_interval_indices.append(i)
            time_bounds.append(last_end_time)

            usable_duration = f2s(usable_length, sample_rate)
            last_end_time += usable_duration

    time_bounds.append(last_end_time)

    return usable_interval_indices, np.array(time_bounds)

def _resolve_start_index_collision(
        start_index, length, used_start_indices, interval_end_index):

    # This method tries to resolve a start index collision, first
    # by looking for a later usable start index and then, if none
    # is available, for an earlier one. For reasonable random clip
    # densities, the next start index will almost always be available
    # so this method will just return that.

    # Find first start index after `start_index` that is unused.
    # This will usually be the next index, but might not be if other
    # collisions for this same start index have already been resolved.
    new_start_index = start_index + 1
    while new_start_index in used_start_indices:
        new_start_index += 1

    if new_start_index + length <= interval_end_index:
        # clip with this start index fits in interval

        return new_start_index

    # If we get here, there are no unused start indices for this interval
    # after `start_index`.

    # Find first start index before `start_index` that is unused.
    new_start_index = start_index - 1
    while new_start_index in used_start_indices:
        new_start_index -= 1

    if new_start_index >= 0:
        return new_start_index

    # If we get here, there are no unused start indices for this interval!
    return None

class ScheduleCache:

    def __init__(self, schedule_spec):
        self._schedule_spec = schedule_spec
        self._schedules = {}

    def get_schedule(self, station):

        try:
            return self._schedules[station.name]

        except KeyError:
            # cache miss

            if _logger is not None:
                _logger.info(
                    f'Compiling schedule for station "{station.name}"...')

            # Compile schedule for this station.
            schedule = Schedule.compile_dict(
                self._schedule_spec, station.latitude, station.longitude,
                station.time_zone)

            # Add schedule to cache.
            self._schedules[station.name] = schedule

            return schedule
<commit_msg>Bulk create clips in random clip creator.<commit_after>"""Module containing class `CreateRandomClipsCommand`."""

from datetime import timedelta as TimeDelta
import logging

import django.db
import numpy as np

from vesper.command.command import Command
from vesper.django.app.models import Clip, Job, Processor, RecordingChannel
from vesper.singleton.archive import archive
from vesper.singleton.preset_manager import preset_manager
from vesper.util.schedule import Interval, Schedule
import vesper.command.command_utils as command_utils
import vesper.util.signal_utils as signal_utils
import vesper.util.time_utils as time_utils

_PROCESSOR_NAME = 'Vesper Random Clip Creator 1.0'
_PROCESSOR_TYPE = 'Clip Creator'
_PROCESSOR_DESCRIPTION = '''
Virtual processor created by Vesper's Create Random Clips command.
The Create Random Clips command assigns this processor as the
creating processor of each clip that it creates.
'''.lstrip()

_logger = logging.getLogger()

# TODO: Consider avoiding collisions across multiple runs of this command.

# TODO: Consider optionally avoiding creating clips that intersect
# specified existing ones, e.g. Call* clips. The command might do this
# by excluding the intervals of the specified clips from consideration
# for random clip creation.

class CreateRandomClipsCommand(Command):

    extension_name = 'create_random_clips'

    def __init__(self, args):

        super().__init__(args)

        get = command_utils.get_required_arg
        self._sm_pair_ui_names = get('station_mics', args)
        self._start_date = get('start_date', args)
        self._end_date = get('end_date', args)
        self._schedule_name = get('schedule', args)
        self._clip_duration = get('clip_duration', args)
        self._clip_count = get('clip_count', args)

        self._schedule_preset = _get_schedule_preset(self._schedule_name)
        if self._schedule_preset is None:
            self._schedule_cache = None
        else:
            self._schedule_cache = _ScheduleCache(self._schedule_preset)

        self._clip_creator = _ClipCreator()

    def execute(self, job_info):

        # django.db.reset_queries()

        self._job_info = job_info

        _logger.info(
            'Gathering information about recording channel intervals in '
            'which to create clips...')

        # Get recording channel intervals in which to create clips,
        # as specified by command arguments.
        sm_pairs = command_utils.get_station_mic_output_pairs(
            self._sm_pair_ui_names)
        intervals = _get_processing_intervals(
            sm_pairs, self._start_date, self._end_date, self._schedule_cache)

        # Get subset of intervals that are *usable* in the sense that
        # they are long enough to contain a clip of duration
        # `self._clip_duration`. Also get time bounds of usable portions
        # of usable intervals in the concatenation of the usable portions.
        # The *usable portion* of a usable interval is the portion of
        # the interval in which a clip of duration `self._clip_duration`
        # can start and still be contained in the interval, i.e. the
        # initial portion ending `self._clip_duration` seconds before
        # the end of the interval. The length of the time bounds array
        # is one more than the number of usable intervals. Element i
        # of the array is the start time of interval i and the end
        # time of interval i - 1, in seconds from the start of the
        # concatenation.
        usable_intervals, time_bounds = \
            _get_concatenated_time_bounds(intervals, self._clip_duration)

        _logger.info('Generating random clip start times...')

        # Get start offsets of random clips in concatenation of usable
        # interval portions, in seconds.
        start_offsets = np.random.uniform(
            time_bounds[0], time_bounds[-1], self._clip_count)

        # Sort start offsets so they are in ascending order. We don't
        # have to do this for the call to `np.searchsorted` below to
        # work, but it seems like a good idea so we can create the
        # clips in a sensible order.
        start_offsets.sort()

        # Get index of usable interval of each clip.
        interval_indices = \
            np.searchsorted(time_bounds, start_offsets, 'right') - 1

        # Create clips.

        creation_time = time_utils.get_utc_now()
        creating_processor = self._get_creating_processor()
        creating_job = Job.objects.get(pk=self._job_info.job_id)

        _logger.info('Creating clips ...')

        # Bookkeeping for start index collision detection and resolution.
        last_interval_num = -1
        used_start_indices = set()

        for i, start_offset in zip(interval_indices, start_offsets):

            station, channel, interval_start_time, interval_end_time = \
                usable_intervals[i]
            recording = channel.recording
            sample_rate = recording.sample_rate

            # Get clip start index in recording.
            start_offset = TimeDelta(seconds=start_offset - time_bounds[i])
            start_time = interval_start_time + start_offset
            start_index = signal_utils.time_to_index(
                start_time, recording.start_time, sample_rate)

            # Get clip length in samples.
            length = signal_utils.seconds_to_frames(
                self._clip_duration, sample_rate)

            if i != last_interval_num:
                # new interval

                last_interval_num = i
                used_start_indices.clear()

            elif start_index in used_start_indices:
                # start index collision

                interval_end_index = signal_utils.time_to_index(
                    interval_end_time, interval_start_time, sample_rate)

                start_index = self._resolve_start_index_collision(
                    start_index, length, used_start_indices,
                    interval_end_index)

                if start_index is None:

                    _logger.warning(
                        'Could not resolve clip start index collision '
                        'while creating random clips in recording channel '
                        f'{str(channel)}. Clip for which collision '
                        'could not be resolved will not be created.')

                    # Move on to next clip.
                    continue

            # Get time of first clip sample.
            start_time = signal_utils.index_to_time(
                start_index, recording.start_time, sample_rate)

            # Get time of last clip sample.
            end_time = \
                signal_utils.get_end_time(start_time, length, sample_rate)

            date = station.get_night(start_time)

            clip = Clip(
                station=station,
                mic_output=channel.mic_output,
                recording_channel=channel,
                start_index=start_index,
                length=length,
                sample_rate=sample_rate,
                start_time=start_time,
                end_time=end_time,
                date=date,
                creation_time=creation_time,
                creating_processor=creating_processor,
                creating_job=creating_job)

            self._clip_creator.add_clip(clip)

            used_start_indices.add(start_index)

        # Make sure to create all clips.
        self._clip_creator.flush()

        # _show_queries('create_random_clips')

        return True

    def _get_creating_processor(self):

        try:
            processor = Processor.objects.get(
                name=_PROCESSOR_NAME, type=_PROCESSOR_TYPE)

        except Processor.DoesNotExist:

            _logger.info(f'Creating processor "{_PROCESSOR_NAME}"...')

            processor = Processor.objects.create(
                name=_PROCESSOR_NAME,
                type=_PROCESSOR_TYPE,
                description=_PROCESSOR_DESCRIPTION
            )

        return processor

def _get_schedule_preset(schedule_name):

    if schedule_name == archive.NULL_CHOICE:
        # no schedule specified

        return None

    else:
        # schedule specified

        preset_path = ('Detection Schedule', schedule_name)
        preset = preset_manager.get_preset(preset_path)
        return preset.data

def _get_processing_intervals(sm_pairs, start_date, end_date, schedule_cache):

    processing_intervals = []

    for station, mic_output in sm_pairs:

        start_time, end_time = \
            station.get_night_interval_utc(start_date, end_date)

        channels = RecordingChannel.objects.filter(
            recording__station=station,
            mic_output=mic_output,
            recording__start_time__lt=end_time,
            recording__end_time__gt=start_time
        ).select_related('recording')

        if schedule_cache is None:
            schedule = None
        else:
            schedule = schedule_cache.get_schedule(station)

        for channel in channels:

            # Get time intervals to process for channel.
            intervals = _get_channel_intervals(channel, schedule)

            # Prepend other information to intervals.
            intervals = [(station, channel) + i for i in intervals]

            processing_intervals.extend(intervals)

    processing_intervals.sort(key=_create_interval_key)

    return processing_intervals

def _get_channel_intervals(channel, schedule):

    recording = channel.recording
    start_time = recording.start_time
    end_time = recording.end_time

    if schedule is None:
        return [(start_time, end_time)]

    else:
        # have schedule

        schedule_intervals = schedule.get_intervals(start_time, end_time)
        recording_interval = Interval(start=start_time, end=end_time)

        # Here we know from the contract of `Schedule.get_intervals`
        # that each interval of `schedule_intervals` intersects the
        # recording interval, so we don't have to worry about
        # `_get_time_intervals_intersection` returning `None`.
        return [
            _get_time_intervals_intersection(i, recording_interval)
            for i in schedule_intervals]

def _get_time_intervals_intersection(a, b):

    if a.end < b.start or b.end < a.start:
        # intervals do not intersect

        return None

    else:
        # intervals intersect

        start = a.start if b.start < a.start else b.start
        end = a.end if a.end < b.end else b.end
        return (start, end)

def _create_interval_key(interval):
    station, channel, start_time, _ = interval
    return (
        station.name, channel.recording.start_time, channel.channel_num,
        start_time)

def _get_concatenated_time_bounds(intervals, clip_duration):

   # The work of this function is divided as it is to make it easier
    # to unit test the `_get_concatenated_time_bounds_aux` function.

    # Prepare intervals for `_get_concatenated_time_bounds_aux`.
    diddled_intervals = _diddle_intervals(intervals)

    # Figure out which intervals are long enough to create clips in,
    # and get the time bounds of the portions of ths intervals in
    # which clips can start in their concatenation.
    usable_interval_indices, time_bounds = \
        _get_concatenated_time_bounds_aux(diddled_intervals, clip_duration)

    # Get usable intervals from indices.
    usable_intervals = [intervals[i] for i in usable_interval_indices]

    return usable_intervals, time_bounds

def _diddle_intervals(intervals):
    return [_diddle_interval(i) for i in intervals]

def _diddle_interval(interval):
    _, channel, start_time, end_time = interval
    recording = channel.recording
    sample_rate = recording.sample_rate
    recording_start_time = recording.start_time
    return recording_start_time, sample_rate, start_time, end_time

def _get_concatenated_time_bounds_aux(intervals, clip_duration):

    s2f = signal_utils.seconds_to_frames
    f2s = signal_utils.get_duration

    usable_interval_indices = []
    time_bounds = []
    last_end_time = 0.

    for i, interval in enumerate(intervals):

        recording_start_time, sample_rate, start_time, end_time = interval

        # Get usable subinterval length in samples. Work in samples
        # instead of in seconds to ensure that a clip of the desired
        # length will be entirely contained in the interval if it
        # starts in what we decide is the usable subinterval.

        offset = (start_time - recording_start_time).total_seconds()
        start_index = s2f(offset, sample_rate)

        offset = (end_time - recording_start_time).total_seconds()
        end_index = s2f(offset, sample_rate)

        interval_length = end_index - start_index

        clip_length = s2f(clip_duration, sample_rate)

        usable_length = interval_length - clip_length

        if usable_length > 0:
            # interval is longer than clip length

            usable_interval_indices.append(i)
            time_bounds.append(last_end_time)

            usable_duration = f2s(usable_length, sample_rate)
            last_end_time += usable_duration

    time_bounds.append(last_end_time)

    return usable_interval_indices, np.array(time_bounds)

def _resolve_start_index_collision(
        start_index, length, used_start_indices, interval_end_index):

    # This method tries to resolve a start index collision, first
    # by looking for a later usable start index and then, if none
    # is available, for an earlier one. For reasonable random clip
    # densities, the next start index will almost always be available
    # so this method will just return that.

    # Find first start index after `start_index` that is unused.
    # This will usually be the next index, but might not be if other
    # collisions for this same start index have already been resolved.
    new_start_index = start_index + 1
    while new_start_index in used_start_indices:
        new_start_index += 1

    if new_start_index + length <= interval_end_index:
        # clip with this start index fits in interval

        return new_start_index

    # If we get here, there are no unused start indices for this interval
    # after `start_index`.

    # Find first start index before `start_index` that is unused.
    new_start_index = start_index - 1
    while new_start_index in used_start_indices:
        new_start_index -= 1

    if new_start_index >= 0:
        return new_start_index

    # If we get here, there are no unused start indices for this interval!
    return None

def _show_queries(name):
    connection = django.db.connection
    queries = connection.queries
    print(f'{name} performed {len(queries)} queries:')
    for i, query in enumerate(connection.queries):
        print(f'    {i}: {query}')

class _ScheduleCache:

    def __init__(self, schedule_spec):
        self._schedule_spec = schedule_spec
        self._schedules = {}

    def get_schedule(self, station):

        try:
            return self._schedules[station.name]

        except KeyError:
            # cache miss

            if _logger is not None:
                _logger.info(
                    f'Compiling schedule for station "{station.name}"...')

            # Compile schedule for this station.
            schedule = Schedule.compile_dict(
                self._schedule_spec, station.latitude, station.longitude,
                station.time_zone)

            # Add schedule to cache.
            self._schedules[station.name] = schedule

            return schedule

class _ClipCreator:

    def __init__(self, batch_size=50):
        self._batch_size = batch_size
        self._clips = []

    def add_clip(self, clip):
        self._clips.append(clip)
        if len(self._clips) == self._batch_size:
            self.flush()

    def flush(self):
        if len(self._clips) != 0:
            Clip.objects.bulk_create(self._clips)
            self._clips.clear()
<|endoftext|>


<commit_before>"""TensorFlow Lite Support Library Helper Rules for iOS"""

# When the static framework is built with bazel, the all header files are moved
# to the "Headers" directory with no header path prefixes. This auxiliary rule
# is used for stripping the path prefix to the C/iOS API header files included by
# other C/iOS API header files.
# In case of C header files includes start with a keyword of "#include'.
# Imports in iOS header files start with a keyword of '#import'.
def strip_api_include_path_prefix(name, hdr_labels, prefix = ""):
    """Create modified header files with the import path stripped out.

    Args:
      name: The name to be used as a prefix to the generated genrules.
      hdr_labels: List of header labels to strip out the include path. Each
          label must end with a colon followed by the header file name.
      prefix: Optional prefix path to prepend to the header inclusion path.
    """
    for hdr_label in hdr_labels:
        hdr_filename = hdr_label.split(":")[-1]
        # The last path component of iOS header files can be sources/some_file.h
        # or Sources/some_file.h. Hence it wiill contain a '/'.
        if "/" in hdr_filename:
            hdr_filename = hdr_filename.split("/")[-1]

        hdr_basename = hdr_filename.split(".")[0]
        native.genrule(
            name = "{}_{}".format(name, hdr_basename),
            srcs = [hdr_label],
            outs = [hdr_filename],
            cmd = """
            sed 's|#\\([a-z]*\\) ".*/\\([^/]\\{{1,\\}}\\.h\\)"|#\\1 "{}\\2"|'\
            "$(location {})"\
            > "$@"
            """.format(prefix, hdr_label),
        )

<commit_msg>Updated comments<commit_after>"""TensorFlow Lite Support Library Helper Rules for iOS"""

# When the static framework is built with bazel, the all header files are moved
# to the "Headers" directory with no header path prefixes. This auxiliary rule
# is used for stripping the path prefix to the C/iOS API header files included by
# other C/iOS API header files.
# In case of C header files includes start with a keyword of "#include'.
# Imports in iOS header files start with a keyword of '#import'.
def strip_api_include_path_prefix(name, hdr_labels, prefix = ""):
    """Create modified header files with the import path stripped out.

    Args:
      name: The name to be used as a prefix to the generated genrules.
      hdr_labels: List of header labels to strip out the include path. Each
          label must end with a colon followed by the header file name.
      prefix: Optional prefix path to prepend to the header inclusion path.
    """
    for hdr_label in hdr_labels:
        hdr_filename = hdr_label.split(":")[-1]
        # The last path component of iOS header files can be sources/some_file.h
        # or Sources/some_file.h. Hence it wiill contain a '/'. So the string
        # can be split at / to get the header file name.
        if "/" in hdr_filename:
            hdr_filename = hdr_filename.split("/")[-1]

        hdr_basename = hdr_filename.split(".")[0]
        native.genrule(
            name = "{}_{}".format(name, hdr_basename),
            srcs = [hdr_label],
            outs = [hdr_filename],
            cmd = """
            sed 's|#\\([a-z]*\\) ".*/\\([^/]\\{{1,\\}}\\.h\\)"|#\\1 "{}\\2"|'\
            "$(location {})"\
            > "$@"
            """.format(prefix, hdr_label),
        )

<|endoftext|>