<commit_before>#SPARQL_BAMS_Store_Persist_Example.py
#This program is used to open up BAMS data, persist it to a store for speed up of querying

#For Parsing
import rdflib
from rdflib import plugin

#for getting the length of the files
import os

#for working with tempfiles
import os.path as op
import tempfile

#For Unzipping
import zipfile
from StringIO import StringIO

plugin.register(
    'sparql', rdflib.query.Processor,
    'rdfextras.sparql.processor', 'Processor')
plugin.register(
    'sparql', rdflib.query.Result,
    'rdfextras.sparql.query', 'SPARQLQueryResult')

zipdata = StringIO()

# open the file using a relative path
#r = open("../Data/BAMS1.zip")

# adding the BAMS Thesaurus instead of the more limited set of data:
r = open("../Data/bams_thesaurus_2013-09-24_17-12-40.xml.zip")

# zipdata is a buffer holding the contents of the zip file in memory
zipdata.write(r.read())

print("~40 seconds for zip to open...")

#myzipfile opens the contents of the zip file as an object that knows how to unzip
myzipfile = zipfile.ZipFile(zipdata)

#grab the contents out of myzipfile by name
#foofile = myzipfile.open('bams_ontology_2013-07-10_03-20-00.xml')

#changing the foofile to be the file we upen above^^^^^ in r = open()....etc.
foofile = myzipfile.open('bams_thesaurus_2013-09-24_17-12-40.xml')

print("loading up the BAMS file in memory...")

#Get a Graph object using a Sleepycat persistent store
g = rdflib.Graph('Sleepycat',identifier='BAMS')

# first time create the store
#  put the store in a temp directory so it doesn't get confused with stuff we should commit
tempStore = op.join( tempfile.gettempdir(), 'myRDF_BAMS_Store')
g.open(tempStore, create = True)

#pull in the BAMS RDF document, parse, and store.
result = g.parse(file=myzipfile.open('bams_ontology_2013-07-10_03-20-00.xml'), format="application/rdf+xml")

foofile.close()

# when done!
g.close()

print("Graph stored to disk")<commit_msg>Storing BAMS Thesaurus to Disk<commit_after>#SPARQL_BAMS_Store_Persist_Example.py
#This program is used to open up BAMS data, persist it to a store for speed up of querying

#For Parsing
import rdflib
from rdflib import plugin

#for getting the length of the files
import os

#for working with tempfiles
import os.path as op
import tempfile

#For Unzipping
import zipfile
from StringIO import StringIO

plugin.register(
    'sparql', rdflib.query.Processor,
    'rdfextras.sparql.processor', 'Processor')
plugin.register(
    'sparql', rdflib.query.Result,
    'rdfextras.sparql.query', 'SPARQLQueryResult')

zipdata = StringIO()

# open the file using a relative path
#r = open("../Data/BAMS1.zip")

# adding the BAMS Thesaurus instead of the more limited set of data:
r = open("../Data/bams_thesaurus_2013-09-24_17-12-40.xml.zip")

# zipdata is a buffer holding the contents of the zip file in memory
zipdata.write(r.read())

print("~40 seconds for zip to open...")

#myzipfile opens the contents of the zip file as an object that knows how to unzip
myzipfile = zipfile.ZipFile(zipdata)

#grab the contents out of myzipfile by name
#foofile = myzipfile.open('bams_ontology_2013-07-10_03-20-00.xml')

#changing the foofile to be the file we upen above^^^^^ in r = open()....etc.
foofile = myzipfile.open('bams_thesaurus_2013-09-24_17-12-40.xml')

print("loading up the BAMS file in memory...")

#Get a Graph object using a Sleepycat persistent store
g = rdflib.Graph('Sleepycat',identifier='BAMS')

# first time create the store
#  put the store in a temp directory so it doesn't get confused with stuff we should commit
tempStore = op.join( tempfile.gettempdir(), 'myRDF_BAMS_Store')
g.open(tempStore, create = True)

#pull in the BAMS RDF document, parse, and store.
#result = g.parse(file=myzipfile.open('bams_ontology_2013-07-10_03-20-00.xml'), format="application/rdf+xml")

#do the same thing but with the BAMS thesaurus file
result = g.parse(file=myzipfile.open('bams_thesaurus_2013-09-24_17-12-40.xml'), format="application/rdf+xml")

foofile.close()

# when done!
g.close()

print("Graph stored to disk")<|endoftext|>


<commit_before>import importlib

class LammpsPotential:
    def __init__(self,
                 potential_data,
                 structure,
                 potential_filename='None'):

        self._names = [site.name for site in structure.kinds]
        self._type = potential_data.dict.pair_style
        self._data = potential_data.dict.data

        #self._type = potential_data['pair_style']
        #self._data = potential_data['data']
        #self._names = None

        self._potential_filename = potential_filename

        try:
            self._potential_module = importlib.import_module('.tersoff')
            #import tersoff
            #self._potential_module = tersoff

        except ImportError:
            raise ImportError('This lammps potential is not implemented')

#        if self._type == 'tersoff':
#            from tersoff import _generate_LAMMPS_potential, _get_input_potential_lines
#        elif self._type == 'lennard_jones':
#            from lennard_jones import _generate_LAMMPS_potential, _get_input_potential_lines
#        else:
#            raise ValueError('This lammps potential is not implemented')

#        self._generate_LAMMPS_potential = _generate_LAMMPS_potential
#        self._get_input_potential_lines = _get_input_potential_lines

    def get_potential_file(self):
        return self._potential_module.generate_LAMMPS_potential(self._data)

    def get_input_potential_lines(self):
        return self._potential_module.get_input_potential_lines(self._data,
                                                                potential_filename=self._potential_filename,
                                                                names=self._names)

if __name__ == "__main__":
    potential = {'pair_style': 'lennard_jones',
                 #                 epsilon,  sigma, cutoff
                 'data': {'1  1': '0.01029   3.4    2.5',
                          # '2  2':   '1.0      1.0    2.5',
                          # '1  2':   '1.0      1.0    2.5'
                          }}
    structure = None
    test = LammpsPotential(potential, structure)
    print test.get_potential_file()
    print test.get_input_potential_lines()<commit_msg>testing better importing system<commit_after>import importlib

class LammpsPotential:
    def __init__(self,
                 potential_data,
                 structure,
                 potential_filename='None'):

        self._names = [site.name for site in structure.kinds]
        self._type = potential_data.dict.pair_style
        self._data = potential_data.dict.data

        #self._type = potential_data['pair_style']
        #self._data = potential_data['data']
        #self._names = None

        self._potential_filename = potential_filename

        try:
            self._potential_module = importlib.import_module('.tersoff', __name__)
            #import tersoff
            #self._potential_module = tersoff

        except ImportError:
            raise ImportError('This lammps potential is not implemented')

#        if self._type == 'tersoff':
#            from tersoff import _generate_LAMMPS_potential, _get_input_potential_lines
#        elif self._type == 'lennard_jones':
#            from lennard_jones import _generate_LAMMPS_potential, _get_input_potential_lines
#        else:
#            raise ValueError('This lammps potential is not implemented')

#        self._generate_LAMMPS_potential = _generate_LAMMPS_potential
#        self._get_input_potential_lines = _get_input_potential_lines

    def get_potential_file(self):
        return self._potential_module.generate_LAMMPS_potential(self._data)

    def get_input_potential_lines(self):
        return self._potential_module.get_input_potential_lines(self._data,
                                                                potential_filename=self._potential_filename,
                                                                names=self._names)

if __name__ == "__main__":
    potential = {'pair_style': 'lennard_jones',
                 #                 epsilon,  sigma, cutoff
                 'data': {'1  1': '0.01029   3.4    2.5',
                          # '2  2':   '1.0      1.0    2.5',
                          # '1  2':   '1.0      1.0    2.5'
                          }}
    structure = None
    test = LammpsPotential(potential, structure)
    print test.get_potential_file()
    print test.get_input_potential_lines()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import json
import operator as op

from jsonschema import validate

MOD_URL = 'http://modomics.genesilico.pl/sequences/list/{id}'

TRUSTED_DB = set([
    'gtrnadb',
    'lncrnadb',
    'mirbase',
    'modomics',
    'pdbe',
    'snopy'
    'srpdb',
    'tmrna website',
])

def external_id(data):
    if data['database'] == 'PDBe':
        return '%s_%s' % (data['external_id'], data['optional_id'])
    if data['database'] == 'Modomics':
        return MOD_URL.format(id=data['external_id'])
    return data['external_id']

def is_high_quality(data):
    name = data['database'].lower()
    if name in TRUSTED_DB:
        return True
    if name == 'rfam':
        return data['molecule_type'] == 'seed'
    return False

def as_xref(xref):
    return {'database': xref['database'], 'id': external_id(xref)}

def builder(data):
    result = dict(data)
    xrefs = []
    seen = set()
    key = op.itemgetter('database', 'id')
    for xref in data['xrefs']:
        if not is_high_quality(xref):
            continue

        updated = as_xref(xref)
        value = key(updated)
        if value not in seen:
            xrefs.append(updated)
            seen.add(value)

    result['xrefs'] = xrefs
    return result

def generate_file(raw, output, schema_file=None):
    results = [builder(json.loads(l)) for l in raw]

    if schema_file:
        with open(schema_file, 'r') as raw:
            validate(results, json.load(raw))

    json.dump(results, output)
<commit_msg>Deal with incorrect entries in the database<commit_after># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import json
import operator as op

from jsonschema import validate

MOD_URL = 'http://modomics.genesilico.pl/sequences/list/{id}'

TRUSTED_DB = set([
    'gtrnadb',
    'lncrnadb',
    'mirbase',
    'modomics',
    'pdbe',
    'snopy'
    'srpdb',
    'tmrna website',
])

def external_id(data):
    if data['database'] == 'PDBe':
        return '%s_%s' % (data['external_id'], data['optional_id'])
    if data['database'] == 'Modomics':
        return MOD_URL.format(id=data['external_id'])
    return data['external_id']

def is_high_quality(data):
    name = data['database'].lower()
    if name in TRUSTED_DB:
        return True
    if name == 'rfam':
        return data['molecule_type'] == 'seed'
    return False

def as_xref(xref):
    return {'database': xref['database'], 'id': external_id(xref)}

def builder(data):
    result = dict(data)
    xrefs = []
    seen = set()
    key = op.itemgetter('database', 'id')
    for xref in data['xrefs']:
        if not is_high_quality(xref):
            continue

        updated = as_xref(xref)
        value = key(updated)
        if value not in seen:
            xrefs.append(updated)
            seen.add(value)

    result['sequence'] = result['sequence'].upper().replace('U', 'T')
    result['xrefs'] = xrefs
    return result

def generate_file(raw, output, schema_file=None):
    results = [builder(json.loads(l)) for l in raw]

    if schema_file:
        with open(schema_file, 'r') as raw:
            validate(results, json.load(raw))

    json.dump(results, output)
<|endoftext|>


<commit_before># Copyright 2012-2013 Johannes Reinhardt <jreinhardt@ist-dein-freund.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from common import BackendData, BackendExporter
from os import listdir,makedirs
from os.path import join, exists, basename,splitext
from shutil import rmtree,copy,copytree
from errors import *
import yaml
import importlib

_freecad_base_specification = {
	"file-function" : (["filename","author","license","type","functions"],[]),
	"file-fcstd" : (["filename","author","license","type","objects"],[]),
	"function" : (["name","classids"],["baseid"]),
	"object" : (["objectlabel","classids"],["baseid"])
}

class FreeCADBase:
	def __init__(self,basefile,collname,backend_root):
		self.collection = collname
		self.filename = join(backend_root,collname,basefile["filename"])
		self.path = join(collname,self.filename)
		self.author = basefile["author"]
		self.license = basefile["license"]
	def add_part(self,params,doc):
		raise NotImplementedError

class BaseFunction(FreeCADBase):
	def __init__(self,function,basefile,collname,backend_root):
		self._check_conformity(function,basefile)
		FreeCADBase.__init__(self,basefile,collname,backend_root)
		self.name = function["name"]
		self.baseid = self.name
		if "baseid" in function:
			self.baseid = function["baseid"]
		self.classids = function["classids"]
		self.module_name = splitext(basename(self.filename))[0]
	def _check_conformity(self,function, basefile):
		spec = _freecad_base_specification
		check_dict(function,spec["function"])
		check_dict(basefile,spec["file-function"])
	def add_part(self,params,doc):
		module = importlib.import_module("BOLTS.freecad.%s.%s" % (self.collection,self.module_name))
		module.__dict__[self.name](params,doc)

class BaseFcstd(FreeCADBase):
	def __init__(self,obj,basefile, collname,backend_root):
		self._check_conformity(obj,basefile)
		FreeCADBase.__init__(self,basefile,collname,backend_root)
		self.objectlabel = obj["objectlabel"]
		self.baseid = self.objectlabel
		if "baseid" in obj:
			self.baseid = obj["baseid"]

	def _check_conformity(self,obj,basefile):
		spec = _freecad_base_specification
		check_dict(basefile,spec["file-fcstd"])
		check_dict(obj,spec["object"])

	def _recursive_copy(self,obj,doc):
		doc.copyObject(obj)
		obj_copy = doc.getObject(obj.Name)
		for child in obj.OutList:
			child_copy = self._recursive_copy(child,doc)
			obj_copy.OutList.append(child_copy)
			child_copy.InList.append(obj_copy)
		return obj_copy

	def add_part(self,params,doc):
		import FreeCAD
		newdoc = FreeCAD.openDocument(self.filename)
		obj = newdoc.getObjectsByLabel(self.objectlabel)
		if len(obj) != 1:
			raise MalformedBaseError("No file %s found" % self.filename)
		obj_copy = self._recursive_copy(obj[0],doc)
		FreeCAD.setActiveDocument(doc.Name)
		FreeCAD.closeDocument(newdoc.Name)

class FreeCADData(BackendData):
	def __init__(self,path):
		BackendData.__init__(self,"freecad",path)
		self.getbase = {}

		for coll in listdir(self.backend_root):
			basename = join(self.backend_root,coll,"%s.base" % coll)
			if not exists(basename):
				#skip directory that is no collection
				continue
			base_info =  list(yaml.load_all(open(basename)))
			if len(base_info) != 1:
				raise MalformedCollectionError(
						"Not exactly one YAML document found in file %s" % bltname)
			base_info = base_info[0]
			for basefile in base_info:
				if basefile["type"] == "function":
					basepath = join(self.backend_root,coll,"%s.py" % coll)
					if not exists(basepath):
						raise MalformedBaseError("Python module %s does not exist" % basepath)
					for func in basefile["functions"]:
						try:
							function = BaseFunction(func,basefile,coll,self.backend_root)
							for id in func["classids"]:
								self.getbase[id] = function
						except ParsingError as e:
							e.set_base(basefile["filename"])
							e.set_collection(coll)
							raise e
#				elif basefile["type"] == "fcstd":
#					basepath = join(self.backend_root,coll,basefile["filename"])
#					if not exists(basepath):
#						raise MalformedBaseError("Fcstd file %s does not exist" % basepath)
#					for obj in basefile["objects"]:
#						try:
#							fcstd = BaseFcstd(obj,basefile,coll,self.backend_root)
#							for id in obj["classids"]:
#								self.getbase[id] = fcstd
#						except ParsingError as e:
#							e.set_base(basefile["filename"])
#							e.set_collection(coll)
#							raise e

class FreeCADExporter(BackendExporter):
	def write_output(self,repo):
		freecad = repo.freecad

		repo_path = freecad.repo_root
		out_path = freecad.out_root
		bolts_path = join(out_path,"BOLTS")

		self.clear_output_dir(freecad)

		#generate macro
		start_macro = open(join(out_path,"start_bolts.py"),"w")
		start_macro.write("import BOLTS\n")
		start_macro.close()

		#copy files
		copytree(join(repo_path,"data"),join(bolts_path,"data"))
		copytree(join(repo_path,"bolttools"),join(bolts_path,"bolttools"))
		copytree(join(repo_path,"drawings"),join(bolts_path,"drawings"))
		copytree(join(repo_path,"freecad"),join(bolts_path,"freecad"))
		copy(join(repo_path,"__init__.py"),bolts_path)
<commit_msg>copying objects from Fcstd bases now works<commit_after># Copyright 2012-2013 Johannes Reinhardt <jreinhardt@ist-dein-freund.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from common import BackendData, BackendExporter
from os import listdir,makedirs
from os.path import join, exists, basename,splitext
from shutil import rmtree,copy,copytree
from errors import *
import yaml
import importlib

_freecad_base_specification = {
	"file-function" : (["filename","author","license","type","functions"],[]),
	"file-fcstd" : (["filename","author","license","type","objects"],[]),
	"function" : (["name","classids"],["baseid"]),
	"object" : (["objectlabel","classids"],["baseid"])
}

class FreeCADBase:
	def __init__(self,basefile,collname,backend_root):
		self.collection = collname
		self.filename = join(backend_root,collname,basefile["filename"])
		self.path = join(collname,self.filename)
		self.author = basefile["author"]
		self.license = basefile["license"]
	def add_part(self,params,doc):
		raise NotImplementedError

class BaseFunction(FreeCADBase):
	def __init__(self,function,basefile,collname,backend_root):
		self._check_conformity(function,basefile)
		FreeCADBase.__init__(self,basefile,collname,backend_root)
		self.name = function["name"]
		self.baseid = self.name
		if "baseid" in function:
			self.baseid = function["baseid"]
		self.classids = function["classids"]
		self.module_name = splitext(basename(self.filename))[0]
	def _check_conformity(self,function, basefile):
		spec = _freecad_base_specification
		check_dict(function,spec["function"])
		check_dict(basefile,spec["file-function"])
	def add_part(self,params,doc):
		module = importlib.import_module("BOLTS.freecad.%s.%s" % (self.collection,self.module_name))
		module.__dict__[self.name](params,doc)

class BaseFcstd(FreeCADBase):
	def __init__(self,obj,basefile, collname,backend_root):
		self._check_conformity(obj,basefile)
		FreeCADBase.__init__(self,basefile,collname,backend_root)
		self.objectlabel = obj["objectlabel"]
		self.baseid = self.objectlabel
		if "baseid" in obj:
			self.baseid = obj["baseid"]

	def _check_conformity(self,obj,basefile):
		spec = _freecad_base_specification
		check_dict(basefile,spec["file-fcstd"])
		check_dict(obj,spec["object"])

	def _recursive_copy(self,obj,doc):
		doc.copyObject(obj)
		obj_copy = doc.getObjectsByLabel(obj.Label)
		if len(obj_copy) != 1:
			raise ValueError("Something went wrong")
		obj_copy = obj_copy[0]
		for prop_name in obj.PropertiesList:
			prop = obj.getPropertyByName(prop_name)
			prop_copy = obj_copy.getPropertyByName(prop_name)
			if prop_copy is None and (not prop is None):
				setattr(obj_copy,prop_name,self._recursive_copy(prop,doc))
		return obj_copy

	def add_part(self,params,doc):
		import FreeCAD
		newdoc = FreeCAD.openDocument(self.filename)
		obj = newdoc.getObjectsByLabel(self.objectlabel)
		if len(obj) != 1:
			raise MalformedBaseError("No file %s found" % self.filename)
		obj_copy = self._recursive_copy(obj[0],doc)
		obj_copy.touch()
		FreeCAD.setActiveDocument(doc.Name)
		FreeCAD.closeDocument(newdoc.Name)

class FreeCADData(BackendData):
	def __init__(self,path):
		BackendData.__init__(self,"freecad",path)
		self.getbase = {}

		for coll in listdir(self.backend_root):
			basename = join(self.backend_root,coll,"%s.base" % coll)
			if not exists(basename):
				#skip directory that is no collection
				continue
			base_info =  list(yaml.load_all(open(basename)))
			if len(base_info) != 1:
				raise MalformedCollectionError(
						"Not exactly one YAML document found in file %s" % bltname)
			base_info = base_info[0]
			for basefile in base_info:
				if basefile["type"] == "function":
					basepath = join(self.backend_root,coll,"%s.py" % coll)
					if not exists(basepath):
						raise MalformedBaseError("Python module %s does not exist" % basepath)
					for func in basefile["functions"]:
						try:
							function = BaseFunction(func,basefile,coll,self.backend_root)
							for id in func["classids"]:
								self.getbase[id] = function
						except ParsingError as e:
							e.set_base(basefile["filename"])
							e.set_collection(coll)
							raise e
				elif basefile["type"] == "fcstd":
					basepath = join(self.backend_root,coll,basefile["filename"])
					if not exists(basepath):
						raise MalformedBaseError("Fcstd file %s does not exist" % basepath)
					for obj in basefile["objects"]:
						try:
							fcstd = BaseFcstd(obj,basefile,coll,self.backend_root)
							for id in obj["classids"]:
								self.getbase[id] = fcstd
						except ParsingError as e:
							e.set_base(basefile["filename"])
							e.set_collection(coll)
							raise e

class FreeCADExporter(BackendExporter):
	def write_output(self,repo):
		freecad = repo.freecad

		repo_path = freecad.repo_root
		out_path = freecad.out_root
		bolts_path = join(out_path,"BOLTS")

		self.clear_output_dir(freecad)

		#generate macro
		start_macro = open(join(out_path,"start_bolts.py"),"w")
		start_macro.write("import BOLTS\n")
		start_macro.close()

		#copy files
		copytree(join(repo_path,"data"),join(bolts_path,"data"))
		copytree(join(repo_path,"bolttools"),join(bolts_path,"bolttools"))
		copytree(join(repo_path,"drawings"),join(bolts_path,"drawings"))
		copytree(join(repo_path,"freecad"),join(bolts_path,"freecad"))
		copy(join(repo_path,"__init__.py"),bolts_path)
<|endoftext|>


<commit_before># Jacob Martin
# CS 311

import json

class Grammar:

	def __init__(self, source):
		self.desc  = ""
		self.rules = dict()
		try:
			grammar   = json.load(open(source))
			self.desc = grammar["desc"]
			for variable, rules in grammar["rules"].items():
				self.rules[variable] = rules
		except FileNotFoundError:
			pass

	def get_desc(self):
		return self.desc

	def produces(self, variable):
		if variable not in self.rules:
			return False

		return self.rules[variable]

	def get_rule(self, variable, input_symbol):
		if len(input_symbol) != 1:
			return False

		productions = self.produces(variable)
		if not productions:
			return False

		for production in productions:
			if production[0] == input_symbol:
				return production

		return False
<commit_msg>Fix file warning<commit_after># Jacob Martin
# CS 311

import json

class Grammar:

	def __init__(self, source):
		self.desc  = ""
		self.rules = dict()
		try:
			with open(source) as json_file:
				grammar = json.load(json_file)
			self.desc = grammar["desc"]
			for variable, rules in grammar["rules"].items():
				self.rules[variable] = rules
		except FileNotFoundError:
			pass

	def get_desc(self):
		return self.desc

	def produces(self, variable):
		if variable not in self.rules:
			return False

		return self.rules[variable]

	def get_rule(self, variable, input_symbol):
		if len(input_symbol) != 1:
			return False

		productions = self.produces(variable)
		if not productions:
			return False

		for production in productions:
			if production[0] == input_symbol:
				return production

		return False
<|endoftext|>


<commit_before>from keywordgroup import KeywordGroup

class _CookieKeywords(KeywordGroup):

    def delete_all_cookies(self):
        self._current_browser().delete_all_cookies()

    def get_cookies(self):
        pairs = []
        for cookie in self._current_browser().get_cookies():
            pairs.append(cookie['name'] + "=" + cookie['value'])
        return '; '.join(pairs)

    def get_cookie_value(self, name):
        cookie = self._current_browser().get_cookie(name)
        if cookie is not None:
            return cookie['value']
        return None

    def delete_cookie(self, name):
        self._current_browser().delete_cookie(name)
<commit_msg>Documentation for cookie keywords<commit_after>from keywordgroup import KeywordGroup

class _CookieKeywords(KeywordGroup):

    def delete_all_cookies(self):
        """Deletes all cookies."""
        self._current_browser().delete_all_cookies()

    def delete_cookie(self, name):
        """Deletes cookie matching `name`.

        If the cookie is not found, nothing happens.
        """
        self._current_browser().delete_cookie(name)

    def get_cookies(self):
        """Returns all cookies of the current page."""
        pairs = []
        for cookie in self._current_browser().get_cookies():
            pairs.append(cookie['name'] + "=" + cookie['value'])
        return '; '.join(pairs)

    def get_cookie_value(self, name):
        """Returns value of cookie found with `name`.

        If no cookie is found with `name`, this keyword fails.
        """
        cookie = self._current_browser().get_cookie(name)
        if cookie is not None:
            return cookie['value']
        raise ValueError("Cookie with name %s not found." % name)
<|endoftext|>


<commit_before>#!/usr/local/bin/python

import CommonGraphDiffer as cgd
import pygraphviz as pgv
import argparse

def parseArgs():
    parser = argparse.ArgumentParser()
    parser.add_argument("cg1", help="This is a .CGX (CommonGraph) file")
    parser.add_argument("ds", help="This is a .DSX (DiffSet) file")
    parser.add_argument("png", help="This is the output filename of a .png file")
    return parser.parse_args()

def graphAddCGX(G, CGX, col='black'):
    # add all nodes from CGA
    G.edge_attr['color'] = col
    G.node_attr['color'] = col
    for (nodeid, name, position) in CGX.getNodesForGraphviz():
        print (nodeid, name, position)
        G.add_node(nodeid, color=col, label=name, pin="true", pos=position + "!")
    for (src, dst) in CGX.getEdgePairs():
        print (src, dst)
        G.add_edge(src,dst)
    return G

def graphApplyDS(G, CGX,  ds, addedColor, removedColor, changedColor):
    for thisNodeChange in [change for change in ds.Changes if change.__class__.__name__ == "NodeChange"]:
        print thisNodeChange
        if(thisNodeChange.Status == "added"):
            (nodeid, name, position) = cgd.Node.getGraphVizRep(thisNodeChange)
            G.add_node(nodeid, color=addedColor, label=name, pin="true", pos=position + "!")
        if(thisNodeChange.Status == "removed"):
            G.get_node(thisNodeChange.InstanceGuid).attr['color']=removedColor
        if(thisNodeChange.Status == "changed"):
            G.get_node(thisNodeChange.InstanceGuid).attr['color']=changedColor

    for thisEdgeChange in [change for change in ds.Changes if change.__class__.__name__ == "EdgeChange"]:
        print thisEdgeChange
        if(thisEdgeChange.Status == "added"):
            # try to find parentEdge from CGX
            parentEdge = cgd.Edge.parentEdge(CGX, thisEdgeChange, ds)
            G.add_edge(*parentEdge, color=addedColor)
        if(thisEdgeChange.Status == "removed"):
            thisEdge = [ed for ed in CGX.Edges if ed.InstanceGuid == thisEdgeChange.InstanceGuid][0]
            G.get_edge(*cgd.Edge.parentEdge(CGX, thisEdge)).attr['color']= removedColor
    return G

def main():
    args = parseArgs()
    CGA = cgd.CgxToObject(args.cg1)
    ds =  cgd.XMLToDS(args.ds)

    G = pgv.AGraph()

    G.node_attr['fontsize'] =  5.0

    G = graphAddCGX(G, CGA, 'grey')

    G = graphApplyDS(G, CGA, ds, addedColor='blue', removedColor='red', changedColor='orange')

    print G.string()
    G.layout() # layout with default (neato)
    G.draw(args.png)

if __name__ == "__main__":
    main()
<commit_msg>multiple edges, directed graphs<commit_after>#!/usr/local/bin/python

import CommonGraphDiffer as cgd
import pygraphviz as pgv
import argparse

def parseArgs():
    parser = argparse.ArgumentParser()
    parser.add_argument("cg1", help="This is a .CGX (CommonGraph) file")
    parser.add_argument("ds", help="This is a .DSX (DiffSet) file")
    parser.add_argument("png", help="This is the output filename of a .png file")
    return parser.parse_args()

def graphAddCGX(G, CGX, col='black'):
    # add all nodes from CGA
    G.edge_attr['color'] = col
    G.node_attr['color'] = col
    for (nodeid, name, position) in CGX.getNodesForGraphviz():
        print (nodeid, name, position)
        G.add_node(nodeid, color=col, label=name, pin="true", pos=position + "!")
    for (src, dst) in CGX.getEdgePairs():
        print (src, dst)
        G.add_edge(src,dst)
    return G

def graphApplyDS(G, CGX,  ds, addedColor, removedColor, changedColor):
    for thisNodeChange in [change for change in ds.Changes if change.__class__.__name__ == "NodeChange"]:
        print thisNodeChange
        if(thisNodeChange.Status == "added"):
            (nodeid, name, position) = cgd.Node.getGraphVizRep(thisNodeChange)
            G.add_node(nodeid, color=addedColor, label=name, pin="true", pos=position + "!")
        if(thisNodeChange.Status == "removed"):
            G.get_node(thisNodeChange.InstanceGuid).attr['color']=removedColor
        if(thisNodeChange.Status == "changed"):
            G.get_node(thisNodeChange.InstanceGuid).attr['color']=changedColor

    for thisEdgeChange in [change for change in ds.Changes if change.__class__.__name__ == "EdgeChange"]:
        print thisEdgeChange
        if(thisEdgeChange.Status == "added"):
            # try to find parentEdge from CGX
            parentEdge = cgd.Edge.parentEdge(CGX, thisEdgeChange, ds)
            G.add_edge(*parentEdge, color=addedColor)
        if(thisEdgeChange.Status == "removed"):
            thisEdge = [ed for ed in CGX.Edges if ed.InstanceGuid == thisEdgeChange.InstanceGuid][0]
            G.get_edge(*cgd.Edge.parentEdge(CGX, thisEdge)).attr['color']= removedColor
    return G

def main():
    args = parseArgs()
    CGA = cgd.CgxToObject(args.cg1)
    ds =  cgd.XMLToDS(args.ds)

    G = pgv.AGraph(strict=False, directed=True)

    G.node_attr['fontsize'] =  5.0

    G = graphAddCGX(G, CGA, 'grey')

    G = graphApplyDS(G, CGA, ds, addedColor='blue', removedColor='red', changedColor='orange')

    print G.string()
    G.layout() # layout with default (neato)
    G.draw(args.png)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from ggame import App, Color, LineStyle, Sprite
from ggame import RectangleAsset, CircleAsset
import math

SCREEN_WIDTH = 1536
SCREEN_HEIGHT = 1024

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
purple = Color(0xff00ff, 1.0)
thinline = LineStyle(0, black)

xcord = range(0, 1500, 1)

mode = input("Which type of function do you have? \nl for linear, t for Trig, p for Parabola ")
if mode == "l":
    k = float(input("f(x) = kx + b, set the value of k: "))
    b = float(input("f(x) = kx + b, set the value of b: "))
elif mode == "t":
    a = int(input("f(x) = a sin h (x + k) , set the value of a: "))

class Dot(Sprite):

    asset = RectangleAsset(2,2, thinline, red)

    def __init__(self, position):
        super().__init__(Dot.asset, position)
        self.x = 0

    def l(self, position):
        return 450-k*(self.x-750)-100*b

    def step(self):
        self.x += 1
        if self.x < 1500:
            if mode == "l":
                self.y = l(self.x)
                self.y1 = l(self.x+1)
                n = m = (self.y1 - self.y)/2
                while n > 0:
                    sprite = Sprite(Dot.asset, (self.x, self.y))
                    self.y += 1
                    n -= 1
                while m > 0:
                    sprite = Sprite(Dot.asset, (self.x+1, self.y))
                    m -= 1
                    self.y += 1

            elif mode == "t":
                sprite = Sprite(Dot.asset, (self.x, 450-a*100*math.tan((self.x-750)/100)))
                self.x += 1

class Grapher(App):
    def __init__(self, width, height):
        super().__init__(width, height)

        black = Color(0x000000, 1.0)
        thinline = LineStyle(0, black)
        xaxis = RectangleAsset(1500,2, thinline, black)
        yaxis = RectangleAsset(2,1050, thinline, black)
        sprite1 = Sprite(xaxis, (0, 450))
        sprite2 = Sprite(yaxis, (750, 0))

        Dot((0,450))
        #sprite0 = [Sprite(Dot.asset, (x, 450-k*(x-750)-100*b)) for x in xcord]

    def step(self):
        for dot in self.getSpritesbyClass(Dot):
            dot.step()
        for dot in self.getSpritesbyClass(Dot):
            dot.step()

myapp = Grapher(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()
<commit_msg>Updated from Brython Server: 11/19/2015 3:06:32 PM<commit_after>from ggame import App, Color, LineStyle, Sprite
from ggame import RectangleAsset, CircleAsset
import math

SCREEN_WIDTH = 1536
SCREEN_HEIGHT = 1024

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
purple = Color(0xff00ff, 1.0)
thinline = LineStyle(0, black)

xcord = range(0, 1500, 1)

mode = input("Which type of function do you have? \nl for linear, t for Trig, p for Parabola ")
if mode == "l":
    k = float(input("f(x) = kx + b, set the value of k: "))
    b = float(input("f(x) = kx + b, set the value of b: "))
elif mode == "t":
    a = int(input("f(x) = a sin h (x + k) , set the value of a: "))

class Dot(Sprite):

    asset = RectangleAsset(2,2, thinline, red)

    def __init__(self, position):
        super().__init__(Dot.asset, position)
        self.x = 0

    def l(self, position):
        return 450-k*(self.x-750)-100*b

    def step(self):
        self.x += 1
        if self.x < 1500:
            if mode == "l":
                self.y = self.l(self.x)
                self.y1 = self.l(self.x+1)
                n = m = (self.y1 - self.y)/2
                while n > 0:
                    sprite = Sprite(Dot.asset, (self.x, self.y))
                    self.y += 1
                    n -= 1
                while m > 0:
                    sprite = Sprite(Dot.asset, (self.x+1, self.y))
                    m -= 1
                    self.y += 1

            elif mode == "t":
                sprite = Sprite(Dot.asset, (self.x, 450-a*100*math.tan((self.x-750)/100)))
                self.x += 1

class Grapher(App):
    def __init__(self, width, height):
        super().__init__(width, height)

        black = Color(0x000000, 1.0)
        thinline = LineStyle(0, black)
        xaxis = RectangleAsset(1500,2, thinline, black)
        yaxis = RectangleAsset(2,1050, thinline, black)
        sprite1 = Sprite(xaxis, (0, 450))
        sprite2 = Sprite(yaxis, (750, 0))

        Dot((0,450))
        #sprite0 = [Sprite(Dot.asset, (x, 450-k*(x-750)-100*b)) for x in xcord]

    def step(self):
        for dot in self.getSpritesbyClass(Dot):
            dot.step()
        for dot in self.getSpritesbyClass(Dot):
            dot.step()

myapp = Grapher(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# _*_ coding:utf-8 _*_
from mimetypes import guess_type
import os
import subprocess

from flask import Blueprint, abort, make_response, render_template
from jinja2 import Environment, FileSystemLoader
from jinja2.exceptions import TemplateNotFound

import app_config
import copytext
import oauth
from render_utils import load_graphic_config, make_context, render_with_context, smarty_filter

graphic = Blueprint('graphic', __name__)

@graphic.route('/<slug>/')
@oauth.oauth_required
def _graphics_detail(slug):
    """
    Renders a parent.html index with child.html embedded as iframe.
    """
    from flask import request, g

    alt_path = getattr(g, 'alt_path', None)
    if alt_path:
        graphic_path = alt_path
    else:
        graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug)

    # NOTE: Parent must load pym.js from same source as child to prevent version conflicts!
    context = make_context(asset_depth=2, root_path=graphic_path)
    context['slug'] = slug
    context['var_name'] = slug.replace('-', '_')

    # Use local_pym for legacy graphics
    local_pym = getattr(g, 'local_pym', None)
    context['LOCAL_PYM'] = local_pym
    # warning message
    custom_location = getattr(g, 'custom_location', None)
    context['CUSTOM_LOCATION'] = custom_location

    template = 'parent.html'

    try:
        graphic_config = load_graphic_config(graphic_path)
        context.update(graphic_config.__dict__)

        if hasattr(graphic_config, 'COPY_GOOGLE_DOC_KEY') and graphic_config.COPY_GOOGLE_DOC_KEY:
            copy_path = '%s/%s.xlsx' % (graphic_path, slug)

            if request.args.get('refresh'):
                oauth.get_document(graphic_config.COPY_GOOGLE_DOC_KEY, copy_path)

            context['COPY'] = copytext.Copy(filename=copy_path)
    except IOError:
        pass

    try:
        env = Environment(loader=FileSystemLoader(graphic_path))
        template = env.get_template('parent.html')
        return make_response(template.render(**context))
    except TemplateNotFound:
        return make_response(render_template(template, **context))

@graphic.route('/<slug>/child.html')
@oauth.oauth_required
def _graphics_child(slug):
    """
    Renders a child.html for embedding.
    """
    from flask import g
    alt_path = getattr(g, 'alt_path', None)
    if alt_path:
        graphic_path = alt_path
    else:
        graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug)
    print graphic_path

    # Fallback for legacy projects w/o child templates
    if not os.path.exists('%s/child_template.html' % graphic_path):
        with open('%s/child.html' % graphic_path) as f:
            contents = f.read()

        return contents

    context = make_context(asset_depth=2, root_path=graphic_path)
    context['slug'] = slug
    context['var_name'] = slug.replace('-', '_')

    env = Environment(loader=FileSystemLoader(graphic_path))

    try:
        graphic_config = load_graphic_config(graphic_path)
        context.update(graphic_config.__dict__)

        if hasattr(graphic_config, 'JINJA_FILTER_FUNCTIONS'):
            for func in graphic_config.JINJA_FILTER_FUNCTIONS:
                env.filters[func.__name__] = func

        if hasattr(graphic_config, 'COPY_GOOGLE_DOC_KEY') and graphic_config.COPY_GOOGLE_DOC_KEY:
            copy_path = '%s/%s.xlsx' % (graphic_path, slug)

            context['COPY'] = copytext.Copy(filename=copy_path)
    except IOError:
        pass

    env.globals.update(render=render_with_context)
    env.filters['smarty'] = smarty_filter
    template = env.get_template('child_template.html')

    return make_response(template.render(**context))

# Render graphic LESS files on-demand
@graphic.route('/<slug>/css/<filename>.less')
def _graphic_less(slug, filename):
    """
    Compiles LESS for a graphic.
    """
    from flask import g
    alt_path = getattr(g, 'alt_path', None)
    if alt_path:
        graphic_path = alt_path
    else:
        graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug)
    less_path = '%s/css/%s.less' % (graphic_path, filename)

    if not os.path.exists(less_path):
        abort(404)

    r = subprocess.check_output(['node_modules/less/bin/lessc', less_path])

    return make_response(r, 200, { 'Content-Type': 'text/css' })

# Serve arbitrary static files on-demand
@graphic.route('/<slug>/<path:path>')
def _static(slug, path):
    from flask import g
    alt_path = getattr(g, 'alt_path', None)
    if alt_path:
        graphic_path = alt_path
    else:
        graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug)
    real_path = '%s/%s' % (graphic_path, path)

    try:
        with open(real_path) as f:
            return f.read(), 200, { 'Content-Type': guess_type(real_path)[0] }
    except IOError:
        abort(404)
<commit_msg>remove spurious print command<commit_after>#!/usr/bin/env python
# _*_ coding:utf-8 _*_
from mimetypes import guess_type
import os
import subprocess

from flask import Blueprint, abort, make_response, render_template
from jinja2 import Environment, FileSystemLoader
from jinja2.exceptions import TemplateNotFound

import app_config
import copytext
import oauth
from render_utils import load_graphic_config, make_context, render_with_context, smarty_filter

graphic = Blueprint('graphic', __name__)

@graphic.route('/<slug>/')
@oauth.oauth_required
def _graphics_detail(slug):
    """
    Renders a parent.html index with child.html embedded as iframe.
    """
    from flask import request, g

    alt_path = getattr(g, 'alt_path', None)
    if alt_path:
        graphic_path = alt_path
    else:
        graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug)

    # NOTE: Parent must load pym.js from same source as child to prevent version conflicts!
    context = make_context(asset_depth=2, root_path=graphic_path)
    context['slug'] = slug
    context['var_name'] = slug.replace('-', '_')

    # Use local_pym for legacy graphics
    local_pym = getattr(g, 'local_pym', None)
    context['LOCAL_PYM'] = local_pym
    # warning message
    custom_location = getattr(g, 'custom_location', None)
    context['CUSTOM_LOCATION'] = custom_location

    template = 'parent.html'

    try:
        graphic_config = load_graphic_config(graphic_path)
        context.update(graphic_config.__dict__)

        if hasattr(graphic_config, 'COPY_GOOGLE_DOC_KEY') and graphic_config.COPY_GOOGLE_DOC_KEY:
            copy_path = '%s/%s.xlsx' % (graphic_path, slug)

            if request.args.get('refresh'):
                oauth.get_document(graphic_config.COPY_GOOGLE_DOC_KEY, copy_path)

            context['COPY'] = copytext.Copy(filename=copy_path)
    except IOError:
        pass

    try:
        env = Environment(loader=FileSystemLoader(graphic_path))
        template = env.get_template('parent.html')
        return make_response(template.render(**context))
    except TemplateNotFound:
        return make_response(render_template(template, **context))

@graphic.route('/<slug>/child.html')
@oauth.oauth_required
def _graphics_child(slug):
    """
    Renders a child.html for embedding.
    """
    from flask import g
    alt_path = getattr(g, 'alt_path', None)
    if alt_path:
        graphic_path = alt_path
    else:
        graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug)

    # Fallback for legacy projects w/o child templates
    if not os.path.exists('%s/child_template.html' % graphic_path):
        with open('%s/child.html' % graphic_path) as f:
            contents = f.read()

        return contents

    context = make_context(asset_depth=2, root_path=graphic_path)
    context['slug'] = slug
    context['var_name'] = slug.replace('-', '_')

    env = Environment(loader=FileSystemLoader(graphic_path))

    try:
        graphic_config = load_graphic_config(graphic_path)
        context.update(graphic_config.__dict__)

        if hasattr(graphic_config, 'JINJA_FILTER_FUNCTIONS'):
            for func in graphic_config.JINJA_FILTER_FUNCTIONS:
                env.filters[func.__name__] = func

        if hasattr(graphic_config, 'COPY_GOOGLE_DOC_KEY') and graphic_config.COPY_GOOGLE_DOC_KEY:
            copy_path = '%s/%s.xlsx' % (graphic_path, slug)

            context['COPY'] = copytext.Copy(filename=copy_path)
    except IOError:
        pass

    env.globals.update(render=render_with_context)
    env.filters['smarty'] = smarty_filter
    template = env.get_template('child_template.html')

    return make_response(template.render(**context))

# Render graphic LESS files on-demand
@graphic.route('/<slug>/css/<filename>.less')
def _graphic_less(slug, filename):
    """
    Compiles LESS for a graphic.
    """
    from flask import g
    alt_path = getattr(g, 'alt_path', None)
    if alt_path:
        graphic_path = alt_path
    else:
        graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug)
    less_path = '%s/css/%s.less' % (graphic_path, filename)

    if not os.path.exists(less_path):
        abort(404)

    r = subprocess.check_output(['node_modules/less/bin/lessc', less_path])

    return make_response(r, 200, { 'Content-Type': 'text/css' })

# Serve arbitrary static files on-demand
@graphic.route('/<slug>/<path:path>')
def _static(slug, path):
    from flask import g
    alt_path = getattr(g, 'alt_path', None)
    if alt_path:
        graphic_path = alt_path
    else:
        graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug)
    real_path = '%s/%s' % (graphic_path, path)

    try:
        with open(real_path) as f:
            return f.read(), 200, { 'Content-Type': guess_type(real_path)[0] }
    except IOError:
        abort(404)
<|endoftext|>


<commit_before>import numpy

from chainer import functions
from chainer import testing

def _decorrelated_batch_normalization(x, mean, projection, groups):
    xs = numpy.split(x, groups, axis=1)
    assert mean.shape[0] == groups
    assert projection.shape[0] == groups
    ys = [
        _decorrelated_batch_normalization_1group(xi, m, p)
        for (xi, m, p) in zip(xs, mean, projection)]
    return numpy.concatenate(ys, axis=1)

def _decorrelated_batch_normalization_1group(x, mean, projection):
    spatial_ndim = len(x.shape[2:])
    spatial_axis = tuple(range(2, 2 + spatial_ndim))
    b, C = x.shape[:2]
    x_hat = x.transpose((1, 0) + spatial_axis).reshape(C, -1)
    y_hat = projection.dot(x_hat - mean[:, None])
    y = y_hat.reshape((C, b) + x.shape[2:]).transpose(
        (1, 0) + spatial_axis)
    return y

def _calc_projection(x, mean, eps, groups):
    xs = numpy.split(x, groups, axis=1)
    assert mean.shape[0] == groups
    projections = [
        _calc_projection_1group(xi, m, eps)
        for (xi, m) in zip(xs, mean)]
    return numpy.concatenate([p[None] for p in projections])

def _calc_projection_1group(x, mean, eps):
    spatial_ndim = len(x.shape[2:])
    spatial_axis = tuple(range(2, 2 + spatial_ndim))
    b, C = x.shape[:2]
    m = b
    for i in spatial_axis:
        m *= x.shape[i]

    x_hat = x.transpose((1, 0) + spatial_axis).reshape(C, -1)
    mean = x_hat.mean(axis=1)
    x_hat = x_hat - mean[:, None]
    cov = x_hat.dot(x_hat.T) / m + eps * numpy.eye(C, dtype=x.dtype)
    eigvals, eigvectors = numpy.linalg.eigh(cov)
    projection = eigvectors.dot(numpy.diag(eigvals ** -0.5)).dot(eigvectors.T)
    return projection

def _calc_mean(x, groups):
    axis = (0,) + tuple(range(2, x.ndim))
    return x.mean(axis=axis).reshape(groups, -1)

@testing.parameterize(*(testing.product({
    'n_channels': [8],
    'ndim': [0, 2],
    'groups': [1, 2],
    'eps': [2e-5, 5e-1],
    'dtype': [numpy.float32],
    'contiguous': ['C', None],
}) + testing.product({
    'n_channels': [8],
    'ndim': [1],
    'groups': [1, 2],
    'eps': [2e-5, 5e-1],
    # NOTE(crcrpar): np.linalg.eigh does not support float16
    'dtype': [numpy.float32, numpy.float64],
    'contiguous': ['C', None],
})))
@testing.backend.inject_backend_tests(
    None,
    # CPU tests
    [{}]
    # GPU tests
    + [{'use_cuda': True}]
    + testing.product({
        'use_chainerx': [True],
        'chainerx_device': ['native:0', 'cuda:0']
    })
)
class TestDecorrelatedBatchNormalization(testing.FunctionTestCase):

    # TODO(crcrpar): Delete this line once double backward of
    # :func:`~chainer.functions.decorrelated_batch_normalization` is
    # implemented.
    skip_double_backward_test = True

    def setUp(self):
        check_forward_options = {'atol': 1e-3, 'rtol': 1e-3}
        check_backward_options = {'atol': 1e-3, 'rtol': 1e-3}
        if self.dtype == numpy.float16:
            check_forward_options = {'atol': 1e-2, 'rtol': 1e-2}
            check_backward_options = {'atol': 1e-2, 'rtol': 1e-2}
        elif self.dtype == numpy.float32:
            check_backward_options = {'atol': 1e-2, 'rtol': 1e-2}
        self.check_forward_options = check_forward_options
        self.check_backward_options = check_backward_options

    def generate_inputs(self):
        dtype = self.dtype
        ndim = self.ndim
        shape = (5, self.n_channels) + (2,) * ndim
        x = numpy.random.uniform(-1, 1, shape).astype(dtype)
        return x,

    def forward(self, inputs, device):
        x, = inputs
        return functions.decorrelated_batch_normalization(
            x, groups=self.groups, eps=self.eps),

    def forward_expected(self, inputs):
        x, = inputs
        groups = self.groups
        mean = _calc_mean(x, groups)
        projection = _calc_projection(x, mean, self.eps, groups)
        return _decorrelated_batch_normalization(
            x, mean, projection, groups),

@testing.parameterize(*(testing.product({
    'n_channels': [8],
    'ndim': [0, 1, 2],
    'groups': [1, 2],
    'eps': [2e-5, 5e-1],
    'dtype': [numpy.float32],
    'contiguous': ['C', None],
}) + testing.product({
    'n_channels': [8],
    'ndim': [1],
    'groups': [1, 2],
    'eps': [2e-5, 5e-1],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
    'contiguous': ['C', None],
})))
@testing.backend.inject_backend_tests(
    None,
    # CPU tests
    [{}]
    # GPU tests
    + [{'use_cuda': True}]
    + testing.product({
        'use_chainerx': [True],
        'chainerx_device': ['native:0', 'cuda:0'],
    })
)
class TestFixedDecorrelatedBatchNormalization(testing.FunctionTestCase):

    # TODO(crcrpar): Delete this line once double backward of
    # :func:`~chainer.functions.fixed_decorrelated_batch_normalization` is
    # implemented.
    skip_double_backward_test = True

    def setUp(self):
        C = self.n_channels // self.groups
        dtype = self.dtype
        self.mean = numpy.random.uniform(
            -1, 1, (self.groups, C)).astype(dtype)
        self.projection = numpy.random.uniform(
            0.5, 1, (self.groups, C, C)).astype(dtype)

        check_forward_options = {'atol': 1e-4, 'rtol': 1e-3}
        check_backward_options = {'atol': 1e-4, 'rtol': 1e-3}
        if self.dtype == numpy.float16:
            check_forward_options = {'atol': 1e-2, 'rtol': 1e-2}
            check_backward_options = {'atol': 1e-2, 'rtol': 1e-2}
        if self.dtype == numpy.float32:
            check_backward_options = {'atol': 1e-2, 'rtol': 1e-2}
        self.check_forward_options = check_forward_options
        self.check_backward_options = check_backward_options

    def generate_inputs(self):
        dtype = self.dtype
        ndim = self.ndim
        shape = (5, self.n_channels) + (2,) * ndim
        x = numpy.random.uniform(-1, 1, shape).astype(dtype)
        return x,

    def forward(self, inputs, device):
        x, = inputs
        mean = device.send_array(self.mean.copy())
        projection = device.send_array(self.projection.copy())
        return functions.fixed_decorrelated_batch_normalization(
            x, mean, projection, groups=self.groups
        ),

    def forward_expected(self, inputs):
        x, = inputs
        mean = self.mean.copy()
        projection = self.projection.copy()
        return _decorrelated_batch_normalization(
            x, mean, projection, self.groups),

testing.run_module(__name__, __file__)
<commit_msg>Avoid unstable inputs in tests of decorrelated batch normalization<commit_after>import numpy

from chainer import functions
from chainer import testing

def _decorrelated_batch_normalization(x, mean, projection, groups):
    xs = numpy.split(x, groups, axis=1)
    assert mean.shape[0] == groups
    assert projection.shape[0] == groups
    ys = [
        _decorrelated_batch_normalization_1group(xi, m, p)
        for (xi, m, p) in zip(xs, mean, projection)]
    return numpy.concatenate(ys, axis=1)

def _decorrelated_batch_normalization_1group(x, mean, projection):
    spatial_ndim = len(x.shape[2:])
    spatial_axis = tuple(range(2, 2 + spatial_ndim))
    b, C = x.shape[:2]
    x_hat = x.transpose((1, 0) + spatial_axis).reshape(C, -1)
    y_hat = projection.dot(x_hat - mean[:, None])
    y = y_hat.reshape((C, b) + x.shape[2:]).transpose(
        (1, 0) + spatial_axis)
    return y

def _calc_projection(x, mean, eps, groups):
    xs = numpy.split(x, groups, axis=1)
    assert mean.shape[0] == groups
    projections = [
        _calc_projection_1group(xi, m, eps)
        for (xi, m) in zip(xs, mean)]
    return numpy.concatenate([p[None] for p in projections])

def _calc_projection_1group(x, mean, eps):
    spatial_ndim = len(x.shape[2:])
    spatial_axis = tuple(range(2, 2 + spatial_ndim))
    b, C = x.shape[:2]
    m = b
    for i in spatial_axis:
        m *= x.shape[i]

    x_hat = x.transpose((1, 0) + spatial_axis).reshape(C, -1)
    mean = x_hat.mean(axis=1)
    x_hat = x_hat - mean[:, None]
    cov = x_hat.dot(x_hat.T) / m + eps * numpy.eye(C, dtype=x.dtype)
    eigvals, eigvectors = numpy.linalg.eigh(cov)
    projection = eigvectors.dot(numpy.diag(eigvals ** -0.5)).dot(eigvectors.T)
    return projection

def _calc_mean(x, groups):
    axis = (0,) + tuple(range(2, x.ndim))
    return x.mean(axis=axis).reshape(groups, -1)

@testing.parameterize(*(testing.product({
    'n_channels': [8],
    'ndim': [0, 2],
    'groups': [1, 2],
    'eps': [2e-5, 5e-1],
    'dtype': [numpy.float32],
    'contiguous': ['C', None],
}) + testing.product({
    'n_channels': [8],
    'ndim': [1],
    'groups': [1, 2],
    'eps': [2e-5, 5e-1],
    # NOTE(crcrpar): np.linalg.eigh does not support float16
    'dtype': [numpy.float32, numpy.float64],
    'contiguous': ['C', None],
})))
@testing.backend.inject_backend_tests(
    None,
    # CPU tests
    [{}]
    # GPU tests
    + [{'use_cuda': True}]
    + testing.product({
        'use_chainerx': [True],
        'chainerx_device': ['native:0', 'cuda:0']
    })
)
class TestDecorrelatedBatchNormalization(testing.FunctionTestCase):

    # TODO(crcrpar): Delete this line once double backward of
    # :func:`~chainer.functions.decorrelated_batch_normalization` is
    # implemented.
    skip_double_backward_test = True

    def setUp(self):
        check_forward_options = {'atol': 1e-3, 'rtol': 1e-3}
        check_backward_options = {'atol': 1e-3, 'rtol': 1e-3}
        if self.dtype == numpy.float16:
            check_forward_options = {'atol': 1e-2, 'rtol': 1e-2}
            check_backward_options = {'atol': 1e-2, 'rtol': 1e-2}
        elif self.dtype == numpy.float32:
            check_backward_options = {'atol': 1e-2, 'rtol': 1e-2}
        self.check_forward_options = check_forward_options
        self.check_backward_options = check_backward_options

    def generate_inputs(self):
        dtype = self.dtype
        ndim = self.ndim
        shape = (5, self.n_channels) + (2,) * ndim
        m = 5 * 2 ** ndim

        # NOTE(kataoka): The current implementation uses linalg.eigh. Small
        # eigenvalues of the correlation matrix, which can be as small as
        # eps=2e-5, cannot be computed with good *relative* accuracy, but
        # the eigenvalues are used later as `eigvals ** -0.5`. Require the
        # following is sufficiently large:
        # min(eigvals[:k]) == min(singular_vals ** 2 / m + eps)
        min_singular_value = 0.1
        # NOTE(kataoka): Decorrelated batch normalization should be free from
        # "stochastic axis swapping". Requiring a gap between singular values
        # just hides mistakes in implementations.
        min_singular_value_gap = 0.001
        g = self.groups
        zca_shape = g, self.n_channels // g, m
        x = numpy.random.uniform(-1, 1, zca_shape)
        mean = x.mean(axis=2, keepdims=True)
        a = x - mean
        u, s, vh = numpy.linalg.svd(a, full_matrices=False)
        # Decrement the latter dim because of the constraint `sum(_) == 0`
        k = min(zca_shape[1], zca_shape[2] - 1)
        s[:, :k] += (
            min_singular_value
            + min_singular_value_gap * numpy.arange(k)
        )[::-1]
        a = numpy.einsum('bij,bj,bjk->bik', u, s, vh)
        x = a + mean

        x = x.reshape((self.n_channels, shape[0]) + shape[2:]).swapaxes(0, 1)
        x = x.astype(dtype)
        return x,

    def forward(self, inputs, device):
        x, = inputs
        return functions.decorrelated_batch_normalization(
            x, groups=self.groups, eps=self.eps),

    def forward_expected(self, inputs):
        x, = inputs
        groups = self.groups
        mean = _calc_mean(x, groups)
        projection = _calc_projection(x, mean, self.eps, groups)
        return _decorrelated_batch_normalization(
            x, mean, projection, groups),

@testing.parameterize(*(testing.product({
    'n_channels': [8],
    'ndim': [0, 1, 2],
    'groups': [1, 2],
    'eps': [2e-5, 5e-1],
    'dtype': [numpy.float32],
    'contiguous': ['C', None],
}) + testing.product({
    'n_channels': [8],
    'ndim': [1],
    'groups': [1, 2],
    'eps': [2e-5, 5e-1],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
    'contiguous': ['C', None],
})))
@testing.backend.inject_backend_tests(
    None,
    # CPU tests
    [{}]
    # GPU tests
    + [{'use_cuda': True}]
    + testing.product({
        'use_chainerx': [True],
        'chainerx_device': ['native:0', 'cuda:0'],
    })
)
class TestFixedDecorrelatedBatchNormalization(testing.FunctionTestCase):

    # TODO(crcrpar): Delete this line once double backward of
    # :func:`~chainer.functions.fixed_decorrelated_batch_normalization` is
    # implemented.
    skip_double_backward_test = True

    def setUp(self):
        C = self.n_channels // self.groups
        dtype = self.dtype
        self.mean = numpy.random.uniform(
            -1, 1, (self.groups, C)).astype(dtype)
        self.projection = numpy.random.uniform(
            0.5, 1, (self.groups, C, C)).astype(dtype)

        check_forward_options = {'atol': 1e-4, 'rtol': 1e-3}
        check_backward_options = {'atol': 1e-4, 'rtol': 1e-3}
        if self.dtype == numpy.float16:
            check_forward_options = {'atol': 1e-2, 'rtol': 1e-2}
            check_backward_options = {'atol': 1e-2, 'rtol': 1e-2}
        if self.dtype == numpy.float32:
            check_backward_options = {'atol': 1e-2, 'rtol': 1e-2}
        self.check_forward_options = check_forward_options
        self.check_backward_options = check_backward_options

    def generate_inputs(self):
        dtype = self.dtype
        ndim = self.ndim
        shape = (5, self.n_channels) + (2,) * ndim
        x = numpy.random.uniform(-1, 1, shape).astype(dtype)
        return x,

    def forward(self, inputs, device):
        x, = inputs
        mean = device.send_array(self.mean.copy())
        projection = device.send_array(self.projection.copy())
        return functions.fixed_decorrelated_batch_normalization(
            x, mean, projection, groups=self.groups
        ),

    def forward_expected(self, inputs):
        x, = inputs
        mean = self.mean.copy()
        projection = self.projection.copy()
        return _decorrelated_batch_normalization(
            x, mean, projection, self.groups),

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>"""
fs.path
=======

Useful functions for FS path manipulation.

This is broadly similar to the standard ``os.path`` module but works with
paths in the canonical format expected by all FS objects (that is, separated
by forward slashes and with an optional leading slash).

"""

import re

_requires_normalization = re.compile(r'/\.\.|\./|\.|//|\\').search
def normpath(path):
    """Normalizes a path to be in the format expected by FS objects.

    This function remove any leading or trailing slashes, collapses
    duplicate slashes, replaces backward with forward slashes, and generally
    tries very hard to return a new path string the canonical FS format.
    If the path is invalid, ValueError will be raised.

    :param path: path to normalize
    :returns: a valid FS path

    >>> normpath(r"foo\\bar\\baz")
    'foo/bar/baz'

    >>> normpath("/foo//bar/frob/../baz")
    '/foo/bar/baz'

    >>> normpath("foo/../../bar")
    Traceback (most recent call last)
        ...
    ValueError: too many backrefs in path 'foo/../../bar'

    """

    if path in ('', '/'):
        return path

    # An early out if there is no need to normalize this path
    if not _requires_normalization(path):
        return path.rstrip('/')

    components = []
    append = components.append
    for comp in [c for c in path.replace('\\','/').split("/") if c not in ('', '.')]:
        if comp == "..":
            try:
                components.pop()
            except IndexError:
                raise ValueError("too many backrefs in path '%s'" % path)
        else:
            append(comp)
    if path[0] in '\\/':
        if not components:
            append("")
        components.insert(0, "")
    return "/".join(components)

def iteratepath(path, numsplits=None):
    """Iterate over the individual components of a path.

    :param path: Path to iterate over
    :numsplits: Maximum number of splits

    """
    path = relpath(normpath(path))
    if not path:
        return []
    if numsplits == None:
        return path.split('/')
    else:
        return path.split('/', numsplits)

def recursepath(path, reverse=False):
    """Returns intermediate paths from the root to the given path

    :param reverse: reverses the order of the paths

    >>> recursepath('a/b/c')
    ['/', u'/a', u'/a/b', u'/a/b/c']

    """

    if path in ('', '/'):
        return [u'/']

    path = abspath(normpath(path)) + '/'

    paths = [u'/']
    find = path.find
    append = paths.append
    pos = 1
    len_path = len(path)

    while pos < len_path:
        pos = find('/', pos)
        append(path[:pos])
        pos += 1

    if reverse:
        return paths[::-1]
    return paths

def abspath(path):
    """Convert the given path to an absolute path.

    Since FS objects have no concept of a 'current directory' this simply
    adds a leading '/' character if the path doesn't already have one.

    """
    if not path.startswith('/'):
        return u'/' + path
    return path

def relpath(path):
    """Convert the given path to a relative path.

    This is the inverse of abspath(), stripping a leading '/' from the
    path if it is present.

    :param path: Path to adjust

    >>> relpath('/a/b')
    'a/b'

    """
    return path.lstrip('/')

def pathjoin(*paths):
    """Joins any number of paths together, returning a new path string.

    :param paths: Paths to join are given in positional arguments

    >>> pathjoin('foo', 'bar', 'baz')
    'foo/bar/baz'

    >>> pathjoin('foo/bar', '../baz')
    'foo/baz'

    >>> pathjoin('foo/bar', '/baz')
    '/baz'

    """
    absolute = False
    relpaths = []
    for p in paths:
        if p:
            if p[0] in '\\/':
                del relpaths[:]
                absolute = True
            relpaths.append(p)

    path = normpath(u"/".join(relpaths))
    if absolute:
        path = abspath(path)
    return path

def join(*paths):
    """Joins any number of paths together, returning a new path string.

    This is a simple alias for the ``pathjoin`` function, allowing it to be
    used as ``fs.path.join`` in direct correspondence with ``os.path.join``.

    :param paths: Paths to join are given in positional arguments
    """
    return pathjoin(*paths)

def pathsplit(path):
    """Splits a path into (head, tail) pair.

    This function splits a path into a pair (head, tail) where 'tail' is the
    last pathname component and 'head' is all preceding components.

    :param path: Path to split

    >>> pathsplit("foo/bar")
    ('foo', 'bar')

    >>> pathsplit("foo/bar/baz")
    ('foo/bar', 'baz')

    >>> pathsplit("/foo/bar/baz")
    ('/foo/bar', 'baz')

    """
    if '/' not in path:
        return ('', path)
    split = path.rsplit('/', 1)
    return (split[0] or '/', split[1])

def split(path):
    """Splits a path into (head, tail) pair.

    This is a simple alias for the ``pathsplit`` function, allowing it to be
    used as ``fs.path.split`` in direct correspondence with ``os.path.split``.

    :param path: Path to split
    """
    return pathsplit(path)

def splitext(path):
    """Splits the extension from the path, and returns the path (up to the last
    '.' and the extension

    :param path: A path to split

    >>> splitext('baz.txt')
    ('baz', 'txt')

    >>> splitext('foo/bar/baz.txt')
    ('foo/bar/baz', 'txt')

    """

    parent_path, pathname = pathsplit(path)
    if '.' not in pathname:
        return path, ''
    pathname, ext = pathname.rsplit('.', 1)
    path = pathjoin(parent_path, pathname)
    return path, '.' + ext

def isdotfile(path):
    """Detects if a path references a dot file, i.e. a resource who's name
    starts with a '.'

    :param path: Path to check

    >>> isdotfile('.baz')
    True

    >>> isdotfile('foo/bar/baz')
    True

    >>> isdotfile('foo/bar.baz').
    False

    """
    return basename(path).startswith('.')

def dirname(path):
    """Returns the parent directory of a path.

    This is always equivalent to the 'head' component of the value returned
    by pathsplit(path).

    :param path: A FS path

    >>> dirname('foo/bar/baz')
    'foo/bar'

    """
    if '/' not in path:
        return ''
    return path.rsplit('/', 1)[0]

def basename(path):
    """Returns the basename of the resource referenced by a path.

    This is always equivalent to the 'head' component of the value returned
    by pathsplit(path).

    :param path: A FS path

    >>> basename('foo/bar/baz')
    'baz'

    """
    if '/' not in path:
        return path
    return path.rsplit('/', 1)[-1]

def issamedir(path1, path2):
    """Return true if two paths reference a resource in the same directory.

    :param path1: An FS path
    :param path2: An FS path

    >>> issamedir("foo/bar/baz.txt", "foo/bar/spam.txt")
    True
    >>> issamedir("foo/bar/baz/txt", "spam/eggs/spam.txt")
    False

    """
    return dirname(normpath(path1)) == dirname(normpath(path2))

def isbase(path1, path2):
    p1 = forcedir(abspath(path1))
    p2 = forcedir(abspath(path2))
    return p1 == p2 or p1.startswith(p2)

def isprefix(path1, path2):
    """Return true is path1 is a prefix of path2.

    :param path1: An FS path
    :param path2: An FS path

    >>> isprefix("foo/bar", "foo/bar/spam.txt")
    True
    >>> isprefix("foo/bar/", "foo/bar")
    True
    >>> isprefix("foo/barry", "foo/baz/bar")
    False
    >>> isprefix("foo/bar/baz/", "foo/baz/bar")
    False

    """
    bits1 = path1.split("/")
    bits2 = path2.split("/")
    while bits1 and bits1[-1] == "":
        bits1.pop()
    if len(bits1) > len(bits2):
        return False
    for (bit1,bit2) in zip(bits1,bits2):
        if bit1 != bit2:
            return False
    return True

def forcedir(path):
    """Ensure the path ends with a trailing /

    :param path: An FS path

    >>> forcedir("foo/bar")
    'foo/bar/'
    >>> forcedir("foo/bar/")
    'foo/bar/'

    """

    if not path.endswith('/'):
        return path + '/'
    return path

def frombase(path1, path2):
    if not isprefix(path1, path2):
        raise ValueError("path1 must be a prefix of path2")
    return path2[len(path1):]

class PathMap(object):
    """Dict-like object with paths for keys.

    A PathMap is like a dictionary where the keys are all FS paths.  It has
    two main advantages over a standard dictionary.  First, keys are normalized
    automatically::

        >>> pm = PathMap()
        >>> pm["hello/world"] = 42
        >>> print pm["/hello/there/../world"]
        42

    Second, various dictionary operations (e.g. listing or clearing values)
    can be efficiently performed on a subset of keys sharing some common
    prefix::

        # list all values in the map
        pm.values()

        # list all values for paths starting with "/foo/bar"
        pm.values("/foo/bar")

    Under the hood, a PathMap is a trie-like structure where each level is
    indexed by path name component.  This allows lookups to be performed in
    O(number of path components) while permitting efficient prefix-based
    operations.
    """

    def __init__(self):
        self._map = {}

    def __getitem__(self,path):
        """Get the value stored under the given path."""
        m = self._map
        for name in iteratepath(path):
            try:
                m = m[name]
            except KeyError:
                raise KeyError(path)
        try:
            return m[""]
        except KeyError:
            raise KeyError(path)

    def __contains__(self,path):
        """Check whether the given path has a value stored in the map."""
        try:
            self[path]
        except KeyError:
            return False
        else:
            return True

    def __setitem__(self,path,value):
        """Set the value stored under the given path."""
        m = self._map
        for name in iteratepath(path):
            try:
                m = m[name]
            except KeyError:
                m = m.setdefault(name,{})
        m[""] = value

    def __delitem__(self,path):
        """Delete the value stored under the given path."""
        ms = [[self._map,None]]
        for name in iteratepath(path):
            try:
                ms.append([ms[-1][0][name],None])
            except KeyError:
                raise KeyError(path)
            else:
                ms[-2][1] = name
        try:
            del ms[-1][0][""]
        except KeyError:
            raise KeyError(path)
        else:
            while len(ms) > 1 and not ms[-1][0]:
                del ms[-1]
                del ms[-1][0][ms[-1][1]]

    def get(self,path,default=None):
        """Get the value stored under the given path, or the given default."""
        try:
            return self[path]
        except KeyError:
            return default

    def pop(self,path,default=None):
        """Pop the value stored under the given path, or the given default."""
        ms = [[self._map,None]]
        for name in iteratepath(path):
            try:
                ms.append([ms[-1][0][name],None])
            except KeyError:
                return default
            else:
                ms[-2][1] = name
        try:
            val = ms[-1][0].pop("")
        except KeyError:
            val = default
        else:
            while len(ms) > 1 and not ms[-1][0]:
                del ms[-1]
                del ms[-1][0][ms[-1][1]]
        return val

    def setdefault(self,path,value):
        m = self._map
        for name in iteratepath(path):
            try:
                m = m[name]
            except KeyError:
                m = m.setdefault(name,{})
        return m.setdefault("",value)

    def clear(self,root="/"):
        """Clear all entries beginning with the given root path."""
        m = self._map
        for name in iteratepath(root):
            try:
                m = m[name]
            except KeyError:
                return
        m.clear()

    def iterkeys(self,root="/",m=None):
        """Iterate over all keys beginning with the given root path."""
        if m is None:
            m = self._map
            for name in iteratepath(root):
                try:
                    m = m[name]
                except KeyError:
                    return
        for (nm,subm) in m.iteritems():
            if not nm:
                yield abspath(normpath(root))
            else:
                k = pathjoin(root,nm)
                for subk in self.iterkeys(k,subm):
                    yield subk

    def __iter__(self):
        return self.iterkeys()

    def keys(self,root="/"):
        return list(self.iterkeys(root))

    def itervalues(self,root="/",m=None):
        """Iterate over all values whose keys begin with the given root path."""
        if m is None:
            m = self._map
            for name in iteratepath(root):
                try:
                    m = m[name]
                except KeyError:
                    return
        for (nm,subm) in m.iteritems():
            if not nm:
                yield subm
            else:
                k = pathjoin(root,nm)
                for subv in self.itervalues(k,subm):
                    yield subv

    def values(self,root="/"):
        return list(self.itervalues(root))

    def iteritems(self,root="/",m=None):
        """Iterate over all (key,value) pairs beginning with the given root."""
        if m is None:
            m = self._map
            for name in iteratepath(root):
                try:
                    m = m[name]
                except KeyError:
                    return
        for (nm,subm) in m.iteritems():
            if not nm:
                yield (abspath(normpath(root)),subm)
            else:
                k = pathjoin(root,nm)
                for (subk,subv) in self.iteritems(k,subm):
                    yield (subk,subv)

    def items(self,root="/"):
        return list(self.iteritems(root))

    def iternames(self,root="/"):
        """Iterate over all names beneath the given root path.

        This is basically the equivalent of listdir() for a PathMap - it yields
        the next level of name components beneath the given path.
        """
        m = self._map
        for name in iteratepath(root):
            try:
                m = m[name]
            except KeyError:
                return
        for (nm,subm) in m.iteritems():
            if nm and subm:
                yield nm

    def names(self,root="/"):
        return list(self.iternames(root))

_wild_chars = frozenset('*?[]!{}')
def iswildcard(path):
    """Check if a path ends with a wildcard

    >>> is_wildcard('foo/bar/baz.*')
    True
    >>> is_wildcard('foo/bar')
    False

    """
    assert path is not None
    base_chars = frozenset(basename(path))
    return bool(base_chars.intersection(_wild_chars))

if __name__ == "__main__":
    print recursepath('a/b/c')
<commit_msg>docstring tweak for basename<commit_after>"""
fs.path
=======

Useful functions for FS path manipulation.

This is broadly similar to the standard ``os.path`` module but works with
paths in the canonical format expected by all FS objects (that is, separated
by forward slashes and with an optional leading slash).

"""

import re

_requires_normalization = re.compile(r'/\.\.|\./|\.|//|\\').search
def normpath(path):
    """Normalizes a path to be in the format expected by FS objects.

    This function remove any leading or trailing slashes, collapses
    duplicate slashes, replaces backward with forward slashes, and generally
    tries very hard to return a new path string the canonical FS format.
    If the path is invalid, ValueError will be raised.

    :param path: path to normalize
    :returns: a valid FS path

    >>> normpath(r"foo\\bar\\baz")
    'foo/bar/baz'

    >>> normpath("/foo//bar/frob/../baz")
    '/foo/bar/baz'

    >>> normpath("foo/../../bar")
    Traceback (most recent call last)
        ...
    ValueError: too many backrefs in path 'foo/../../bar'

    """

    if path in ('', '/'):
        return path

    # An early out if there is no need to normalize this path
    if not _requires_normalization(path):
        return path.rstrip('/')

    components = []
    append = components.append
    for comp in [c for c in path.replace('\\','/').split("/") if c not in ('', '.')]:
        if comp == "..":
            try:
                components.pop()
            except IndexError:
                raise ValueError("too many backrefs in path '%s'" % path)
        else:
            append(comp)
    if path[0] in '\\/':
        if not components:
            append("")
        components.insert(0, "")
    return "/".join(components)

def iteratepath(path, numsplits=None):
    """Iterate over the individual components of a path.

    :param path: Path to iterate over
    :numsplits: Maximum number of splits

    """
    path = relpath(normpath(path))
    if not path:
        return []
    if numsplits == None:
        return path.split('/')
    else:
        return path.split('/', numsplits)

def recursepath(path, reverse=False):
    """Returns intermediate paths from the root to the given path

    :param reverse: reverses the order of the paths

    >>> recursepath('a/b/c')
    ['/', u'/a', u'/a/b', u'/a/b/c']

    """

    if path in ('', '/'):
        return [u'/']

    path = abspath(normpath(path)) + '/'

    paths = [u'/']
    find = path.find
    append = paths.append
    pos = 1
    len_path = len(path)

    while pos < len_path:
        pos = find('/', pos)
        append(path[:pos])
        pos += 1

    if reverse:
        return paths[::-1]
    return paths

def abspath(path):
    """Convert the given path to an absolute path.

    Since FS objects have no concept of a 'current directory' this simply
    adds a leading '/' character if the path doesn't already have one.

    """
    if not path.startswith('/'):
        return u'/' + path
    return path

def relpath(path):
    """Convert the given path to a relative path.

    This is the inverse of abspath(), stripping a leading '/' from the
    path if it is present.

    :param path: Path to adjust

    >>> relpath('/a/b')
    'a/b'

    """
    return path.lstrip('/')

def pathjoin(*paths):
    """Joins any number of paths together, returning a new path string.

    :param paths: Paths to join are given in positional arguments

    >>> pathjoin('foo', 'bar', 'baz')
    'foo/bar/baz'

    >>> pathjoin('foo/bar', '../baz')
    'foo/baz'

    >>> pathjoin('foo/bar', '/baz')
    '/baz'

    """
    absolute = False
    relpaths = []
    for p in paths:
        if p:
            if p[0] in '\\/':
                del relpaths[:]
                absolute = True
            relpaths.append(p)

    path = normpath(u"/".join(relpaths))
    if absolute:
        path = abspath(path)
    return path

def join(*paths):
    """Joins any number of paths together, returning a new path string.

    This is a simple alias for the ``pathjoin`` function, allowing it to be
    used as ``fs.path.join`` in direct correspondence with ``os.path.join``.

    :param paths: Paths to join are given in positional arguments
    """
    return pathjoin(*paths)

def pathsplit(path):
    """Splits a path into (head, tail) pair.

    This function splits a path into a pair (head, tail) where 'tail' is the
    last pathname component and 'head' is all preceding components.

    :param path: Path to split

    >>> pathsplit("foo/bar")
    ('foo', 'bar')

    >>> pathsplit("foo/bar/baz")
    ('foo/bar', 'baz')

    >>> pathsplit("/foo/bar/baz")
    ('/foo/bar', 'baz')

    """
    if '/' not in path:
        return ('', path)
    split = path.rsplit('/', 1)
    return (split[0] or '/', split[1])

def split(path):
    """Splits a path into (head, tail) pair.

    This is a simple alias for the ``pathsplit`` function, allowing it to be
    used as ``fs.path.split`` in direct correspondence with ``os.path.split``.

    :param path: Path to split
    """
    return pathsplit(path)

def splitext(path):
    """Splits the extension from the path, and returns the path (up to the last
    '.' and the extension).

    :param path: A path to split

    >>> splitext('baz.txt')
    ('baz', 'txt')

    >>> splitext('foo/bar/baz.txt')
    ('foo/bar/baz', 'txt')

    """

    parent_path, pathname = pathsplit(path)
    if '.' not in pathname:
        return path, ''
    pathname, ext = pathname.rsplit('.', 1)
    path = pathjoin(parent_path, pathname)
    return path, '.' + ext

def isdotfile(path):
    """Detects if a path references a dot file, i.e. a resource who's name
    starts with a '.'

    :param path: Path to check

    >>> isdotfile('.baz')
    True

    >>> isdotfile('foo/bar/baz')
    True

    >>> isdotfile('foo/bar.baz').
    False

    """
    return basename(path).startswith('.')

def dirname(path):
    """Returns the parent directory of a path.

    This is always equivalent to the 'head' component of the value returned
    by pathsplit(path).

    :param path: A FS path

    >>> dirname('foo/bar/baz')
    'foo/bar'

    """
    if '/' not in path:
        return ''
    return path.rsplit('/', 1)[0]

def basename(path):
    """Returns the basename of the resource referenced by a path.

    This is always equivalent to the 'tail' component of the value returned
    by pathsplit(path).

    :param path: A FS path

    >>> basename('foo/bar/baz')
    'baz'

    >>> basename('foo/bar')
    'bar'

    >>> basename('foo/bar/')
    ''

    """
    if '/' not in path:
        return path
    return path.rsplit('/', 1)[-1]

def issamedir(path1, path2):
    """Return true if two paths reference a resource in the same directory.

    :param path1: An FS path
    :param path2: An FS path

    >>> issamedir("foo/bar/baz.txt", "foo/bar/spam.txt")
    True
    >>> issamedir("foo/bar/baz/txt", "spam/eggs/spam.txt")
    False

    """
    return dirname(normpath(path1)) == dirname(normpath(path2))

def isbase(path1, path2):
    p1 = forcedir(abspath(path1))
    p2 = forcedir(abspath(path2))
    return p1 == p2 or p1.startswith(p2)

def isprefix(path1, path2):
    """Return true is path1 is a prefix of path2.

    :param path1: An FS path
    :param path2: An FS path

    >>> isprefix("foo/bar", "foo/bar/spam.txt")
    True
    >>> isprefix("foo/bar/", "foo/bar")
    True
    >>> isprefix("foo/barry", "foo/baz/bar")
    False
    >>> isprefix("foo/bar/baz/", "foo/baz/bar")
    False

    """
    bits1 = path1.split("/")
    bits2 = path2.split("/")
    while bits1 and bits1[-1] == "":
        bits1.pop()
    if len(bits1) > len(bits2):
        return False
    for (bit1,bit2) in zip(bits1,bits2):
        if bit1 != bit2:
            return False
    return True

def forcedir(path):
    """Ensure the path ends with a trailing /

    :param path: An FS path

    >>> forcedir("foo/bar")
    'foo/bar/'
    >>> forcedir("foo/bar/")
    'foo/bar/'

    """

    if not path.endswith('/'):
        return path + '/'
    return path

def frombase(path1, path2):
    if not isprefix(path1, path2):
        raise ValueError("path1 must be a prefix of path2")
    return path2[len(path1):]

class PathMap(object):
    """Dict-like object with paths for keys.

    A PathMap is like a dictionary where the keys are all FS paths.  It has
    two main advantages over a standard dictionary.  First, keys are normalized
    automatically::

        >>> pm = PathMap()
        >>> pm["hello/world"] = 42
        >>> print pm["/hello/there/../world"]
        42

    Second, various dictionary operations (e.g. listing or clearing values)
    can be efficiently performed on a subset of keys sharing some common
    prefix::

        # list all values in the map
        pm.values()

        # list all values for paths starting with "/foo/bar"
        pm.values("/foo/bar")

    Under the hood, a PathMap is a trie-like structure where each level is
    indexed by path name component.  This allows lookups to be performed in
    O(number of path components) while permitting efficient prefix-based
    operations.
    """

    def __init__(self):
        self._map = {}

    def __getitem__(self,path):
        """Get the value stored under the given path."""
        m = self._map
        for name in iteratepath(path):
            try:
                m = m[name]
            except KeyError:
                raise KeyError(path)
        try:
            return m[""]
        except KeyError:
            raise KeyError(path)

    def __contains__(self,path):
        """Check whether the given path has a value stored in the map."""
        try:
            self[path]
        except KeyError:
            return False
        else:
            return True

    def __setitem__(self,path,value):
        """Set the value stored under the given path."""
        m = self._map
        for name in iteratepath(path):
            try:
                m = m[name]
            except KeyError:
                m = m.setdefault(name,{})
        m[""] = value

    def __delitem__(self,path):
        """Delete the value stored under the given path."""
        ms = [[self._map,None]]
        for name in iteratepath(path):
            try:
                ms.append([ms[-1][0][name],None])
            except KeyError:
                raise KeyError(path)
            else:
                ms[-2][1] = name
        try:
            del ms[-1][0][""]
        except KeyError:
            raise KeyError(path)
        else:
            while len(ms) > 1 and not ms[-1][0]:
                del ms[-1]
                del ms[-1][0][ms[-1][1]]

    def get(self,path,default=None):
        """Get the value stored under the given path, or the given default."""
        try:
            return self[path]
        except KeyError:
            return default

    def pop(self,path,default=None):
        """Pop the value stored under the given path, or the given default."""
        ms = [[self._map,None]]
        for name in iteratepath(path):
            try:
                ms.append([ms[-1][0][name],None])
            except KeyError:
                return default
            else:
                ms[-2][1] = name
        try:
            val = ms[-1][0].pop("")
        except KeyError:
            val = default
        else:
            while len(ms) > 1 and not ms[-1][0]:
                del ms[-1]
                del ms[-1][0][ms[-1][1]]
        return val

    def setdefault(self,path,value):
        m = self._map
        for name in iteratepath(path):
            try:
                m = m[name]
            except KeyError:
                m = m.setdefault(name,{})
        return m.setdefault("",value)

    def clear(self,root="/"):
        """Clear all entries beginning with the given root path."""
        m = self._map
        for name in iteratepath(root):
            try:
                m = m[name]
            except KeyError:
                return
        m.clear()

    def iterkeys(self,root="/",m=None):
        """Iterate over all keys beginning with the given root path."""
        if m is None:
            m = self._map
            for name in iteratepath(root):
                try:
                    m = m[name]
                except KeyError:
                    return
        for (nm,subm) in m.iteritems():
            if not nm:
                yield abspath(normpath(root))
            else:
                k = pathjoin(root,nm)
                for subk in self.iterkeys(k,subm):
                    yield subk

    def __iter__(self):
        return self.iterkeys()

    def keys(self,root="/"):
        return list(self.iterkeys(root))

    def itervalues(self,root="/",m=None):
        """Iterate over all values whose keys begin with the given root path."""
        if m is None:
            m = self._map
            for name in iteratepath(root):
                try:
                    m = m[name]
                except KeyError:
                    return
        for (nm,subm) in m.iteritems():
            if not nm:
                yield subm
            else:
                k = pathjoin(root,nm)
                for subv in self.itervalues(k,subm):
                    yield subv

    def values(self,root="/"):
        return list(self.itervalues(root))

    def iteritems(self,root="/",m=None):
        """Iterate over all (key,value) pairs beginning with the given root."""
        if m is None:
            m = self._map
            for name in iteratepath(root):
                try:
                    m = m[name]
                except KeyError:
                    return
        for (nm,subm) in m.iteritems():
            if not nm:
                yield (abspath(normpath(root)),subm)
            else:
                k = pathjoin(root,nm)
                for (subk,subv) in self.iteritems(k,subm):
                    yield (subk,subv)

    def items(self,root="/"):
        return list(self.iteritems(root))

    def iternames(self,root="/"):
        """Iterate over all names beneath the given root path.

        This is basically the equivalent of listdir() for a PathMap - it yields
        the next level of name components beneath the given path.
        """
        m = self._map
        for name in iteratepath(root):
            try:
                m = m[name]
            except KeyError:
                return
        for (nm,subm) in m.iteritems():
            if nm and subm:
                yield nm

    def names(self,root="/"):
        return list(self.iternames(root))

_wild_chars = frozenset('*?[]!{}')
def iswildcard(path):
    """Check if a path ends with a wildcard

    >>> is_wildcard('foo/bar/baz.*')
    True
    >>> is_wildcard('foo/bar')
    False

    """
    assert path is not None
    base_chars = frozenset(basename(path))
    return bool(base_chars.intersection(_wild_chars))

if __name__ == "__main__":
    print recursepath('a/b/c')
<|endoftext|>