<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyKeras(PythonPackage):
    """Deep Learning library for Python. Convnets, recurrent neural networks,
    and more. Runs on Theano or TensorFlow."""

    homepage = "http://keras.io"
    pypi = "Keras/Keras-1.2.2.tar.gz"

    version('2.2.4', sha256='90b610a3dbbf6d257b20a079eba3fdf2eed2158f64066a7c6f7227023fd60bc9')
    version('2.2.3', sha256='694aee60a6f8e0d3d6d3e4967e063b4623e3ca90032f023fd6d16bb5f81d18de')
    version('2.2.2', sha256='468d98da104ec5c3dbb10c2ef6bb345ab154f6ca2d722d4c250ef4d6105de17a')
    version('2.2.1', sha256='0d3cb14260a3fa2f4a5c4c9efa72226ffac3b4c50135ba6edaf2b3d1d23b11ee')
    version('2.2.0', sha256='5b8499d157af217f1a5ee33589e774127ebc3e266c833c22cb5afbb0ed1734bf')
    version('2.1.6', sha256='c14af1081242c25617ade7eb62121d58d01f16e1e744bae9fc4f1f95a417716e')
    version('2.1.5', sha256='907ad29add1fff27342a9f4fe3e60003d450d3af41a38f22f629c7736fc8399d')
    version('2.1.4', sha256='7ee1fcc79072ac904a4f008d715bcb78c60250ae3cd41d99e268c60ade8d0d3a')
    version('2.1.3', sha256='7ca3a381523bad40a6922e88951a316664cb088fd01cea07e5ec8ada3327e3c7')
    version('2.1.2', sha256='3ee56fc129d9d00b1916046e50056047836f97ada59df029e5661fb34442d5e8')
    version('2.1.1', sha256='f0ca2458c60d9711edf4291230b31795307ad3781cb6232ff4792b53c8f55123')
    version('2.1.0', sha256='67a0d66c20fff99312fc280e34c8f6dc3dbb027d4a33c13c79bec3c1173f6909')
    version('2.0.9', sha256='6b8572cf1b4a22fd0120b7c23382ba4fa04a6f0397e02af1249be9a7309d1767')
    version('2.0.8', sha256='899dc6aaed366f20100b9f80cf1093ea5b43eecc74afd1dc63a4e48dfa776ab9')
    version('2.0.7', sha256='a6c72ee2b94be1ffefe7e77b69582b9827211f0c356b2189459711844d3634c0')
    version('2.0.6', sha256='0519480abe4ad18b2c2d1bc580eab75edd82c95083d341a1157952f4b00019bb')
    version('2.0.5', sha256='cbce24758530e070fe1b403d6d21391cbea78c037b70bf6afc1ca9f1f8269eff')
    version('2.0.4', sha256='1cbe62af6821963321b275d5598fd94e63c11feaa1d4deaa79c9eb9ee0e1d68a')
    version('2.0.3', sha256='398dbd4a95e9d3ab2b2941d3e0c19362d397a2a6c3a667ab89d3d6aad30997f4')
    version('1.2.2', sha256='d2b18c4336eb9c4f0d03469870257efa7980a9b036c9d46dcf4d49e7f4487e2d')
    version('1.2.1', sha256='6adce75b2050608e6683c3046ef938bfdc5bfcd4c6b6c522df5e50d18e0ac7c6')
    version('1.2.0', sha256='33d5297cd0c280640dc5c075466995c05911bc1da35c83ae57b2a48188b605e2')
    version('1.1.2', sha256='cfde0a424961ead4982a7ebefd77d8ca382810b5a69b566fa64c57d8f340eeb4')
    version('1.1.1', sha256='be1b67f62e5119f6f24a239a865dc47e6d9aa93b97b506ba34cab7353dbc23b6')
    version('1.1.0', sha256='36d83b027ba9d2c9da8e1eefc28f600ca93dc03423e033b633cbac9061af8a5d')

    depends_on('py-keras-applications', type='run', when='@2.2:')
    depends_on('py-keras-preprocessing', type='run', when='@2.2:')
    depends_on('py-setuptools', type='build')
    depends_on('py-theano', type=('build', 'run'))
    depends_on('py-pyyaml', type=('build', 'run'))
    depends_on('py-six', type=('build', 'run'))
<commit_msg>py-keras: new version (#22711)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyKeras(PythonPackage):
    """Deep Learning library for Python. Convnets, recurrent neural networks,
    and more. Runs on Theano or TensorFlow."""

    homepage = "http://keras.io"
    pypi = "Keras/Keras-1.2.2.tar.gz"

    version('2.4.3', sha256='fedd729b52572fb108a98e3d97e1bac10a81d3917d2103cc20ab2a5f03beb973')
    version('2.2.4', sha256='90b610a3dbbf6d257b20a079eba3fdf2eed2158f64066a7c6f7227023fd60bc9')
    version('2.2.3', sha256='694aee60a6f8e0d3d6d3e4967e063b4623e3ca90032f023fd6d16bb5f81d18de')
    version('2.2.2', sha256='468d98da104ec5c3dbb10c2ef6bb345ab154f6ca2d722d4c250ef4d6105de17a')
    version('2.2.1', sha256='0d3cb14260a3fa2f4a5c4c9efa72226ffac3b4c50135ba6edaf2b3d1d23b11ee')
    version('2.2.0', sha256='5b8499d157af217f1a5ee33589e774127ebc3e266c833c22cb5afbb0ed1734bf')
    version('2.1.6', sha256='c14af1081242c25617ade7eb62121d58d01f16e1e744bae9fc4f1f95a417716e')
    version('2.1.5', sha256='907ad29add1fff27342a9f4fe3e60003d450d3af41a38f22f629c7736fc8399d')
    version('2.1.4', sha256='7ee1fcc79072ac904a4f008d715bcb78c60250ae3cd41d99e268c60ade8d0d3a')
    version('2.1.3', sha256='7ca3a381523bad40a6922e88951a316664cb088fd01cea07e5ec8ada3327e3c7')
    version('2.1.2', sha256='3ee56fc129d9d00b1916046e50056047836f97ada59df029e5661fb34442d5e8')
    version('2.1.1', sha256='f0ca2458c60d9711edf4291230b31795307ad3781cb6232ff4792b53c8f55123')
    version('2.1.0', sha256='67a0d66c20fff99312fc280e34c8f6dc3dbb027d4a33c13c79bec3c1173f6909')
    version('2.0.9', sha256='6b8572cf1b4a22fd0120b7c23382ba4fa04a6f0397e02af1249be9a7309d1767')
    version('2.0.8', sha256='899dc6aaed366f20100b9f80cf1093ea5b43eecc74afd1dc63a4e48dfa776ab9')
    version('2.0.7', sha256='a6c72ee2b94be1ffefe7e77b69582b9827211f0c356b2189459711844d3634c0')
    version('2.0.6', sha256='0519480abe4ad18b2c2d1bc580eab75edd82c95083d341a1157952f4b00019bb')
    version('2.0.5', sha256='cbce24758530e070fe1b403d6d21391cbea78c037b70bf6afc1ca9f1f8269eff')
    version('2.0.4', sha256='1cbe62af6821963321b275d5598fd94e63c11feaa1d4deaa79c9eb9ee0e1d68a')
    version('2.0.3', sha256='398dbd4a95e9d3ab2b2941d3e0c19362d397a2a6c3a667ab89d3d6aad30997f4')
    version('1.2.2', sha256='d2b18c4336eb9c4f0d03469870257efa7980a9b036c9d46dcf4d49e7f4487e2d')
    version('1.2.1', sha256='6adce75b2050608e6683c3046ef938bfdc5bfcd4c6b6c522df5e50d18e0ac7c6')
    version('1.2.0', sha256='33d5297cd0c280640dc5c075466995c05911bc1da35c83ae57b2a48188b605e2')
    version('1.1.2', sha256='cfde0a424961ead4982a7ebefd77d8ca382810b5a69b566fa64c57d8f340eeb4')
    version('1.1.1', sha256='be1b67f62e5119f6f24a239a865dc47e6d9aa93b97b506ba34cab7353dbc23b6')
    version('1.1.0', sha256='36d83b027ba9d2c9da8e1eefc28f600ca93dc03423e033b633cbac9061af8a5d')

    depends_on('python@3.6:', type=('build', 'run'), when='@2.4:')
    depends_on('py-numpy@1.9.1:', type=('build', 'run'), when='@2.4:')
    depends_on('py-scipy@0.14:', type=('build', 'run'), when='@2.4:')
    depends_on('py-h5py', type=('build', 'run'), when='@2.4:')
    depends_on('py-keras-applications', type='run', when='@2.2')
    depends_on('py-keras-preprocessing', type='run', when='@2.2')
    depends_on('py-setuptools', type='build')
    depends_on('py-theano', type=('build', 'run'), when='@:2.2')
    depends_on('py-pyyaml', type=('build', 'run'))
    depends_on('py-six', type=('build', 'run'), when='@:2.2')
<|endoftext|>


<commit_before>import irclib

import rapidsms
from rapidsms.message import Message

class Backend(rapidsms.backends.Backend):
    def configure(self, host="irc.freenode.net", port=6667,
                        nick="rapidsms", channels=["#rapidsms"]):
        self.host = host
        self.port = port
        self.nick = nick[:16] # 16 char limit for IRC nicks
        self.channels = channels

        self.irc = irclib.IRC()
        self.irc.add_global_handler("privmsg", self.privmsg)
        self.irc.add_global_handler("pubmsg", self.pubmsg)

    def start (self):
        self.info("Connecting to %s as %s", self.host, self.nick)
        self.server = self.irc.server()
        self.server.connect(self.host, self.port, self.nick)

        for channel in self.channels:
            self.info("Joining %s on %s", channel, self.host)
            self.server.join(channel)

    def run (self):
        while self.running:
            if self.message_waiting:
                msg = self.next_message()
                self.outgoing(msg)
            self.irc.process_once(timeout=1.0)

        self.info("Shutting down...")
        self.server.disconnect()

    def outgoing (self, msg):
        try:
            channel = msg.irc_channel
        except AttributeError:
            channel = self.channels[0]
        if channel:
            target = channel
        else:
            target = msg.caller
        response = "%s: %s" % (msg.caller, msg.text)
        self.info("sending to %s: %s", target, response)
        self.server.privmsg(target, response)

    def pubmsg (self, connection, event):
        self.debug("%s -> %s: %r", event.source(), event.target(), event.arguments())
        try:
            nick, txt = map(str.strip, event.arguments()[0].split(":"))
        except ValueError:
            return # not for me
        nick = nick.split("!")[0]
        if nick == self.nick:
            self.info("routing public message from %s", event.source())
            caller = event.source().split("!")[0]
            msg = self.message(caller, txt)
            msg.irc_channel = event.target()
            self.route(msg)

    def privmsg (self, connection, event):
        self.debug("%s -> %s: %r", event.source(), event.target(), event.arguments())
        if event.target() == self.nick:
            self.info("routing private message from %s", event.source())
            caller = event.source().split("!")[0]
            msg = self.message(caller, event.arguments()[0])
            msg.irc_channel = caller
            self.route(msg)
<commit_msg>Make IRC work again (oops), and make nick a mandatory option with no default (so you have to set it, so that we don't get a million bots all called rapidsms).<commit_after>import irclib

import rapidsms
from rapidsms.message import Message

class Backend(rapidsms.backends.Backend):
    def configure(self, host="irc.freenode.net", port=6667,
                        nick=None, channels=["#rapidsms"]):
        self.host = host
        self.port = port
        self.nick = self.config_requires("nick",nick)[:16] # 16 char limit for IRC nicks
        self.channels = self.config_list(channels)

        self.irc = irclib.IRC()
        self.irc.add_global_handler("privmsg", self.privmsg)
        self.irc.add_global_handler("pubmsg", self.pubmsg)

    def run (self):
        self.info("Connecting to %s as %s", self.host, self.nick)
        self.server = self.irc.server()
        self.server.connect(self.host, self.port, self.nick)

        for channel in self.channels:
            self.info("Joining %s on %s", channel, self.host)
            self.server.join(channel)

        while self.running:
            if self.message_waiting:
                msg = self.next_message()
                self.outgoing(msg)
            self.irc.process_once(timeout=1.0)

        self.info("Shutting down...")
        self.server.disconnect()

    def outgoing (self, msg):
        try:
            channel = msg.irc_channel
        except AttributeError:
            channel = self.channels[0]
        if channel:
            target = channel
        else:
            target = msg.caller
        response = "%s: %s" % (msg.caller, msg.text)
        self.info("sending to %s: %s", target, response)
        self.server.privmsg(target, response)

    def pubmsg (self, connection, event):
        self.debug("%s -> %s: %r", event.source(), event.target(), event.arguments())
        try:
            nick, txt = map(str.strip, event.arguments()[0].split(":"))
        except ValueError:
            return # not for me
        nick = nick.split("!")[0]
        if nick == self.nick:
            self.info("routing public message from %s", event.source())
            caller = event.source().split("!")[0]
            msg = self.message(caller, txt)
            msg.irc_channel = event.target()
            self.route(msg)

    def privmsg (self, connection, event):
        self.debug("%s -> %s: %r", event.source(), event.target(), event.arguments())
        if event.target() == self.nick:
            self.info("routing private message from %s", event.source())
            caller = event.source().split("!")[0]
            msg = self.message(caller, event.arguments()[0])
            msg.irc_channel = caller
            self.route(msg)
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class RAffyio(RPackage):
    """Routines for parsing Affymetrix data files based upon file format
       information. Primary focus is on accessing the CEL and CDF file
       formats."""

    homepage = "https://bioconductor.org/packages/affyio/"
    url      = "https://bioconductor.org/packages/3.5/bioc/src/contrib/affyio_1.46.0.tar.gz"

    version('1.46.0', 'e1f7a89ae16940aa29b998a4dbdc0ef9')
    depends_on('r-zlibbioc', type=('build', 'run'))
<commit_msg>r-affyio: added versioned dependency on r (#5661)<commit_after>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class RAffyio(RPackage):
    """Routines for parsing Affymetrix data files based upon file format
       information. Primary focus is on accessing the CEL and CDF file
       formats."""

    homepage = "https://bioconductor.org/packages/affyio/"
    url      = "https://bioconductor.org/packages/3.5/bioc/src/contrib/affyio_1.46.0.tar.gz"

    version('1.46.0', 'e1f7a89ae16940aa29b998a4dbdc0ef9')
    depends_on('r-zlibbioc', type=('build', 'run'))
    depends_on('r@3.4.0:3.4.9', when='@1.46.0')
<|endoftext|>


<commit_before>## dropoff.py
#
# Implements the DROPOFF module of the competition code.

from module import Module
from ..constants import *

class DropoffModule(Module):
    def __init__(self, timer, servo):
        self.timeout = 7000
        self.waitTime = 500     # Time in ms to wait for the door to actually open.
        self.openValue = 90     # Value in degrees the servo should be when the door is open.
        self.closedValue = 180  # Value in degrees the servo should be when the door is closed.
        self.timer = timer
        self.servo = servo

    ## Return True if there was an error in initialization, False otherwise.
    def checkForInitializationErrors(self):
        # TODO: Make sure servo is initialized in the "closed" position.
        return False

    ## Set up the beginning of the dropoff process.
    def start(self):
        self.timer.reset()
        self.servo.write(self.openValue)

    ## Open the back door and drive "forwards," avoiding what needs to be avoided.
    #
    # @return   The value of the next module to return to.
    def run(self):

        # Allow timeout.
        if self.timer.millis() > self.timout:
            print "Timed out from DROPOFF to FIND"
            return MODULE_FIND

        if self.timer.millis() > self.waitTime:
            # TODO: Drive forwards after opening the door.
            raise NotImplementedError

        return MODULE_DROPOFF<commit_msg>Editted drop-off to close door when done<commit_after>## dropoff.py
#
# Implements the DROPOFF module of the competition code.

from module import Module
from ..constants import *

class DropoffModule(Module):
    def __init__(self, timer, servo):
        self.timeout = 7000
        self.waitTime = 500     # Time in ms to wait for the door to actually open.
        self.openValue = 90     # Value in degrees the servo should be when the door is open.
        self.closedValue = 180  # Value in degrees the servo should be when the door is closed.
        self.timer = timer
        self.servo = servo

    ## Return True if there was an error in initialization, False otherwise.
    def checkForInitializationErrors(self):
        # TODO: Make sure servo is initialized in the "closed" position.
        return False

    ## Set up the beginning of the dropoff process.
    def start(self):
        self.timer.reset()
        self.servo.write(self.openValue)

    ## Open the back door and drive "forwards," avoiding what needs to be avoided.
    #
    # @return   The value of the next module to return to.
    def run(self):

        # Allow timeout.
        if self.timer.millis() > self.timout:
            print "Timed out from DROPOFF to FIND"
            return MODULE_FIND

        if self.timer.millis() > self.waitTime:
            # TODO: drive forward a couple inches
            self.servo.write(self.closedValue)
            raise NotImplementedError

        return MODULE_DROPOFF<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (C) 2013 Jason Piper - j.piper@warwick.ac.uk
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import argparse
import pyDNase
import numpy as np
import matplotlib as mpl
from clint.textui import progress, puts
#Required for headless operation
mpl.use('Agg')
import matplotlib.pyplot as plt
from matplotlib import rcParams

parser = argparse.ArgumentParser(description='Plots average profile of DNase activity surrounding a list of regions in a BED file')
parser.add_argument("-w", "--window_size", help="Size of flanking area around centre of the regions to plot (default: 100)",default=100,type=int)
parser.add_argument("-i",action="store_true", help="Ignores any strand information in BED file and plots data relative to reference strand",default=False)
parser.add_argument("-c",action="store_true", help="Combine the strand information into one graph",default=False)
parser.add_argument("regions", help="BED file of the regions you want to generate the average profile for")
parser.add_argument("reads", help="The BAM file containing the DNase-seq data")
parser.add_argument("output", help="filename to write the output to")
args  = parser.parse_args()

reads   = pyDNase.BAMHandler(args.reads)
regions = pyDNase.GenomicIntervalSet(args.regions)

#Set all strands to positive if "ignore strands" is enabled
if args.i:
    for each in regions:
        each.strand = "+"

puts("Resizing Regions to {0}".format(args.window_size))
regions.resizeRegions(args.window_size)

fw = []
rv = []
puts("Reading Data from BAM file...")
for each in progress.bar(regions):
    if sum(reads[each]["+"]) and sum(reads[each]["-"]):
        fw.append(reads[each]["+"])
        rv.append(reads[each]["-"])

if args.c:
    plt.plot(np.add(np.mean(fw,axis=0),np.mean(rv,axis=0)),c="red")
else:
    plt.plot(np.mean(fw,axis=0),c="red")
    plt.plot(np.mean(rv,axis=0),c="blue")

#Pad the axis out reads bit
rcParams['xtick.major.pad'] = 20
rcParams['ytick.major.pad'] = 20

#Sort out the X axis ticks
ticks = [0,args.window_size,args.window_size*2]
labels = [-args.window_size,0,args.window_size]
plt.xticks(ticks, labels)

#Make the yaxis start from 0
plt.gca().set_ylim(0)

#Makes ticks only appear on the left hand side
plt.gca().yaxis.set_ticks_position('left')

#Remove top and right borders
plt.gca().spines['top'].set_visible(False)
plt.gca().spines['right'].set_visible(False)

plt.gca().tick_params(axis='both', which='major', labelsize=32)

plt.gca().set_ylabel('Average DNase\n Activity',size="36", multialignment='center')
plt.savefig(args.output,bbox_inches='tight')
<commit_msg>Enable cut normalisation as fraction of total cuts in region<commit_after>#!/usr/bin/env python

# Copyright (C) 2013 Jason Piper - j.piper@warwick.ac.uk
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import argparse
import pyDNase
import numpy as np
import matplotlib as mpl
from clint.textui import progress, puts
#Required for headless operation
mpl.use('Agg')
import matplotlib.pyplot as plt
from matplotlib import rcParams

parser = argparse.ArgumentParser(description='Plots average profile of DNase activity surrounding a list of regions in a BED file')
parser.add_argument("-w", "--window_size", help="Size of flanking area around centre of the regions to plot (default: 100)",default=100,type=int)
parser.add_argument("-i",action="store_true", help="Ignores any strand information in BED file and plots data relative to reference strand",default=False)
parser.add_argument("-c",action="store_true", help="Combine the strand information into one graph",default=False)
parser.add_argument("-n",action="store_true", help="Normalise cut counts to a fraction peaks",default=False)
parser.add_argument("regions", help="BED file of the regions you want to generate the average profile for")
parser.add_argument("reads", help="The BAM file containing the DNase-seq data")
parser.add_argument("output", help="filename to write the output to")
args  = parser.parse_args()

reads   = pyDNase.BAMHandler(args.reads)
regions = pyDNase.GenomicIntervalSet(args.regions)

#Set all strands to positive if "ignore strands" is enabled
if args.i:
    for each in regions:
        each.strand = "+"

puts("Resizing Regions to {0}".format(args.window_size))
regions.resizeRegions(args.window_size)

fw = []
rv = []
puts("Reading Data from BAM file...")
for each in progress.bar(regions):
    if sum(reads[each]["+"]) and sum(reads[each]["-"]):
        fw.append(reads[each]["+"])
        rv.append(reads[each]["-"])

if args.n:
    fw = [map(float,i)for i in fw]
    rv = [map(float,i) for i in rv]
    fw = [np.divide(np.subtract(i, min(i)), np.subtract(max(i) , min(i))) for i in fw]
    rv = [np.divide(np.subtract(i, min(i)), np.subtract(max(i) , min(i))) for i in rv]

if args.c:
    plt.plot(np.add(np.mean(fw,axis=0),np.mean(rv,axis=0)),c="red")
else:
    plt.plot(np.mean(fw,axis=0),c="red")
    plt.plot(np.mean(rv,axis=0),c="blue")

#Pad the axis out reads bit
rcParams['xtick.major.pad'] = 20
rcParams['ytick.major.pad'] = 20

#Sort out the X axis ticks
ticks = [0,args.window_size,args.window_size*2]
labels = [-args.window_size,0,args.window_size]
plt.xticks(ticks, labels)

#Make the yaxis start from 0
plt.gca().set_ylim(0)

#Makes ticks only appear on the left hand side
plt.gca().yaxis.set_ticks_position('left')

#Remove top and right borders
plt.gca().spines['top'].set_visible(False)
plt.gca().spines['right'].set_visible(False)

plt.gca().tick_params(axis='both', which='major', labelsize=32)

plt.gca().set_ylabel('Average DNase\n Activity',size="36", multialignment='center')
plt.savefig(args.output,bbox_inches='tight')
<|endoftext|>


<commit_before>from base64 import b64encode
from optparse import make_option
import json
import urllib
import urllib2
import argparse

parser = argparse.ArgumentParser(description='Synchronize GeoGig repository with OpenStreetMap (OSM)')
parser.add_argument("url", help="The url to the repository you want to sync.")
parser.add_argument("username", help="The username to use for basic auth requests.")
parser.add_argument("password", help="The password to use for basic auth requests.")
parser.add_argument("authorname", help="The author name to use when merging non-conflicting branches.")
parser.add_argument("authoremail", help="The author email to use when merging non-conflicting branches.")
args = parser.parse_args()

print args.accumulate(args.integers)
<commit_msg>Update geogig_sync_osm.py<commit_after>from base64 import b64encode
from optparse import make_option
import json
import urllib
import urllib2
import argparse

parser = argparse.ArgumentParser(description='Synchronize GeoGig repository with OpenStreetMap (OSM)')
parser.add_argument("url", help="The url to the repository you want to sync.")
parser.add_argument("--username", help="The username to use for basic auth requests.")
parser.add_argument("--password", help="The password to use for basic auth requests.")
parser.add_argument("authorname", help="The author name to use when merging non-conflicting branches.")
parser.add_argument("authoremail", help="The author email to use when merging non-conflicting branches.")
args = parser.parse_args()

print args.accumulate(args.integers)
<|endoftext|>


<commit_before>#!/usr/bin/env python2
# OpenPOWER Automated Test Project
#
# Contributors Listed Below - COPYRIGHT 2017
# [+] International Business Machines Corp.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.

'''
PCI Slot/Location Codes
-----------------------

This test will use the device tree to determine if the pciex is
a root/switch fabric with one or more switch devices.

Slot labels and loc codes are retrieved and if missing or empty
the test will identify for investigation.  Not all test results
reporting missing or empty slot labels or loc codes are problems,
it is the system owner to make that determination.

'''

import unittest
import re
import os

import OpTestConfiguration
from common.OpTestSystem import OpSystemState
from common.Exceptions import CommandFailed

import logging
import OpTestLogger
log = OpTestLogger.optest_logger_glob.get_logger(__name__)

class PciDT(unittest.TestCase):
    '''
    PciDT Class
    --run testcases.PciSlotLocCodes
    '''
    @classmethod
    def setUpClass(cls):
        cls.conf = OpTestConfiguration.conf
        if cls.conf.args.bmc_type in ['qemu', 'mambo']:
            raise unittest.SkipTest("QEMU/Mambo running so skipping tests")
        cls.cv_SYSTEM = cls.conf.system()
        try:
            if cls.desired == OpSystemState.OS:
                cls.c = cls.cv_SYSTEM.cv_HOST.get_ssh_connection()
                cls.cv_SYSTEM.goto_state(OpSystemState.OS)
            else:
                cls.c = cls.cv_SYSTEM.console
                cls.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
        except Exception as e:
            log.debug("Unable to find cls.desired, probably a test code problem")
            cls.c = cls.cv_SYSTEM.console
            cls.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)

    def dump_lspci(self, lspci_dict):
        for key, value in lspci_dict.iteritems():
            log.debug("Dumping lspci value={} key={} ".format(value, key))

    def build_lspci(self, lspci_dict={}):
        bus_dirs = self.c.run_command("find /sys/bus/pci/devices -type l")
        # first build a dict with device info
        # of_base=0000:00:00.0
        # pciex_spec=/pciex@600c3c0000000/pci@0
        # of_key=/proc/device-tree/pciex@600c3c0000000/pci@0
        for directory in sorted(bus_dirs):
            try:
                of_base = os.path.basename(directory)
                of_node_path = directory + "/of_node"
                cmd = "ls -l %s | awk '{print $11}'" % of_node_path
                # goal is to pull off the pciex to build key for dict lookup
                # ../../../firmware/devicetree/base/pciex@600c3c0000000/pci@0
                # /proc/device-tree/pciex@600c3c0000000/pci@0
                of_output = self.c.run_command(cmd)
                pciex_spec = re.sub("^(.*base)", "", of_output[0])
                of_key = "/proc/device-tree" + pciex_spec
                lspci_dict[of_key] = of_base
            except Exception as e:
                log.debug("of_base skipping directory={} Exception={}"
                    .format(directory, e))
        return lspci_dict

    def build_lspci_names(self, lspci_dict):
        self.dump_lspci(lspci_dict) # just logging
        lspci_names = self.c.run_command("lspci -nn")
        lspci_names_dict = {}
        # 0006:00:00.0 Bridge [0680]: IBM Device [1014:04ea] (rev 01)
        for key, value in lspci_dict.iteritems():
            for i in range(len(lspci_names)):
                if value in lspci_names[i]:
                    lspci_names_dict[key] = lspci_names[i]
        return lspci_names_dict

    def build_slot_loc_tables(self):
        '''
        Builds a dictionary to track slots and loc codes and their
        values.
        '''
        node_dirs = self.c.run_command("find /proc/device-tree/ -type d "
                       "| grep -i pciex | grep -i pci@")
        output_dict = {}
        self.slot_failures = 0
        self.loccode_failures = 0
        for directory in sorted(node_dirs):
            matchObj = re.match(".*/pci@\d{1,}$", directory, re.M)
            tracking_dict = {}
            try:
                r = self.c.run_command("find {} -type d".format(directory))
                # empty=1, missing=2
                if len(r) == 1 and matchObj:
                    tracking_dict['slot-label-status'] = 0
                    tracking_dict['loc-code-status'] = 0
                    try:
                        check_list = ["npu"]
                        pciObj = re.search("/(pciex@[0-9a-fA-F]+)/", directory)
                        compat_node = "/proc/device-tree/{}/compatible".format(pciObj.group(1))
                        compat_output = self.c.run_command("cat {}".format(compat_node))
                        matching = [xs for xs in check_list if any(xs in xa for xa in compat_output)]
                        if len(matching) == 0:
                            log.debug("Non-NPU compat_output={} for {}"
                                .format(compat_output, compat_node))
                            res = self.c.run_command("cat {}/ibm,slot-label"
                                .format(directory))
                            if res[0] == "":
                                tracking_dict['slot-label-status'] = 1
                                value = ""
                                self.slot_failures += 1
                            else:
                                value = res[0].rstrip('\x00')
                            tracking_dict['slot-label'] = value
                        else:
                            log.debug("NPU compat_output={} for {}"
                                .format(compat_output, compat_node))
                    except CommandFailed as cf:
                        tracking_dict['slot-label-status'] = 2
                        tracking_dict['slot-label'] = None
                        self.slot_failures += 1
                    try:
                        res = self.c.run_command("cat {}/ibm,loc-code"
                            .format(directory))
                        if res[0] == "":
                            tracking_dict['loc-code-status'] = 1
                            value = ""
                            self.loccode_failures += 1
                        else:
                            value = res[0].rstrip('\x00')
                        tracking_dict['loc-code'] = value
                    except CommandFailed as cf:
                        tracking_dict['loc-code-status'] = 2
                        tracking_dict['loc-code'] = None
                        self.loccode_failures += 1
                    output_dict[directory] = tracking_dict
            except CommandFailed as cf:
                log.debug("Unable to query for pciex info, Exception={}"
                    .format(cf))
        return output_dict

    def doit(self):
        '''
        Performs the building of an lspci like lookup that will
        be used to output helpful information when failures occur
        as well as logging both good and not so good results
        to the debug log for analysis.
        '''
        # build a dictionary of devices, helpfulness in display
        lspci_dict = self.build_lspci()
        # map the devices to names, helpfulness in display
        lspci_mappings = self.build_lspci_names(lspci_dict)
        # build the slot and loc code tables
        output_dict = self.build_slot_loc_tables()
        # package up failures to report
        output_list = []
        count = 1
        for key in output_dict:
            if output_dict[key].get('slot-label-status') != 0 \
                or output_dict[key].get('loc-code-status') != 0:
                    output_list.append("<----------------------"
                        "Investigate #{}---------------------->"
                        .format(count))
                    count += 1
                    output_list.append("PCI Path={}".format(key))
                    output_list.append("{}".format(lspci_mappings[key]))
                    output_list.append("ibm,slot-label={}"
                        .format(output_dict[key].get('slot-label')))
                    output_list.append("ibm,loc-code={}"
                        .format(output_dict[key].get('loc-code')))
                    log.debug("Investigate - PCI Root Path={}".format(key))
                    log.debug("Investigate - {}"
                        .format(lspci_mappings[key]))
                    log.debug("Investigate - lspci={}"
                        .format(lspci_dict.get(key)))
                    log.debug("Investigate - ibm,slot-label={}"
                        .format(output_dict[key].get('slot-label')))
                    log.debug("Investigate - ibm,loc-code={}"
                        .format(output_dict[key].get('loc-code')))
            else:
                # this path just for logging
                log.debug("PCI Root Path={}".format(key))
                log.debug("{}"
                    .format(lspci_mappings[key]))
                log.debug("lspci={}"
                    .format(lspci_dict.get(key)))
                log.debug("ibm,slot-label={}"
                    .format(output_dict[key].get('slot-label')))
                log.debug("ibm,loc-code={}"
                    .format(output_dict[key].get('loc-code')))

        if self.slot_failures != 0 or self.loccode_failures != 0:
            failed_list = '\n'.join(filter(None, output_list))
            self.assertTrue(False, "PCI Root: Slot Label "
                "or Loc Code Failures:\nBased on Platform "
                "Slot Labels may not be present\n{}"
                .format(failed_list))

class SkirootDT(PciDT):
    '''
    SkirootDT Class performs PCI DT checks in skiroot
    --run testcases.PciSlotLocCodes.SkirootDT
    '''
    @classmethod
    def setUpClass(cls):
        cls.desired = OpSystemState.PETITBOOT_SHELL
        super(SkirootDT, cls).setUpClass()

    def runTest(self):
        self.doit()

class HostDT(PciDT):
    '''
    HostDT Class performs PCI DT checks in the Host OS
    --run testcases.PciSlotLocCodes.HostDT
    '''
    @classmethod
    def setUpClass(cls):
        cls.desired = OpSystemState.OS
        super(HostDT, cls).setUpClass()

    def runTest(self):
        self.doit()

<commit_msg>PciSlotLocCodes: permit loc-code but no slot-name<commit_after>#!/usr/bin/env python2
# OpenPOWER Automated Test Project
#
# Contributors Listed Below - COPYRIGHT 2017
# [+] International Business Machines Corp.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.

'''
PCI Slot/Location Codes
-----------------------

This test will use the device tree to determine if the pciex is
a root/switch fabric with one or more switch devices.

Slot labels and loc codes are retrieved and if missing or empty
the test will identify for investigation.  Not all test results
reporting missing or empty slot labels or loc codes are problems,
it is the system owner to make that determination.

'''

import unittest
import re
import os

import OpTestConfiguration
from common.OpTestSystem import OpSystemState
from common.Exceptions import CommandFailed

import logging
import OpTestLogger
log = OpTestLogger.optest_logger_glob.get_logger(__name__)

class PciDT(unittest.TestCase):
    '''
    PciDT Class
    --run testcases.PciSlotLocCodes
    '''
    @classmethod
    def setUpClass(cls):
        cls.conf = OpTestConfiguration.conf
        if cls.conf.args.bmc_type in ['qemu', 'mambo']:
            raise unittest.SkipTest("QEMU/Mambo running so skipping tests")
        cls.cv_SYSTEM = cls.conf.system()
        try:
            if cls.desired == OpSystemState.OS:
                cls.c = cls.cv_SYSTEM.cv_HOST.get_ssh_connection()
                cls.cv_SYSTEM.goto_state(OpSystemState.OS)
            else:
                cls.c = cls.cv_SYSTEM.console
                cls.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
        except Exception as e:
            log.debug("Unable to find cls.desired, probably a test code problem")
            cls.c = cls.cv_SYSTEM.console
            cls.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)

    def dump_lspci(self, lspci_dict):
        for key, value in lspci_dict.iteritems():
            log.debug("Dumping lspci value={} key={} ".format(value, key))

    def build_lspci(self, lspci_dict={}):
        bus_dirs = self.c.run_command("find /sys/bus/pci/devices -type l")
        # first build a dict with device info
        # of_base=0000:00:00.0
        # pciex_spec=/pciex@600c3c0000000/pci@0
        # of_key=/proc/device-tree/pciex@600c3c0000000/pci@0
        for directory in sorted(bus_dirs):
            try:
                of_base = os.path.basename(directory)
                of_node_path = directory + "/of_node"
                cmd = "ls -l %s | awk '{print $11}'" % of_node_path
                # goal is to pull off the pciex to build key for dict lookup
                # ../../../firmware/devicetree/base/pciex@600c3c0000000/pci@0
                # /proc/device-tree/pciex@600c3c0000000/pci@0
                of_output = self.c.run_command(cmd)
                pciex_spec = re.sub("^(.*base)", "", of_output[0])
                of_key = "/proc/device-tree" + pciex_spec
                lspci_dict[of_key] = of_base
            except Exception as e:
                log.debug("of_base skipping directory={} Exception={}"
                    .format(directory, e))
        return lspci_dict

    def build_lspci_names(self, lspci_dict):
        self.dump_lspci(lspci_dict) # just logging
        lspci_names = self.c.run_command("lspci -nn")
        lspci_names_dict = {}
        # 0006:00:00.0 Bridge [0680]: IBM Device [1014:04ea] (rev 01)
        for key, value in lspci_dict.iteritems():
            for i in range(len(lspci_names)):
                if value in lspci_names[i]:
                    lspci_names_dict[key] = lspci_names[i]
        return lspci_names_dict

    def build_slot_loc_tables(self):
        '''
        Builds a dictionary to track slots and loc codes and their
        values.
        '''
        node_dirs = self.c.run_command("find /proc/device-tree/ -type d "
                       "| grep -i pciex | grep -i pci@")
        output_dict = {}
        self.slot_failures = 0
        self.loccode_failures = 0
        for directory in sorted(node_dirs):
            matchObj = re.match(".*/pci@\d{1,}$", directory, re.M)
            tracking_dict = {}
            try:
                r = self.c.run_command("find {} -type d".format(directory))
                # empty=1, missing=2
                if len(r) == 1 and matchObj:
                    tracking_dict['slot-label-status'] = 0
                    tracking_dict['loc-code-status'] = 0
                    try:
                        check_list = ["npu"]
                        pciObj = re.search("/(pciex@[0-9a-fA-F]+)/", directory)
                        compat_node = "/proc/device-tree/{}/compatible".format(pciObj.group(1))
                        compat_output = self.c.run_command("cat {}".format(compat_node))
                        matching = [xs for xs in check_list if any(xs in xa for xa in compat_output)]
                        if len(matching) == 0:
                            log.debug("Non-NPU compat_output={} for {}"
                                .format(compat_output, compat_node))
                            res = self.c.run_command("cat {}/ibm,slot-label"
                                .format(directory))
                            if res[0] == "":
                                tracking_dict['slot-label-status'] = 1
                                value = ""
                                self.slot_failures += 1
                            else:
                                value = res[0].rstrip('\x00')
                            tracking_dict['slot-label'] = value
                        else:
                            log.debug("NPU compat_output={} for {}"
                                .format(compat_output, compat_node))
                    except CommandFailed as cf:
                        tracking_dict['slot-label-status'] = 2
                        tracking_dict['slot-label'] = None
                        self.slot_failures += 1
                    try:
                        res = self.c.run_command("cat {}/ibm,loc-code"
                            .format(directory))
                        if res[0] == "":
                            tracking_dict['loc-code-status'] = 1
                            value = ""
                            self.loccode_failures += 1
                        else:
                            value = res[0].rstrip('\x00')
                        tracking_dict['loc-code'] = value
                    except CommandFailed as cf:
                        tracking_dict['loc-code-status'] = 2
                        tracking_dict['loc-code'] = None
                        self.loccode_failures += 1
                    output_dict[directory] = tracking_dict
            except CommandFailed as cf:
                log.debug("Unable to query for pciex info, Exception={}"
                    .format(cf))
        return output_dict

    def doit(self):
        '''
        Performs the building of an lspci like lookup that will
        be used to output helpful information when failures occur
        as well as logging both good and not so good results
        to the debug log for analysis.
        '''
        # build a dictionary of devices, helpfulness in display
        lspci_dict = self.build_lspci()
        # map the devices to names, helpfulness in display
        lspci_mappings = self.build_lspci_names(lspci_dict)
        # build the slot and loc code tables
        output_dict = self.build_slot_loc_tables()
        # package up failures to report
        output_list = []
        count = 1
        for key in output_dict:
            if output_dict[key].get('slot-label-status') != 0 \
               and output_dict[key].get('loc-code-status') != 0:
                    output_list.append("<----------------------"
                        "Investigate #{}---------------------->"
                        .format(count))
                    count += 1
                    output_list.append("PCI Path={}".format(key))
                    output_list.append("{}".format(lspci_mappings[key]))
                    output_list.append("ibm,slot-label={}"
                        .format(output_dict[key].get('slot-label')))
                    output_list.append("ibm,loc-code={}"
                        .format(output_dict[key].get('loc-code')))
                    log.debug("Investigate - PCI Root Path={}".format(key))
                    log.debug("Investigate - {}"
                        .format(lspci_mappings[key]))
                    log.debug("Investigate - lspci={}"
                        .format(lspci_dict.get(key)))
                    log.debug("Investigate - ibm,slot-label={}"
                        .format(output_dict[key].get('slot-label')))
                    log.debug("Investigate - ibm,loc-code={}"
                        .format(output_dict[key].get('loc-code')))
            else:
                # this path just for logging
                log.debug("PCI Root Path={}".format(key))
                log.debug("{}"
                    .format(lspci_mappings[key]))
                log.debug("lspci={}"
                    .format(lspci_dict.get(key)))
                log.debug("ibm,slot-label={}"
                    .format(output_dict[key].get('slot-label')))
                log.debug("ibm,loc-code={}"
                    .format(output_dict[key].get('loc-code')))

        if len(output_list):
            failed_list = '\n'.join(filter(None, output_list))
            self.assertTrue(False, "PCI Root: Slot Label "
                "or Loc Code Failures:\nBased on Platform "
                "Slot Labels may not be present\n{}"
                .format(failed_list))

class SkirootDT(PciDT):
    '''
    SkirootDT Class performs PCI DT checks in skiroot
    --run testcases.PciSlotLocCodes.SkirootDT
    '''
    @classmethod
    def setUpClass(cls):
        cls.desired = OpSystemState.PETITBOOT_SHELL
        super(SkirootDT, cls).setUpClass()

    def runTest(self):
        self.doit()

class HostDT(PciDT):
    '''
    HostDT Class performs PCI DT checks in the Host OS
    --run testcases.PciSlotLocCodes.HostDT
    '''
    @classmethod
    def setUpClass(cls):
        cls.desired = OpSystemState.OS
        super(HostDT, cls).setUpClass()

    def runTest(self):
        self.doit()

<|endoftext|>


<commit_before>from base64 import b64encode
from optparse import make_option
import json
import urllib
import urllib2
import argparse
import time

def make_request(url, params, auth=None):
    """
    Prepares a request from a url, params, and optionally authentication.
    """
    req = urllib2.Request(url + urllib.urlencode(params))

    if auth:
        req.add_header('AUTHORIZATION', 'Basic ' + auth)

    return urllib2.urlopen(req)

def beginTransaction(url, auth):
    print('Starting transaction...')
    params = {'output_format': 'JSON'}
    request = make_request(url=url+'beginTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("BeginTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on beginTransaction: {0}".format(response['response']['error']))

    print('Transaction started.')
    transactionId = response['response']['Transaction']['ID']

    return transactionId;

def endTransaction(url, auth, cancel, transactionId):
    print('Ending transaction...')
    params = {'output_format': 'JSON', 'cancel': cancel, 'transactionId': transactionId}
    request = make_request(url=url+'endTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("EndTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on endTransaction: {0}".format(response['response']['error']))

    print('Transaction ended.')

def checkout(url, auth, branch, transactionId):
    print "Checking out "+branch+" branch..."
    params = {'output_format': 'JSON', 'branch': branch, 'transactionId':transactionId}
    request = make_request(url=url+'checkout.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Checkout for branch "+branch+" failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())
    #print response
    if response['response']['success']:
        newBranch = response['response']['NewTarget']
        print "Checked out "+newBranch+' branch.'
        return newBranch
    else:
        print "----"
        print "Checkout for "+branch+" branch failed."
        print "Error Message: "+response['response']['error']
        return None

def pollTask(url, auth, taskID):
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json', params=params, auth=auth)

    #print request.getcode()
    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    status = response['task']['status']

    progress = None
    try:
        progress = response['task']['progress']
    except:
        progress = None

    result = None
    try:
        result = response['task']['result']
    except:
        result = None

    errorMessage = None
    try:
        errorMessage = response['task']['error']['message']
    except:
        errorMessage = None

    return status, progress, result, errorMessage

def printTaskStatus(taskID, status, progress, result, errorMessage):

    if status == "RUNNING":
        print "----"
        print "++Task "+str(taskID)+" is running."
        if progress:
            print "Current Step: "+str(progress['task'])
            print "Entities Processed: "+str(progress['amount'])
        if result:
            print "Entities Processed: "+result['OSMReport']['processedEntities']
    elif status == "FAILED":
        print "----"
        print "++Task "+str(taskID)+" failed with error message: "+errorMessage+"."
    elif status == "FINISHED":
        print "----"
        print response
        print "++Task "+str(taskID)+" is finished."
        print "Entities Processed: "+result['OSMReport']['processedEntities']
    elif status == "CANCELLED":
        print "----"
        print response
        print "++Task "+str(taskID)+" was cancelled."
        print "Entities Processed: "+result['OSMReport']['processedEntities']
    else:
        print "----"
        print "++Task "+str(taskID)+" is "+status+"."

def cancelTask(url, auth, taskID, printStatus):
    print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json?cancel=true', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Could not cancel task: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskStatus = response['task']['status']

    if printStatus:
        if taskStatus == "CANCELLED":
            print "++Task "+str(taskID)+" was cancelled."
        else:
            print "++Error.  Could not cancel task "+str(taskID)+".  Task is "+str(taskStatus)+"."

    return taskStatus;

def waitOnTask(url, auth, taskID):
    maxTime = 30
    timeSlept = 0
    sleepCycle = 5
    taskStatus = None
    print "----------------------------------"
    print "Waiting for task "+str(taskID)+"..."
    print "Maximum wait time is "+str(maxTime)+" seconds."
    while timeSlept < maxTime:
        taskStatus, taskProgress, taskResult, errorMessage = pollTask(url, auth, taskID)
        printTaskStatus(taskID, taskStatus, taskProgress, taskResult, errorMessage)
        if not (taskStatus in ['WAITING','RUNNING']):
            break
        #print "Time Slept: "+str(timeSlept)
        time.sleep(sleepCycle)
        timeSlept += sleepCycle

    if taskStatus in ['WAITING','RUNNING']:
        print "Task "+str(taskID)+" timed out after "+str(timeSlept)+" seconds."
        print "Attempting to cancel task "+str(taskID)
        maxTime = 30
        timeSlept = 0
        sleepCycle = 1
        while timeSlept < maxTime:
            taskStatus = cancelTask(url, auth, taskID, True)
            if not (taskStatus in ['WAITING','RUNNING']):
                break
            time.sleep(sleepCycle)
            timeSlept += sleepCycle

    print "Task "+str(taskID)+" is done"

def downloadFromOSM(url, auth, transactionId, update, mapping, bbox):
    print('Downloading from OpenStreetMap ...')
    params = {'output_format': 'JSON', 'update': update, 'mapping': mapping, 'bbox': bbox, 'transactionId':transactionId}
    request = make_request(url=url+'osm/download.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("OSM Download failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskID = response['task']['id']

    #print response
    if response['task']['status'] == 'FAILED':
        raise Exception("An error occurred when pulling new data from OSM: {0}".format(response['task']['status']))

    if response['task']['status'] == 'WAITING':
        print('Download from OpenStreetMap is waiting to be processed.  Task ID is '+str(taskID)+'.')

    if response['task']['status'] == 'RUNNING':
        print('Download from OpenStreetMap is being processed.  Task ID is '+str(taskID)+'.')

    return taskID;

def getRepoID(geoserver, auth, workspace, datastore):
    params = {}
    url = geoserver+"rest/workspaces/"+workspace+"/datastores/"+datastore+".json"
    request = make_request(url=url, params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())
    repoID = None
    for entry in response['dataStore']['connectionParameters']['entry']:
        if entry['@key'] == 'geogig_repository':
            repoID = entry['$']
            break

    return repoID;

def parse_url(url):

    if (url is None) or len(url) == 0:
        return None

    index = url.rfind('/')

    if index != (len(url)-1):
        url += '/'

    return url

def parse_bbox(extent):
    file_extent = "/opt/cybergis-osm-mappings.git/extents/"+extent.replace(":","/")+".txt"
    bbox = None
    with open (file_extent, "r") as f:
        bbox = f.read().replace('\n', '').replace(' ',',')
    return bbox

def parse_mapping(ns_mapping):
    file_mapping = "/opt/cybergis-osm-mappings.git/mappings/"+ns_mapping.replace(":","/")+".json"
    return file_mapping

def run(args):
    #==#
    geoserver = parse_url(args.geoserver)
    repo = args.repo
    datastore = args.datastore
    workspace = args.workspace
    #==#
    authorname = args.authorname
    authoremail = args.authoremail
    #==#
    auth = None
    if args.username and args.password:
      auth = b64encode('{0}:{1}'.format(args.username, args.password))
    #==#
    update = args.update in ["1","y","t","true"]
    bbox = parse_bbox(args.extent)
    mapping = parse_mapping(args.mapping)
    print "=================================="
    print "#==#"
    print "CyberGIS Script / geogig_sync_osm.py"
    print "Downloading Updates from OpenStreetMap"
    print "#==#"

    if repo:
        pass
    elif geoserver and workspace and datastore:
        repo = getRepoID(geoserver, auth, workspace, datastore)
    else:
        print "You need to include the repo id or the datastore name to sync"
    #==#
    url_repo = geoserver+'geogig/'+repo+'/'
    url_tasks = geoserver+'geogig/tasks'
    #==#
    if update:
        pass
    elif bbox and mapping:
       pass
    else:
        return "Update is false and no new data will be brought in because the extent and mapping aren't specified"

    transID = -1
    try:
        transID = beginTransaction(url_repo, auth)
    except Exception:
        transID = -1
        raise

    if transID != -1:
        taskID = -1
        #==#
        #Checkout master branch.  See: https://github.com/boundlessgeo/GeoGig/issues/788
        try:
            branch = checkout(url_repo, auth, 'master', transID)
            if not branch:
                raise Exception('An error occurred when checking out master.  Cancelling task.')
            taskID = downloadFromOSM(url_repo, auth, transID, update, mapping, bbox)
        except Exception:
            taskID = -1
            endTransaction(url_repo, auth, True, transID)
            raise

        if taskID != -1:
            waitOnTask(url_tasks, auth, taskID)

        #==#
        #Checkout master branch.  See: https://github.com/boundlessgeo/GeoGig/issues/788
        try:
            checkout(url_repo, auth, 'master', transID)
        except Exception:
            pass

    try:
        endTransaction(url_repo, auth, False, transID)
    except Exception:
        pass

    print "=================================="

parser = argparse.ArgumentParser(description='Synchronize GeoGig repository with OpenStreetMap (OSM)')

parser.add_argument("update", help="true/false.  Update existing features only or download new features.  If false, extent and mapping are required.")

#
parser.add_argument("--workspace", help="The workspace of the GeoServer data store of the GeoGig repository you want to sync.")
parser.add_argument("--datastore", help="The name of the GeoServer data store of the GeoGig repository you want to sync.")
parser.add_argument("--repo", help="The GeoServer id of the GeoGig repository you want to sync.")

parser.add_argument("--geoserver", help="The url of the GeoServer servicing the GeoGig repository.")
parser.add_argument("--username", help="The username to use for basic auth requests.")
parser.add_argument("--password", help="The password to use for basic auth requests.")
parser.add_argument("--authorname", help="The author name to use when merging non-conflicting branches.")
parser.add_argument("--authoremail", help="The author email to use when merging non-conflicting branches.")
parser.add_argument("--extent", help="The extent of the OpenStreetMap extract. For example, basic:buildings_and_roads.")
parser.add_argument("--mapping", help="The mapping of the OpenStreetMap extract.  For example, dominican_republic:santo_domingo.")

args = parser.parse_args()
run(args)
<commit_msg>Update geogig_sync_osm.py<commit_after>from base64 import b64encode
from optparse import make_option
import json
import urllib
import urllib2
import argparse
import time

def make_request(url, params, auth=None):
    """
    Prepares a request from a url, params, and optionally authentication.
    """
    req = urllib2.Request(url + urllib.urlencode(params))

    if auth:
        req.add_header('AUTHORIZATION', 'Basic ' + auth)

    return urllib2.urlopen(req)

def beginTransaction(url, auth):
    print('Starting transaction...')
    params = {'output_format': 'JSON'}
    request = make_request(url=url+'beginTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("BeginTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on beginTransaction: {0}".format(response['response']['error']))

    print('Transaction started.')
    transactionId = response['response']['Transaction']['ID']

    return transactionId;

def endTransaction(url, auth, cancel, transactionId):
    print('Ending transaction...')
    params = {'output_format': 'JSON', 'cancel': cancel, 'transactionId': transactionId}
    request = make_request(url=url+'endTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("EndTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on endTransaction: {0}".format(response['response']['error']))

    print('Transaction ended.')

def checkout(url, auth, branch, transactionId):
    print "Checking out "+branch+" branch..."
    params = {'output_format': 'JSON', 'branch': branch, 'transactionId':transactionId}
    request = make_request(url=url+'checkout.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Checkout for branch "+branch+" failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())
    #print response
    if response['response']['success']:
        newBranch = response['response']['NewTarget']
        print "Checked out "+newBranch+' branch.'
        return newBranch
    else:
        print "----"
        print "Checkout for "+branch+" branch failed."
        print "Error Message: "+response['response']['error']
        return None

def pollTask(url, auth, taskID):
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json', params=params, auth=auth)

    #print request.getcode()
    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    status = response['task']['status']

    progress = None
    try:
        progress = response['task']['progress']
    except:
        progress = None

    result = None
    try:
        result = response['task']['result']
    except:
        result = None

    errorMessage = None
    try:
        errorMessage = response['task']['error']['message']
    except:
        errorMessage = None

    return status, progress, result, errorMessage

def printTaskStatus(taskID, status, progress, result, errorMessage):

    if status == "RUNNING":
        print "----"
        print "++Task "+str(taskID)+" is running."
        if progress:
            print "Current Step: "+str(progress['task'])
            print "Entities Processed: "+str(progress['amount'])
        if result:
            print "Entities Processed: "+result['OSMReport']['processedEntities']
    elif status == "FAILED":
        print "----"
        print "++Task "+str(taskID)+" failed with error message: "+errorMessage+"."
    elif status == "FINISHED":
        print "----"
        print response
        print "++Task "+str(taskID)+" is finished."
        print "Entities Processed: "+result['OSMReport']['processedEntities']
    elif status == "CANCELLED":
        print "----"
        print response
        print "++Task "+str(taskID)+" was cancelled."
        print "Entities Processed: "+result['OSMReport']['processedEntities']
    else:
        print "----"
        print "++Task "+str(taskID)+" is "+status+"."

def cancelTask(url, auth, taskID, printStatus):
    print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json?cancel=true', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Could not cancel task: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskStatus = response['task']['status']

    if printStatus:
        if taskStatus == "CANCELLED":
            print "++Task "+str(taskID)+" was cancelled."
        else:
            print "++Error.  Could not cancel task "+str(taskID)+".  Task is "+str(taskStatus)+"."

    return taskStatus;

def waitOnTask(url, auth, taskID, timeout):
    maxTime = timeout
    timeSlept = 0
    sleepCycle = 5
    taskStatus = None
    print "----------------------------------"
    print "Waiting for task "+str(taskID)+"..."
    print "Maximum wait time is "+str(maxTime)+" seconds."
    while timeSlept < maxTime:
        taskStatus, taskProgress, taskResult, errorMessage = pollTask(url, auth, taskID)
        printTaskStatus(taskID, taskStatus, taskProgress, taskResult, errorMessage)
        if not (taskStatus in ['WAITING','RUNNING']):
            break
        #print "Time Slept: "+str(timeSlept)
        time.sleep(sleepCycle)
        timeSlept += sleepCycle

    if taskStatus in ['WAITING','RUNNING']:
        print "Task "+str(taskID)+" timed out after "+str(timeSlept)+" seconds."
        print "Attempting to cancel task "+str(taskID)
        maxTime = 30
        timeSlept = 0
        sleepCycle = 1
        while timeSlept < maxTime:
            taskStatus = cancelTask(url, auth, taskID, True)
            if not (taskStatus in ['WAITING','RUNNING']):
                break
            time.sleep(sleepCycle)
            timeSlept += sleepCycle

    print "Task "+str(taskID)+" is done"

def downloadFromOSM(url, auth, transactionId, update, mapping, bbox):
    print('Downloading from OpenStreetMap ...')
    params = {'output_format': 'JSON', 'update': update, 'mapping': mapping, 'bbox': bbox, 'transactionId':transactionId}
    request = make_request(url=url+'osm/download.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("OSM Download failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskID = response['task']['id']

    #print response
    if response['task']['status'] == 'FAILED':
        raise Exception("An error occurred when pulling new data from OSM: {0}".format(response['task']['status']))

    if response['task']['status'] == 'WAITING':
        print('Download from OpenStreetMap is waiting to be processed.  Task ID is '+str(taskID)+'.')

    if response['task']['status'] == 'RUNNING':
        print('Download from OpenStreetMap is being processed.  Task ID is '+str(taskID)+'.')

    return taskID;

def getRepoID(geoserver, auth, workspace, datastore):
    params = {}
    url = geoserver+"rest/workspaces/"+workspace+"/datastores/"+datastore+".json"
    request = make_request(url=url, params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())
    repoID = None
    for entry in response['dataStore']['connectionParameters']['entry']:
        if entry['@key'] == 'geogig_repository':
            repoID = entry['$']
            break

    return repoID;

def parse_url(url):

    if (url is None) or len(url) == 0:
        return None

    index = url.rfind('/')

    if index != (len(url)-1):
        url += '/'

    return url

def parse_bbox(extent):
    file_extent = "/opt/cybergis-osm-mappings.git/extents/"+extent.replace(":","/")+".txt"
    bbox = None
    with open (file_extent, "r") as f:
        bbox = f.read().replace('\n', '').replace(' ',',')
    return bbox

def parse_mapping(ns_mapping):
    file_mapping = "/opt/cybergis-osm-mappings.git/mappings/"+ns_mapping.replace(":","/")+".json"
    return file_mapping

def run(args):
    #==#
    geoserver = parse_url(args.geoserver)
    repo = args.repo
    datastore = args.datastore
    workspace = args.workspace
    timeout = args.timeout || 30
    #==#
    authorname = args.authorname
    authoremail = args.authoremail
    #==#
    auth = None
    if args.username and args.password:
      auth = b64encode('{0}:{1}'.format(args.username, args.password))
    #==#
    update = args.update in ["1","y","t","true"]
    bbox = parse_bbox(args.extent)
    mapping = parse_mapping(args.mapping)
    print "=================================="
    print "#==#"
    print "CyberGIS Script / geogig_sync_osm.py"
    print "Downloading Updates from OpenStreetMap"
    print "#==#"

    if repo:
        pass
    elif geoserver and workspace and datastore:
        repo = getRepoID(geoserver, auth, workspace, datastore)
    else:
        print "You need to include the repo id or the datastore name to sync"
    #==#
    url_repo = geoserver+'geogig/'+repo+'/'
    url_tasks = geoserver+'geogig/tasks'
    #==#
    if update:
        pass
    elif bbox and mapping:
       pass
    else:
        return "Update is false and no new data will be brought in because the extent and mapping aren't specified"

    transID = -1
    try:
        transID = beginTransaction(url_repo, auth)
    except Exception:
        transID = -1
        raise

    if transID != -1:
        taskID = -1
        #==#
        #Checkout master branch.  See: https://github.com/boundlessgeo/GeoGig/issues/788
        try:
            branch = checkout(url_repo, auth, 'master', transID)
            if not branch:
                raise Exception('An error occurred when checking out master.  Cancelling task.')
            taskID = downloadFromOSM(url_repo, auth, transID, update, mapping, bbox)
        except Exception:
            taskID = -1
            endTransaction(url_repo, auth, True, transID)
            raise

        if taskID != -1:
            waitOnTask(url_tasks, auth, taskID)

        #==#
        #Checkout master branch.  See: https://github.com/boundlessgeo/GeoGig/issues/788
        try:
            checkout(url_repo, auth, 'master', transID)
        except Exception:
            pass

    try:
        endTransaction(url_repo, auth, False, transID)
    except Exception:
        pass

    print "=================================="

parser = argparse.ArgumentParser(description='Synchronize GeoGig repository with OpenStreetMap (OSM)')

parser.add_argument("update", help="true/false.  Update existing features only or download new features.  If false, extent and mapping are required.")

#
parser.add_argument("--workspace", help="The workspace of the GeoServer data store of the GeoGig repository you want to sync.")
parser.add_argument("--datastore", help="The name of the GeoServer data store of the GeoGig repository you want to sync.")
parser.add_argument("--repo", help="The GeoServer id of the GeoGig repository you want to sync.")

parser.add_argument("--geoserver", help="The url of the GeoServer servicing the GeoGig repository.")
parser.add_argument("--username", help="The username to use for basic auth requests.")
parser.add_argument("--password", help="The password to use for basic auth requests.")
parser.add_argument("--authorname", help="The author name to use when merging non-conflicting branches.")
parser.add_argument("--authoremail", help="The author email to use when merging non-conflicting branches.")
parser.add_argument("--extent", help="The extent of the OpenStreetMap extract. For example, basic:buildings_and_roads.")
parser.add_argument("--mapping", help="The mapping of the OpenStreetMap extract.  For example, dominican_republic:santo_domingo.")
parser.add_argument("--timeout", help="The number of seconds to wait for the osm download task to complete before cancelling.")

args = parser.parse_args()
run(args)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
This file is part of pyCMBS. (c) 2012-2014
For COPYING and LICENSE details, please refer to the file
COPYRIGHT.md
"""

import unittest
from pycmbs.benchmarking import models

from pycmbs.data import Data
import os
import scipy as sc
import matplotlib.pylab as pl
import numpy as np
from scipy import stats
from dateutil.rrule import rrule
from dateutil.rrule import MONTHLY
import datetime

from nose.tools import assert_raises

class TestPycmbsBenchmarkingModels(unittest.TestCase):

    def setUp(self):
        n=1000  # slows down significantly! constraint is percentile  test
        x = sc.randn(n)*100.  # generate dummy data
        self.D = Data(None, None)
        d=np.ones((n, 1, 1))
        self.D.data = d
        self.D.data[:,0,0]=x
        self.D.data = np.ma.array(self.D.data, mask=self.D.data != self.D.data)
        self.D.verbose = True
        self.D.unit = 'myunit'
        self.D.label = 'testlabel'
        self.D.filename = 'testinputfilename.nc'
        self.D.varname = 'testvarname'
        self.D.long_name = 'This is the longname'
        self.D.time = np.arange(n) + pl.datestr2num('2001-01-01')
        self.D.time_str = "days since 0001-01-01 00:00:00"
        self.D.calendar = 'gregorian'
        self.D.oldtime=False

        # generate dummy Model object
        data_dir = './test/'
        varmethods = {'albedo':'get_albedo()', 'sis': 'get_sis()'}
        self.model = models.Model(data_dir, varmethods, name='testmodel', intervals='monthly')

        sis = self.D.copy()
        sis.mulc(5., copy=False)
        sis.label='sisdummy'

        alb = self.D.copy()
        alb.label='albedodummy'

        # add some dummy data variable
        self.model.variables = {'albedo':alb, 'sis':sis}

    def test_save_prefix_missing(self):
        m = self.model
        odir = './odir/'
        with self.assertRaises(ValueError):
            m.save(odir)

    def test_save_create_odir(self):
        m = self.model
        odir = './odir/'
        if os.path.exists(odir):
            os.system('rm -rf ' + odir)
        m.save(odir, prefix='test')
        self.assertTrue(os.path.exists(odir))
        os.system('rm -rf ' + odir)

    def test_save(self):
        m = self.model
        odir = './odir/'

        sisfile = odir + 'testoutput_SIS.nc'
        albfile = odir + 'testoutput_ALBEDO.nc'
        if os.path.exists(sisfile):
            os.remove(sisfile)
        if os.path.exists(albfile):
            os.remove(albfile)

        m.save(odir, prefix='testoutput')
        self.assertTrue(os.path.exists(sisfile))
        self.assertTrue(os.path.exists(albfile))

        if os.path.exists(sisfile):
            os.remove(sisfile)
        if os.path.exists(albfile):
            os.remove(albfile)
        os.system('rm -rf ' + odir)

if __name__ == "__main__":
    unittest.main()
# vim: expandtab shiftwidth=4 softtabstop=4
<commit_msg>adapted test_models.py<commit_after># -*- coding: utf-8 -*-
"""
This file is part of pyCMBS. (c) 2012-2014
For COPYING and LICENSE details, please refer to the file
COPYRIGHT.md
"""

import unittest
from pycmbs.benchmarking import models

from pycmbs.data import Data
import os
import scipy as sc
import matplotlib.pylab as pl
import numpy as np
from scipy import stats
from dateutil.rrule import rrule
from dateutil.rrule import MONTHLY
import datetime

from nose.tools import assert_raises

import tempfile

class TestPycmbsBenchmarkingModels(unittest.TestCase):

    def setUp(self):
        n=1000  # slows down significantly! constraint is percentile  test
        x = sc.randn(n)*100.  # generate dummy data
        self.D = Data(None, None)
        d=np.ones((n, 1, 1))
        self.D.data = d
        self.D.data[:,0,0]=x
        self.D.data = np.ma.array(self.D.data, mask=self.D.data != self.D.data)
        self.D.verbose = True
        self.D.unit = 'myunit'
        self.D.label = 'testlabel'
        self.D.filename = 'testinputfilename.nc'
        self.D.varname = 'testvarname'
        self.D.long_name = 'This is the longname'
        self.D.time = np.arange(n) + pl.datestr2num('2001-01-01')
        self.D.time_str = "days since 0001-01-01 00:00:00"
        self.D.calendar = 'gregorian'
        self.D.oldtime=False

        # generate dummy Model object
        data_dir = './test/'
        varmethods = {'albedo':'get_albedo()', 'sis': 'get_sis()'}
        self.model = models.Model(data_dir, varmethods, name='testmodel', intervals='monthly')

        sis = self.D.copy()
        sis.mulc(5., copy=False)
        sis.label='sisdummy'

        alb = self.D.copy()
        alb.label='albedodummy'

        # add some dummy data variable
        self.model.variables = {'albedo':alb, 'sis':sis}

    def test_save_prefix_missing(self):
        m = self.model
        odir = tempfile.mkdtemp() + os.sep
        with self.assertRaises(ValueError):
            m.save(odir)

    def test_save_create_odir(self):
        m = self.model
        odir = tempfile.mkdtemp() + os.sep
        if os.path.exists(odir):
            os.system('rm -rf ' + odir)
        m.save(odir, prefix='test')
        self.assertTrue(os.path.exists(odir))
        os.system('rm -rf ' + odir)

    def test_save(self):
        m = self.model
        odir = tempfile.mkdtemp() + os.sep

        sisfile = odir + 'testoutput_SIS.nc'
        albfile = odir + 'testoutput_ALBEDO.nc'
        if os.path.exists(sisfile):
            os.remove(sisfile)
        if os.path.exists(albfile):
            os.remove(albfile)

        m.save(odir, prefix='testoutput')
        self.assertTrue(os.path.exists(sisfile))
        self.assertTrue(os.path.exists(albfile))

        if os.path.exists(sisfile):
            os.remove(sisfile)
        if os.path.exists(albfile):
            os.remove(albfile)
        os.system('rm -rf ' + odir)

if __name__ == "__main__":
    unittest.main()

<|endoftext|>


<commit_before>from django.template.loader import render_to_string

class Sitemap(object):
    template = 'wagtailsitemaps/sitemap.xml'

    def __init__(self, site):
        self.site = site

    def get_pages(self):
        return self.site.root_page.get_descendants(inclusive=True).live().public().order_by('path')

    def get_urls(self):
        for page in self.get_pages():
            for url in page.get_sitemap_urls():
                yield url

    def render(self):
        return render_to_string(self.template, {
            'urlset': self.get_urls()
        })
<commit_msg>Fixed a bug that prevented sitemap url generation override in a Page subclass<commit_after>from django.template.loader import render_to_string

class Sitemap(object):
    template = 'wagtailsitemaps/sitemap.xml'

    def __init__(self, site):
        self.site = site

    def get_pages(self):
        return self.site.root_page.get_descendants(inclusive=True).live().public().order_by('path')

    def get_urls(self):
        for page in self.get_pages():
            for url in page.specific.get_sitemap_urls():
                yield url

    def render(self):
        return render_to_string(self.template, {
            'urlset': self.get_urls()
        })
<|endoftext|>


<commit_before>import json
from HTMLParser import HTMLParser
from re import findall, match, DOTALL, sub, compile

def normalize_special_characters(text):
    """Replace special unicode characters in text"""
    entities = {
        u'\u00A0': ' ',
        u'"': '"     ',
        u'\u0022': '"',
        u'\u00AB': '"',
        u'\u00BB': '"',
        u'\u02BA': '"',
        u'\u02DD': '"',
        u'\u030B': '"',
        u'\u030E': '"',
        u'\u05F4': '"',
        u'\u201C': '"',
        u'\u201D': '"',
        u'\u2036': '"',
        u'\u2033': '"',
        u'\u275D': '"',
        u'\u275E': '"',
        u'\u3003': '"',
        u'\u301D': '"',
        u'\u301E': '"',
        u'\U0001F676': '"',
        u'\U0001F677': '"',
    }
    for c, r in entities.iteritems():
        if c in text:
            text = text.replace(c, r)
    return text

def markdown_to_whitespace(markdown_txt):
    """Convert a text from markdown whitespace padded unicode."""
    resu = u''
    RE_MOVE_DOT = compile(r"(\s)([^\s]+)\\(\.\w)")
    RE_MOVE_DASH = compile(r"(\s)([^\s]+)\\(-\w)")

    # remove table
    match_table = findall(ur'(
.+?
)', markdown_txt, DOTALL)
    for match_t in match_table:
        markdown_txt = markdown_txt.replace(match_t, u' ' * len(match_t))

    for i, line in enumerate(markdown_txt.split('\n')):
        if i != 0:
            resu += u'\n'

        if len(line) > 0:
            # remove blockquote formatting
            if line.startswith(u'> '):
                line = u'  ' + line[2:]

            # remove headline formatting
            match_head = match(ur'^(#+) (.+)$', line)
            if match_head:
                sz = len(match_head.groups()[0])
                if sz < 7:
                    line = (' ' * sz) + line[sz:]

            # remove emphasized formatting
            emphs = findall(ur'((?<!\\)\*(.+?)(?<!\\)\*)', line)
            for emph in emphs:
                line = line.replace(emph[0], emph[0].replace('*', ' '))

            # remove important formatting
            imps = findall(ur'((?<!\\)__(.+?)(?<!\\)__)', line)
            for imp in imps:
                line = line.replace(imp[0], imp[0].replace('_', ' '))

            # remove link formatting
            links_txt = u''
            links = findall(ur'((?<!\\)\[(.+?)(?<!\\)\](?<!\\)(\(\S+(?<!\\)\)))', line)
            for link in links:
                line = line.replace(link[0], ' ' + link[1] + ' ' + (' ' * len(link[2])))
                links_txt = links_txt + link[1]

            # U\.S\. -> U.S., instead of U. S.
            line = sub(RE_MOVE_DOT, r"\1 \2\3", line)
            line = sub(RE_MOVE_DASH, r"\1 \2\3", line)

            left_replace = u"\\`*_{[(#+-!"
            right_replace = u"}])."

            # remove extra backslash
            for char in left_replace:
                line = line.replace(u"\\" + char, " " + char)
            for char in right_replace:
                line = line.replace(u"\\" + char, char + " ")

            # add the text if it contains more than a link
            resu += line

    # remove special unicode characters
    return normalize_special_characters(resu)

class MarkupStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self._pd_char = ' '
        self._pd = []
        self._pd_len = 0
        self._line_offsets = [0]

    def feed_data(self, data):
        offset = len(self.rawdata)

        while True:
            offset = data.find('\n', offset)
            if offset != -1:
                offset += 1
                self._line_offsets.append(offset)
            else:
                break

        self.feed(data)

    def get_raw_offset(self):
        return self._line_offsets[self.lineno-1] + self.offset

    def append_data(self, data):
        self._pd.append(data)
        self._pd_len += len(data)

    def append_padding(self):
        offset = self.get_raw_offset()
        if self._pd_len != offset:
            self.append_data(self._pd_char * (offset - self._pd_len))

    def handle_entityref(self, name):
        self.append_data('&'+name+';')

    def handle_charref(self, ref):
        self.append_data('&#'+ref+';')

    def handle_data(self, data):
        self.append_padding()
        self.append_data(data)

    def handle_endtag(self, tag):
        self.append_padding()

    def get_padded_data(self):
        self.append_padding()
        return ''.join(self._pd)

def markup_to_whitespace(ml):
    parser = MarkupStripper()
    parser.feed_data(ml)
    return parser.get_padded_data()

def mention_to_neleval(doc, chain, mention):
    default_probability = 1.0
    default_type = 'UNK'

    result = u'\t'.join([
        doc.id,
        str(mention.begin),
        str(mention.end),
        chain.resolution.id if chain.resolution else 'NIL',
        str(default_probability),
        mention.tag if mention.tag else 'UNK'
    ])

    return result

def inject_html_links(raw, linked_doc, kb_prefix=None):
    return inject_links(raw, linked_doc, u'{mention}', kb_prefix=kb_prefix)

def inject_markdown_links(raw, linked_doc, kb_prefix=None):
    return inject_links(raw, linked_doc, u'[{mention}]({kb_pfx}{entity})', kb_prefix=kb_prefix)

def inject_links(raw, linked_doc, link_format, kb_prefix=None):
    kb_prefix = 'https://en.wikipedia.org/wiki/' if kb_prefix == None else kb_prefix

    mentions_by_offset = sorted(
        (m.begin, m, c.resolution.id)
        for c in linked_doc.chains
            for m in c.mentions
                if c.resolution != None)

    output = []
    last_offset = None
    for offset, mention, entity_id in mentions_by_offset:
        output.append(raw[last_offset:offset])

        # todo: handle escaping
        output.append(
            link_format.format(
                mention = mention.text,
                kb_pfx = kb_prefix,
                entity = entity_id))

        last_offset = mention.end

    output.append(raw[last_offset:None])
    return ''.join(output)

def to_neleval(doc):
    return u'\n'.join(mention_to_neleval(doc, chain, m) for chain in doc.chains for m in chain.mentions)

def to_json(doc):
    return json.dumps({
        'id':doc.id,
        'mentions':[{
            'begin': m.begin,
            'length': len(m.text),
            'text': m.text,
            'resolution': None if chain.resolution == None else {
                'entity': chain.resolution.id,
                'features': chain.resolution.features
            }
        } for chain in doc.chains for m in chain.mentions]})
<commit_msg>Add function for formatting store docs in sift format<commit_after>import json
from HTMLParser import HTMLParser
from re import findall, match, DOTALL, sub, compile

def normalize_special_characters(text):
    """Replace special unicode characters in text"""
    entities = {
        u'\u00A0': ' ',
        u'"': '"     ',
        u'\u0022': '"',
        u'\u00AB': '"',
        u'\u00BB': '"',
        u'\u02BA': '"',
        u'\u02DD': '"',
        u'\u030B': '"',
        u'\u030E': '"',
        u'\u05F4': '"',
        u'\u201C': '"',
        u'\u201D': '"',
        u'\u2036': '"',
        u'\u2033': '"',
        u'\u275D': '"',
        u'\u275E': '"',
        u'\u3003': '"',
        u'\u301D': '"',
        u'\u301E': '"',
        u'\U0001F676': '"',
        u'\U0001F677': '"',
    }
    for c, r in entities.iteritems():
        if c in text:
            text = text.replace(c, r)
    return text

def markdown_to_whitespace(markdown_txt):
    """Convert a text from markdown whitespace padded unicode."""
    resu = u''
    RE_MOVE_DOT = compile(r"(\s)([^\s]+)\\(\.\w)")
    RE_MOVE_DASH = compile(r"(\s)([^\s]+)\\(-\w)")

    # remove table
    match_table = findall(ur'(
.+?
)', markdown_txt, DOTALL)
    for match_t in match_table:
        markdown_txt = markdown_txt.replace(match_t, u' ' * len(match_t))

    for i, line in enumerate(markdown_txt.split('\n')):
        if i != 0:
            resu += u'\n'

        if len(line) > 0:
            # remove blockquote formatting
            if line.startswith(u'> '):
                line = u'  ' + line[2:]

            # remove headline formatting
            match_head = match(ur'^(#+) (.+)$', line)
            if match_head:
                sz = len(match_head.groups()[0])
                if sz < 7:
                    line = (' ' * sz) + line[sz:]

            # remove emphasized formatting
            emphs = findall(ur'((?<!\\)\*(.+?)(?<!\\)\*)', line)
            for emph in emphs:
                line = line.replace(emph[0], emph[0].replace('*', ' '))

            # remove important formatting
            imps = findall(ur'((?<!\\)__(.+?)(?<!\\)__)', line)
            for imp in imps:
                line = line.replace(imp[0], imp[0].replace('_', ' '))

            # remove link formatting
            links_txt = u''
            links = findall(ur'((?<!\\)\[(.+?)(?<!\\)\](?<!\\)(\(\S+(?<!\\)\)))', line)
            for link in links:
                line = line.replace(link[0], ' ' + link[1] + ' ' + (' ' * len(link[2])))
                links_txt = links_txt + link[1]

            # U\.S\. -> U.S., instead of U. S.
            line = sub(RE_MOVE_DOT, r"\1 \2\3", line)
            line = sub(RE_MOVE_DASH, r"\1 \2\3", line)

            left_replace = u"\\`*_{[(#+-!"
            right_replace = u"}])."

            # remove extra backslash
            for char in left_replace:
                line = line.replace(u"\\" + char, " " + char)
            for char in right_replace:
                line = line.replace(u"\\" + char, char + " ")

            # add the text if it contains more than a link
            resu += line

    # remove special unicode characters
    return normalize_special_characters(resu)

class MarkupStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self._pd_char = ' '
        self._pd = []
        self._pd_len = 0
        self._line_offsets = [0]

    def feed_data(self, data):
        offset = len(self.rawdata)

        while True:
            offset = data.find('\n', offset)
            if offset != -1:
                offset += 1
                self._line_offsets.append(offset)
            else:
                break

        self.feed(data)

    def get_raw_offset(self):
        return self._line_offsets[self.lineno-1] + self.offset

    def append_data(self, data):
        self._pd.append(data)
        self._pd_len += len(data)

    def append_padding(self):
        offset = self.get_raw_offset()
        if self._pd_len != offset:
            self.append_data(self._pd_char * (offset - self._pd_len))

    def handle_entityref(self, name):
        self.append_data('&'+name+';')

    def handle_charref(self, ref):
        self.append_data('&#'+ref+';')

    def handle_data(self, data):
        self.append_padding()
        self.append_data(data)

    def handle_endtag(self, tag):
        self.append_padding()

    def get_padded_data(self):
        self.append_padding()
        return ''.join(self._pd)

def markup_to_whitespace(ml):
    parser = MarkupStripper()
    parser.feed_data(ml)
    return parser.get_padded_data()

def mention_to_neleval(doc, chain, mention):
    default_probability = 1.0
    default_type = 'UNK'

    result = u'\t'.join([
        doc.id,
        str(mention.begin),
        str(mention.end),
        chain.resolution.id if chain.resolution else 'NIL',
        str(default_probability),
        mention.tag if mention.tag else 'UNK'
    ])

    return result

def inject_html_links(raw, linked_doc, kb_prefix=None):
    return inject_links(raw, linked_doc, u'{mention}', kb_prefix=kb_prefix)

def inject_markdown_links(raw, linked_doc, kb_prefix=None):
    return inject_links(raw, linked_doc, u'[{mention}]({kb_pfx}{entity})', kb_prefix=kb_prefix)

def inject_links(raw, linked_doc, link_format, kb_prefix=None):
    kb_prefix = 'https://en.wikipedia.org/wiki/' if kb_prefix == None else kb_prefix

    mentions_by_offset = sorted(
        (m.begin, m, c.resolution.id)
        for c in linked_doc.chains
            for m in c.mentions
                if c.resolution != None)

    output = []
    last_offset = None
    for offset, mention, entity_id in mentions_by_offset:
        output.append(raw[last_offset:offset])

        # todo: handle escaping
        output.append(
            link_format.format(
                mention = mention.text,
                kb_pfx = kb_prefix,
                entity = entity_id))

        last_offset = mention.end

    output.append(raw[last_offset:None])
    return ''.join(output)

def to_neleval(doc):
    return u'\n'.join(mention_to_neleval(doc, chain, m) for chain in doc.chains for m in chain.mentions)

def to_json(doc):
    return json.dumps({
        'id':doc.id,
        'mentions':[{
            'begin': m.begin,
            'length': len(m.text),
            'text': m.text,
            'resolution': None if chain.resolution == None else {
                'entity': chain.resolution.id,
                'features': chain.resolution.features
            }
        } for chain in doc.chains for m in chain.mentions]})

def to_sift(doc):
    return json.dumps({
        '_id': doc.id,
        'text': doc.text,
        'links': [{
            'target': m.resolution.id,
            'start': m.span.start,
            'stop': m.span.stop
        } for chain in doc.chains for m in chain.mentions]
    })
<|endoftext|>