<commit_before>from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
import pygrib
import numpy as np
try:
    import spharm
except:
    raise ImportError("requires pyspharm (python spherical harmonic module) from http://code.google.com/p/pyspharm")

grbs = pygrib.open('../sampledata/spherical_pressure_level.grib1')
g = grbs[1]
fld = g.values

# ECMWF normalizes the spherical harmonic coeffs differently than NCEP.
# (m=0,n=0 is global mean, instead of sqrt(2)/2 times global mean)
fld = 2.*fld/np.sqrt(2.)
fldr = fld[0::2]
fldi = fld[1::2]
fld = np.zeros(fldr.shape,'F')
fld.real = fldr
fld.imag = fldi
nlons = 360;  nlats = 181
s = spharm.Spharmt(nlons,nlats)
data = s.spectogrd(fld)
lons = (360./nlons)*np.arange(nlons)
lats = 90.-(180./(nlats-1))*np.arange(nlats)
lons, lats = np.meshgrid(lons, lats)
# stack grids side-by-side (in longitiudinal direction), so
# any range of longitudes (between -360 and 360) may be plotted on a world map.
lons = np.concatenate((lons-360,lons),1)
lats = np.concatenate((lats,lats),1)
data = np.concatenate((data,data),1)
# setup miller cylindrical map projection.
m = Basemap(llcrnrlon=-180.,llcrnrlat=-90,urcrnrlon=180.,urcrnrlat=90.,\
            resolution='l',area_thresh=10000.,projection='mill')
x, y = m(lons,lats)
CS = m.contourf(x,y,data,15,cmap=plt.cm.jet)
ax = plt.gca()
pos = ax.get_position()
l, b, w, h = pos.bounds
cax = plt.axes([l+w+0.025, b, 0.025, h]) # setup colorbar axes
plt.colorbar(drawedges=True, cax=cax) # draw colorbar
plt.axes(ax)  # make the original axes current again
m.drawcoastlines()
# draw parallels
delat = 30.
circles = np.arange(-90.,90.+delat,delat)
m.drawparallels(circles,labels=[1,0,0,0])
# draw meridians
delon = 60.
meridians = np.arange(-180,180,delon)
m.drawmeridians(meridians,labels=[0,0,0,1])
plt.title(repr(g['level'])+' '+g['typeOfLevel']+' '+g['name']+' from Spherical Harmonic Coeffs')
plt.show()
<commit_msg>basemap --> cartopy<commit_after>import matplotlib.pyplot as plt
import pygrib
import numpy as np
from cartopy.util import add_cyclic_point
import cartopy.crs as ccrs
try:
    import spharm
except:
    raise ImportError("requires pyspharm (python spherical harmonic module) from http://code.google.com/p/pyspharm")

grbs = pygrib.open('../sampledata/spherical_pressure_level.grib1')
g = grbs[1]
fld = g.values

# ECMWF normalizes the spherical harmonic coeffs differently than NCEP.
# (m=0,n=0 is global mean, instead of sqrt(2)/2 times global mean)
fld = 2.*fld/np.sqrt(2.)
fldr = fld[0::2]
fldi = fld[1::2]
fld = np.zeros(fldr.shape,'F')
fld.real = fldr
fld.imag = fldi
nlons = 360;  nlats = 181
s = spharm.Spharmt(nlons,nlats)
data = s.spectogrd(fld)
lons = (360./nlons)*np.arange(nlons)
lats = 90.-(180./(nlats-1))*np.arange(nlats)
# add cyclic (wrap-around) point to global grid
data,lons = add_cyclic_point(data, coord=lons)
lons,lats = np.meshgrid(lons, lats)

# setup mercator map projection.
ax = plt.axes(projection=ccrs.Mercator(central_longitude=0))
cs = ax.contourf(lons,lats,data,15,cmap=plt.cm.jet,transform=ccrs.PlateCarree())
ax.coastlines()
gl = ax.gridlines(draw_labels=True)
gl.ylabels_top = False; gl.xlabels_top = False
gl.ylabels_right = False; gl.xlabels_right = False
plt.colorbar(cs,shrink=0.9)
plt.title(repr(g.level)+' '+g.typeOfLevel+' '+g.name+' from Spherical Harmonic Coeffs',fontsize=9)
plt.show()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Script to process a Wikipedia dump.
By Braddock Gaskill, 22 Jan 2013

EXAMPLE USAGE:
    sudo scripts/make_wiki.py -l es -u wikimirror \
            -p wikimirror -r 'whatever' \
            -x /knowledge/data/wikipedia/eswiki-20121227-pages-articles.xml.bz2
"""

from subprocess import Popen, PIPE, call
from optparse import OptionParser
import string
import os
import sys
from time import time

def call2(cmd):
    print "CALL: " + string.join(cmd, ' ')
    call(cmd)

parser = OptionParser()
parser.add_option("-l", "--language",
                  help="Language code for wiki")
parser.add_option("-d", "--destination",
                  help="Apache destination directory",
                  default="/var/www")
parser.add_option("-u", "--user",
                  help="MySQL user name",
                  default="wikimirror")
parser.add_option("-p", "--password",
                  help="MySQL password for user")
parser.add_option("-r", "--rootpassword",
                  help="MySQL password for root")
parser.add_option("-x", "--xml",
                  help="Wikipedia XML bz2 dump file")
parser.add_option("-k", "--knowledge",
                  help="Knowledge directory",
                  default="/knowledge")
parser.add_option("--mediawiki",
                  help="Location of mediawiki tarball")

(options, args) = parser.parse_args()
if not options.language:
    parser.error('Language not given (-l)')
if not options.password:
    parser.error('MySQL password not given (-p)')
if not options.rootpassword:
    parser.error('MySQL password not given for MySQL root (-r)')
if not options.xml:
    parser.error('XML xml file not given (-d)')
if not options.mediawiki:
    options.mediawiki = os.path.join(options.knowledge, "packages", "mediawiki-1.20.2.tar.gz")
if not os.path.exists(options.knowledge):
    print 'ERROR: ' + options.knowledge + " does not exist"
    sys.exit(-1)
if not os.path.exists(options.destination):
    print 'ERROR: ' + options.destination + " does not exist"
    sys.exit(-1)
if not os.path.exists(options.xml):
    print 'ERROR: ' + options.xml + " does not exist"
    sys.exit(-1)

dbname = options.language + 'wiki'
wikidir = os.path.join(options.destination, dbname)

# Drop and create new user and database
p = Popen(['mysql', '-u', 'root',
           '--password=' + options.rootpassword],
          stdin=PIPE)
#       "GRANT USAGE ON *.* TO '" + options.user + "'@'localhost';",
#       "DROP USER '" + options.user + "'@'localhost';",
#       "CREATE USER '" + options.user + "'@'localhost' IDENTIFIED BY '"
#       + options.password + "';",
sql = ["DROP DATABASE IF EXISTS " + dbname + ";",
       "CREATE DATABASE " + dbname + ";",
       "GRANT USAGE ON * . * TO '" + options.user + "'@'localhost' IDENTIFIED BY '"
       + options.password + "' WITH MAX_QUERIES_PER_HOUR 0 MAX_CONNECTIONS_PER_HOUR 0 MAX_UPDATES_PER_HOUR 0 MAX_USER_CONNECTIONS 0;",
       "GRANT ALL PRIVILEGES ON " + dbname + ".* TO '" + options.user + "'@'localhost' WITH GRANT OPTION;"
       ]
sql = string.join(sql, "\n")
p.communicate(sql)
p.wait()

# Install wikimedia
untardir = os.path.join(options.destination, 'mediawiki-1.20.2')
if os.path.exists(untardir):
    print "ERROR: " + untardir + " already exists, refusing to proceed"
    sys.exit(-2)
    #call2(['rm', '-r', untardir])
if os.path.exists(wikidir):
    print "ERROR: " + wikidir + " already exists, refusing to proceed"
    sys.exit(-2)
    #call2(['rm', '-r', wikidir])
call2(['tar', '-C', options.destination, '-xzf', options.mediawiki])
os.rename(os.path.join(options.destination, 'mediawiki-1.20.2'), wikidir)

# Configure mediawiki
os.chdir(wikidir)
call2(['php', 'maintenance/install.php', '--wiki', dbname,
       '--dbuser', options.user, '--dbpass', options.password,
       '--pass', options.password,
       '--dbname', dbname, '--lang', 'en',
       '--scriptpath', '/' + dbname,
       dbname, options.user])
call2(['chown', '-R', 'www-data.www-data', wikidir])

# Clean up tables created by MediaWiki install
# This solves duplicate key errors
p = Popen(['mysql', '-u', options.user,
           '--password=' + options.password, dbname],
           stdin=PIPE)
sql = ['DELETE FROM page;',
       'DELETE FROM text;']
sql = string.join(sql, "\n")
p.communicate(sql)
p.wait()

# Import XML Dump file
cmd = ["java -cp /usr/share/java/commons-compress-1.2.jar:/knowledge/packages/mwdumper-1.16.jar",
       "org.mediawiki.dumper.Dumper '" + options.xml + "' --format=sql:1.5",
       "| mysql -u " + options.user + " --password=" + options.password + " eswiki"]
cmd = string.join(cmd, ' ')
print cmd
t0 = time()
call(cmd, shell=True)
print "mwimport time = " + str(time() - t0) + " seconds"
<commit_msg>fix to wiki name - must be Wikipedia<commit_after>#!/usr/bin/env python
"""Script to process a Wikipedia dump.
By Braddock Gaskill, 22 Jan 2013

EXAMPLE USAGE:
    sudo scripts/make_wiki.py -l es -u wikimirror \
            -p wikimirror -r 'whatever' \
            -x /knowledge/data/wikipedia/eswiki-20121227-pages-articles.xml.bz2
"""

from subprocess import Popen, PIPE, call
from optparse import OptionParser
import string
import os
import sys
from time import time

def call2(cmd):
    print "CALL: " + string.join(cmd, ' ')
    call(cmd)

parser = OptionParser()
parser.add_option("-l", "--language",
                  help="Language code for wiki")
parser.add_option("-d", "--destination",
                  help="Apache destination directory",
                  default="/var/www")
parser.add_option("-u", "--user",
                  help="MySQL user name",
                  default="wikimirror")
parser.add_option("-p", "--password",
                  help="MySQL password for user")
parser.add_option("-r", "--rootpassword",
                  help="MySQL password for root")
parser.add_option("-x", "--xml",
                  help="Wikipedia XML bz2 dump file")
parser.add_option("-k", "--knowledge",
                  help="Knowledge directory",
                  default="/knowledge")
parser.add_option("--mediawiki",
                  help="Location of mediawiki tarball")

(options, args) = parser.parse_args()
if not options.language:
    parser.error('Language not given (-l)')
if not options.password:
    parser.error('MySQL password not given (-p)')
if not options.rootpassword:
    parser.error('MySQL password not given for MySQL root (-r)')
if not options.xml:
    parser.error('XML xml file not given (-d)')
if not options.mediawiki:
    options.mediawiki = os.path.join(options.knowledge, "packages", "mediawiki-1.20.2.tar.gz")
if not os.path.exists(options.knowledge):
    print 'ERROR: ' + options.knowledge + " does not exist"
    sys.exit(-1)
if not os.path.exists(options.destination):
    print 'ERROR: ' + options.destination + " does not exist"
    sys.exit(-1)
if not os.path.exists(options.xml):
    print 'ERROR: ' + options.xml + " does not exist"
    sys.exit(-1)

dbname = options.language + 'wiki'
wikidir = os.path.join(options.destination, dbname)

# Drop and create new user and database
p = Popen(['mysql', '-u', 'root',
           '--password=' + options.rootpassword],
          stdin=PIPE)
#       "GRANT USAGE ON *.* TO '" + options.user + "'@'localhost';",
#       "DROP USER '" + options.user + "'@'localhost';",
#       "CREATE USER '" + options.user + "'@'localhost' IDENTIFIED BY '"
#       + options.password + "';",
sql = ["DROP DATABASE IF EXISTS " + dbname + ";",
       "CREATE DATABASE " + dbname + ";",
       "GRANT USAGE ON * . * TO '" + options.user + "'@'localhost' IDENTIFIED BY '"
       + options.password + "' WITH MAX_QUERIES_PER_HOUR 0 MAX_CONNECTIONS_PER_HOUR 0 MAX_UPDATES_PER_HOUR 0 MAX_USER_CONNECTIONS 0;",
       "GRANT ALL PRIVILEGES ON " + dbname + ".* TO '" + options.user + "'@'localhost' WITH GRANT OPTION;"
       ]
sql = string.join(sql, "\n")
p.communicate(sql)
p.wait()

# Install wikimedia
untardir = os.path.join(options.destination, 'mediawiki-1.20.2')
if os.path.exists(untardir):
    print "ERROR: " + untardir + " already exists, refusing to proceed"
    sys.exit(-2)
    #call2(['rm', '-r', untardir])
if os.path.exists(wikidir):
    print "ERROR: " + wikidir + " already exists, refusing to proceed"
    sys.exit(-2)
    #call2(['rm', '-r', wikidir])
call2(['tar', '-C', options.destination, '-xzf', options.mediawiki])
os.rename(os.path.join(options.destination, 'mediawiki-1.20.2'), wikidir)

# Configure mediawiki
os.chdir(wikidir)
call2(['php', 'maintenance/install.php', '--wiki', 'Wikipedia',
       '--dbuser', options.user, '--dbpass', options.password,
       '--pass', options.password,
       '--dbname', dbname, '--lang', 'en',
       '--scriptpath', '/' + dbname,
       dbname, options.user])
call2(['chown', '-R', 'www-data.www-data', wikidir])

# Clean up tables created by MediaWiki install
# This solves duplicate key errors
p = Popen(['mysql', '-u', options.user,
           '--password=' + options.password, dbname],
           stdin=PIPE)
sql = ['DELETE FROM page;',
       'DELETE FROM text;']
sql = string.join(sql, "\n")
p.communicate(sql)
p.wait()

# Import XML Dump file
cmd = ["java -cp /usr/share/java/commons-compress-1.2.jar:/knowledge/packages/mwdumper-1.16.jar",
       "org.mediawiki.dumper.Dumper '" + options.xml + "' --format=sql:1.5",
       "| mysql -u " + options.user + " --password=" + options.password + " eswiki"]
cmd = string.join(cmd, ' ')
print cmd
t0 = time()
call(cmd, shell=True)
print "mwimport time = " + str(time() - t0) + " seconds"
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import synthtool as s
import synthtool.gcp as gcp
import logging
import subprocess

logging.basicConfig(level=logging.DEBUG)

gapic = gcp.GAPICGenerator()
versions = ['v2', 'v2beta1']
for version in versions:
    library = gapic.node_library('dialogflow', version)
    s.copy(library, excludes=['package.json', 'README.md', 'src/index.js'])

common_templates = gcp.CommonTemplates()
templates = common_templates.node_library()
s.copy(templates, excludes=["README.md", "samples/README.md"])

# dialogflow publishes to npm with no scope.
s.replace("src/**/*",
        "@google-cloud\/dialogflow",
        "dialogflow")

subprocess.run(['npm', 'install'])
subprocess.run(['npm', 'run', 'fix'])
subprocess.run(['npx', 'compileProtos', 'src'])
<commit_msg>chore: fix artman config location for synthtool (#440)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import synthtool as s
import synthtool.gcp as gcp
import logging
import subprocess

logging.basicConfig(level=logging.DEBUG)

gapic = gcp.GAPICGenerator()
versions = ['v2', 'v2beta1']
for version in versions:
    library = gapic.node_library('dialogflow', version, config_path=version + '/artman_dialogflow_' + version + '.yaml')
    s.copy(library, excludes=['package.json', 'README.md', 'src/index.js'])

common_templates = gcp.CommonTemplates()
templates = common_templates.node_library()
s.copy(templates, excludes=["README.md", "samples/README.md"])

# dialogflow publishes to npm with no scope.
s.replace("src/**/*",
        "@google-cloud\/dialogflow",
        "dialogflow")

subprocess.run(['npm', 'install'])
subprocess.run(['npm', 'run', 'fix'])
subprocess.run(['npx', 'compileProtos', 'src'])
<|endoftext|>


<commit_before>"""The device class used to represent a particular function of an accelerator
element.

A physical element in an accelerator may have multiple devices: an example at
DLS is a sextupole magnet that contains also horizontal and vertical corrector
magnets and a skew quadrupole.
"""
import pytac

class HandleException(Exception):
    """Exception associated with requests with invalid handles.
    """
    pass

class DeviceException(Exception):
    """Exception associated with Device misconfiguration or invalid requests.
    """
    pass

class Device(object):
    """A device attached to an element.

    Contains a control system, readback and setpoint PVs. A readback
    or setpoint PV is required when creating a device otherwise a
    DeviceException is raised. The device is enabled by default.

    **Attributes:**

    Attributes:
        name (str): The prefix of EPICS PVs for this device.
        rb_pv (str): The EPICS readback PV.
        sp_pv (str): The EPICS setpoint PV.

    .. Private Attributes:
           _cs (ControlSystem): The control system object used to get and set
                                 the value of a PV.
           _enabled (bool-like): Whether the device is enabled. May be a
                                  PvEnabler object.
    """
    def __init__(self, name, cs, enabled=True, rb_pv=None, sp_pv=None):
        """.. The constructor method for the class, called whenever a 'Device'
               object is constructed.

        Args:
            name (str): The prefix of EPICS PV for this device.
            cs (ControlSystem): The control system object used to get and set
                                 the value of a PV.
            enabled (bool-like): Whether the device is enabled. May be a
                                  PvEnabler object.
            rb_pv (str): The EPICS readback PV.
            sp_pv (str): The EPICS setpoint PV.

        **Methods:**
        """
        self.name = name
        self._cs = cs
        self.rb_pv = rb_pv
        self.sp_pv = sp_pv
        self._enabled = enabled

    def is_enabled(self):
        """Whether the device is enabled.

        Returns:
            bool: whether the device is enabled.
        """
        return bool(self._enabled)

    def set_value(self, value):
        """Set the device value.

        Args:
            value (float): The value to set on the PV.

        Raises:
            DeviceException: if no setpoint PV exists.
        """
        if self.sp_pv is None:
            raise HandleException("""Device {0} has no setpoint PV."""
                              .format(self.name))
        self._cs.put(self.sp_pv, value)

    def get_value(self, handle):
        """Read the value of a readback or setpoint PV.

        Args:
            handle (str): The handle used to get the value off a readback or
                           setpoint PV.

        Returns:
            float: The value of the PV.

        Raises:
            DeviceException: if the requested PV doesn't exist.
        """
        print('getting {}'.format('handle'))
        if handle == pytac.RB and self.rb_pv:
            return self._cs.get(self.rb_pv)
        elif handle == pytac.SP and self.sp_pv:
            return self._cs.get(self.sp_pv)

        raise HandleException("""Device {0} has no {1} PV."""
                          .format(self.name, handle))

    def get_pv_name(self, handle):
        """Get a PV name on a specified handle.

        Args:
            handle (str): The readback or setpoint handle to be returned.

        Returns:
            str: A readback or setpoint PV.

        Raises:
            DeviceException: if the PV doesn't exist.
        """
        if handle == pytac.RB and self.rb_pv:
            return self.rb_pv
        elif handle == pytac.SP and self.sp_pv:
            return self.sp_pv

        raise HandleException("""Device {0} has no {1} PV."""
                          .format(self.name, handle))

    def get_cs(self):
        """The control system object used to get and set the value of a PV.

        Returns:
            ControlSystem: The control system object used to get and set the
            value of a PV.
        """
        return self._cs

class PvEnabler(object):
    """A PvEnabler class to check whether a device is enabled.

    The class will behave like True if the PV value equals enabled_value,
    and False otherwise.

    .. Private Attributes:
           _pv (str): The PV name.
           _enabled_value (str): The value for PV for which the device should
                                  be considered enabled.
           _cs (ControlSystem): The control system object.
    """
    def __init__(self, pv, enabled_value, cs):
        """.. The constructor method for the class, called whenever a
               'PvEnabler' object is constructed.

        Args:
            pv (str): The PV name.
            enabled_value (str): The value for PV for which the device should
                                  be considered enabled.
            cs (ControlSystem): The control system object.
        """
        self._pv = pv
        self._enabled_value = str(int(float(enabled_value)))
        self._cs = cs

    def __nonzero__(self):
        """Used to override the 'if object' clause.

        Support for Python 2.7.

        Returns:
            bool: True if the device should be considered enabled.
        """
        pv_value = self._cs.get(self._pv)
        return self._enabled_value == str(int(float(pv_value)))

    def __bool__(self):
        """Used to override the 'if object' clause.

        Support for Python 3.x.

        Returns:
            bool: True if the device should be considered enabled.
        """
        return self.__nonzero__()
<commit_msg>Correct DeviceException to HandleException in comments where relevant.<commit_after>"""The device class used to represent a particular function of an accelerator
element.

A physical element in an accelerator may have multiple devices: an example at
DLS is a sextupole magnet that contains also horizontal and vertical corrector
magnets and a skew quadrupole.
"""
import pytac

class HandleException(Exception):
    """Exception associated with requests with invalid handles.
    """
    pass

class DeviceException(Exception):
    """Exception associated with Device misconfiguration or invalid requests.
    """
    pass

class Device(object):
    """A device attached to an element.

    Contains a control system, readback and setpoint PVs. A readback
    or setpoint PV is required when creating a device otherwise a
    DeviceException is raised. The device is enabled by default.

    **Attributes:**

    Attributes:
        name (str): The prefix of EPICS PVs for this device.
        rb_pv (str): The EPICS readback PV.
        sp_pv (str): The EPICS setpoint PV.

    .. Private Attributes:
           _cs (ControlSystem): The control system object used to get and set
                                 the value of a PV.
           _enabled (bool-like): Whether the device is enabled. May be a
                                  PvEnabler object.
    """
    def __init__(self, name, cs, enabled=True, rb_pv=None, sp_pv=None):
        """.. The constructor method for the class, called whenever a 'Device'
               object is constructed.

        Args:
            name (str): The prefix of EPICS PV for this device.
            cs (ControlSystem): The control system object used to get and set
                                 the value of a PV.
            enabled (bool-like): Whether the device is enabled. May be a
                                  PvEnabler object.
            rb_pv (str): The EPICS readback PV.
            sp_pv (str): The EPICS setpoint PV.

        **Methods:**
        """
        self.name = name
        self._cs = cs
        self.rb_pv = rb_pv
        self.sp_pv = sp_pv
        self._enabled = enabled

    def is_enabled(self):
        """Whether the device is enabled.

        Returns:
            bool: whether the device is enabled.
        """
        return bool(self._enabled)

    def set_value(self, value):
        """Set the device value.

        Args:
            value (float): The value to set on the PV.

        Raises:
            HandleException: if no setpoint PV exists.
        """
        if self.sp_pv is None:
            raise HandleException("""Device {0} has no setpoint PV."""
                              .format(self.name))
        self._cs.put(self.sp_pv, value)

    def get_value(self, handle):
        """Read the value of a readback or setpoint PV.

        Args:
            handle (str): The handle used to get the value off a readback or
                           setpoint PV.

        Returns:
            float: The value of the PV.

        Raises:
            HandleException: if the requested PV doesn't exist.
        """
        print('getting {}'.format('handle'))
        if handle == pytac.RB and self.rb_pv:
            return self._cs.get(self.rb_pv)
        elif handle == pytac.SP and self.sp_pv:
            return self._cs.get(self.sp_pv)

        raise HandleException("""Device {0} has no {1} PV."""
                          .format(self.name, handle))

    def get_pv_name(self, handle):
        """Get a PV name on a specified handle.

        Args:
            handle (str): The readback or setpoint handle to be returned.

        Returns:
            str: A readback or setpoint PV.

        Raises:
            HandleException: if the PV doesn't exist.
        """
        if handle == pytac.RB and self.rb_pv:
            return self.rb_pv
        elif handle == pytac.SP and self.sp_pv:
            return self.sp_pv

        raise HandleException("""Device {0} has no {1} PV."""
                          .format(self.name, handle))

    def get_cs(self):
        """The control system object used to get and set the value of a PV.

        Returns:
            ControlSystem: The control system object used to get and set the
            value of a PV.
        """
        return self._cs

class PvEnabler(object):
    """A PvEnabler class to check whether a device is enabled.

    The class will behave like True if the PV value equals enabled_value,
    and False otherwise.

    .. Private Attributes:
           _pv (str): The PV name.
           _enabled_value (str): The value for PV for which the device should
                                  be considered enabled.
           _cs (ControlSystem): The control system object.
    """
    def __init__(self, pv, enabled_value, cs):
        """.. The constructor method for the class, called whenever a
               'PvEnabler' object is constructed.

        Args:
            pv (str): The PV name.
            enabled_value (str): The value for PV for which the device should
                                  be considered enabled.
            cs (ControlSystem): The control system object.
        """
        self._pv = pv
        self._enabled_value = str(int(float(enabled_value)))
        self._cs = cs

    def __nonzero__(self):
        """Used to override the 'if object' clause.

        Support for Python 2.7.

        Returns:
            bool: True if the device should be considered enabled.
        """
        pv_value = self._cs.get(self._pv)
        return self._enabled_value == str(int(float(pv_value)))

    def __bool__(self):
        """Used to override the 'if object' clause.

        Support for Python 3.x.

        Returns:
            bool: True if the device should be considered enabled.
        """
        return self.__nonzero__()
<|endoftext|>


<commit_before>"""Hyperparameter tuning - mitosis detection"""
import argparse
import os
import random

import numpy as np
import tensorflow as tf
import keras.backend as K

import train_mitoses

def main(args=None):
  # parse args
  parser = argparse.ArgumentParser()
  parser.add_argument("--patches_path", required=True,
      help="path to the generated image patches containing `train` & `val` folders ")
  parser.add_argument("--exp_parent_path",
      default=os.path.join("experiments", "mitoses", "hyp"),
      help="parent path in which to store experiment folders (default: %(default)s)")
  parser.add_argument("--models", nargs='*', default=["vgg", "resnet"],
      help="list of names of models to use, where the names can be selected from ['logreg', "\
           "'vgg', 'vgg19', 'resnet'] (default: %(default)s)")
  parser.add_argument("--train_batch_sizes", nargs='*', type=int, default=[32],
      help="list of training batch sizes (default: %(default)s)")
  parser.add_argument("--val_batch_size", type=int, default=32,
      help="validation batch size for all experiments (default: %(default)s)")
  parser.add_argument("--clf_epochs", type=int, default=5,
      help="number of epochs for which to train the new classifier layers (default: %(default)s)")
  parser.add_argument("--finetune_epochs", type=int, default=5,
      help="number of epochs for which to fine-tune the unfrozen layers (default: %(default)s)")
  parser.add_argument("--clf_lr_range", nargs=2, type=float, default=(1e-5, 1e-2),
      help="half-open interval for the learning rate for training the new classifier layers "\
           "(default: %(default)s)")
  parser.add_argument("--finetune_lr_range", nargs=2, type=float, default=(1e-7, 1e-2),
      help="half-open interval for the learning rate for fine-tuning the unfrozen layers "\
           "(default: %(default)s)")
  parser.add_argument("--finetune_momentum_range", nargs=2, type=float, default=(0.85, 0.95),
      help="half-open interval for the momentum rate for fine-tuning the unfrozen layers "\
           "(default: %(default)s)")
  parser.add_argument("--finetune_layers", nargs='*', type=int, default=[0, -1],
      help="list of the number of layers at the end of the pretrained portion of the model to "\
           "fine-tune (note: the new classifier layers will still be trained during fine-tuning "\
           "as well) (default: %(default)s)")
  parser.add_argument("--l2_range", nargs=2, type=float, default=[0, 1e-2],
      help="half-closed interval for the amount of l2 weight regularization (default: %(default)s)")
  augment_parser = parser.add_mutually_exclusive_group(required=False)
  augment_parser.add_argument("--augment", dest="augment", action="store_true",
      help="apply random augmentation to the training images (default: True)")
  augment_parser.add_argument("--no_augment", dest="augment", action="store_false",
      help="do not apply random augmentation to the training images (default: False)")
  parser.set_defaults(augment=True)
  parser.add_argument("--marginalize", default=False, action="store_true",
      help="use noise marginalization when evaluating the validation set. if this is set, then "\
           "the validation batch_size must be divisible by 4, or equal to 1 for no augmentation "\
           "(default: %(default)s)")
  parser.add_argument("--threads", type=int, default=5,
      help="number of threads for dataset parallel processing; note: this will cause "\
           "non-reproducibility for values > 1 (default: %(default)s)")
  parser.add_argument("--prefetch_batches", type=int, default=100,
      help="number of batches to prefetch (default: %(default)s)")
  parser.add_argument("--log_interval", type=int, default=100,
      help="number of steps between logging during training (default: %(default)s)")
  parser.add_argument("--num_experiments", type=int, default=100,
      help="number of experiments to run (default: %(default)s)")

  args = parser.parse_args(args)

  # hyperparameter search
  for i in range(args.num_experiments):
    # NOTE: as a quick POC, we will use the command-line interface of the training script
    # TODO: extract experiment setup code in the training script main function into a class so that
    # we can reuse it from here
    train_args = []
    train_args.append(f"--patches_path={args.patches_path}")

    train_args.append(f"--exp_parent_path={args.exp_parent_path}")

    model = random.choice(args.models)
    train_args.append(f"--model={model}")

    train_batch_size = random.choice(args.train_batch_sizes)
    train_args.append(f"--train_batch_size={train_batch_size}")

    train_args.append(f"--val_batch_size={args.val_batch_size}")

    train_args.append(f"--clf_epochs={args.clf_epochs}")

    train_args.append(f"--finetune_epochs={args.finetune_epochs}")

    clf_lr_lb, clf_lr_ub = args.clf_lr_range
    clf_lr = np.random.uniform(clf_lr_lb, clf_lr_ub)
    train_args.append(f"--clf_lr={clf_lr}")

    finetune_lr_lb, finetune_lr_ub = args.finetune_lr_range
    finetune_lr = np.random.uniform(finetune_lr_lb, finetune_lr_ub)
    train_args.append(f"--finetune_lr={finetune_lr}")

    finetune_momentum_lb, finetune_momentum_ub = args.finetune_momentum_range
    finetune_momentum = np.random.uniform(finetune_momentum_lb, finetune_momentum_ub)
    train_args.append(f"--finetune_momentum={finetune_momentum}")

    finetune_layers = random.choice(args.finetune_layers)
    train_args.append(f"--finetune_layers={finetune_layers}")

    l2_lb, l2_ub = args.l2_range
    l2 = np.random.uniform(l2_lb, l2_ub)
    train_args.append(f"--l2={l2}")

    if args.augment:
      train_args.append("--augment")
    else:
      train_args.append("--no_augment")

    if args.marginalize:
      train_args.append("--marginalize")

    train_args.append(f"--threads={args.threads}")

    train_args.append(f"--prefetch_batches={args.prefetch_batches}")

    train_args.append(f"--log_interval={args.log_interval}")

    # train!
    try:
      train_mitoses.main(train_args)
    except tf.errors.InvalidArgumentError:  # if values become nan or inf
      print("Experiment failed!")

    # it is necessary to completely reset everything in between experiments
    K.clear_session()

if __name__ == "__main__":
  main()

<commit_msg>Add an initial model weights option to the hyperparamter tuning script<commit_after>"""Hyperparameter tuning - mitosis detection"""
import argparse
import os
import random

import numpy as np
import tensorflow as tf
import keras.backend as K

import train_mitoses

def main(args=None):
  # parse args
  parser = argparse.ArgumentParser()
  parser.add_argument("--patches_path", required=True,
      help="path to the generated image patches containing `train` & `val` folders ")
  parser.add_argument("--exp_parent_path",
      default=os.path.join("experiments", "mitoses", "hyp"),
      help="parent path in which to store experiment folders (default: %(default)s)")
  parser.add_argument("--models", nargs='*', default=["vgg", "resnet"],
      help="list of names of models to use, where the names can be selected from ['logreg', "\
           "'vgg', 'vgg19', 'resnet'] (default: %(default)s)")
  parser.add_argument("--model_weights", default=None,
      help="optional hdf5 file containing the initial weights of the model. if not supplied, the "\
           "model will start with pretrained weights from imagenet. if this is set, the `models` "\
           "list must contain a single model that is compatible with these weights "\
           "(default: %(default)s)")
  parser.add_argument("--train_batch_sizes", nargs='*', type=int, default=[32],
      help="list of training batch sizes (default: %(default)s)")
  parser.add_argument("--val_batch_size", type=int, default=32,
      help="validation batch size for all experiments (default: %(default)s)")
  parser.add_argument("--clf_epochs", type=int, default=5,
      help="number of epochs for which to train the new classifier layers (default: %(default)s)")
  parser.add_argument("--finetune_epochs", type=int, default=5,
      help="number of epochs for which to fine-tune the unfrozen layers (default: %(default)s)")
  parser.add_argument("--clf_lr_range", nargs=2, type=float, default=(1e-5, 1e-2),
      help="half-open interval for the learning rate for training the new classifier layers "\
           "(default: %(default)s)")
  parser.add_argument("--finetune_lr_range", nargs=2, type=float, default=(1e-7, 1e-2),
      help="half-open interval for the learning rate for fine-tuning the unfrozen layers "\
           "(default: %(default)s)")
  parser.add_argument("--finetune_momentum_range", nargs=2, type=float, default=(0.85, 0.95),
      help="half-open interval for the momentum rate for fine-tuning the unfrozen layers "\
           "(default: %(default)s)")
  parser.add_argument("--finetune_layers", nargs='*', type=int, default=[0, -1],
      help="list of the number of layers at the end of the pretrained portion of the model to "\
           "fine-tune (note: the new classifier layers will still be trained during fine-tuning "\
           "as well) (default: %(default)s)")
  parser.add_argument("--l2_range", nargs=2, type=float, default=[0, 1e-2],
      help="half-closed interval for the amount of l2 weight regularization (default: %(default)s)")
  augment_parser = parser.add_mutually_exclusive_group(required=False)
  augment_parser.add_argument("--augment", dest="augment", action="store_true",
      help="apply random augmentation to the training images (default: True)")
  augment_parser.add_argument("--no_augment", dest="augment", action="store_false",
      help="do not apply random augmentation to the training images (default: False)")
  parser.set_defaults(augment=True)
  parser.add_argument("--marginalize", default=False, action="store_true",
      help="use noise marginalization when evaluating the validation set. if this is set, then "\
           "the validation batch_size must be divisible by 4, or equal to 1 for no augmentation "\
           "(default: %(default)s)")
  parser.add_argument("--threads", type=int, default=5,
      help="number of threads for dataset parallel processing; note: this will cause "\
           "non-reproducibility for values > 1 (default: %(default)s)")
  parser.add_argument("--prefetch_batches", type=int, default=100,
      help="number of batches to prefetch (default: %(default)s)")
  parser.add_argument("--log_interval", type=int, default=100,
      help="number of steps between logging during training (default: %(default)s)")
  parser.add_argument("--num_experiments", type=int, default=100,
      help="number of experiments to run (default: %(default)s)")

  args = parser.parse_args(args)

  # hyperparameter search
  for i in range(args.num_experiments):
    # NOTE: as a quick POC, we will use the command-line interface of the training script
    # TODO: extract experiment setup code in the training script main function into a class so that
    # we can reuse it from here
    train_args = []
    train_args.append(f"--patches_path={args.patches_path}")

    train_args.append(f"--exp_parent_path={args.exp_parent_path}")

    model = random.choice(args.models)
    train_args.append(f"--model={model}")

    if args.model_weights:
      train_args.append(f"--model_weights={args.model_weights}")

    train_batch_size = random.choice(args.train_batch_sizes)
    train_args.append(f"--train_batch_size={train_batch_size}")

    train_args.append(f"--val_batch_size={args.val_batch_size}")

    train_args.append(f"--clf_epochs={args.clf_epochs}")

    train_args.append(f"--finetune_epochs={args.finetune_epochs}")

    clf_lr_lb, clf_lr_ub = args.clf_lr_range
    clf_lr = np.random.uniform(clf_lr_lb, clf_lr_ub)
    train_args.append(f"--clf_lr={clf_lr}")

    finetune_lr_lb, finetune_lr_ub = args.finetune_lr_range
    finetune_lr = np.random.uniform(finetune_lr_lb, finetune_lr_ub)
    train_args.append(f"--finetune_lr={finetune_lr}")

    finetune_momentum_lb, finetune_momentum_ub = args.finetune_momentum_range
    finetune_momentum = np.random.uniform(finetune_momentum_lb, finetune_momentum_ub)
    train_args.append(f"--finetune_momentum={finetune_momentum}")

    finetune_layers = random.choice(args.finetune_layers)
    train_args.append(f"--finetune_layers={finetune_layers}")

    l2_lb, l2_ub = args.l2_range
    l2 = np.random.uniform(l2_lb, l2_ub)
    train_args.append(f"--l2={l2}")

    if args.augment:
      train_args.append("--augment")
    else:
      train_args.append("--no_augment")

    if args.marginalize:
      train_args.append("--marginalize")

    train_args.append(f"--threads={args.threads}")

    train_args.append(f"--prefetch_batches={args.prefetch_batches}")

    train_args.append(f"--log_interval={args.log_interval}")

    # train!
    try:
      train_mitoses.main(train_args)
    except tf.errors.InvalidArgumentError:  # if values become nan or inf
      print("Experiment failed!")

    # it is necessary to completely reset everything in between experiments
    K.clear_session()

if __name__ == "__main__":
  main()

<|endoftext|>


<commit_before>import pandas as pd

def parse_sheet_index(df, index):
    """Parse sheet index into df index"""
    if index and len(df.columns) > index:
        df = df.set_index(df.columns[index - 1])
        # if it was multi-index, the name is tuple;
        # choose last value in tuple since that is more common
        if type(df.index.name) == tuple:
            df.index.name = df.index.name[-1]
        # get rid of falsey index names
        df.index.name = df.index.name or None
    return df

def parse_df_col_names(df, include_index):
    """Parse column names from a df into sheet headers"""
    headers = df.columns.tolist()

    # handle multi-index headers
    if len(headers) > 0 and type(headers[0]) == tuple:
        headers = [list(row) for row in zip(*headers)]

        # Pandas sets index name as top level col name with reset_index
        # Switch to low level since that is more natural
        if include_index:
            headers[-1][0] = headers[0][0]
            headers[0][0] = ''
    # handle regular columns
    else:
        headers = [headers]

    return headers

def parse_sheet_headers(vals, header_rows):
    """Parse headers from a sheet into df columns"""
    col_names = None
    if header_rows:
        headers = vals[:header_rows]
        if header_rows > 1:
            _fix_sheet_header_level(headers)
            col_names = pd.MultiIndex.from_arrays(headers)
        elif header_rows == 1:
            col_names = headers

    return col_names

def _fix_sheet_header_level(header_names):
    for col_ix in range(len(header_names[0])):
        _shift_header_up(header_names, col_ix)

    return header_names

def _shift_header_up(header_names, col_index, row_index=0,
                     shift_val=0, found_first=False):
    """Recursively shift headers up so that top level is not empty"""
    rows = len(header_names)
    if row_index < rows:
        current_value = header_names[row_index][col_index]

        if current_value == '' and not found_first:
            shift_val += 1
        else:
            found_first = True

        shift_val = _shift_header_up(header_names, col_index, row_index + 1,
                                     shift_val, found_first)
        if shift_val <= row_index:
            header_names[row_index - shift_val][col_index] = current_value

        if row_index + shift_val >= rows:
            header_names[row_index][col_index] = ''
    return shift_val

def chunks(lst, chunk_size):
    """Chunk a list into specified chunk sizes"""
    for i in range(0, len(lst), chunk_size):
        yield lst[i:i + chunk_size]

def deprecate(message):
    """Display message about deprecation"""
    import warnings
    warnings.warn(message, DeprecationWarning, stacklevel=2)

def create_frozen_request(sheet_id, rows=None, cols=None):
    """
    Create v4 API request to freeze rows and/or columns for a
    given worksheet.
    """
    grid_properties = {}

    if rows >= 0:
        grid_properties['frozen_row_count'] = rows

    if cols >= 0:
        grid_properties['frozen_column_count'] = cols

    changed_props = grid_properties.keys()

    return {
        'update_sheet_properties': {
            'properties': {
                'sheet_id': sheet_id,
                'grid_properties': grid_properties
            },
            'fields': 'grid_properties({0})'.format(', '.join(changed_props))
        }
    }

def fillna(df, fill_value=''):
    """
    Replace null values with `fill_value`. Also replaces in categorical columns.
    """
    for col in df.dtypes[df.dtypes == 'category'].index:
        df[col].cat.add_categories([fill_value])
    return df.fillna(fill_value)
<commit_msg>Fix for previous headers fix<commit_after>import pandas as pd

def parse_sheet_index(df, index):
    """Parse sheet index into df index"""
    if index and len(df.columns) > index:
        df = df.set_index(df.columns[index - 1])
        # if it was multi-index, the name is tuple;
        # choose last value in tuple since that is more common
        if type(df.index.name) == tuple:
            df.index.name = df.index.name[-1]
        # get rid of falsey index names
        df.index.name = df.index.name or None
    return df

def parse_df_col_names(df, include_index):
    """Parse column names from a df into sheet headers"""
    headers = df.columns.tolist()

    # handle multi-index headers
    if len(headers) > 0 and type(headers[0]) == tuple:
        headers = [list(row) for row in zip(*headers)]

        # Pandas sets index name as top level col name with reset_index
        # Switch to low level since that is more natural
        if include_index:
            headers[-1][0] = headers[0][0]
            headers[0][0] = ''
    # handle regular columns
    else:
        headers = [headers]

    return headers

def parse_sheet_headers(vals, header_rows):
    """Parse headers from a sheet into df columns"""
    col_names = None
    if header_rows:
        headers = vals[:header_rows]
        if header_rows > 1:
            _fix_sheet_header_level(headers)
            col_names = pd.MultiIndex.from_arrays(headers)
        elif header_rows == 1:
            col_names = headers[0]

    return col_names

def _fix_sheet_header_level(header_names):
    for col_ix in range(len(header_names[0])):
        _shift_header_up(header_names, col_ix)

    return header_names

def _shift_header_up(header_names, col_index, row_index=0,
                     shift_val=0, found_first=False):
    """Recursively shift headers up so that top level is not empty"""
    rows = len(header_names)
    if row_index < rows:
        current_value = header_names[row_index][col_index]

        if current_value == '' and not found_first:
            shift_val += 1
        else:
            found_first = True

        shift_val = _shift_header_up(header_names, col_index, row_index + 1,
                                     shift_val, found_first)
        if shift_val <= row_index:
            header_names[row_index - shift_val][col_index] = current_value

        if row_index + shift_val >= rows:
            header_names[row_index][col_index] = ''
    return shift_val

def chunks(lst, chunk_size):
    """Chunk a list into specified chunk sizes"""
    for i in range(0, len(lst), chunk_size):
        yield lst[i:i + chunk_size]

def deprecate(message):
    """Display message about deprecation"""
    import warnings
    warnings.warn(message, DeprecationWarning, stacklevel=2)

def create_frozen_request(sheet_id, rows=None, cols=None):
    """
    Create v4 API request to freeze rows and/or columns for a
    given worksheet.
    """
    grid_properties = {}

    if rows >= 0:
        grid_properties['frozen_row_count'] = rows

    if cols >= 0:
        grid_properties['frozen_column_count'] = cols

    changed_props = grid_properties.keys()

    return {
        'update_sheet_properties': {
            'properties': {
                'sheet_id': sheet_id,
                'grid_properties': grid_properties
            },
            'fields': 'grid_properties({0})'.format(', '.join(changed_props))
        }
    }

def fillna(df, fill_value=''):
    """
    Replace null values with `fill_value`. Also replaces in categorical columns.
    """
    for col in df.dtypes[df.dtypes == 'category'].index:
        df[col].cat.add_categories([fill_value])
    return df.fillna(fill_value)
<|endoftext|>


<commit_before>import sys
import requests
import pandas as pd
import json
import time
from pytaxize.refactor import Refactor
import os

class NoResultException(Exception):
    pass

def gnr_datasources(todf=True):
    '''
    Get data sources for the Global Names Resolver.

    Retrieve data sources used in Global Names Index, see
       http://gni.globalnames.org/ for information.

    :param todf: logical; Should a DataFrame be returned?

    Usage:
    # all data sources
    >>> import pytaxize
    >>> out = pytaxize.gnr_datasources()
    >>> out.head()
       id              title
    0   1  Catalogue of Life
    1   2        Wikispecies
    2   3               ITIS
    3   4               NCBI
    4   5     Index Fungorum

    # give me the id for EOL
    >>> out = pytaxize.gnr_datasources()
    >>> out.ix[out['title'] == 'EOL']
        id title
    11  12   EOL

    # Output json
    >>> pytaxize.gnr_datasources(False)
    '''
    url = "http://resolver.globalnames.org/data_sources.json"
    if(todf):
        out = Refactor(url, payload={}, request='get').json()
        data = []
        for i in range(len(out)):
            data.append([out[i]['id'],out[i]['title']])
        df = pd.DataFrame(data, columns=['id','title'])
    else:
        df = Refactor(url, payload={}, request='get').json()
    return df

def gnr_resolve(names='Homo sapiens', source=None, format='json', resolve_once='false',
    with_context='false', best_match_only='false', header_only='false', preferred_data_sources='false', http='get'):
    if names.__class__.__name__ != 'list':
        return _gnr_resolve(names, source, format, resolve_once,
    with_context, best_match_only, header_only, preferred_data_sources, http)

    maxlen = 1000
    #splitting list to smaller lists of size <= 1000
    names_sublists = [names[x:x+maxlen] for x in range(0, len(names), maxlen)]
    data = []
    for sublist in names_sublists:
        data.extend(_gnr_resolve(sublist, source, format, resolve_once,
    with_context, best_match_only, header_only, preferred_data_sources, http))

    return data

def _gnr_resolve(names='Homo sapiens', source=None, format='json', resolve_once='false',
    with_context='false', best_match_only='false', header_only='false', preferred_data_sources='false', http='get'):
    '''
    Uses the Global Names Resolver to resolve scientific names
    :param names: List of taxonomic names
    :param source: Source to pull from, one of x, y, z
    :param format: One of json or xml
    :param resolve_once: Logical, true or false
    :param with_context: Return context with taxonomic names
    :param best_match_only: Logical, if true (default) return the best match only
    :param header_only: Return header only, logical
    :param preferred_data_sources: Return only preferred data sources.
    :param http: The HTTP method to use, one of "get" or "post". Default="get"
    Usage:
    >>> import pytaxize
    >>> pytaxize.gnr_resolve('Helianthus annus')
    [{u'classification_path': u'', u'data_source_title': u'EOL', u'match_type': 1, u'score': 0.988, u'url': u'http://eol.org/pages/468106/names/synonyms', u'classification_path_ranks': u'', u'name_string': u'Helianthus annus', u'prescore': u'3|0|0', u'canonical_form': u'Helianthus annus', u'classification_path_ids': u'', u'local_id': u'468106', u'data_source_id': 12, u'taxon_id': u's_5106367', u'gni_uuid': u'f5674e32-00cc-57e3-b632-6a0b89fa4df4'}, {u'classification_path': u'|Helianthus annus', u'data_source_title': u'uBio NameBank', u'match_type': 1, u'score': 0.988, u'url': u'http://www.ubio.org/browser/details.php?namebankID=10130157', u'classification_path_ranks': u'kingdom|', u'name_string': u'Helianthus annus', u'global_id': u'urn:lsid:ubio.org:namebank:10130157', u'prescore': u'3|0|0', u'canonical_form': u'Helianthus annus', u'classification_path_ids': u'', u'local_id': u'urn:lsid:ubio.org:namebank:10130157', u'data_source_id': 169, u'taxon_id': u'102910884', u'gni_uuid': u'f5674e32-00cc-57e3-b632-6a0b89fa4df4'}, {u'classification_path': u'', u'data_source_title': u'EOL', u'match_type': 2, u'score': 0.988, u'url': u'http://eol.org/pages/468106', u'classification_path_ranks': u'', u'name_string': u'Helianthus annus L.', u'prescore': u'3|0|0', u'canonical_form': u'Helianthus annus', u'classification_path_ids': u'', u'local_id': u'468106', u'data_source_id': 12, u'taxon_id': u'20584982', u'gni_uuid': u'e757b3c1-421f-5bb9-a27f-d56259baaf3d'}]
    >>> pytaxize.gnr_resolve(['Helianthus annus','Poa annua'])
    '''
    url = 'http://resolver.globalnames.org/name_resolvers'
    payload = {'data_source_ids': source, 'format': format,
                'resolve_once': resolve_once, 'with_context': with_context,
                'best_match_only': best_match_only, 'header_only': header_only,
                'preferred_data_sources': preferred_data_sources}
    if names.__class__.__name__ == 'list':
        if len(names) > 300 and http == 'get':
            http = 'post'
        else:
            names = "|".join(names)
            payload['names'] = names
    else:
        payload['names'] = names
    if http == 'get':
        result_json = Refactor(url, payload, request='get').json()
    else:
        if names.__class__.__name__ != 'list':
           result_json = Refactor(url, payload, request='post').json()
        else:
            with open('names_list.txt', 'w') as f:
                for name in names:
                    f.write(name+"\n")
            f.close()
            result_json = Refactor(url, payload, request='post').json(files = {'file': open('names_list.txt', 'rb')})
            while result_json['status'] == 'working':
                result_url = result_json['url']
                time.sleep(10)
                result_json = Refactor(result_url, payload={},request='get').json()

            os.remove('names_list.txt')

    data = []
    for each_result in result_json['data']:
        data.append(each_result['results'] if 'results' in each_result else [])
    if data == [[]]:
        sys.exit('No matching results to the query')
    return data

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<commit_msg>fix bug, no matching results warning should be in gnr_resolve instead of _gnr_resolve<commit_after>import sys
import requests
import pandas as pd
import json
import time
from pytaxize.refactor import Refactor
import os

class NoResultException(Exception):
    pass

def gnr_datasources(todf=True):
    '''
    Get data sources for the Global Names Resolver.

    Retrieve data sources used in Global Names Index, see
       http://gni.globalnames.org/ for information.

    :param todf: logical; Should a DataFrame be returned?

    Usage:
    # all data sources
    >>> import pytaxize
    >>> out = pytaxize.gnr_datasources()
    >>> out.head()
       id              title
    0   1  Catalogue of Life
    1   2        Wikispecies
    2   3               ITIS
    3   4               NCBI
    4   5     Index Fungorum

    # give me the id for EOL
    >>> out = pytaxize.gnr_datasources()
    >>> out.ix[out['title'] == 'EOL']
        id title
    11  12   EOL

    # Output json
    >>> pytaxize.gnr_datasources(False)
    '''
    url = "http://resolver.globalnames.org/data_sources.json"
    if(todf):
        out = Refactor(url, payload={}, request='get').json()
        data = []
        for i in range(len(out)):
            data.append([out[i]['id'],out[i]['title']])
        df = pd.DataFrame(data, columns=['id','title'])
    else:
        df = Refactor(url, payload={}, request='get').json()
    return df

def gnr_resolve(names='Homo sapiens', source=None, format='json', resolve_once='false',
    with_context='false', best_match_only='false', header_only='false', preferred_data_sources='false', http='get'):
    if names.__class__.__name__ != 'list':
        return _gnr_resolve(names, source, format, resolve_once,
    with_context, best_match_only, header_only, preferred_data_sources, http)

    maxlen = 1000
    #splitting list to smaller lists of size <= 1000
    names_sublists = [names[x:x+maxlen] for x in range(0, len(names), maxlen)]
    data = []
    for sublist in names_sublists:
        data.extend(_gnr_resolve(sublist, source, format, resolve_once,
    with_context, best_match_only, header_only, preferred_data_sources, http))
    if data == [[]]:
        sys.exit('No matching results to the query')

    return data

def _gnr_resolve(names='Homo sapiens', source=None, format='json', resolve_once='false',
    with_context='false', best_match_only='false', header_only='false', preferred_data_sources='false', http='get'):
    '''
    Uses the Global Names Resolver to resolve scientific names
    :param names: List of taxonomic names
    :param source: Source to pull from, one of x, y, z
    :param format: One of json or xml
    :param resolve_once: Logical, true or false
    :param with_context: Return context with taxonomic names
    :param best_match_only: Logical, if true (default) return the best match only
    :param header_only: Return header only, logical
    :param preferred_data_sources: Return only preferred data sources.
    :param http: The HTTP method to use, one of "get" or "post". Default="get"
    Usage:
    >>> import pytaxize
    >>> pytaxize.gnr_resolve('Helianthus annus')
    [{u'classification_path': u'', u'data_source_title': u'EOL', u'match_type': 1, u'score': 0.988, u'url': u'http://eol.org/pages/468106/names/synonyms', u'classification_path_ranks': u'', u'name_string': u'Helianthus annus', u'prescore': u'3|0|0', u'canonical_form': u'Helianthus annus', u'classification_path_ids': u'', u'local_id': u'468106', u'data_source_id': 12, u'taxon_id': u's_5106367', u'gni_uuid': u'f5674e32-00cc-57e3-b632-6a0b89fa4df4'}, {u'classification_path': u'|Helianthus annus', u'data_source_title': u'uBio NameBank', u'match_type': 1, u'score': 0.988, u'url': u'http://www.ubio.org/browser/details.php?namebankID=10130157', u'classification_path_ranks': u'kingdom|', u'name_string': u'Helianthus annus', u'global_id': u'urn:lsid:ubio.org:namebank:10130157', u'prescore': u'3|0|0', u'canonical_form': u'Helianthus annus', u'classification_path_ids': u'', u'local_id': u'urn:lsid:ubio.org:namebank:10130157', u'data_source_id': 169, u'taxon_id': u'102910884', u'gni_uuid': u'f5674e32-00cc-57e3-b632-6a0b89fa4df4'}, {u'classification_path': u'', u'data_source_title': u'EOL', u'match_type': 2, u'score': 0.988, u'url': u'http://eol.org/pages/468106', u'classification_path_ranks': u'', u'name_string': u'Helianthus annus L.', u'prescore': u'3|0|0', u'canonical_form': u'Helianthus annus', u'classification_path_ids': u'', u'local_id': u'468106', u'data_source_id': 12, u'taxon_id': u'20584982', u'gni_uuid': u'e757b3c1-421f-5bb9-a27f-d56259baaf3d'}]
    >>> pytaxize.gnr_resolve(['Helianthus annus','Poa annua'])
    '''
    url = 'http://resolver.globalnames.org/name_resolvers'
    payload = {'data_source_ids': source, 'format': format,
                'resolve_once': resolve_once, 'with_context': with_context,
                'best_match_only': best_match_only, 'header_only': header_only,
                'preferred_data_sources': preferred_data_sources}
    if names.__class__.__name__ == 'list':
        if len(names) > 300 and http == 'get':
            http = 'post'
        else:
            names = "|".join(names)
            payload['names'] = names
    else:
        payload['names'] = names
    if http == 'get':
        result_json = Refactor(url, payload, request='get').json()
    else:
        if names.__class__.__name__ != 'list':
           result_json = Refactor(url, payload, request='post').json()
        else:
            with open('names_list.txt', 'w') as f:
                for name in names:
                    f.write(name+"\n")
            f.close()
            result_json = Refactor(url, payload, request='post').json(files = {'file': open('names_list.txt', 'rb')})
            while result_json['status'] == 'working':
                result_url = result_json['url']
                time.sleep(10)
                result_json = Refactor(result_url, payload={},request='get').json()

            os.remove('names_list.txt')

    data = []
    for each_result in result_json['data']:
        data.append(each_result['results'] if 'results' in each_result else [])

    return data

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<|endoftext|>


<commit_before>from Configuration import Configuration
# import pUtil
import sys
import os

def set_environment():

    env = Configuration()

    # env['version'] = pUtil.getPilotVersion()
    env['pilot_version_tag'] = 'PR' # 'RC'
    env['force_devpilot'] = False

    # get the python executable path, to be used by child job
    env['pyexe'] = sys.executable

    # default job dispatcher web server
    env['pshttpurl'] = "pandaserver.cern.ch"

    # global variables with some default (test) values
    env['psport'] = 25443                      # PanDA server listening port
    env['uflag'] = None                        # Flag if this pilot is to only get a user analysis job or ordinary job
    env['abortDQSpace'] = 10                   # Unit is in GB
    env['warnDQSpace'] = 100                   # Unit is in GB
    env['localsizelimit_stdout'] = 2*1024**2   # Size limit of payload stdout size during running. unit is in kB
    env['localspacelimit'] = 2*1024**2         # Space limit of remaining local disk size during running. unit is in kB
    env['localspacelimit_user'] = 7*1024**2    # Maximum size of user work area. unit is in kB
    env['localspacelimit0'] = 5*1024**2        # Initial space limit before asking for a job. unit is in kB
    env['outputlimit'] = 500*1024**3           # Maximum allowed size of an output file. unit is B
    env['pilotId'] = 'xtestP001'               # Pilot id
    env['jobSchedulerId'] = 'xtestJS001'       # Scheduler id
    env['heartbeatPeriod'] = 1 #30*60             # 30 minutes limit for checking job state file
    env['maxjobrecDefault'] = 20               # Default maximum number of job recoveries
    env['maxjobrec'] = env['maxjobrecDefault'] # Maximum number of job recoveries
    env['zombieJobList'] = []                  # Zombie list
    env['prodJobDone'] = False                 # Will be set to true when production job has finished
    env['rmwkdir'] = None                      # If set to true, workdir will always be deleted
    env['jobrec'] = True                       # Perform job recovery per default
    env['jobRequestFlag'] = True               # Ask server for initial job (read from file otherwise)
    env['debugLevel'] = 0                      # 0: debug info off, 1: display function name when called, 2: full debug info
    env['maxNumberOfRecoveryAttempts'] = 15    # As attempted by the job recovery
    env['stagein'] = False                     # Set to True during stagein phase
    env['stageout'] = False                    # Set to True during stageout phase
    env['queuename'] = ""                      # Name of queue used to download config info
    env['loopingLimitDefaultProd'] = 60 # 12*3600   # If job does not write anything in N hours, it is considered a looping job (production jobs)
    env['loopingLimitDefaultUser'] = 3*3600    # If job does not write anything in N hours, it is considered a looping job (user analysis jobs)
    env['loopingLimitMinDefault'] = 60 # 2*3600     # Minimum allow looping limit
    env['stageinretry'] = 2                    # Number of stage-in tries
    env['stageoutretry'] = 2                   # Number of stage-out tries
    env['logFileDir'] = ""                     # Log file directory
    env['testLevel'] = "0"                     # Test suite control variable (0: no test, 1: put error, 2: ...)
    env['loggingMode'] = None                  # True puts the pilot in logging mode (will not run a job), False is a normal pilot. None means run as before
    env['updateServerFlag'] = True             # Switch on/off Panda server updates (for testing only), default is True
    env['globalSite'] = None                   # Site object
    env['globalWorkNode'] = None               # Work node object
    env['globalJob'] = None                    # Job object
    env['memory'] = None                       # For memory restrictions when asking the dispatcher for a new job (MB)
    env['proxycheckFlag'] = True               # True (default): perform proxy validity checks, False: no check
    env['wrapperFlag'] = False                 # True for wrappers that expect an exit code via return, False (default) when exit() can be used
    env['cleanupLimit'] = 2                    # Cleanup time limit in hours, see Cleaner.py
    env['logTransferred'] = False              # Boolean to keep track of whether the log has been transferred or not
    env['errorLabel'] = "WARNING"              # Set to FAILED when job recovery is not used
    env['nSent'] = 0                           # Throttle variable
    env['proxyguard'] = None                   # Global proxyguard object needed for the signal handler to be able to restore the proxy if necessary
    env['jobRecoveryMode'] = None              # When true, the pilot only runs job recovery, no payload download
    env['pilotToken'] = None                   # Pilot authentication token
    env['countryGroup'] = ""                   # Country group selector for getJob request
    env['workingGroup'] = ""                   # Working group selector for getJob request
    env['allowOtherCountry'] = False           #
    env['inputDir'] = ""                       # Location of input files (source for mv site mover)
    env['outputDir'] = ""                      # Location of output files (destination for mv site mover)
    env['jobIds'] = []                         # Global job id list
    env['stageoutStartTime'] = None            # Set when pilot receives "stageout" state message, used by looping job killer
    env['lfcRegistration'] = True              # Should the pilot perform LFC registration?
    env['timefloor_default'] = None            # Time limit for multi-jobs in minutes (turned off by default)
    env['useCoPilot'] = False                  # CERNVM Co-Pilot framework (on: let Co-Pilot finish job, off: let pilot finish job (default))
    env['update_freq_server'] = 3*60          # Server update frequency, 30 minutes
    env['experiment'] = "ATLAS"                # Current experiment (can be set with pilot option -F <experiment>)
    env['getjobmaxtime'] = 3*60                # Maximum time the pilot will attempt to download a single job (seconds)
    env['pandaJobDataFileName'] = "pandaJobData.out" # Job definition file name
    env['verifySoftwareDirectory'] = True      # Normally the softwre directory should be verified, but potentially not on an HPC system
    env['workerNode'] = None
    env['hasQueueData'] = None
    env['stdout_tail'] = ""

    # In case the PILOT_INITDIR file has not been created yet, which means that this module is being
    # used by pilot.py, it will be created here using the current directory as init dir
    path = os.path.join(os.getcwd(), "PILOT_INITDIR")
    if not os.path.exists(path):
        writeToFile(path, os.getcwd())
    env['pilot_initdir'] = readStringFromFile("PILOT_INITDIR")

    # data structure for job information in main loop
    # jobDic = {'prod/analy':[pid,job,os.getpgrp]}
    # (analysis label currently not used)
    env['jobDic'] = {}

    # some default values
    env['sitename'] = "testsite"
    env['workdir'] = "/tmp"
    env['appdir'] = ""
    env['dq2url'] = ""
    env['return_code'] = None
    env['return'] = 0

    return env

def readStringFromFile(filename):
    """ read exit code from file <workdir>/EXITCODE """

    s = ""
    if os.path.exists(filename):
        try:
            f = open(filename, "r")
        except Exception, e:
            print "Failed to open %s: %s" % (filename, str(e))
        else:
            s = f.read()
            print "Found string %s in file %s" % (s, filename)
            f.close()
    else:
        print "No string to report (file %s does not exist)" % (filename)

    return s

def storePilotInitdir(targetdir, pilot_initdir):
    """ Store the pilot launch directory in a file used by environment.py """

    # This function is used to store the location of the init directory in the init directory itself as well as in the
    # site work directory. The location file is used by environment.py to set the global env['pilot_initdir'] used
    # by the pilot and the Monitor

    # This function must be called before the global env variable is instantiated in the pilot

    path = os.path.join(targetdir, "PILOT_INITDIR")
    writeToFile(path, pilot_initdir)

def writeToFile(filename, s):
    """ write string s to file """

    try:
        f = open(filename, "w")
    except Exception, e:
        print "!!WARNING!!2990!! Could not open: %s, %s" % (filename, str(e))
    else:
        f.write("%s" % (s))
        f.close()
        print 'Wrote string "%s" to file: %s' % (s, filename)
<commit_msg>Updated environment.py<commit_after>from Configuration import Configuration
# import pUtil
import sys
import os

def set_environment():

    env = Configuration()

    # env['version'] = pUtil.getPilotVersion()
    env['pilot_version_tag'] = 'PR' # 'RC'
    env['force_devpilot'] = False

    # get the python executable path, to be used by child job
    env['pyexe'] = sys.executable

    # default job dispatcher web server
    env['pshttpurl'] = "pandaserver.cern.ch"

    # global variables with some default (test) values
    env['psport'] = 25443                      # PanDA server listening port
    env['uflag'] = None                        # Flag if this pilot is to only get a user analysis job or ordinary job
    env['abortDQSpace'] = 10                   # Unit is in GB
    env['warnDQSpace'] = 100                   # Unit is in GB
    env['localsizelimit_stdout'] = 2*1024**2   # Size limit of payload stdout size during running. unit is in kB
    env['localspacelimit'] = 2*1024**2         # Space limit of remaining local disk size during running. unit is in kB
    env['localspacelimit_user'] = 7*1024**2    # Maximum size of user work area. unit is in kB
    env['localspacelimit0'] = 5*1024**2        # Initial space limit before asking for a job. unit is in kB
    env['outputlimit'] = 500*1024**3           # Maximum allowed size of an output file. unit is B
    env['pilotId'] = 'xtestP001'               # Pilot id
    env['jobSchedulerId'] = 'xtestJS001'       # Scheduler id
    env['heartbeatPeriod'] = 1 #30*60             # 30 minutes limit for checking job state file
    env['maxjobrecDefault'] = 20               # Default maximum number of job recoveries
    env['maxjobrec'] = env['maxjobrecDefault'] # Maximum number of job recoveries
    env['zombieJobList'] = []                  # Zombie list
    env['prodJobDone'] = False                 # Will be set to true when production job has finished
    env['rmwkdir'] = None                      # If set to true, workdir will always be deleted
    env['jobrec'] = True                       # Perform job recovery per default
    env['jobRequestFlag'] = True               # Ask server for initial job (read from file otherwise)
    env['debugLevel'] = 0                      # 0: debug info off, 1: display function name when called, 2: full debug info
    env['maxNumberOfRecoveryAttempts'] = 15    # As attempted by the job recovery
    env['stagein'] = False                     # Set to True during stagein phase
    env['stageout'] = False                    # Set to True during stageout phase
    env['queuename'] = ""                      # Name of queue used to download config info
    env['loopingLimitDefaultProd'] = 12*3600   # If job does not write anything in N hours, it is considered a looping job (production jobs)
    env['loopingLimitDefaultUser'] = 3*3600    # If job does not write anything in N hours, it is considered a looping job (user analysis jobs)
    env['loopingLimitMinDefault'] = 2*3600     # Minimum allow looping limit
    env['stageinretry'] = 2                    # Number of stage-in tries
    env['stageoutretry'] = 2                   # Number of stage-out tries
    env['logFileDir'] = ""                     # Log file directory
    env['testLevel'] = "0"                     # Test suite control variable (0: no test, 1: put error, 2: ...)
    env['loggingMode'] = None                  # True puts the pilot in logging mode (will not run a job), False is a normal pilot. None means run as before
    env['updateServerFlag'] = True             # Switch on/off Panda server updates (for testing only), default is True
    env['globalSite'] = None                   # Site object
    env['globalWorkNode'] = None               # Work node object
    env['globalJob'] = None                    # Job object
    env['memory'] = None                       # For memory restrictions when asking the dispatcher for a new job (MB)
    env['proxycheckFlag'] = True               # True (default): perform proxy validity checks, False: no check
    env['wrapperFlag'] = False                 # True for wrappers that expect an exit code via return, False (default) when exit() can be used
    env['cleanupLimit'] = 2                    # Cleanup time limit in hours, see Cleaner.py
    env['logTransferred'] = False              # Boolean to keep track of whether the log has been transferred or not
    env['errorLabel'] = "WARNING"              # Set to FAILED when job recovery is not used
    env['nSent'] = 0                           # Throttle variable
    env['proxyguard'] = None                   # Global proxyguard object needed for the signal handler to be able to restore the proxy if necessary
    env['jobRecoveryMode'] = None              # When true, the pilot only runs job recovery, no payload download
    env['pilotToken'] = None                   # Pilot authentication token
    env['countryGroup'] = ""                   # Country group selector for getJob request
    env['workingGroup'] = ""                   # Working group selector for getJob request
    env['allowOtherCountry'] = False           #
    env['inputDir'] = ""                       # Location of input files (source for mv site mover)
    env['outputDir'] = ""                      # Location of output files (destination for mv site mover)
    env['jobIds'] = []                         # Global job id list
    env['stageoutStartTime'] = None            # Set when pilot receives "stageout" state message, used by looping job killer
    env['lfcRegistration'] = True              # Should the pilot perform LFC registration?
    env['timefloor_default'] = None            # Time limit for multi-jobs in minutes (turned off by default)
    env['useCoPilot'] = False                  # CERNVM Co-Pilot framework (on: let Co-Pilot finish job, off: let pilot finish job (default))
    env['update_freq_server'] = 30*60          # Server update frequency, 30 minutes
    env['experiment'] = "ATLAS"                # Current experiment (can be set with pilot option -F <experiment>)
    env['getjobmaxtime'] = 3*60                # Maximum time the pilot will attempt to download a single job (seconds)
    env['pandaJobDataFileName'] = "pandaJobData.out" # Job definition file name
    env['verifySoftwareDirectory'] = True      # Normally the softwre directory should be verified, but potentially not on an HPC system
    env['workerNode'] = None
    env['hasQueueData'] = None
    env['stdout_tail'] = ""

    # In case the PILOT_INITDIR file has not been created yet, which means that this module is being
    # used by pilot.py, it will be created here using the current directory as init dir
    path = os.path.join(os.getcwd(), "PILOT_INITDIR")
    if not os.path.exists(path):
        writeToFile(path, os.getcwd())
    env['pilot_initdir'] = readStringFromFile("PILOT_INITDIR")

    # data structure for job information in main loop
    # jobDic = {'prod/analy':[pid,job,os.getpgrp]}
    # (analysis label currently not used)
    env['jobDic'] = {}

    # some default values
    env['sitename'] = "testsite"
    env['workdir'] = "/tmp"
    env['appdir'] = ""
    env['dq2url'] = ""
    env['return_code'] = None
    env['return'] = 0

    return env

def readStringFromFile(filename):
    """ read exit code from file <workdir>/EXITCODE """

    s = ""
    if os.path.exists(filename):
        try:
            f = open(filename, "r")
        except Exception, e:
            print "Failed to open %s: %s" % (filename, str(e))
        else:
            s = f.read()
            print "Found string %s in file %s" % (s, filename)
            f.close()
    else:
        print "No string to report (file %s does not exist)" % (filename)

    return s

def storePilotInitdir(targetdir, pilot_initdir):
    """ Store the pilot launch directory in a file used by environment.py """

    # This function is used to store the location of the init directory in the init directory itself as well as in the
    # site work directory. The location file is used by environment.py to set the global env['pilot_initdir'] used
    # by the pilot and the Monitor

    # This function must be called before the global env variable is instantiated in the pilot

    path = os.path.join(targetdir, "PILOT_INITDIR")
    writeToFile(path, pilot_initdir)

def writeToFile(filename, s):
    """ write string s to file """

    try:
        f = open(filename, "w")
    except Exception, e:
        print "!!WARNING!!2990!! Could not open: %s, %s" % (filename, str(e))
    else:
        f.write("%s" % (s))
        f.close()
        print 'Wrote string "%s" to file: %s' % (s, filename)
<|endoftext|>