<commit_before>#! /usr/bin/env python
import os
import time
import logging
import warnings
from copy import deepcopy
try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse
import jsonpickle
import requests
try:
    from tcptest import tcp_test
except ImportError:
    from .tcptest import tcp_test
from yapsy.PluginManager import PluginManager

from . import COLOURS

FORMAT = '%(asctime)-15s %(name)s [%(levelname)s]: %(message)s'
THIS_DIR = os.path.dirname(os.path.realpath(__file__))

logging.basicConfig(format=FORMAT, level=logging.ERROR, datefmt="%Y-%m-%d %H:%M:%S")
LOG = logging.getLogger('smolder')
REQUESTS_LOG = logging.getLogger('requests')
REQUESTS_LOG.setLevel(logging.ERROR)

logging.getLogger('yapsy').setLevel(logging.INFO)
manager = PluginManager()
manager.setPluginPlaces([THIS_DIR, "~/.smolder_plugins"])
manager.collectPlugins()

OUTPUT_WIDTH = 108

def deepupdate(original, update):
    """
    Recursively update a dict.
    Subdict's won't be overwritten but also updated.
    """
    for key, value in original.items():
        if key not in update:
            update[key] = value
        elif isinstance(value, dict):
            deepupdate(value, update[key])
    return update

class Charcoal(object):
    def __init__(self, test, host):
        """

        :rtype : object
        """

        self.passed = 0
        self.failed = 0

        try:
            self.port = test["port"]
        except (AttributeError, KeyError):
            print("Warning: No port definition found in the first test, using port 80 as default.")
            try:
                if test["protocol"] == "https":
                    self.port = 443
                else:
                    self.port = 80
            except (AttributeError, KeyError):
                self.port = 80

        LOG.debug("Test: {0}".format(test))
        test_defaults = dict(inputs=dict(allow_redirects=False, timeout=30), protocol="http", port=self.port, method="get",
                             outcomes=dict(expect_status_code=200, colour_output=True))
        intermediate_dict = deepupdate(test_defaults, test)

        host_overrides_object = urlparse(host)
        host_overrides = dict()
        if host_overrides_object.scheme is not None:
            host_overrides["protocol"] = host_overrides_object.scheme
        if host_overrides_object.port is not None:
            host_overrides["port"] = host_overrides_object.port
        if host_overrides_object.hostname is not None:
            self.host = host_overrides_object.hostname
        else:
            self.host = host
        host_overrides = dict()
        final_dict = deepupdate(intermediate_dict, host_overrides)
        if "tcp_test" in test and test["tcp_test"]:
            tcp_test(self.host, self.port)
        try:
            if type(final_dict["inputs"]["verify"]) == bool:
                self.verify = final_dict["inputs"]["verify"]
            elif final_dict["inputs"]["verify"] == "True":
                self.verify = True
            elif final_dict["inputs"]["verify"] == "False":
                self.verify = False
            elif type(final_dict["inputs"]["verify"]) == str:
                LOG.debug("Fucking requests having one argument which can accept input of multiple types")
                self.verify = final_dict["inputs"]["verify"]
            else:
                raise TypeError("Not sure what you want here")
            self.verify_specified = True
            del(final_dict["inputs"]["verify"])
        except (AttributeError, KeyError):
            if 'https' in final_dict['protocol']:
                self.verify = True
                self.verify_specified = False
            else:
                self.verify = False
                self.verify_specified = False
        self.test = deepcopy(final_dict)
        LOG.debug("Test with defaults: {0}".format(self.test))
        self.inputs = deepcopy(self.test['inputs'])
        request_url_format = '{protocol}://{host}:{port}{uri}'
        self.output = ("-" * OUTPUT_WIDTH)
        self.inputs['url'] = request_url_format.format(protocol=self.test['protocol'], host=self.host,
                                                       port=self.test['port'],
                                                       uri=self.test['uri'])
        LOG.debug("Testing {0}".format(self.inputs['url']))
        self.output = ("-" * OUTPUT_WIDTH)

        this_name = ("{0:^" + str(OUTPUT_WIDTH) + "}").format(self.test['name'][:OUTPUT_WIDTH])
        self.output = "\n".join([self.output, this_name])
        this_url = ("{0:^" + str(OUTPUT_WIDTH) + "}").format(self.inputs["url"][:OUTPUT_WIDTH])
        self.output = "\n".join([self.output, this_url])
        self.output = "\n".join([self.output, self.__repr__()])
        self.output = "\n".join([self.output, ("-" * OUTPUT_WIDTH)])
        with warnings.catch_warnings():
            warnings.simplefilter("error")
            try:
                LOG.debug("Verify is a: {0}, with value: {1}".format(type(self.verify), self.verify))
                start = int(round(time.time() * 1000))
                self.req = getattr(requests, self.test['method'].lower())(verify=self.verify, **self.inputs)
                end = int(round(time.time() * 1000))
                self.duration_ms = end - start
            except Exception:
                warnings.simplefilter("ignore")
                start = int(round(time.time() * 1000))
                self.req = getattr(requests, self.test['method'].lower())(verify=self.verify, **self.inputs)
                end = int(round(time.time() * 1000))
                self.duration_ms = end - start
                if not self.verify_specified:
                    message, status = self.fail_test("Insecure request not ignored by inputs['verify']")
                    self.add_output("SecureRequest", message, status)
                else:
                    if self.verify:
                        message, status = self.fail_test("Insecure request made")
                        self.add_output("SecureRequest", message, status)
                    else:
                        message, status = self.warn_test("Insecure request made and ignored")
                        self.add_output("SecureRequest", message, status)

        if 'show_body' in self.test:
            try:
                req_content = self.req.content.decode()
            except UnicodeDecodeError:
                req_content = self.req.content
            self.output = "\n".join([self.output, req_content])

        for plugin_info in manager.getAllPlugins():
            for outcome in self.test['outcomes']:
                if plugin_info.name == outcome:
                    manager.activatePluginByName(plugin_info.name)
                    message, status = plugin_info.plugin_object.run(self)
                    self.add_output(plugin_info.name, message, status)
                    manager.deactivatePluginByName(plugin_info.name)

        self.output = "\n".join([self.output, "\n"])

    def __str__(self):
        return self.output

    def __repr__(self):
        LOG.debug(self.inputs)
        if "headers" in self.inputs:
            if self.inputs["headers"] != '':
                header_list = ['"{0}: {1}"'.format(k, v) for k, v in list(self.inputs["headers"].items())]
                header = "-H " + (" -H ".join(header_list))
            else:
                header = self.inputs["headers"]
        else:
            header = ''
        if "data" in self.inputs:
            data = '-d' + str(self.inputs["data"])
        else:
            data = ''

        if not self.verify and self.test["protocol"] == "https":
            curl_insecure = '--insecure'
        else:
            curl_insecure = ''

        # Adding curl output to allow simple debugging of the requests
        command = 'curl {curl_insecure} -v -s -o /dev/null {headers} {data} -X {method} "{uri}"'
        output = command.format(method=str(self.test['method']).upper(), headers=header, data=data,
                                uri=self.inputs['url'], curl_insecure=curl_insecure)
        return output

    def pass_test(self, message):
        self.passed += 1
        status = "[PASS]"
        if self.test["outcomes"]["colour_output"]:
            status = COLOURS.to_green(status)
        return (message, status)

    def fail_test(self, message):
        self.failed += 1
        status = "[FAIL]"
        if self.test["outcomes"]["colour_output"]:
            status = COLOURS.to_red(status)
        return (message, status)

    def warn_test(self, message):
        self.passed += 1
        status = "[WARN]"
        if self.test["outcomes"]["colour_output"]:
            status = COLOURS.to_yellow(status)
        return (message, status)

    def add_output(self, name, message, status):
        test_out = name + ": " + message + "." * (OUTPUT_WIDTH - len(name) - len(message) - 8) + status.rjust(8)
        self.output = "\n".join([self.output, test_out])

    def _to_json(self):
        return jsonpickle.encode(self)
<commit_msg>Host overrides<commit_after>#! /usr/bin/env python
import os
import time
import logging
import warnings
from copy import deepcopy

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse
import jsonpickle
import requests

try:
    from tcptest import tcp_test
except ImportError:
    from .tcptest import tcp_test
from yapsy.PluginManager import PluginManager

from . import COLOURS

FORMAT = '%(asctime)-15s %(name)s [%(levelname)s]: %(message)s'
THIS_DIR = os.path.dirname(os.path.realpath(__file__))

logging.basicConfig(format=FORMAT, level=logging.ERROR, datefmt="%Y-%m-%d %H:%M:%S")
LOG = logging.getLogger('smolder')
REQUESTS_LOG = logging.getLogger('requests')
REQUESTS_LOG.setLevel(logging.ERROR)

logging.getLogger('yapsy').setLevel(logging.INFO)
manager = PluginManager()
manager.setPluginPlaces([THIS_DIR, "~/.smolder_plugins"])
manager.collectPlugins()

OUTPUT_WIDTH = 108

def deepupdate(original, update):
    """
    Recursively update a dict.
    Subdict's won't be overwritten but also updated.
    """
    for key, value in original.items():
        if key not in update:
            update[key] = value
        elif isinstance(value, dict):
            deepupdate(value, update[key])
    return update

class Charcoal(object):
    def __init__(self, test, host):
        """

        :rtype : object
        """

        self.passed = 0
        self.failed = 0

        try:
            self.port = test["port"]
        except (AttributeError, KeyError):
            print("Warning: No port definition found in the first test, using port 80 as default.")
            try:
                if test["protocol"] == "https":
                    self.port = 443
                else:
                    self.port = 80
            except (AttributeError, KeyError):
                self.port = 80

        LOG.debug("Test: {0}".format(test))
        test_defaults = dict(inputs=dict(allow_redirects=False, timeout=30), protocol="http", port=self.port,
                             method="get", outcomes=dict(expect_status_code=200, colour_output=True))

        host_overrides_object = urlparse(host)
        host_overrides = dict()
        if host_overrides_object.scheme is not None and host_overrides_object.scheme is not "":
            host_overrides["protocol"] = host_overrides_object.scheme
        else:
            host_overrides["protocol"] = test_defaults["protocol"]
        if host_overrides_object.port is not None:
            host_overrides["port"] = host_overrides_object.port
        else:
            host_overrides["port"] = test_defaults["port"]
        if host_overrides_object.hostname is not None:
            self.host = host_overrides_object.hostname
        else:
            self.host = host
        intermediate_dict = deepupdate(test_defaults, host_overrides)

        final_dict = deepupdate(intermediate_dict, test)
        if "tcp_test" in test and test["tcp_test"]:
            tcp_test(self.host, self.port)
        try:
            if type(final_dict["inputs"]["verify"]) == bool:
                self.verify = final_dict["inputs"]["verify"]
            elif final_dict["inputs"]["verify"] == "True":
                self.verify = True
            elif final_dict["inputs"]["verify"] == "False":
                self.verify = False
            elif type(final_dict["inputs"]["verify"]) == str:
                LOG.debug("Fucking requests having one argument which can accept input of multiple types")
                self.verify = final_dict["inputs"]["verify"]
            else:
                raise TypeError("Not sure what you want here")
            self.verify_specified = True
            del (final_dict["inputs"]["verify"])
        except (AttributeError, KeyError):
            if 'https' in final_dict['protocol']:
                self.verify = True
                self.verify_specified = False
            else:
                self.verify = False
                self.verify_specified = False
        self.test = deepcopy(final_dict)
        LOG.debug("Test with defaults: {0}".format(self.test))
        self.inputs = deepcopy(self.test['inputs'])
        request_url_format = '{protocol}://{host}:{port}{uri}'
        self.output = ("-" * OUTPUT_WIDTH)
        self.inputs['url'] = request_url_format.format(protocol=self.test['protocol'], host=self.host,
                                                       port=self.test['port'],
                                                       uri=self.test['uri'])
        LOG.debug("Testing {0}".format(self.inputs['url']))
        self.output = ("-" * OUTPUT_WIDTH)

        this_name = ("{0:^" + str(OUTPUT_WIDTH) + "}").format(self.test['name'][:OUTPUT_WIDTH])
        self.output = "\n".join([self.output, this_name])
        this_url = ("{0:^" + str(OUTPUT_WIDTH) + "}").format(self.inputs["url"][:OUTPUT_WIDTH])
        self.output = "\n".join([self.output, this_url])
        self.output = "\n".join([self.output, self.__repr__()])
        self.output = "\n".join([self.output, ("-" * OUTPUT_WIDTH)])
        with warnings.catch_warnings():
            warnings.simplefilter("error")
            try:
                LOG.debug("Verify is a: {0}, with value: {1}".format(type(self.verify), self.verify))
                start = int(round(time.time() * 1000))
                self.req = getattr(requests, self.test['method'].lower())(verify=self.verify, **self.inputs)
                end = int(round(time.time() * 1000))
                self.duration_ms = end - start
            except Exception:
                warnings.simplefilter("ignore")
                start = int(round(time.time() * 1000))
                self.req = getattr(requests, self.test['method'].lower())(verify=self.verify, **self.inputs)
                end = int(round(time.time() * 1000))
                self.duration_ms = end - start
                if not self.verify_specified:
                    message, status = self.fail_test("Insecure request not ignored by inputs['verify']")
                    self.add_output("SecureRequest", message, status)
                else:
                    if self.verify:
                        message, status = self.fail_test("Insecure request made")
                        self.add_output("SecureRequest", message, status)
                    else:
                        message, status = self.warn_test("Insecure request made and ignored")
                        self.add_output("SecureRequest", message, status)

        if 'show_body' in self.test:
            try:
                req_content = self.req.content.decode()
            except UnicodeDecodeError:
                req_content = self.req.content
            self.output = "\n".join([self.output, req_content])

        for plugin_info in manager.getAllPlugins():
            for outcome in self.test['outcomes']:
                if plugin_info.name == outcome:
                    manager.activatePluginByName(plugin_info.name)
                    message, status = plugin_info.plugin_object.run(self)
                    self.add_output(plugin_info.name, message, status)
                    manager.deactivatePluginByName(plugin_info.name)

        self.output = "\n".join([self.output, "\n"])

    def __str__(self):
        return self.output

    def __repr__(self):
        LOG.debug(self.inputs)
        if "headers" in self.inputs:
            if self.inputs["headers"] != '':
                header_list = ['"{0}: {1}"'.format(k, v) for k, v in list(self.inputs["headers"].items())]
                header = "-H " + (" -H ".join(header_list))
            else:
                header = self.inputs["headers"]
        else:
            header = ''
        if "data" in self.inputs:
            data = '-d' + str(self.inputs["data"])
        else:
            data = ''

        if not self.verify and self.test["protocol"] == "https":
            curl_insecure = '--insecure'
        else:
            curl_insecure = ''

        # Adding curl output to allow simple debugging of the requests
        command = 'curl {curl_insecure} -v -s -o /dev/null {headers} {data} -X {method} "{uri}"'
        output = command.format(method=str(self.test['method']).upper(), headers=header, data=data,
                                uri=self.inputs['url'], curl_insecure=curl_insecure)
        return output

    def pass_test(self, message):
        self.passed += 1
        status = "[PASS]"
        if self.test["outcomes"]["colour_output"]:
            status = COLOURS.to_green(status)
        return (message, status)

    def fail_test(self, message):
        self.failed += 1
        status = "[FAIL]"
        if self.test["outcomes"]["colour_output"]:
            status = COLOURS.to_red(status)
        return (message, status)

    def warn_test(self, message):
        self.passed += 1
        status = "[WARN]"
        if self.test["outcomes"]["colour_output"]:
            status = COLOURS.to_yellow(status)
        return (message, status)

    def add_output(self, name, message, status):
        test_out = name + ": " + message + "." * (OUTPUT_WIDTH - len(name) - len(message) - 8) + status.rjust(8)
        self.output = "\n".join([self.output, test_out])

    def _to_json(self):
        return jsonpickle.encode(self)
<|endoftext|>


<commit_before><commit_msg>browser/base/browser_theme_provider_unittest.cc => browser/browser_theme_provider_unittest.cc<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
from xml.sax import make_parser
from xml.sax.handler import ContentHandler
import smallsmilhandler

Lector = smallsmilhandler.SmallSMILHandler

if __name__ == "__main__":
    try:
        fichero = sys.argv[1]
    except IndexError:
        sys.exit("Usage: python3 karaoke.py file.smil.")
    parser = make_parser()
    cHandler = smallsmilhandler.SmallSMILHandler()
    parser.setContentHandler(cHandler)
    parser.parse(open('karaoke.smil'))
<commit_msg>Correccion<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
from xml.sax import make_parser
from xml.sax.handler import ContentHandler
import smallsmilhandler

def crearlinea(misDatos):
    linea=''
    for sublista in misDatos:
        linea = linea + sublista[0]
        dicc = sublista[1]
        for atributo in dicc:
            if dicc[atributo] != "":
                linea = linea + "\t" + atributo + "=" + (dicc[atributo] + " ")
    print(linea)

if __name__ == "__main__":
    try:
        fichero = sys.argv[1]
    except IndexError:
        sys.exit("Usage: python3 karaoke.py file.smil")
    parser = make_parser()
    cHandler = smallsmilhandler.SmallSMILHandler()
    parser.setContentHandler(cHandler)
    parser.parse(open(fichero))
    misDatos = cHandler.get_tags()
    crearlinea(misDatos)
<|endoftext|>


<commit_before><commit_msg>Have 'browser_tests_dll' use the common chrome_dll_version.rc generated by the (new) 'chrome_dll_version' target, not by rolling its own. BUG=none TEST=none Review URL: http://codereview.chromium.org/125017<commit_after><|endoftext|>


<commit_before><commit_msg>Enable browser commands unit test on linux.<commit_after><|endoftext|>


<commit_before>import numpy as np
import sys
from matplotlib import pyplot as plt
import kepio, kepmsg, kepkey
import re

__svnid__ = "$Id: kepdraw.py 6165 2014-03-26 21:16:27Z mstill $"
__url__ = "$URL: svn+ssh://mstill@murzim.amn.nasa.gov/data-repo/trunk/data/flight/go/PyKE/kepler/kepdraw.py $"

def kepdraw(infile,outfile,datacol,ploterr,errcol,quality,
	    lcolor,lwidth,fcolor,falpha,labelsize,ticksize,
	    xsize,ysize,fullrange,chooserange,y1,y2,plotgrid,
            ylabel,plottype,verbose,logfile,status,cmdLine=False):

# log the call

    hashline = '----------------------------------------------------------------------------'
    kepmsg.log(logfile,hashline,verbose)
    call = 'KEPDRAW -- '
    call += 'infile='+infile+' '
    call += 'outfile='+outfile+' '
    call += 'datacol='+datacol+' '
    perr = 'n'
    if (ploterr): perr = 'y'
    call += 'ploterr='+perr+ ' '
    call += 'errcol='+errcol+' '
    qual = 'n'
    if (quality): qual = 'y'
    call += 'quality='+qual+ ' '
    call += 'lcolor='+str(lcolor)+' '
    call += 'lwidth='+str(lwidth)+' '
    call += 'fcolor='+str(fcolor)+' '
    call += 'falpha='+str(falpha)+' '
    call += 'labelsize='+str(labelsize)+' '
    call += 'ticksize='+str(ticksize)+' '
    call += 'xsize='+str(xsize)+' '
    call += 'ysize='+str(ysize)+' '
    frange = 'n'
    if (fullrange): frange = 'y'
    call += 'fullrange='+frange+ ' '
    crange = 'n'
    if (chooserange): crange = 'y'
    call += 'chooserange='+crange+ ' '
    call += 'ymin='+str(y1)+' '
    call += 'ymax='+str(y2)+' '
    pgrid = 'n'
    if (plotgrid): pgrid = 'y'
    call += 'plotgrid='+pgrid+ ' '
    call += 'ylabel='+str(ylabel)+' '
    call += 'plottype='+plottype+' '
    chatter = 'n'
    if (verbose): chatter = 'y'
    call += 'verbose='+chatter+' '
    call += 'logfile='+logfile
    kepmsg.log(logfile,call+'\n',verbose)

# start time

    kepmsg.clock('KEPDRAW started at',logfile,verbose)

# test log file

    logfile = kepmsg.test(logfile)

# open input file

    if status == 0:
        struct, status = kepio.openfits(infile,'readonly',logfile,verbose)
    if status == 0:
        tstart, tstop, bjdref, cadence, status = kepio.timekeys(struct,infile,logfile,verbose,status)

# read table structure

    if status == 0:
	table, status = kepio.readfitstab(infile,struct[1],logfile,verbose)

# read table columns

    if status == 0:
        intime, status = kepio.readtimecol(infile,table,logfile,verbose)
        intime += bjdref
	indata, status = kepio.readfitscol(infile,table,datacol,logfile,verbose)
        indataerr, status = kepio.readfitscol(infile,table,errcol,logfile,verbose)
        qualty, status = kepio.readfitscol(infile,table,'SAP_QUALITY',logfile,verbose)

# close infile

    if status == 0:
	status = kepio.closefits(struct,logfile,verbose)

# remove infinities and bad data

    if status == 0:
        if np.isnan(np.nansum(indataerr)):
            indataerr[:] = 1.0e-5
        work1 = np.array([intime, indata, indataerr, qualty],dtype='float64')
        work1 = np.rot90(work1,3)
        work1 = work1[~np.isnan(work1).any(1)]
        work1 = work1[~np.isinf(work1).any(1)]
        if quality:
            work1 = work1[work1[:,0] == 0.0]
        barytime = np.array(work1[:,3],dtype='float64')
        data = np.array(work1[:,2],dtype='float32')
        dataerr = np.array(work1[:,1],dtype='float32')
        if len(barytime) == 0:
            message = 'ERROR -- KEPDRAW: Plotting arrays are full of NaN'
            status = kepmsg.err(logfile,message,verbose)

# clean up x-axis unit

    if status == 0:
	barytime0 = float(int(tstart / 100) * 100.0)
	barytime -= barytime0
        xlab = 'BJD $-$ %d' % barytime0

# clean up y-axis units

        nrm = 0
        try:
            nrm = len(str(int(np.nanmax(data))))-1
        except:
            nrm = 0
	data = data / 10**nrm
        if 'e$^-$ s$^{-1}$' in ylabel or 'default' in ylabel:
            if nrm == 0:
                ylab1 = 'e$^-$ s$^{-1}$'
            else:
                ylab1 = '10$^{%d}$ e$^-$ s$^{-1}$' % nrm
        else:
            ylab1 = re.sub('_','-',ylabel)

# data limits

	xmin = np.nanmin(barytime)
	xmax = np.nanmax(barytime)
	ymin = np.nanmin(data)
	ymax = np.nanmax(data)
	xr = xmax - xmin
	yr = ymax - ymin
        barytime = np.insert(barytime,[0],[barytime[0]])
        barytime = np.append(barytime,[barytime[-1]])
        data = np.insert(data,[0],[-10000.0])
        data = np.append(data,-10000.0)

# define plot formats

        try:
            params = {'backend': 'png',
                      'axes.linewidth': 2.5,
                      'axes.labelsize': labelsize,
                      'axes.font': 'sans-serif',
                      'axes.fontweight' : 'bold',
                      'text.fontsize': 12,
                      'legend.fontsize': 12,
                      'xtick.labelsize': ticksize,
                      'ytick.labelsize': ticksize}
            plt.rcParams.update(params)
        except:
            pass

# define size of plot on monitor screen

	plt.figure(figsize=[xsize,ysize])

# delete any fossil plots in the matplotlib window

        plt.clf()

# position axes inside the plotting window

#        ax = plt.axes([0.1,0.11,0.89,0.87])
	ax = plt.subplot(111)
	plt.subplots_adjust(0.06,0.15,0.92,0.83)

# force tick labels to be absolute rather than relative

        plt.gca().xaxis.set_major_formatter(plt.ScalarFormatter(useOffset=False))
        plt.gca().yaxis.set_major_formatter(plt.ScalarFormatter(useOffset=False))
        ax.yaxis.set_major_locator(plt.MaxNLocator(5))

# rotate y labels by 90 deg

        labels = ax.get_yticklabels()
        plt.setp(labels, 'rotation', 90, fontsize=ticksize)

# if plot type is 'fast' plot data time series as points

        if plottype == 'fast':
            plt.plot(barytime,data,'o',color=lcolor)

# if plot type is 'pretty' plot data time series as an unbroken line, retaining data gaps

        else:
            ltime = np.array([],dtype='float64')
            ldata = np.array([],dtype='float32')
            dt = 0
            work1 = 2.0 * cadence / 86400
            for i in range(1,len(data)-1):
                dt = barytime[i] - barytime[i-1]
                if dt < work1:
                    ltime = np.append(ltime,barytime[i])
                    ldata = np.append(ldata,data[i])
                else:
                    plt.plot(ltime,ldata,color=lcolor,linestyle='-',linewidth=lwidth)
                    ltime = np.array([],dtype='float64')
                    ldata = np.array([],dtype='float32')
            plt.plot(ltime,ldata,color=lcolor,linestyle='-',linewidth=lwidth)

# plot the fill color below data time series, with no data gaps

	plt.fill(barytime,data,fc=fcolor,linewidth=0.0,alpha=falpha)

# define plot x and y limits

	plt.xlim(xmin-xr*0.01,xmax+xr*0.01)
	if ymin-yr*0.01 <= 0.0 or fullrange:
            plt.ylim(1.0e-10,ymax+yr*0.01)
	else:
            plt.ylim(ymin-yr*0.01,ymax+yr*0.01)
        if chooserange:
            plt.ylim(y1,y2)

# plot labels

	plt.xlabel(xlab, {'color' : 'k'})
        try:
            plt.ylabel(ylab1, {'color' : 'k'})
        except:
            ylab1 = '10**%d e-/s' % nrm
            plt.ylabel(ylab1, {'color' : 'k'})

        ax.minorticks_on()
        ax.tick_params('both', length=20, width=2, which='major')
        ax.tick_params('both', length=10, width=1, which='minor')

# save plot to file

    if status == 0 and outfile.lower() != 'none':
	plt.savefig(outfile)

# render plot
        plt.ion()
        plt.show()
# end time

    if (status == 0):
        message = 'KEPDRAW completed at'
    else:
        message = '\nKEPDRAW aborted at'
    kepmsg.clock(message,logfile,verbose)

# -----------------------------------------------------------
# main

if '--shell' in sys.argv:
    import argparse

    parser = argparse.ArgumentParser(description='Interactive plotting of Kepler time series data')
    parser.add_argument('--shell', action='store_true', help='Are we running from the shell?')

    parser.add_argument('infile', help='Name of input file', type=str)
    parser.add_argument('outfile', help='name of output PNG file', type=str)
    parser.add_argument('--datacol', default='SAP_FLUX', help='Name of data column to plot', type=str)
    parser.add_argument('--ploterr', action='store_true', help='Plot data error bars?')
    parser.add_argument('--errcol', default='SAP_FLUX_ERR', help='Name of data error column', type=str)
    parser.add_argument('--quality', action='store_true', help='Ignore cadences where the data quality is questionable?')
    parser.add_argument('--lcolor', default='#0000ff', help='HTML color of data line within plot', type=str)
    parser.add_argument('--lwidth', default=1.0, help='type of image intensity scale', type=float)
    parser.add_argument('--fcolor', default='#ffff00', help='HTML color of data line within plot', type=str)
    parser.add_argument('--falpha', default=0.2, help='type of image intensity scale', type=float)
    parser.add_argument('--labelsize', default=24., help='Fontsize of axis labels', type=float)
    parser.add_argument('--ticksize', default=16., help='Fontsize of numeric tick labels', type=float)
    parser.add_argument('--xsize', default=18., help='X-dimension size of plot', type=float)
    parser.add_argument('--ysize', default=6., help='Y-dimension size of plot', type=float)
    parser.add_argument('--fullrange', action='store_true', help='Plot flux range from 0.0 e-/sec?')
    parser.add_argument('--chooserange', action='store_true', help='Choose Y-axis range?')
    parser.add_argument('--ymin', default=0., help='Low limit of the Y-axis range to plot [e-/s]', type=float)
    parser.add_argument('--ymax', default=100000., help='High limit of the Y-axis range to plot [e-/s]', type=float)
    parser.add_argument('--plotgrid', action='store_true', help='Plot axis grid?')
    parser.add_argument('--ylabel', default='e$^-$ s$^{-1}$', help='Plot axis label', type=str)
    parser.add_argument('--plottype', default='fast', help='plot type', type=str, choices=['fast','pretty'])
    parser.add_argument('--verbose', action='store_true', help='Write to a log file?')
    parser.add_argument('--logfile', '-l', help='Name of ascii log file', default='kepcotrend.log', dest='logfile', type=str)
    parser.add_argument('--status', '-e', help='Exit status (0=good)', default=0, dest='status', type=int)

    args = parser.parse_args()
    cmdLine=True
    kepdraw(args.infile,args.outfile,args.datacol,args.ploterr,args.errcol,args.quality,
            args.lcolor,args.lwidth,args.fcolor,args.falpha,args.labelsize,args.ticksize,
            args.xsize,args.ysize,args.fullrange,args.chooserange,args.ymin,args.ymax,
            args.plotgrid,args.ylabel,args.plottype,args.verbose,args.logfile,args.status,
            cmdLine)

else:
    from pyraf import iraf
    parfile = iraf.osfn("kepler$kepdraw.par")
    t = iraf.IrafTaskFactory(taskname="kepdraw", value=parfile, function=kepdraw)
<commit_msg>remove ploting style in kepdraw<commit_after>import numpy as np
import sys
from matplotlib import pyplot as plt
import kepio, kepmsg, kepkey
import re

__svnid__ = "$Id: kepdraw.py 6165 2014-03-26 21:16:27Z mstill $"
__url__ = "$URL: svn+ssh://mstill@murzim.amn.nasa.gov/data-repo/trunk/data/flight/go/PyKE/kepler/kepdraw.py $"

def kepdraw(infile,outfile,datacol,ploterr,errcol,quality,
	    lcolor,lwidth,fcolor,falpha,labelsize,ticksize,
	    xsize,ysize,fullrange,chooserange,y1,y2,plotgrid,
            ylabel,plottype,verbose,logfile,status,cmdLine=False):

# log the call

    hashline = '----------------------------------------------------------------------------'
    kepmsg.log(logfile,hashline,verbose)
    call = 'KEPDRAW -- '
    call += 'infile='+infile+' '
    call += 'outfile='+outfile+' '
    call += 'datacol='+datacol+' '
    perr = 'n'
    if (ploterr): perr = 'y'
    call += 'ploterr='+perr+ ' '
    call += 'errcol='+errcol+' '
    qual = 'n'
    if (quality): qual = 'y'
    call += 'quality='+qual+ ' '
    call += 'lcolor='+str(lcolor)+' '
    call += 'lwidth='+str(lwidth)+' '
    call += 'fcolor='+str(fcolor)+' '
    call += 'falpha='+str(falpha)+' '
    call += 'labelsize='+str(labelsize)+' '
    call += 'ticksize='+str(ticksize)+' '
    call += 'xsize='+str(xsize)+' '
    call += 'ysize='+str(ysize)+' '
    frange = 'n'
    if (fullrange): frange = 'y'
    call += 'fullrange='+frange+ ' '
    crange = 'n'
    if (chooserange): crange = 'y'
    call += 'chooserange='+crange+ ' '
    call += 'ymin='+str(y1)+' '
    call += 'ymax='+str(y2)+' '
    pgrid = 'n'
    if (plotgrid): pgrid = 'y'
    call += 'plotgrid='+pgrid+ ' '
    call += 'ylabel='+str(ylabel)+' '
    call += 'plottype='+plottype+' '
    chatter = 'n'
    if (verbose): chatter = 'y'
    call += 'verbose='+chatter+' '
    call += 'logfile='+logfile
    kepmsg.log(logfile,call+'\n',verbose)

# start time

    kepmsg.clock('KEPDRAW started at',logfile,verbose)

# test log file

    logfile = kepmsg.test(logfile)

# open input file

    if status == 0:
        struct, status = kepio.openfits(infile,'readonly',logfile,verbose)
    if status == 0:
        tstart, tstop, bjdref, cadence, status = kepio.timekeys(struct,infile,logfile,verbose,status)

# read table structure

    if status == 0:
	table, status = kepio.readfitstab(infile,struct[1],logfile,verbose)

# read table columns

    if status == 0:
        intime, status = kepio.readtimecol(infile,table,logfile,verbose)
        intime += bjdref
	indata, status = kepio.readfitscol(infile,table,datacol,logfile,verbose)
        indataerr, status = kepio.readfitscol(infile,table,errcol,logfile,verbose)
        qualty, status = kepio.readfitscol(infile,table,'SAP_QUALITY',logfile,verbose)

# close infile

    if status == 0:
	status = kepio.closefits(struct,logfile,verbose)

# remove infinities and bad data

    if status == 0:
        if np.isnan(np.nansum(indataerr)):
            indataerr[:] = 1.0e-5
        work1 = np.array([intime, indata, indataerr, qualty],dtype='float64')
        work1 = np.rot90(work1,3)
        work1 = work1[~np.isnan(work1).any(1)]
        work1 = work1[~np.isinf(work1).any(1)]
        if quality:
            work1 = work1[work1[:,0] == 0.0]
        barytime = np.array(work1[:,3],dtype='float64')
        data = np.array(work1[:,2],dtype='float32')
        dataerr = np.array(work1[:,1],dtype='float32')
        if len(barytime) == 0:
            message = 'ERROR -- KEPDRAW: Plotting arrays are full of NaN'
            status = kepmsg.err(logfile,message,verbose)

# clean up x-axis unit

    if status == 0:
	barytime0 = float(int(tstart / 100) * 100.0)
	barytime -= barytime0
        xlab = 'BJD $-$ %d' % barytime0

# clean up y-axis units

        nrm = 0
        try:
            nrm = len(str(int(np.nanmax(data))))-1
        except:
            nrm = 0
	data = data / 10**nrm
        if 'e$^-$ s$^{-1}$' in ylabel or 'default' in ylabel:
            if nrm == 0:
                ylab1 = 'e$^-$ s$^{-1}$'
            else:
                ylab1 = '10$^{%d}$ e$^-$ s$^{-1}$' % nrm
        else:
            ylab1 = re.sub('_','-',ylabel)

# data limits

	xmin = np.nanmin(barytime)
	xmax = np.nanmax(barytime)
	ymin = np.nanmin(data)
	ymax = np.nanmax(data)
	xr = xmax - xmin
	yr = ymax - ymin
        barytime = np.insert(barytime,[0],[barytime[0]])
        barytime = np.append(barytime,[barytime[-1]])
        data = np.insert(data,[0],[-10000.0])
        data = np.append(data,-10000.0)

# define size of plot on monitor screen

	plt.figure(figsize=[xsize,ysize])

# delete any fossil plots in the matplotlib window

        plt.clf()

# position axes inside the plotting window

#        ax = plt.axes([0.1,0.11,0.89,0.87])
	ax = plt.subplot(111)
	plt.subplots_adjust(0.06,0.15,0.92,0.83)

# force tick labels to be absolute rather than relative

        plt.gca().xaxis.set_major_formatter(plt.ScalarFormatter(useOffset=False))
        plt.gca().yaxis.set_major_formatter(plt.ScalarFormatter(useOffset=False))
        ax.yaxis.set_major_locator(plt.MaxNLocator(5))

# rotate y labels by 90 deg

        labels = ax.get_yticklabels()
        plt.setp(labels, 'rotation', 90, fontsize=ticksize)

# if plot type is 'fast' plot data time series as points

        if plottype == 'fast':
            plt.plot(barytime,data,'o',color=lcolor)

# if plot type is 'pretty' plot data time series as an unbroken line, retaining data gaps

        else:
            ltime = np.array([],dtype='float64')
            ldata = np.array([],dtype='float32')
            dt = 0
            work1 = 2.0 * cadence / 86400
            for i in range(1,len(data)-1):
                dt = barytime[i] - barytime[i-1]
                if dt < work1:
                    ltime = np.append(ltime,barytime[i])
                    ldata = np.append(ldata,data[i])
                else:
                    plt.plot(ltime,ldata,color=lcolor,linestyle='-',linewidth=lwidth)
                    ltime = np.array([],dtype='float64')
                    ldata = np.array([],dtype='float32')
            plt.plot(ltime,ldata,color=lcolor,linestyle='-',linewidth=lwidth)

# plot the fill color below data time series, with no data gaps

	plt.fill(barytime,data,fc=fcolor,linewidth=0.0,alpha=falpha)

# define plot x and y limits

	plt.xlim(xmin-xr*0.01,xmax+xr*0.01)
	if ymin-yr*0.01 <= 0.0 or fullrange:
            plt.ylim(1.0e-10,ymax+yr*0.01)
	else:
            plt.ylim(ymin-yr*0.01,ymax+yr*0.01)
        if chooserange:
            plt.ylim(y1,y2)

# plot labels

	plt.xlabel(xlab, {'color' : 'k'})
        try:
            plt.ylabel(ylab1, {'color' : 'k'})
        except:
            ylab1 = '10**%d e-/s' % nrm
            plt.ylabel(ylab1, {'color' : 'k'})

        ax.minorticks_on()
        ax.tick_params('both', length=20, width=2, which='major')
        ax.tick_params('both', length=10, width=1, which='minor')

# save plot to file

    if status == 0 and outfile.lower() != 'none':
	plt.savefig(outfile)

# render plot
        plt.ion()
        plt.show()
# end time

    if (status == 0):
        message = 'KEPDRAW completed at'
    else:
        message = '\nKEPDRAW aborted at'
    kepmsg.clock(message,logfile,verbose)

# -----------------------------------------------------------
# main

if '--shell' in sys.argv:
    import argparse

    parser = argparse.ArgumentParser(description='Interactive plotting of Kepler time series data')
    parser.add_argument('--shell', action='store_true', help='Are we running from the shell?')

    parser.add_argument('infile', help='Name of input file', type=str)
    parser.add_argument('outfile', help='name of output PNG file', type=str)
    parser.add_argument('--datacol', default='SAP_FLUX', help='Name of data column to plot', type=str)
    parser.add_argument('--ploterr', action='store_true', help='Plot data error bars?')
    parser.add_argument('--errcol', default='SAP_FLUX_ERR', help='Name of data error column', type=str)
    parser.add_argument('--quality', action='store_true', help='Ignore cadences where the data quality is questionable?')
    parser.add_argument('--lcolor', default='#0000ff', help='HTML color of data line within plot', type=str)
    parser.add_argument('--lwidth', default=1.0, help='type of image intensity scale', type=float)
    parser.add_argument('--fcolor', default='#ffff00', help='HTML color of data line within plot', type=str)
    parser.add_argument('--falpha', default=0.2, help='type of image intensity scale', type=float)
    parser.add_argument('--labelsize', default=24., help='Fontsize of axis labels', type=float)
    parser.add_argument('--ticksize', default=16., help='Fontsize of numeric tick labels', type=float)
    parser.add_argument('--xsize', default=18., help='X-dimension size of plot', type=float)
    parser.add_argument('--ysize', default=6., help='Y-dimension size of plot', type=float)
    parser.add_argument('--fullrange', action='store_true', help='Plot flux range from 0.0 e-/sec?')
    parser.add_argument('--chooserange', action='store_true', help='Choose Y-axis range?')
    parser.add_argument('--ymin', default=0., help='Low limit of the Y-axis range to plot [e-/s]', type=float)
    parser.add_argument('--ymax', default=100000., help='High limit of the Y-axis range to plot [e-/s]', type=float)
    parser.add_argument('--plotgrid', action='store_true', help='Plot axis grid?')
    parser.add_argument('--ylabel', default='e$^-$ s$^{-1}$', help='Plot axis label', type=str)
    parser.add_argument('--plottype', default='fast', help='plot type', type=str, choices=['fast','pretty'])
    parser.add_argument('--verbose', action='store_true', help='Write to a log file?')
    parser.add_argument('--logfile', '-l', help='Name of ascii log file', default='kepcotrend.log', dest='logfile', type=str)
    parser.add_argument('--status', '-e', help='Exit status (0=good)', default=0, dest='status', type=int)

    args = parser.parse_args()
    cmdLine=True
    kepdraw(args.infile,args.outfile,args.datacol,args.ploterr,args.errcol,args.quality,
            args.lcolor,args.lwidth,args.fcolor,args.falpha,args.labelsize,args.ticksize,
            args.xsize,args.ysize,args.fullrange,args.chooserange,args.ymin,args.ymax,
            args.plotgrid,args.ylabel,args.plottype,args.verbose,args.logfile,args.status,
            cmdLine)

else:
    from pyraf import iraf
    parfile = iraf.osfn("kepler$kepdraw.par")
    t = iraf.IrafTaskFactory(taskname="kepdraw", value=parfile, function=kepdraw)
<|endoftext|>


<commit_before><commit_msg>Convert additional targets to gyp generation: image_diff.exe, startup_tests.exe, page_cycler_tests.exe, plugin_tests.exe, security_tests.dll, test_chrome_plugin.dll, theme/default.dll. BUG=none TEST=none Review URL: http://codereview.chromium.org/118264<commit_after><|endoftext|>


<commit_before><commit_msg>Add convert_dict and flush_cache targets. Review URL: http://codereview.chromium.org/45050<commit_after><|endoftext|>


<commit_before>import os

import sql
import common
import utils

class ConfigDir:
    def __init__(self, config_path):
        self.config_path = os.path.abspath(config_path)
        if not os.path.isdir(self.config_path): os.makedirs(self.config_path)

        self.db_name = "dartui.db"
        self.db_path = os.path.join(self.config_path, self.db_name)

        self.refresh()

        # quick and dirty way to see if config file is using the old db system from v1.0.0
        # (this can be removed sometime in the distant future)
        db = self.get_db()
        tables = db._get_tables()
        if "settings" in tables and "table_versions" not in tables:
            print("old db style found, update settings table")
            update_settings_table = True
        else:
            update_settings_table = False

        self.table_versions = self.get_table_versions()

        # check if tables are up to date, and update them if not
        for table in sql.tables:
            if table != "table_versions" and table in self.table_versions: # already handled by get_table_versions
                db = self.get_db(sql.tables[table]) # this will create the table if it doesn't already exist
                if sql.tables[table].version < self.table_versions[table]: db.update_table_struct()
                elif table == "settings" and update_settings_table: db.update_table_struct()

    def get_table_versions(self):
        table_versions = {}
        db = self.get_db(sql.tables["table_versions"])
        db.insert_defaults() # insert new tables (if any)
        for row in db.get_table_contents():
            table_versions[row["name"]] = row["version"]
        return(table_versions)

    def has_db_conn(self):
        if self.db_conn is not None: return(True)
        else: return(False)

    def _set_default_values(self):
        self.settings = {}
        self.rt = None

    def update_settings(self, settings):
        settings["show_welcome"] = False # no need to show welcome screen after user has updated settings
        db = self.get_db(sql.tables["settings"])
        db.update_table_contents(settings)
        self.settings = db.get_table_contents()

    def refresh(self):
        self._set_default_values()

        db = self.get_db(sql.tables["settings"])
        self.settings = db.get_table_contents()

        self.get_rt_connection()

    def get_rt_url(self):
        self.rt_url = utils.build_url(
                        self.settings["host"],
                        self.settings["port"],
                        self.settings["username"],
                        self.settings["password"],
                        )

        return(self.rt_url)

    def get_rt_connection(self):
        self.rt = utils.get_rtorrent_connection(self.get_rt_url())
        self.tracker_cache = {}
        self.old_torrent_cache = []
        self.torrent_cache = []
        return(self.rt)

    def get_db(self, table_obj=None):
        return(sql.Database(self.db_path, table_obj))

    def get_rt(self):
        return(self.rt)<commit_msg>fix for CannotSendRequest and ResponseNotReady exceptions<commit_after>import os

import sql
import common
import utils

class ConfigDir:
    def __init__(self, config_path):
        self.config_path = os.path.abspath(config_path)
        if not os.path.isdir(self.config_path): os.makedirs(self.config_path)

        self.db_name = "dartui.db"
        self.db_path = os.path.join(self.config_path, self.db_name)

        self.refresh()

        # quick and dirty way to see if config file is using the old db system from v1.0.0
        # (this can be removed sometime in the distant future)
        db = self.get_db()
        tables = db._get_tables()
        if "settings" in tables and "table_versions" not in tables:
            print("old db style found, update settings table")
            update_settings_table = True
        else:
            update_settings_table = False

        self.table_versions = self.get_table_versions()

        # check if tables are up to date, and update them if not
        for table in sql.tables:
            if table != "table_versions" and table in self.table_versions: # already handled by get_table_versions
                db = self.get_db(sql.tables[table]) # this will create the table if it doesn't already exist
                if sql.tables[table].version < self.table_versions[table]: db.update_table_struct()
                elif table == "settings" and update_settings_table: db.update_table_struct()

    def get_table_versions(self):
        table_versions = {}
        db = self.get_db(sql.tables["table_versions"])
        db.insert_defaults() # insert new tables (if any)
        for row in db.get_table_contents():
            table_versions[row["name"]] = row["version"]
        return(table_versions)

    def has_db_conn(self):
        if self.db_conn is not None: return(True)
        else: return(False)

    def _set_default_values(self):
        self.settings = {}
        self.rt = None

    def update_settings(self, settings):
        settings["show_welcome"] = False # no need to show welcome screen after user has updated settings
        db = self.get_db(sql.tables["settings"])
        db.update_table_contents(settings)
        self.settings = db.get_table_contents()

    def refresh(self):
        self._set_default_values()

        db = self.get_db(sql.tables["settings"])
        self.settings = db.get_table_contents()

        self.get_rt_connection()

    def get_rt_url(self):
        self.rt_url = utils.build_url(
                        self.settings["host"],
                        self.settings["port"],
                        self.settings["username"],
                        self.settings["password"],
                        )

        return(self.rt_url)

    def get_rt_connection(self): # TODO: rename?
        self.rt = utils.get_rtorrent_connection(self.get_rt_url())
        self.tracker_cache = {}
        self.old_torrent_cache = []
        self.torrent_cache = []
        return(self.rt)

    def get_db(self, table_obj=None):
        return(sql.Database(self.db_path, table_obj))

    def get_rt(self):
        # httplib (via xmlrpclib) throws CannotSendRequest and ResponseNotReady exceptions
        # because HttpRequest objects shouldn't be shared as they in rtorrent-python
        # (see: http://stackoverflow.com/questions/3231543/python-httplib-responsenotready)
        # because of this, we'll have to work around that until rtorrent-python is fixed
        # by creating a new RTorrent object everytime
        return(utils.get_rtorrent_connection(self.get_rt_url()))<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from .__about__ import __version__
<commit_msg>Ignore missing import for __version__<commit_after># -*- coding: utf-8 -*-
from .__about__ import __version__  # NOQA: F40
<|endoftext|>


<commit_before>"""
Code shared by LocalCKAN, RemoteCKAN and TestCKAN
"""

import json

from ckanapi.errors import (CKANAPIError, NotAuthorized, NotFound,
    ValidationError, SearchQueryError, SearchError, SearchIndexError)

class ActionShortcut(object):
    """
    ActionShortcut(foo).bar(baz=2) <=> foo.call_action('bar', {'baz':2})

    An instance of this class is used as the .action attribute of
    LocalCKAN and RemoteCKAN instances to provide a short way to call
    actions, e.g::

        demo = RemoteCKAN('http://demo.ckan.org')
        pkg = demo.action.package_show(id='adur_district_spending')

    instead of::

        demo = RemoteCKAN('http://demo.ckan.org')
        pkg = demo.call_action('package_show', {'id':'adur_district_spending'})

    """
    def __init__(self, ckan):
        self._ckan = ckan

    def __getattr__(self, name):
        def action(apikey=None, **kwargs):
            return self._ckan.call_action(name, data_dict=kwargs,
                                          apikey=apikey)
        return action

def prepare_action(action, data_dict=None, apikey=None, files=None):
    """
    Return action_url, data_json, http_headers
    """
    if not data_dict:
        data_dict = {}
    headers = {}
    if not files:
        data_dict = json.dumps(data_dict).encode('ascii')
        headers['Content-Type'] = 'application/json'
    if apikey:
        apikey = str(apikey)
        headers['X-CKAN-API-Key'] = apikey
        headers['Authorization'] = apikey
    url = 'api/action/' + action
    return url, data_dict, headers

def reverse_apicontroller_action(url, status, response):
    """
    Make an API call look like a direct action call by reversing the
    exception -> HTTP response translation that ApiController.action does
    """
    try:
        parsed = json.loads(response)
        if parsed.get('success'):
            return parsed['result']
        if hasattr(parsed, 'get'):
            err = parsed.get('error', {})
        else:
            err = {}
    except ValueError:
        err = {}

    etype = err.get('__type')
    emessage = err.get('message', '').split(': ', 1)[-1]
    if etype == 'Search Query Error':
        # I refuse to eval(emessage), even if it would be more correct
        raise SearchQueryError(emessage)
    elif etype == 'Search Error':
        # I refuse to eval(emessage), even if it would be more correct
        raise SearchError(emessage)
    elif etype == 'Search Index Error':
        raise SearchIndexError(emessage)
    elif etype == 'Validation Error':
        raise ValidationError(err)
    elif etype == 'Not Found Error':
        raise NotFound(emessage)
    elif etype == 'Authorization Error':
        raise NotAuthorized(err)

    # don't recognize the error
    raise CKANAPIError(url, status, response)
<commit_msg>handle non-json object responses<commit_after>"""
Code shared by LocalCKAN, RemoteCKAN and TestCKAN
"""

import json

from ckanapi.errors import (CKANAPIError, NotAuthorized, NotFound,
    ValidationError, SearchQueryError, SearchError, SearchIndexError)

class ActionShortcut(object):
    """
    ActionShortcut(foo).bar(baz=2) <=> foo.call_action('bar', {'baz':2})

    An instance of this class is used as the .action attribute of
    LocalCKAN and RemoteCKAN instances to provide a short way to call
    actions, e.g::

        demo = RemoteCKAN('http://demo.ckan.org')
        pkg = demo.action.package_show(id='adur_district_spending')

    instead of::

        demo = RemoteCKAN('http://demo.ckan.org')
        pkg = demo.call_action('package_show', {'id':'adur_district_spending'})

    """
    def __init__(self, ckan):
        self._ckan = ckan

    def __getattr__(self, name):
        def action(apikey=None, **kwargs):
            return self._ckan.call_action(name, data_dict=kwargs,
                                          apikey=apikey)
        return action

def prepare_action(action, data_dict=None, apikey=None, files=None):
    """
    Return action_url, data_json, http_headers
    """
    if not data_dict:
        data_dict = {}
    headers = {}
    if not files:
        data_dict = json.dumps(data_dict).encode('ascii')
        headers['Content-Type'] = 'application/json'
    if apikey:
        apikey = str(apikey)
        headers['X-CKAN-API-Key'] = apikey
        headers['Authorization'] = apikey
    url = 'api/action/' + action
    return url, data_dict, headers

def reverse_apicontroller_action(url, status, response):
    """
    Make an API call look like a direct action call by reversing the
    exception -> HTTP response translation that ApiController.action does
    """
    try:
        parsed = json.loads(response)
        if parsed.get('success'):
            return parsed['result']
        if hasattr(parsed, 'get'):
            err = parsed.get('error', {})
        else:
            err = {}
    except (AttributeError, ValueError):
        err = {}

    etype = err.get('__type')
    emessage = err.get('message', '').split(': ', 1)[-1]
    if etype == 'Search Query Error':
        # I refuse to eval(emessage), even if it would be more correct
        raise SearchQueryError(emessage)
    elif etype == 'Search Error':
        # I refuse to eval(emessage), even if it would be more correct
        raise SearchError(emessage)
    elif etype == 'Search Index Error':
        raise SearchIndexError(emessage)
    elif etype == 'Validation Error':
        raise ValidationError(err)
    elif etype == 'Not Found Error':
        raise NotFound(emessage)
    elif etype == 'Authorization Error':
        raise NotAuthorized(err)

    # don't recognize the error
    raise CKANAPIError(url, status, response)
<|endoftext|>


<commit_before>from itertools import count
from .compatibility import zip_longest
from .core import (istask, get_dependencies, subs, toposort, flatten,
                   reverse_dict, add, inc, ishashable, preorder_traversal)
from .rewrite import END
from toolz import identity

def cull(dsk, keys):
    """ Return new dask with only the tasks required to calculate keys.

    In other words, remove unnecessary tasks from dask.
    ``keys`` may be a single key or list of keys.

    Examples
    --------

    >>> d = {'x': 1, 'y': (inc, 'x'), 'out': (add, 'x', 10)}
    >>> cull(d, 'out')  # doctest: +SKIP
    {'x': 1, 'out': (add, 'x', 10)}
    """
    if not isinstance(keys, list):
        keys = [keys]
    nxt = set(flatten(keys))
    seen = nxt
    while nxt:
        cur = nxt
        nxt = set()
        for item in cur:
            for dep in get_dependencies(dsk, item):
                if dep not in seen:
                    nxt.add(dep)
        seen.update(nxt)
    return dict((k, v) for k, v in dsk.items() if k in seen)

def fuse(dsk, keys=None):
    """ Return new dask with linear sequence of tasks fused together.

    If specified, the keys in ``keys`` keyword argument are *not* fused.

    This may be used as an optimization step.

    Examples
    --------

    >>> d = {'a': 1, 'b': (inc, 'a'), 'c': (inc, 'b')}
    >>> fuse(d)  # doctest: +SKIP
    {'c': (inc, (inc, 1))}
    >>> fuse(d, keys=['b'])  # doctest: +SKIP
    {'b': (inc, 1), 'c': (inc, 'b')}
    """
    if isinstance(keys, (set, list)):
        keys = set(keys)
    elif keys is not None:
        keys = set([keys])
    # locate all members of linear chains
    parents = {}
    deadbeats = set()
    for parent in dsk:
        deps = get_dependencies(dsk, parent, as_list=True)
        for child in deps:
            if child in parents:
                del parents[child]
                deadbeats.add(child)
            elif len(deps) > 1 or (keys is not None and child in keys):
                deadbeats.add(child)
            elif child not in deadbeats:
                parents[child] = parent

    # construct the chains from ancestor to descendant
    chains = []
    children = dict(map(reversed, parents.items()))
    while parents:
        child, parent = parents.popitem()
        chain = [child, parent]
        while parent in parents:
            parent = parents.pop(parent)
            del children[parent]
            chain.append(parent)
        chain.reverse()
        while child in children:
            child = children.pop(child)
            del parents[child]
            chain.append(child)
        chains.append(chain)

    # create a new dask with fused chains
    rv = {}
    fused = set()
    for chain in chains:
        child = chain.pop()
        val = dsk[child]
        while chain:
            parent = chain.pop()
            val = subs(dsk[parent], child, val)
            fused.add(child)
            child = parent
        fused.add(child)
        rv[child] = val

    for key, val in dsk.items():
        if key not in fused:
            rv[key] = val
    return rv

def inline(dsk, keys=None, inline_constants=True):
    """ Return new dask with the given keys inlined with their values.

    Inlines all constants if ``inline_constants`` keyword is True.

    Examples
    --------

    >>> d = {'x': 1, 'y': (inc, 'x'), 'z': (add, 'x', 'y')}
    >>> inline(d)  # doctest: +SKIP
    {'y': (inc, 1), 'z': (add, 1, 'y')}

    >>> inline(d, keys='y')  # doctest: +SKIP
    {'z': (add, 1, (inc, 1))}

    >>> inline(d, keys='y', inline_constants=False)  # doctest: +SKIP
    {'x': 1, 'z': (add, 'x', (inc, 'x'))}
    """
    if keys is None:
        keys  = set()
    elif isinstance(keys, (set, list)):
        keys = set(keys)
    else:
        keys = set([keys])
    if inline_constants:
        keys.update(k for k, v in dsk.items() if not istask(v))

    # Keys may depend on other keys, so determine replace order with toposort.
    # The values stored in `keysubs` do not include other keys.
    replaceorder = toposort(dict((k, dsk[k]) for k in keys if k in dsk))
    keysubs = {}
    for key in replaceorder:
        val = dsk[key]
        for dep in keys & get_dependencies(dsk, key):
            if dep in keysubs:
                replace = keysubs[dep]
            else:
                replace = dsk[dep]
            val = subs(val, dep, replace)
        keysubs[key] = val

    # Make new dask with substitutions
    rv = {}
    for key, val in dsk.items():
        if key in keys:
            continue
        for item in keys & get_dependencies(dsk, key):
            val = subs(val, item, keysubs[item])
        rv[key] = val
    return rv

def inline_functions(dsk, fast_functions=None, inline_constants=False):
    """ Inline cheap functions into larger operations

    >>> dsk = {'out': (add, 'i', 'd'),  # doctest: +SKIP
    ...        'i': (inc, 'x'),
    ...        'd': (double, 'y'),
    ...        'x': 1, 'y': 1}
    >>> inline_functions(dsk, [inc])  # doctest: +SKIP
    {'out': (add, (inc, 'x'), 'd'),
     'd': (double, 'y'),
     'x': 1, 'y': 1}
    """
    if not fast_functions:
        return dsk
    fast_functions = set(fast_functions)

    dependencies = dict((k, get_dependencies(dsk, k)) for k in dsk)
    dependents = reverse_dict(dependencies)

    keys = [k for k, v in dsk.items()
              if istask(v)
              and functions_of(v).issubset(fast_functions)
              and dependents[k]]
    if keys:
        return inline(dsk, keys, inline_constants=inline_constants)
    else:
        return dsk

def functions_of(task):
    """ Set of functions contained within nested task

    >>> task = (add, (mul, 1, 2), (inc, 3))  # doctest: +SKIP
    >>> functions_of(task)  # doctest: +SKIP
    set([add, mul, inc])
    """
    result = set()
    if istask(task):
        args = set.union(*map(functions_of, task[1:])) if task[1:] else set()
        return set([unwrap_partial(task[0])]) | args
    if isinstance(task, (list, tuple)):
        if not task:
            return set()
        return set.union(*map(functions_of, task))
    return set()

def unwrap_partial(func):
    while hasattr(func, 'func'):
        func = func.func
    return func

def dealias(dsk):
    """ Remove aliases from dask

    Removes and renames aliases using ``inline``.  Keeps aliases at the top of
    the DAG to ensure entry points stay the same.

    Aliases are not expected by schedulers.  It's unclear that this is a legal
    state.

    Examples
    --------

    >>> dsk = {'a': (range, 5),
    ...        'b': 'a',
    ...        'c': 'b',
    ...        'd': (sum, 'c'),
    ...        'e': 'd',
    ...        'f': (inc, 'd')}

    >>> dealias(dsk)  # doctest: +SKIP
    {'a': (range, 5),
     'd': (sum, 'a'),
     'e': (identity, 'd'),
     'f': (inc, 'd')}
    """
    dependencies = dict((k, get_dependencies(dsk, k)) for k in dsk)
    dependents = reverse_dict(dependencies)

    aliases = set((k for k, task in dsk.items() if ishashable(task) and task in dsk))
    roots = set((k for k, v in dependents.items() if not v))

    dsk2 = inline(dsk, aliases - roots, inline_constants=False)
    dsk3 = dsk2.copy()

    dependencies = dict((k, get_dependencies(dsk2, k)) for k in dsk2)
    dependents = reverse_dict(dependencies)

    for k in roots & aliases:
        k2 = dsk3[k]
        if len(dependents[k2]) == 1:
            dsk3[k] = dsk3[k2]
            del dsk3[k2]
        else:
            dsk3[k] = (identity, k2)
    return dsk3

def equivalent(term1, term2, subs=None):
    """Determine if two terms are equivalent, modulo variable substitution.

    Equivalent to applying substitutions in `subs` to `term2`, then checking if
    `term1 == term2`.

    If a subterm doesn't support comparison (i.e. `term1 == term2` errors),
    returns `False`.

    Parameters
    ----------
    term1, term2 : terms
    subs : dict, optional
        Mapping of substitutions from `term2` to `term1`

    Examples
    --------
    >>> from operator import add
    >>> term1 = (add, 'a', 'b')
    >>> term2 = (add, 'x', 'y')
    >>> subs = {'x': 'a', 'y': 'b'}
    >>> equivalent(term1, term2, subs)
    True
    >>> subs = {'x': 'a'}
    >>> equivalent(term1, term2, subs)
    False
    """

    # Quick escape for special cases
    head_type = type(term1)
    if type(term2) != head_type:
        # If terms aren't same type, fail
        return False
    elif head_type not in (tuple, list):
        # For literals, just compare
        try:
            # `is` is tried first, to allow objects that don't implement `==`
            # to work for cases where term1 is term2. If `is` returns False,
            # and `==` errors, then the only thing we can do is return False.
            return term1 is term2 or term1 == term2
        except:
            return False

    pot1 = preorder_traversal(term1)
    pot2 = preorder_traversal(term2)
    subs = {} if subs is None else subs

    for t1, t2 in zip_longest(pot1, pot2, fillvalue=END):
        if t1 is END or t2 is END:
            # If terms aren't same length: fail
            return False
        elif ishashable(t2) and t2 in subs:
            val = subs[t2]
        else:
            val = t2
        try:
            if t1 is not t2 and t1 != val:
                return False
        except:
            return False
    return True

def dependency_dict(dsk):
    """Create a dict matching ordered dependencies to keys.

    Examples
    --------
    >>> from operator import add
    >>> dsk = {'a': 1, 'b': 2, 'c': (add, 'a', 'a'), 'd': (add, 'b', 'a')}
    >>> dependency_dict(dsk)    # doctest: +SKIP
    {(): ['a', 'b'], ('a', 'a'): ['c'], ('b', 'a'): ['d']}
    """

    dep_dict = {}
    for key in dsk:
        deps = tuple(get_dependencies(dsk, key, True))
        dep_dict.setdefault(deps, []).append(key)
    return dep_dict

def _possible_matches(dep_dict, deps, subs):
    deps2 = []
    for d in deps:
        v = subs.get(d, None)
        if v is not None:
            deps2.append(v)
        else:
            return []
    deps2 = tuple(deps2)
    return dep_dict.get(deps2, [])

def _sync_keys(dsk1, dsk2, dsk2_topo):
    dep_dict1 = dependency_dict(dsk1)
    subs = {}

    for key2 in toposort(dsk2):
        deps = tuple(get_dependencies(dsk2, key2, True))
        # List of keys in dsk1 that have terms that *may* match key2
        possible_matches = _possible_matches(dep_dict1, deps, subs)
        if possible_matches:
            val2 = dsk2[key2]
            for key1 in possible_matches:
                val1 = dsk1[key1]
                if equivalent(val1, val2, subs):
                    subs[key2] = key1
                    break
    return subs

def sync_keys(dsk1, dsk2):
    """Return a dict matching keys in `dsk2` to equivalent keys in `dsk1`.

    Parameters
    ----------
    dsk1, dsk2 : dict

    Examples
    --------
    >>> from operator import add, mul
    >>> dsk1 = {'a': 1, 'b': (add, 'a', 10), 'c': (mul, 'b', 5)}
    >>> dsk2 = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)}
    >>> sync_keys(dsk1, dsk2)   # doctest: +SKIP
    {'x': 'a', 'y': 'b'}
    """

    return _sync_keys(dsk1, dsk2, toposort(dsk2))

def merge_sync(dsk1, dsk2):
    """Merge two dasks together, combining equivalent tasks.

    If a task in `dsk2` exists in `dsk1`, the task and key from `dsk1` is used.
    If a task in `dsk2` has the same key as a task in `dsk1` (and they aren't
    equivalent tasks), then a new key is created for the task in `dsk2`. This
    prevents name conflicts.

    Parameters
    ----------
    dsk1, dsk2 : dict
        Variable names in `dsk2` are replaced with equivalent ones in `dsk1`
        before merging.

    Returns
    -------
    new_dsk : dict
        The merged dask.
    key_map : dict
        A mapping between the keys from `dsk2` to their new names in `new_dsk`.

    Examples
    -------
    >>> from operator import add, mul
    >>> dsk1 = {'a': 1, 'b': (add, 'a', 10), 'c': (mul, 'b', 5)}
    >>> dsk2 = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)}
    >>> new_dsk, key_map = merge_sync(dsk1, dsk2)
    >>> new_dsk     # doctest: +SKIP
    {'a': 1, 'b': (add, 'a', 10), 'c': (mul, 'b', 5), 'z': (mul, 'b', 2)}
    >>> key_map     # doctest: +SKIP
    {'x': 'a', 'y': 'b', 'z': 'z'}

    Conflicting names are replaced with auto-generated names upon merging.

    >>> dsk1 = {'a': 1, 'res': (add, 'a', 1)}
    >>> dsk2 = {'x': 1, 'res': (add, 'x', 2)}
    >>> new_dsk, key_map = merge_sync(dsk1, dsk2)
    >>> new_dsk     # doctest: +SKIP
    {'a': 1, 'res': (add, 'a', 1), 'merge_1': (add, 'a', 2)}
    >>> key_map     # doctest: +SKIP
    {'x': 'a', 'res': 'merge_1'}
    """

    dsk2_topo = toposort(dsk2)
    sd = _sync_keys(dsk1, dsk2, dsk2_topo)
    new_dsk = dsk1.copy()
    for key in dsk2_topo:
        if key in sd:
            new_key = sd[key]
        else:
            if key in dsk1:
                new_key = next(merge_sync.names)
            else:
                new_key = key
            sd[key] = new_key
        task = dsk2[key]
        for a, b in sd.items():
            task = subs(task, a, b)
        new_dsk[new_key] = task
    return new_dsk, sd

# store the name iterator in the function
merge_sync.names = ('merge_%d' % i for i in count(1))
<commit_msg>More consistent API of dask.optimize functions that accept `keys` argument.<commit_after>from itertools import count
from .compatibility import zip_longest
from .core import (istask, get_dependencies, subs, toposort, flatten,
                   reverse_dict, add, inc, ishashable, preorder_traversal)
from .rewrite import END
from toolz import identity

def cull(dsk, keys):
    """ Return new dask with only the tasks required to calculate keys.

    In other words, remove unnecessary tasks from dask.
    ``keys`` may be a single key or list of keys.

    Examples
    --------

    >>> d = {'x': 1, 'y': (inc, 'x'), 'out': (add, 'x', 10)}
    >>> cull(d, 'out')  # doctest: +SKIP
    {'x': 1, 'out': (add, 'x', 10)}
    """
    if not isinstance(keys, (list, set)):
        keys = [keys]
    nxt = set(flatten(keys))
    seen = nxt
    while nxt:
        cur = nxt
        nxt = set()
        for item in cur:
            for dep in get_dependencies(dsk, item):
                if dep not in seen:
                    nxt.add(dep)
        seen.update(nxt)
    return dict((k, v) for k, v in dsk.items() if k in seen)

def fuse(dsk, keys=None):
    """ Return new dask with linear sequence of tasks fused together.

    If specified, the keys in ``keys`` keyword argument are *not* fused.

    This may be used as an optimization step.

    Examples
    --------

    >>> d = {'a': 1, 'b': (inc, 'a'), 'c': (inc, 'b')}
    >>> fuse(d)  # doctest: +SKIP
    {'c': (inc, (inc, 1))}
    >>> fuse(d, keys=['b'])  # doctest: +SKIP
    {'b': (inc, 1), 'c': (inc, 'b')}
    """
    if keys is not None and not isinstance(keys, set):
        if not isinstance(keys, list):
            keys = [keys]
        keys = set(flatten(keys))

    # locate all members of linear chains
    child2parent = {}
    unfusible = set()
    for parent in dsk:
        deps = get_dependencies(dsk, parent, as_list=True)
        has_many_children = len(deps) > 1
        for child in deps:
            if keys is not None and child in keys:
                unfusible.add(child)
            elif child in child2parent:
                del child2parent[child]
                unfusible.add(child)
            elif has_many_children:
                unfusible.add(child)
            elif child not in unfusible:
                child2parent[child] = parent

    # construct the chains from ancestor to descendant
    chains = []
    parent2child = dict(map(reversed, child2parent.items()))
    while child2parent:
        child, parent = child2parent.popitem()
        chain = [child, parent]
        while parent in child2parent:
            parent = child2parent.pop(parent)
            del parent2child[parent]
            chain.append(parent)
        chain.reverse()
        while child in parent2child:
            child = parent2child.pop(child)
            del child2parent[child]
            chain.append(child)
        chains.append(chain)

    # create a new dask with fused chains
    rv = {}
    fused = set()
    for chain in chains:
        child = chain.pop()
        val = dsk[child]
        while chain:
            parent = chain.pop()
            val = subs(dsk[parent], child, val)
            fused.add(child)
            child = parent
        fused.add(child)
        rv[child] = val

    for key, val in dsk.items():
        if key not in fused:
            rv[key] = val
    return rv

def inline(dsk, keys=None, inline_constants=True):
    """ Return new dask with the given keys inlined with their values.

    Inlines all constants if ``inline_constants`` keyword is True.

    Examples
    --------

    >>> d = {'x': 1, 'y': (inc, 'x'), 'z': (add, 'x', 'y')}
    >>> inline(d)  # doctest: +SKIP
    {'y': (inc, 1), 'z': (add, 1, 'y')}

    >>> inline(d, keys='y')  # doctest: +SKIP
    {'z': (add, 1, (inc, 1))}

    >>> inline(d, keys='y', inline_constants=False)  # doctest: +SKIP
    {'x': 1, 'z': (add, 'x', (inc, 'x'))}
    """
    if keys is None:
        keys = []
    elif not isinstance(keys, (list, set)):
        keys = [keys]
    keys = set(flatten(keys))

    if inline_constants:
        keys.update(k for k, v in dsk.items() if not istask(v))

    # Keys may depend on other keys, so determine replace order with toposort.
    # The values stored in `keysubs` do not include other keys.
    replaceorder = toposort(dict((k, dsk[k]) for k in keys if k in dsk))
    keysubs = {}
    for key in replaceorder:
        val = dsk[key]
        for dep in keys & get_dependencies(dsk, key):
            if dep in keysubs:
                replace = keysubs[dep]
            else:
                replace = dsk[dep]
            val = subs(val, dep, replace)
        keysubs[key] = val

    # Make new dask with substitutions
    rv = {}
    for key, val in dsk.items():
        if key in keys:
            continue
        for item in keys & get_dependencies(dsk, key):
            val = subs(val, item, keysubs[item])
        rv[key] = val
    return rv

def inline_functions(dsk, fast_functions=None, inline_constants=False):
    """ Inline cheap functions into larger operations

    >>> dsk = {'out': (add, 'i', 'd'),  # doctest: +SKIP
    ...        'i': (inc, 'x'),
    ...        'd': (double, 'y'),
    ...        'x': 1, 'y': 1}
    >>> inline_functions(dsk, [inc])  # doctest: +SKIP
    {'out': (add, (inc, 'x'), 'd'),
     'd': (double, 'y'),
     'x': 1, 'y': 1}
    """
    if not fast_functions:
        return dsk
    fast_functions = set(fast_functions)

    dependencies = dict((k, get_dependencies(dsk, k)) for k in dsk)
    dependents = reverse_dict(dependencies)

    keys = [k for k, v in dsk.items()
              if istask(v)
              and functions_of(v).issubset(fast_functions)
              and dependents[k]]
    if keys:
        return inline(dsk, keys, inline_constants=inline_constants)
    else:
        return dsk

def functions_of(task):
    """ Set of functions contained within nested task

    >>> task = (add, (mul, 1, 2), (inc, 3))  # doctest: +SKIP
    >>> functions_of(task)  # doctest: +SKIP
    set([add, mul, inc])
    """
    result = set()
    if istask(task):
        args = set.union(*map(functions_of, task[1:])) if task[1:] else set()
        return set([unwrap_partial(task[0])]) | args
    if isinstance(task, (list, tuple)):
        if not task:
            return set()
        return set.union(*map(functions_of, task))
    return set()

def unwrap_partial(func):
    while hasattr(func, 'func'):
        func = func.func
    return func

def dealias(dsk):
    """ Remove aliases from dask

    Removes and renames aliases using ``inline``.  Keeps aliases at the top of
    the DAG to ensure entry points stay the same.

    Aliases are not expected by schedulers.  It's unclear that this is a legal
    state.

    Examples
    --------

    >>> dsk = {'a': (range, 5),
    ...        'b': 'a',
    ...        'c': 'b',
    ...        'd': (sum, 'c'),
    ...        'e': 'd',
    ...        'f': (inc, 'd')}

    >>> dealias(dsk)  # doctest: +SKIP
    {'a': (range, 5),
     'd': (sum, 'a'),
     'e': (identity, 'd'),
     'f': (inc, 'd')}
    """
    dependencies = dict((k, get_dependencies(dsk, k)) for k in dsk)
    dependents = reverse_dict(dependencies)

    aliases = set((k for k, task in dsk.items() if ishashable(task) and task in dsk))
    roots = set((k for k, v in dependents.items() if not v))

    dsk2 = inline(dsk, aliases - roots, inline_constants=False)
    dsk3 = dsk2.copy()

    dependencies = dict((k, get_dependencies(dsk2, k)) for k in dsk2)
    dependents = reverse_dict(dependencies)

    for k in roots & aliases:
        k2 = dsk3[k]
        if len(dependents[k2]) == 1:
            dsk3[k] = dsk3[k2]
            del dsk3[k2]
        else:
            dsk3[k] = (identity, k2)
    return dsk3

def equivalent(term1, term2, subs=None):
    """Determine if two terms are equivalent, modulo variable substitution.

    Equivalent to applying substitutions in `subs` to `term2`, then checking if
    `term1 == term2`.

    If a subterm doesn't support comparison (i.e. `term1 == term2` errors),
    returns `False`.

    Parameters
    ----------
    term1, term2 : terms
    subs : dict, optional
        Mapping of substitutions from `term2` to `term1`

    Examples
    --------
    >>> from operator import add
    >>> term1 = (add, 'a', 'b')
    >>> term2 = (add, 'x', 'y')
    >>> subs = {'x': 'a', 'y': 'b'}
    >>> equivalent(term1, term2, subs)
    True
    >>> subs = {'x': 'a'}
    >>> equivalent(term1, term2, subs)
    False
    """

    # Quick escape for special cases
    head_type = type(term1)
    if type(term2) != head_type:
        # If terms aren't same type, fail
        return False
    elif head_type not in (tuple, list):
        # For literals, just compare
        try:
            # `is` is tried first, to allow objects that don't implement `==`
            # to work for cases where term1 is term2. If `is` returns False,
            # and `==` errors, then the only thing we can do is return False.
            return term1 is term2 or term1 == term2
        except:
            return False

    pot1 = preorder_traversal(term1)
    pot2 = preorder_traversal(term2)
    subs = {} if subs is None else subs

    for t1, t2 in zip_longest(pot1, pot2, fillvalue=END):
        if t1 is END or t2 is END:
            # If terms aren't same length: fail
            return False
        elif ishashable(t2) and t2 in subs:
            val = subs[t2]
        else:
            val = t2
        try:
            if t1 is not t2 and t1 != val:
                return False
        except:
            return False
    return True

def dependency_dict(dsk):
    """Create a dict matching ordered dependencies to keys.

    Examples
    --------
    >>> from operator import add
    >>> dsk = {'a': 1, 'b': 2, 'c': (add, 'a', 'a'), 'd': (add, 'b', 'a')}
    >>> dependency_dict(dsk)    # doctest: +SKIP
    {(): ['a', 'b'], ('a', 'a'): ['c'], ('b', 'a'): ['d']}
    """

    dep_dict = {}
    for key in dsk:
        deps = tuple(get_dependencies(dsk, key, True))
        dep_dict.setdefault(deps, []).append(key)
    return dep_dict

def _possible_matches(dep_dict, deps, subs):
    deps2 = []
    for d in deps:
        v = subs.get(d, None)
        if v is not None:
            deps2.append(v)
        else:
            return []
    deps2 = tuple(deps2)
    return dep_dict.get(deps2, [])

def _sync_keys(dsk1, dsk2, dsk2_topo):
    dep_dict1 = dependency_dict(dsk1)
    subs = {}

    for key2 in toposort(dsk2):
        deps = tuple(get_dependencies(dsk2, key2, True))
        # List of keys in dsk1 that have terms that *may* match key2
        possible_matches = _possible_matches(dep_dict1, deps, subs)
        if possible_matches:
            val2 = dsk2[key2]
            for key1 in possible_matches:
                val1 = dsk1[key1]
                if equivalent(val1, val2, subs):
                    subs[key2] = key1
                    break
    return subs

def sync_keys(dsk1, dsk2):
    """Return a dict matching keys in `dsk2` to equivalent keys in `dsk1`.

    Parameters
    ----------
    dsk1, dsk2 : dict

    Examples
    --------
    >>> from operator import add, mul
    >>> dsk1 = {'a': 1, 'b': (add, 'a', 10), 'c': (mul, 'b', 5)}
    >>> dsk2 = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)}
    >>> sync_keys(dsk1, dsk2)   # doctest: +SKIP
    {'x': 'a', 'y': 'b'}
    """

    return _sync_keys(dsk1, dsk2, toposort(dsk2))

def merge_sync(dsk1, dsk2):
    """Merge two dasks together, combining equivalent tasks.

    If a task in `dsk2` exists in `dsk1`, the task and key from `dsk1` is used.
    If a task in `dsk2` has the same key as a task in `dsk1` (and they aren't
    equivalent tasks), then a new key is created for the task in `dsk2`. This
    prevents name conflicts.

    Parameters
    ----------
    dsk1, dsk2 : dict
        Variable names in `dsk2` are replaced with equivalent ones in `dsk1`
        before merging.

    Returns
    -------
    new_dsk : dict
        The merged dask.
    key_map : dict
        A mapping between the keys from `dsk2` to their new names in `new_dsk`.

    Examples
    -------
    >>> from operator import add, mul
    >>> dsk1 = {'a': 1, 'b': (add, 'a', 10), 'c': (mul, 'b', 5)}
    >>> dsk2 = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)}
    >>> new_dsk, key_map = merge_sync(dsk1, dsk2)
    >>> new_dsk     # doctest: +SKIP
    {'a': 1, 'b': (add, 'a', 10), 'c': (mul, 'b', 5), 'z': (mul, 'b', 2)}
    >>> key_map     # doctest: +SKIP
    {'x': 'a', 'y': 'b', 'z': 'z'}

    Conflicting names are replaced with auto-generated names upon merging.

    >>> dsk1 = {'a': 1, 'res': (add, 'a', 1)}
    >>> dsk2 = {'x': 1, 'res': (add, 'x', 2)}
    >>> new_dsk, key_map = merge_sync(dsk1, dsk2)
    >>> new_dsk     # doctest: +SKIP
    {'a': 1, 'res': (add, 'a', 1), 'merge_1': (add, 'a', 2)}
    >>> key_map     # doctest: +SKIP
    {'x': 'a', 'res': 'merge_1'}
    """

    dsk2_topo = toposort(dsk2)
    sd = _sync_keys(dsk1, dsk2, dsk2_topo)
    new_dsk = dsk1.copy()
    for key in dsk2_topo:
        if key in sd:
            new_key = sd[key]
        else:
            if key in dsk1:
                new_key = next(merge_sync.names)
            else:
                new_key = key
            sd[key] = new_key
        task = dsk2[key]
        for a, b in sd.items():
            task = subs(task, a, b)
        new_dsk[new_key] = task
    return new_dsk, sd

# store the name iterator in the function
merge_sync.names = ('merge_%d' % i for i in count(1))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"CLA Public app"

import logging
import logging.config
import os
from flask import Flask, render_template
from flask.ext.babel import Babel
from flask.ext.cache import Cache
from flask.ext.mail import Mail
from raven.contrib.flask import Sentry
import urllib3.contrib.pyopenssl

from cla_public.django_to_jinja import change_jinja_templates
from cla_public.apps.addressfinder_proxy.views import addressfinder
from cla_public.apps.base.views import base
from cla_public.apps.contact.views import contact
from cla_public.apps.checker.views import checker
from cla_public.apps.scope.urls import scope
from cla_public.apps.checker.session import CheckerSessionInterface, \
    CustomJSONEncoder
from cla_public.middleware import StatsdMiddleware
from cla_public.libs import honeypot
from cla_public.libs.utils import get_locale

def create_app(config_file=None):
    app = Flask(__name__)
    app = change_jinja_templates(app)
    if config_file:
        app.config.from_pyfile(config_file)
    else:
        app.config.from_envvar('CLA_PUBLIC_CONFIG')

    if app.config.get('SENTRY_DSN'):
        app.sentry = Sentry(
            app,
            dsn=app.config.get('SENTRY_DSN'),
            logging=True,
            level=logging.ERROR
        )

    app.babel = Babel(app)
    app.babel.localeselector(get_locale)

    app.cache = Cache(app)

    app.mail = Mail(app)

    for extension in app.config['EXTENSIONS']:
        extension.init_app(app)

    app.session_interface = CheckerSessionInterface()
    app.json_encoder = CustomJSONEncoder

    register_error_handlers(app)

    app.add_template_global(
        honeypot.FIELD_NAME,
        name='honeypot_field_name')

    app.register_blueprint(base)
    app.register_blueprint(addressfinder)
    app.register_blueprint(contact)
    app.register_blueprint(scope)
    if not app.config.get('CONTACT_ONLY'):
        app.register_blueprint(checker)

    logging.config.dictConfig(app.config['LOGGING'])
    # quiet markdown module
    logging.getLogger('MARKDOWN').setLevel(logging.WARNING)

    app.wsgi_app = StatsdMiddleware(app.wsgi_app, app.config)

    if app.debug:
        from werkzeug.debug import DebuggedApplication
        app.wsgi_app = DebuggedApplication(app.wsgi_app, True)

    return app

def register_error_handlers(app):
    """
    Assign error page templates to all error status codes we care about
    """

    error_handlers = {
        '404.html': [404],
        '4xx.html': [401, 402, 405, 406, 407, 408, 409],
        '5xx.html': [500, 501, 502, 503, 504, 505]}

    def make_handler(code, template):

        def handler(e):
            return render_template(os.path.join('errors', template)), code

        return handler

    for template, codes in error_handlers.iteritems():
        for code in codes:
            app.register_error_handler(code, make_handler(code, template))

    return app

urllib3.contrib.pyopenssl.inject_into_urllib3()
<commit_msg>BE: Pass response error code to template<commit_after># -*- coding: utf-8 -*-
"CLA Public app"

import logging
import logging.config
import os
from flask import Flask, render_template
from flask.ext.babel import Babel
from flask.ext.cache import Cache
from flask.ext.mail import Mail
from raven.contrib.flask import Sentry
import urllib3.contrib.pyopenssl

from cla_public.django_to_jinja import change_jinja_templates
from cla_public.apps.addressfinder_proxy.views import addressfinder
from cla_public.apps.base.views import base
from cla_public.apps.contact.views import contact
from cla_public.apps.checker.views import checker
from cla_public.apps.scope.urls import scope
from cla_public.apps.checker.session import CheckerSessionInterface, \
    CustomJSONEncoder
from cla_public.middleware import StatsdMiddleware
from cla_public.libs import honeypot
from cla_public.libs.utils import get_locale

def create_app(config_file=None):
    app = Flask(__name__)
    app = change_jinja_templates(app)
    if config_file:
        app.config.from_pyfile(config_file)
    else:
        app.config.from_envvar('CLA_PUBLIC_CONFIG')

    if app.config.get('SENTRY_DSN'):
        app.sentry = Sentry(
            app,
            dsn=app.config.get('SENTRY_DSN'),
            logging=True,
            level=logging.ERROR
        )

    app.babel = Babel(app)
    app.babel.localeselector(get_locale)

    app.cache = Cache(app)

    app.mail = Mail(app)

    for extension in app.config['EXTENSIONS']:
        extension.init_app(app)

    app.session_interface = CheckerSessionInterface()
    app.json_encoder = CustomJSONEncoder

    register_error_handlers(app)

    app.add_template_global(
        honeypot.FIELD_NAME,
        name='honeypot_field_name')

    app.register_blueprint(base)
    app.register_blueprint(addressfinder)
    app.register_blueprint(contact)
    app.register_blueprint(scope)
    if not app.config.get('CONTACT_ONLY'):
        app.register_blueprint(checker)

    logging.config.dictConfig(app.config['LOGGING'])
    # quiet markdown module
    logging.getLogger('MARKDOWN').setLevel(logging.WARNING)

    app.wsgi_app = StatsdMiddleware(app.wsgi_app, app.config)

    if app.debug:
        from werkzeug.debug import DebuggedApplication
        app.wsgi_app = DebuggedApplication(app.wsgi_app, True)

    return app

def register_error_handlers(app):
    """
    Assign error page templates to all error status codes we care about
    """

    error_handlers = {
        '404.html': [404],
        '4xx.html': [401, 402, 405, 406, 407, 408, 409],
        '5xx.html': [500, 501, 502, 503, 504, 505]}

    def make_handler(code, template):

        def handler(e):
            return render_template(os.path.join('errors', template), code=code), code

        return handler

    for template, codes in error_handlers.iteritems():
        for code in codes:
            app.register_error_handler(code, make_handler(code, template))

    return app

urllib3.contrib.pyopenssl.inject_into_urllib3()
<|endoftext|>