<commit_before>import logging
LOGFMT = '%(asctime)s %(name)-30s %(levelname)-8s %(message).320s'
logging.basicConfig(level = logging.INFO,
                    format = LOGFMT)
log = logging.getLogger()
import datetime
from dateutil.relativedelta import *
import requests
import json
import pytz
import urlparse
import os
import os.path
import socket
import sys

server_url = 'http://amm-csr2:4242/api'

def convert_time(ts):
    if isinstance(ts, datetime.date):
        ts = datetime.datetime(ts.year, ts.month, ts.day)
    return int((ts - datetime.datetime(1970,1,1) +
                datetime.timedelta(hours=4)).total_seconds())

def unconvert_time(uxtime):
    return datetime.datetime(1970,1,1,tzinfo=pytz.timezone('US/Eastern')) + \
            datetime.timedelta(seconds=uxtime) - \
            datetime.timedelta(hours=4)

def get_tags_from_metric(m):
    r = requests.get("{}/search/lookup?m={}".format(server_url, m))

    if r.status_code == requests.codes.ok:
        rsp = json.loads(r.text)
        log.debug(rsp)
        if len(rsp['results']) == 0: # metric has no tags
            return []
        else:
            tags = []
            for result in rsp['results']:
                tags += result['']
    else:
        log.error("Error processing request")
        log.error(r.text)

# this gets all the metrics:
#r = requests.get("{}/suggest?type=metrics&q=&max=999999".format(server_url))

def get_day(day,
            span = datetime.timedelta(days=1),
            group = None,
            outdir = './archive',
            downsample = None):
    assert group in [None, 'daily', 'monthly']

    # construct output filename, to see if we already have the file
    if not os.path.exists(outdir):
        os.mkdir(outdir)

    if downsample:
        resolution = downsample
    else:
        resolution = "raw"

    if group:
        full_path = os.path.join(outdir,
                                 group,
                                 str(day.year),
                                 resolution)
    else:
        full_path = os.path.join(outdir,
                                 str(day.year),
                                 resolution)
    if not os.path.exists(full_path):
        os.makedirs(full_path)

    if group == None or group == group == 'daily':
        outputfilename = os.path.join(full_path, '{}-data-{}.csv'.format(day.isoformat(), resolution))
    elif group == 'monthly':
        outputfilename = os.path.join(full_path, '{}-data-{}.csv'.format(day.strftime("%Y-%B"), resolution))

    if os.path.exists(outputfilename):
        logging.info("Skipping, file already exists: {}".format(outputfilename))
        return

    queries = [{
                'metric': 'Table',
                'aggregator': 'avg',
                'downsample': downsample,
                'tags': {'sensor_name': x}
                } for x in ['AirTemp', 'RelHum', 'WaterLevel', 'WindDir', 'WindSpeed', 'Solar_kW', 'BattV', 'WaterTemp', 'PTemp_C', 'Rain_Tot', 'Solar_MJ_Tot']]
    queries += [{
                'metric': 'CR800_1',
                'aggregator': 'avg',
                'downsample': downsample,
                'tags': {'sensor_name': x}
                } for x in ['WaterLevel', 'WaterTemp', 'PTemp_C', 'BattV']]

    queries += [{
                'metric': 'CR800_2',
                'aggregator': 'avg',
                'downsample': downsample,
                'tags': {'sensor_name': x}
                } for x in ['WaterLevel', 'WaterTemp', 'PTemp_C', 'BattV']]

    queries += [{
                'metric': 'lwbp1',
                'aggregator': 'avg',
                'downsample': downsample,
                }]
    begin = day
    end = begin + span

    if end >= datetime.datetime.now().date():
        log.warn("End time is in the future, skipping!")
        return

    query = {"start": convert_time(begin),
             "end": convert_time(end),
             "queries": queries}
    url = "{}/{}".format(server_url, "query")

    log.info("requesting: {}, data: {}".format(url,
                                               json.dumps(query)))
    r = requests.post(url,
                      data=json.dumps(query),
                      timeout = 60*60) # set timeout to 1 hr

    rename = {"Table": "WeatherStation"}

    if r.status_code == requests.codes.ok:

        text = r.text
        rslt = json.loads(text)

        rows = {} ## key is the timestamp
        #cols= ['Timestamp']
        cols = []

        for res in rslt:

            if 'sensor_name' in res['tags']:
                m = res['metric']
                if m in rename:
                    m = rename[m]

                name = "{}.{} ({})".format(m, res['tags']['sensor_name'], res['tags']['sensor_units'])
            elif res['metric'] == 'lwbp1':
                name = "{} ({})".format(res['metric'], res['tags']['units'])

            cols.append(name)

        #log.info(cols)

        for res in rslt:

            if 'sensor_name' in res['tags']:
                m = res['metric']
                if m in rename:
                    m = rename[m]

                name = "{}.{} ({})".format(m, res['tags']['sensor_name'], res['tags']['sensor_units'])
            elif res['metric'] == 'lwbp1':
                name = "{} ({})".format(res['metric'], res['tags']['units'])

            if "dps" in res:
                k = list(res["dps"].keys())
                k.sort()
                for i in k:

                    ts = unconvert_time(int(i))
                    if ts not in rows:
                        rows[ts] = {}
                    rows[ts][name] = res["dps"][i]

        times = rows.keys()
        times.sort()

        with open(outputfilename, 'wb') as f:

            line = ", ".join(['Timestamp'] + cols)
            f.write(line)
            f.write("\n")
            #print(line)
            logging.debug(line)
            for t in times:
                row_array = [t.isoformat()]
                for colname in cols:
                    if colname in rows[t]:
                        row_array += ["{}".format(rows[t][colname])]
                    else:
                        row_array += [""]
                line = ", ".join(row_array)
                f.write(line)
                f.write("\n")
                #print(line)
                logging.debug(line)

    else:

        log.error("error in request")
        log.error(r.text)

if __name__ == "__main__":
    # full download
    begin = datetime.date(2014, 8, 1)

    step = datetime.timedelta(days=1)
    end = datetime.date.today()

    resolutions = [None, '5m-avg', '15m-avg', '60m-avg']
    groups = [('daily', relativedelta(days=1)),
              ('monthly', relativedelta(months=1))]

    if len(sys.argv) == 2 and os.path.exists(sys.argv[1]):
        outdir = sys.argv[1]
    else:
        outdir = './archive'

    at = begin
    while at < end:
        for resolution in resolutions:
            for group, span in groups:

                # only do one monthly a month!
                if group == 'monthly' and at.day > 1:
                    continue

                logging.info("Downloading {} data from: {} to {} with resolution = {}".format(group,
                                                                                at.isoformat(),
                                                                                (at + span).isoformat(),
                                                                                resolution))
                get_day(at,
                        span = span,
                        group = group,
                        outdir = outdir,
                        downsample = resolution)

        at += step

<commit_msg>changed warning to info about future dates<commit_after>import logging
LOGFMT = '%(asctime)s %(name)-30s %(levelname)-8s %(message).320s'
logging.basicConfig(level = logging.INFO,
                    format = LOGFMT)
log = logging.getLogger()
import datetime
from dateutil.relativedelta import *
import requests
import json
import pytz
import urlparse
import os
import os.path
import socket
import sys

server_url = 'http://amm-csr2:4242/api'

def convert_time(ts):
    if isinstance(ts, datetime.date):
        ts = datetime.datetime(ts.year, ts.month, ts.day)
    return int((ts - datetime.datetime(1970,1,1) +
                datetime.timedelta(hours=4)).total_seconds())

def unconvert_time(uxtime):
    return datetime.datetime(1970,1,1,tzinfo=pytz.timezone('US/Eastern')) + \
            datetime.timedelta(seconds=uxtime) - \
            datetime.timedelta(hours=4)

def get_tags_from_metric(m):
    r = requests.get("{}/search/lookup?m={}".format(server_url, m))

    if r.status_code == requests.codes.ok:
        rsp = json.loads(r.text)
        log.debug(rsp)
        if len(rsp['results']) == 0: # metric has no tags
            return []
        else:
            tags = []
            for result in rsp['results']:
                tags += result['']
    else:
        log.error("Error processing request")
        log.error(r.text)

# this gets all the metrics:
#r = requests.get("{}/suggest?type=metrics&q=&max=999999".format(server_url))

def get_day(day,
            span = datetime.timedelta(days=1),
            group = None,
            outdir = './archive',
            downsample = None):
    assert group in [None, 'daily', 'monthly']

    # construct output filename, to see if we already have the file
    if not os.path.exists(outdir):
        os.mkdir(outdir)

    if downsample:
        resolution = downsample
    else:
        resolution = "raw"

    if group:
        full_path = os.path.join(outdir,
                                 group,
                                 str(day.year),
                                 resolution)
    else:
        full_path = os.path.join(outdir,
                                 str(day.year),
                                 resolution)
    if not os.path.exists(full_path):
        os.makedirs(full_path)

    if group == None or group == group == 'daily':
        outputfilename = os.path.join(full_path, '{}-data-{}.csv'.format(day.isoformat(), resolution))
    elif group == 'monthly':
        outputfilename = os.path.join(full_path, '{}-data-{}.csv'.format(day.strftime("%Y-%B"), resolution))

    if os.path.exists(outputfilename):
        logging.info("Skipping, file already exists: {}".format(outputfilename))
        return

    queries = [{
                'metric': 'Table',
                'aggregator': 'avg',
                'downsample': downsample,
                'tags': {'sensor_name': x}
                } for x in ['AirTemp', 'RelHum', 'WaterLevel', 'WindDir', 'WindSpeed', 'Solar_kW', 'BattV', 'WaterTemp', 'PTemp_C', 'Rain_Tot', 'Solar_MJ_Tot']]
    queries += [{
                'metric': 'CR800_1',
                'aggregator': 'avg',
                'downsample': downsample,
                'tags': {'sensor_name': x}
                } for x in ['WaterLevel', 'WaterTemp', 'PTemp_C', 'BattV']]

    queries += [{
                'metric': 'CR800_2',
                'aggregator': 'avg',
                'downsample': downsample,
                'tags': {'sensor_name': x}
                } for x in ['WaterLevel', 'WaterTemp', 'PTemp_C', 'BattV']]

    queries += [{
                'metric': 'lwbp1',
                'aggregator': 'avg',
                'downsample': downsample,
                }]
    begin = day
    end = begin + span

    if end >= datetime.datetime.now().date():
        log.info("End time is in the future, skipping!")
        return

    query = {"start": convert_time(begin),
             "end": convert_time(end),
             "queries": queries}
    url = "{}/{}".format(server_url, "query")

    log.info("requesting: {}, data: {}".format(url,
                                               json.dumps(query)))
    r = requests.post(url,
                      data=json.dumps(query),
                      timeout = 60*60) # set timeout to 1 hr

    rename = {"Table": "WeatherStation"}

    if r.status_code == requests.codes.ok:

        text = r.text
        rslt = json.loads(text)

        rows = {} ## key is the timestamp
        #cols= ['Timestamp']
        cols = []

        for res in rslt:

            if 'sensor_name' in res['tags']:
                m = res['metric']
                if m in rename:
                    m = rename[m]

                name = "{}.{} ({})".format(m, res['tags']['sensor_name'], res['tags']['sensor_units'])
            elif res['metric'] == 'lwbp1':
                name = "{} ({})".format(res['metric'], res['tags']['units'])

            cols.append(name)

        #log.info(cols)

        for res in rslt:

            if 'sensor_name' in res['tags']:
                m = res['metric']
                if m in rename:
                    m = rename[m]

                name = "{}.{} ({})".format(m, res['tags']['sensor_name'], res['tags']['sensor_units'])
            elif res['metric'] == 'lwbp1':
                name = "{} ({})".format(res['metric'], res['tags']['units'])

            if "dps" in res:
                k = list(res["dps"].keys())
                k.sort()
                for i in k:

                    ts = unconvert_time(int(i))
                    if ts not in rows:
                        rows[ts] = {}
                    rows[ts][name] = res["dps"][i]

        times = rows.keys()
        times.sort()

        with open(outputfilename, 'wb') as f:

            line = ", ".join(['Timestamp'] + cols)
            f.write(line)
            f.write("\n")
            #print(line)
            logging.debug(line)
            for t in times:
                row_array = [t.isoformat()]
                for colname in cols:
                    if colname in rows[t]:
                        row_array += ["{}".format(rows[t][colname])]
                    else:
                        row_array += [""]
                line = ", ".join(row_array)
                f.write(line)
                f.write("\n")
                #print(line)
                logging.debug(line)

    else:

        log.error("error in request")
        log.error(r.text)

if __name__ == "__main__":
    # full download
    begin = datetime.date(2014, 8, 1)

    step = datetime.timedelta(days=1)
    end = datetime.date.today()

    resolutions = [None, '5m-avg', '15m-avg', '60m-avg']
    groups = [('daily', relativedelta(days=1)),
              ('monthly', relativedelta(months=1))]

    if len(sys.argv) == 2 and os.path.exists(sys.argv[1]):
        outdir = sys.argv[1]
    else:
        outdir = './archive'

    at = begin
    while at < end:
        for resolution in resolutions:
            for group, span in groups:

                # only do one monthly a month!
                if group == 'monthly' and at.day > 1:
                    continue

                logging.info("Downloading {} data from: {} to {} with resolution = {}".format(group,
                                                                                at.isoformat(),
                                                                                (at + span).isoformat(),
                                                                                resolution))
                get_day(at,
                        span = span,
                        group = group,
                        outdir = outdir,
                        downsample = resolution)

        at += step

<|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

import frappe, json
from frappe.model.meta import is_single
from frappe.modules import load_doctype_module
import frappe.desk.form.meta
import frappe.desk.form.load
from six import string_types

@frappe.whitelist()
def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
	if isinstance(linkinfo, string_types):
		# additional fields are added in linkinfo
		linkinfo = json.loads(linkinfo)

	results = {}

	if not linkinfo:
		return results

	if for_doctype:
		links = frappe.get_doc(doctype, name).get_link_filters(for_doctype)

		if links:
			linkinfo = links

		if for_doctype in linkinfo:
			# only get linked with for this particular doctype
			linkinfo = { for_doctype: linkinfo.get(for_doctype) }
		else:
			return results

	me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)

	for dt, link in linkinfo.items():
		link["doctype"] = dt
		link_meta_bundle = frappe.desk.form.load.get_meta_bundle(dt)
		linkmeta = link_meta_bundle[0]
		if not linkmeta.get("issingle"):
			fields = [d.fieldname for d in linkmeta.get("fields", {"in_list_view":1,
				"fieldtype": ["not in", ["Image", "HTML", "Button", "Table"]]})] \
				+ ["name", "modified", "docstatus"]

			if link.get("add_fields"):
				fields += link["add_fields"]

			fields = ["`tab{dt}`.`{fn}`".format(dt=dt, fn=sf.strip()) for sf in fields if sf
				and "`tab" not in sf]

			try:
				if link.get("filters"):
					ret = frappe.get_list(doctype=dt, fields=fields, filters=link.get("filters"))

				elif link.get("get_parent"):
					if me and me.parent and me.parenttype == dt:
						ret = frappe.get_list(doctype=dt, fields=fields,
							filters=[[dt, "name", '=', me.parent]])
					else:
						ret = None

				elif link.get("child_doctype"):
					filters = [[link.get('child_doctype'), link.get("fieldname"), '=', name]]

					# dynamic link
					if link.get("doctype_fieldname"):
						filters.append([link.get('child_doctype'), link.get("doctype_fieldname"), "=", doctype])

					ret = frappe.get_list(doctype=dt, fields=fields, filters=filters, distinct=True)

				else:
					if link.get("fieldname"):
						filters = [[dt, link.get("fieldname"), '=', name]]
						# dynamic link
						if link.get("doctype_fieldname"):
							filters.append([dt, link.get("doctype_fieldname"), "=", doctype])
						ret = frappe.get_list(doctype=dt, fields=fields, filters=filters)

					else:
						ret = None

			except frappe.PermissionError:
				if frappe.local.message_log:
					frappe.local.message_log.pop()

				continue

			if ret:
				results[dt] = ret

	return results

@frappe.whitelist()
def get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False):
	"""add list of doctypes this doctype is 'linked' with.

	Example, for Customer:

		{"Address": {"fieldname": "customer"}..}
	"""
	if(without_ignore_user_permissions_enabled):
		return frappe.cache().hget("linked_doctypes_without_ignore_user_permissions_enabled",
			doctype, lambda: _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled))
	else:
		return frappe.cache().hget("linked_doctypes", doctype, lambda: _get_linked_doctypes(doctype))

def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False):
	ret = {}
	# find fields where this doctype is linked
	ret.update(get_linked_fields(doctype, without_ignore_user_permissions_enabled))
	ret.update(get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled))

	filters=[['fieldtype','=','Table'], ['options', '=', doctype]]
	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
	# find links of parents
	links = frappe.get_all("DocField", fields=["parent as dt"], filters=filters)
	links+= frappe.get_all("Custom Field", fields=["dt"], filters=filters)

	for dt, in links:
		if dt in ret: continue
		ret[dt] = {"get_parent": True}

	for dt in list(ret):
		try:
			doctype_module = load_doctype_module(dt)
		except ImportError:
			# in case of Custom DocType
			continue

		if getattr(doctype_module, "exclude_from_linked_with", False):
			del ret[dt]

	return ret

def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False):

	filters=[['fieldtype','=', 'Link'], ['options', '=', doctype]]
	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])

	# find links of parents
	links = frappe.get_all("DocField", fields=["parent", "fieldname"], filters=filters, as_list=1)
	links+= frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1)

	links = dict(links)

	ret = {}

	if not links: return ret

	for doctype_name in links:
		ret[doctype_name] = { "fieldname": links.get(doctype_name) }
	table_doctypes = frappe.get_all("DocType", filters=[["istable", "=", "1"], ["name", "in",  tuple(links)]])
	child_filters = [['fieldtype','=', 'Table'], ['options', 'in', tuple(doctype.name for doctype in table_doctypes)]]
	if without_ignore_user_permissions_enabled: child_filters.append(['ignore_user_permissions', '!=', 1])

	# find out if linked in a child table
	for parent, options in frappe.get_all("DocField", fields=["parent", "options"], filters=child_filters, as_list=1):
		ret[parent] = { "child_doctype": options, "fieldname": links[options]}
		if options in ret: del ret[options]

	return ret

def get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
	ret = {}

	filters=[['fieldtype','=', 'Dynamic Link']]
	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])

	# find dynamic links of parents
	links = frappe.get_all("DocField", fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
	links+= frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)

	for df in links:
		if is_single(df.doctype): continue

		# optimized to get both link exists and parenttype
		possible_link = frappe.db.sql("""select distinct `{doctype_fieldname}`, parenttype
			from `tab{doctype}` where `{doctype_fieldname}`=%s""".format(**df), doctype, as_dict=True)

		if not possible_link: continue

		for d in possible_link:
			# is child
			if d.parenttype:
				ret[d.parenttype] = {
					"child_doctype": df.doctype,
					"fieldname": df.fieldname,
					"doctype_fieldname": df.doctype_fieldname
				}
			else:
				ret[df.doctype] = {
					"fieldname": df.fieldname,
					"doctype_fieldname": df.doctype_fieldname
				}

	return ret<commit_msg>[fix] linked with if there are multiple link field in the same doctype (#5587)<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

import frappe, json
from frappe.model.meta import is_single
from frappe.modules import load_doctype_module
import frappe.desk.form.meta
import frappe.desk.form.load
from six import string_types
from collections import defaultdict

@frappe.whitelist()
def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
	if isinstance(linkinfo, string_types):
		# additional fields are added in linkinfo
		linkinfo = json.loads(linkinfo)

	results = {}

	if not linkinfo:
		return results

	if for_doctype:
		links = frappe.get_doc(doctype, name).get_link_filters(for_doctype)

		if links:
			linkinfo = links

		if for_doctype in linkinfo:
			# only get linked with for this particular doctype
			linkinfo = { for_doctype: linkinfo.get(for_doctype) }
		else:
			return results

	me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)

	for dt, link in linkinfo.items():
		filters = []
		link["doctype"] = dt
		link_meta_bundle = frappe.desk.form.load.get_meta_bundle(dt)
		linkmeta = link_meta_bundle[0]
		if not linkmeta.get("issingle"):
			fields = [d.fieldname for d in linkmeta.get("fields", {"in_list_view":1,
				"fieldtype": ["not in", ["Image", "HTML", "Button", "Table"]]})] \
				+ ["name", "modified", "docstatus"]

			if link.get("add_fields"):
				fields += link["add_fields"]

			fields = ["`tab{dt}`.`{fn}`".format(dt=dt, fn=sf.strip()) for sf in fields if sf
				and "`tab" not in sf]

			try:
				if link.get("filters"):
					ret = frappe.get_list(doctype=dt, fields=fields, filters=link.get("filters"))

				elif link.get("get_parent"):
					if me and me.parent and me.parenttype == dt:
						ret = frappe.get_list(doctype=dt, fields=fields,
							filters=[[dt, "name", '=', me.parent]])
					else:
						ret = None

				elif link.get("child_doctype"):
					or_filters = [[link.get('child_doctype'), link_fieldnames, '=', name] for link_fieldnames in link.get("fieldname")]

					# dynamic link
					if link.get("doctype_fieldname"):
						filters.append([link.get('child_doctype'), link.get("doctype_fieldname"), "=", doctype])

					ret = frappe.get_list(doctype=dt, fields=fields, filters=filters, or_filters=or_filters, distinct=True)

				else:
					link_fieldnames = link.get("fieldname")
					if link_fieldnames:
						if isinstance(link_fieldnames, string_types): link_fieldnames = [link_fieldnames]
						or_filters = [[dt, fieldname, '=', name] for fieldname in link_fieldnames]
						# dynamic link
						if link.get("doctype_fieldname"):
							filters.append([dt, link.get("doctype_fieldname"), "=", doctype])
						ret = frappe.get_list(doctype=dt, fields=fields, filters=filters, or_filters=or_filters)

					else:
						ret = None

			except frappe.PermissionError:
				if frappe.local.message_log:
					frappe.local.message_log.pop()

				continue

			if ret:
				results[dt] = ret

	return results

@frappe.whitelist()
def get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False):
	"""add list of doctypes this doctype is 'linked' with.

	Example, for Customer:

		{"Address": {"fieldname": "customer"}..}
	"""
	if(without_ignore_user_permissions_enabled):
		return frappe.cache().hget("linked_doctypes_without_ignore_user_permissions_enabled",
			doctype, lambda: _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled))
	else:
		return frappe.cache().hget("linked_doctypes", doctype, lambda: _get_linked_doctypes(doctype))

def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False):
	ret = {}
	# find fields where this doctype is linked
	ret.update(get_linked_fields(doctype, without_ignore_user_permissions_enabled))
	ret.update(get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled))

	filters=[['fieldtype','=','Table'], ['options', '=', doctype]]
	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
	# find links of parents
	links = frappe.get_all("DocField", fields=["parent as dt"], filters=filters)
	links+= frappe.get_all("Custom Field", fields=["dt"], filters=filters)

	for dt, in links:
		if dt in ret: continue
		ret[dt] = {"get_parent": True}

	for dt in list(ret):
		try:
			doctype_module = load_doctype_module(dt)
		except ImportError:
			# in case of Custom DocType
			continue

		if getattr(doctype_module, "exclude_from_linked_with", False):
			del ret[dt]

	return ret

def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False):

	filters=[['fieldtype','=', 'Link'], ['options', '=', doctype]]
	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])

	# find links of parents
	links = frappe.get_all("DocField", fields=["parent", "fieldname"], filters=filters, as_list=1)
	links+= frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1)

	ret = {}

	if not links: return ret

	links_dict = defaultdict(list)
	for doctype, fieldname in links:
		links_dict[doctype].append(fieldname)

	for doctype_name in links_dict:
		ret[doctype_name] = { "fieldname": links_dict.get(doctype_name) }
	table_doctypes = frappe.get_all("DocType", filters=[["istable", "=", "1"], ["name", "in", tuple(links_dict)]])
	child_filters = [['fieldtype','=', 'Table'], ['options', 'in', tuple(doctype.name for doctype in table_doctypes)]]
	if without_ignore_user_permissions_enabled: child_filters.append(['ignore_user_permissions', '!=', 1])

	# find out if linked in a child table
	for parent, options in frappe.get_all("DocField", fields=["parent", "options"], filters=child_filters, as_list=1):
		ret[parent] = { "child_doctype": options, "fieldname": links_dict[options]}
		if options in ret: del ret[options]

	return ret

def get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
	ret = {}

	filters=[['fieldtype','=', 'Dynamic Link']]
	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])

	# find dynamic links of parents
	links = frappe.get_all("DocField", fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
	links+= frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)

	for df in links:
		if is_single(df.doctype): continue

		# optimized to get both link exists and parenttype
		possible_link = frappe.db.sql("""select distinct `{doctype_fieldname}`, parenttype
			from `tab{doctype}` where `{doctype_fieldname}`=%s""".format(**df), doctype, as_dict=True)

		if not possible_link: continue

		for d in possible_link:
			# is child
			if d.parenttype:
				ret[d.parenttype] = {
					"child_doctype": df.doctype,
					"fieldname": [df.fieldname],
					"doctype_fieldname": df.doctype_fieldname
				}
			else:
				ret[df.doctype] = {
					"fieldname": [df.fieldname],
					"doctype_fieldname": df.doctype_fieldname
				}

	return ret<|endoftext|>


<commit_before>import unittest

from hamcrest import assert_that, equal_to, is_not

from google.appengine.ext import testbed
from guestbook import get_user_url

class GetUserUrlTestCase(unittest.TestCase):

    def setUp(self):
        # First, create an instance of the Testbed class.
        self.testbed = testbed.Testbed()
        # Then activate the testbed, which prepares the service stubs for use.
        self.testbed.activate()
        # Next, declare which service stubs you want to use.
        self.testbed.init_user_stub()
        # Set up expected values
        self.user_login_url = {
            'url': 'https://www.google.com/accounts/Login?continue=http%3A//testbed.example.com/',
            'url_linktext': 'Login'
        }

    def tearDown(self):
        self.testbed.deactivate()

    def setCurrentUser(self, email, user_id, is_admin=False):
        email_to_set = email or ''
        id_to_set = user_id or ''
        admin_to_set = '1' if is_admin else '0'
        self.testbed.setup_env(USER_EMAIL=email_to_set, overwrite=True)
        self.testbed.setup_env(USER_ID=id_to_set, overwrite=True)
        self.testbed.setup_env(USER_IS_ADMIN=admin_to_set, overwrite=True)

    def testNoLoginUser(self):
        self.setCurrentUser(None, None)
        result = get_user_url('/')
        assert_that(result, equal_to(self.user_login_url))
        assert_that(result['url_linktext'], is_not('Logout'))

    # TODO: test login with a user
<commit_msg>finish get_user_url test<commit_after>import unittest

from hamcrest import assert_that, equal_to, is_not

from google.appengine.ext import testbed
from guestbook import get_user_url

class GetUserUrlTestCase(unittest.TestCase):

    def setUp(self):
        # First, create an instance of the Testbed class.
        self.testbed = testbed.Testbed()
        # Then activate the testbed, which prepares the service stubs for use.
        self.testbed.activate()
        # Next, declare which service stubs you want to use.
        self.testbed.init_user_stub()
        # Set up expected values
        self.user_login_url = {
            'url': 'https://www.google.com/accounts/Login?continue=http%3A//testbed.example.com/',
            'url_linktext': 'Login'
        }
        self.normal_user_logout_url = {
            'url': 'https://www.google.com/accounts/Logout?continue=http%3A//testbed.example.com/',
            'url_linktext': 'Logout'
        }
        self.admin_user_logout_url = {
            'url': 'https://www.google.com/accounts/Logout?continue=http%3A//testbed.example.com/',
            'url_linktext': 'Logout'
        }

    def tearDown(self):
        self.testbed.deactivate()

    def set_current_user(self, email, user_id, is_admin=False):
        email_to_set = email or ''
        id_to_set = user_id or ''
        admin_to_set = '1' if is_admin else '0'
        self.testbed.setup_env(USER_EMAIL=email_to_set, overwrite=True)
        self.testbed.setup_env(USER_ID=id_to_set, overwrite=True)
        self.testbed.setup_env(USER_IS_ADMIN=admin_to_set, overwrite=True)

    def test_no_login_user(self):
        self.set_current_user(None, None)
        result = get_user_url('/')
        assert_that(result, equal_to(self.user_login_url))
        assert_that(result['url_linktext'], is_not('Logout'))

    def test_normal_login_user(self):
        self.set_current_user('test@test.com', 'test')
        result = get_user_url('/')
        assert_that(result, equal_to(self.normal_user_logout_url))
        assert_that(result['url_linktext'], is_not('Login'))

    def test_admin_login_user(self):
        self.set_current_user('admin@test.com', 'admin', True)
        result = get_user_url('/')
        assert_that(result, equal_to(self.admin_user_logout_url))
        assert_that(result['url_linktext'], is_not('Login'))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import time
import argparse
import httplib2
from config import *
from apiclient.discovery import build
from apiclient.errors import HttpError
from oauth2client.client import flow_from_clientsecrets
from oauth2client.file import Storage
from oauth2client.tools import argparser, run_flow

# Creates the resource object for interacting with the YouTube API
# Sets up OAuth 2.0 for authorized requests if necessary
def create_resource_obj():
    global youtube

    if (args.id or args.username):
        youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
            developerKey=DEVELOPER_KEY)
    else:
        flow = flow_from_clientsecrets(CLIENT_SECRETS_FILE,
            message=MISSING_CLIENT_SECRETS_MESSAGE,
            scope=YOUTUBE_READONLY_SCOPE)

        storage = Storage("%s-oauth2.json" % sys.argv[0])
        credentials = storage.get()

        if credentials is None or credentials.invalid:
            credentials = run_flow(flow, storage, args)

        youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
            http=credentials.authorize(httplib2.Http()))

# Creates the initial playlists request used in backup_playlists()
# Checks the commandline arguments and assembles the correct request
def setup_request():
    if (args.id):
        request = youtube.playlists().list(
            part="id,snippet",
            fields="items(id,snippet/title),nextPageToken,pageInfo/totalResults",
            channelId=args.id,
            maxResults=50
        )
    elif (args.username):
        # Create channel request to obtain channel id from YouTube username
        channel_request = youtube.channels().list(
            part="id",
            forUsername=args.username,
            maxResults=50
        )

        channel_response = channel_request.execute()

        try:
            request = youtube.playlists().list(
                part="id,snippet",
                fields="items(id,snippet/title),nextPageToken,pageInfo/totalResults",
                channelId=channel_response["items"][0]["id"],
                maxResults=50
            )
        except IndexError:
            sys.exit("No channel found for {}".format(args.username))
    else:
        request = youtube.playlists().list(
            part="id,snippet",
            fields="items(id,snippet/title),nextPageToken,pageInfo/totalResults",
            mine="true",
            maxResults=50
        )

    return request

# Backs-up playlists using the provided request
def backup_playlists(playlists_request):
    timestamp = time.strftime("%Y%m%d-%H%M%S")

    path = os.path.join(args.directory, timestamp) if args.directory else timestamp

    try:
        os.mkdir(path)
    except OSError as e:
        sys.exit("{}: '{}'".format(e.strerror, e.filename))

    playlist_page = 0

    # Fetch pages of playlists until end
    while playlists_request:
        playlists_response = playlists_request.execute()

        for i, playlist in enumerate(playlists_response["items"], start=1):
            print "Saving playlist {} of {}".format(i + playlist_page * 50, playlists_response["pageInfo"]["totalResults"]), "\r",
            sys.stdout.flush()

            # Create new file for playlist
            # Assemble request for videos in each playlist
            playlist_items_request = youtube.playlistItems().list(
                part="id,snippet",
                fields="items(id,snippet/title),nextPageToken",
                playlistId=playlist["id"],
                maxResults=50
            )

            with open(os.path.join(path, playlist["snippet"]["title"]), 'w') as f:

                # Fetch pages of videos until end
                while playlist_items_request:
                    playlist_items_response = playlist_items_request.execute()

                    # Print videos in each playlist
                    for i, video in enumerate(playlist_items_response["items"], start=1):
                        f.write("{}. ".format(i) + video["snippet"]["title"].encode("utf-8") + '\n')

                    # Request next page of videos
                    playlist_items_request = youtube.playlistItems().list_next(
                        playlist_items_request, playlist_items_response)

            f.close()

        # Request next page of playlists
        playlists_request = youtube.playlists().list_next(
            playlists_request, playlists_response)

        playlist_page += 1

    print "\r"

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
        parents=[argparser])

    parser.add_argument("-d", "--directory", help="Parent directory to save backup")
    parser.add_argument("-i", "--id", help="Retrieve playlists using channel ID")
    parser.add_argument("-u", "--username", help="Retrieve playlists using legacy YouTube username")
    args = parser.parse_args()

    youtube = None

    try:
        create_resource_obj()
        req = setup_request()
        backup_playlists(req)
    except HttpError, e:
        print "An HTTP error %d occurred:\n%s" % (e.resp.status, e.content)

<commit_msg>Updated exception handler syntax for HttpError<commit_after>#!/usr/bin/env python

import os
import sys
import time
import argparse
import httplib2
from config import *
from apiclient.discovery import build
from apiclient.errors import HttpError
from oauth2client.client import flow_from_clientsecrets
from oauth2client.file import Storage
from oauth2client.tools import argparser, run_flow

# Creates the resource object for interacting with the YouTube API
# Sets up OAuth 2.0 for authorized requests if necessary
def create_resource_obj():
    global youtube

    if (args.id or args.username):
        youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
            developerKey=DEVELOPER_KEY)
    else:
        flow = flow_from_clientsecrets(CLIENT_SECRETS_FILE,
            message=MISSING_CLIENT_SECRETS_MESSAGE,
            scope=YOUTUBE_READONLY_SCOPE)

        storage = Storage("%s-oauth2.json" % sys.argv[0])
        credentials = storage.get()

        if credentials is None or credentials.invalid:
            credentials = run_flow(flow, storage, args)

        youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
            http=credentials.authorize(httplib2.Http()))

# Creates the initial playlists request used in backup_playlists()
# Checks the commandline arguments and assembles the correct request
def setup_request():
    if (args.id):
        request = youtube.playlists().list(
            part="id,snippet",
            fields="items(id,snippet/title),nextPageToken,pageInfo/totalResults",
            channelId=args.id,
            maxResults=50
        )
    elif (args.username):
        # Create channel request to obtain channel id from YouTube username
        channel_request = youtube.channels().list(
            part="id",
            forUsername=args.username,
            maxResults=50
        )

        channel_response = channel_request.execute()

        try:
            request = youtube.playlists().list(
                part="id,snippet",
                fields="items(id,snippet/title),nextPageToken,pageInfo/totalResults",
                channelId=channel_response["items"][0]["id"],
                maxResults=50
            )
        except IndexError:
            sys.exit("No channel found for {}".format(args.username))
    else:
        request = youtube.playlists().list(
            part="id,snippet",
            fields="items(id,snippet/title),nextPageToken,pageInfo/totalResults",
            mine="true",
            maxResults=50
        )

    return request

# Backs-up playlists using the provided request
def backup_playlists(playlists_request):
    timestamp = time.strftime("%Y%m%d-%H%M%S")

    path = os.path.join(args.directory, timestamp) if args.directory else timestamp

    try:
        os.mkdir(path)
    except OSError as e:
        sys.exit("{}: '{}'".format(e.strerror, e.filename))

    playlist_page = 0

    # Fetch pages of playlists until end
    while playlists_request:
        playlists_response = playlists_request.execute()

        for i, playlist in enumerate(playlists_response["items"], start=1):
            print "Saving playlist {} of {}".format(i + playlist_page * 50, playlists_response["pageInfo"]["totalResults"]), "\r",
            sys.stdout.flush()

            # Create new file for playlist
            # Assemble request for videos in each playlist
            playlist_items_request = youtube.playlistItems().list(
                part="id,snippet",
                fields="items(id,snippet/title),nextPageToken",
                playlistId=playlist["id"],
                maxResults=50
            )

            with open(os.path.join(path, playlist["snippet"]["title"]), 'w') as f:

                # Fetch pages of videos until end
                while playlist_items_request:
                    playlist_items_response = playlist_items_request.execute()

                    # Print videos in each playlist
                    for i, video in enumerate(playlist_items_response["items"], start=1):
                        f.write("{}. ".format(i) + video["snippet"]["title"].encode("utf-8") + '\n')

                    # Request next page of videos
                    playlist_items_request = youtube.playlistItems().list_next(
                        playlist_items_request, playlist_items_response)

            f.close()

        # Request next page of playlists
        playlists_request = youtube.playlists().list_next(
            playlists_request, playlists_response)

        playlist_page += 1

    print "\r"

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
        parents=[argparser])

    parser.add_argument("-d", "--directory", help="Parent directory to save backup")
    parser.add_argument("-i", "--id", help="Retrieve playlists using channel ID")
    parser.add_argument("-u", "--username", help="Retrieve playlists using legacy YouTube username")
    args = parser.parse_args()

    youtube = None

    try:
        create_resource_obj()
        req = setup_request()
        backup_playlists(req)
    except HttpError as e:
        print "An HTTP error %d occurred:\n%s" % (e.resp.status, e.content)

<|endoftext|>


<commit_before>#!/usr/bin/env python3

from urllib.request import urlopen
from geojson import Feature, Point, FeatureCollection, dumps

def get_users():
    wiki = urlopen("https://wiki.archlinux.org/index.php/ArchMap/List")

    wiki_source = wiki.read().decode()

    wiki_text_start = wiki_source.find('
', wiki_source.find('
') + 1) + 6
    wiki_text_end = wiki_source.find('
', wiki_source.find('
') + 1) - 1
    wiki_text = wiki_source[wiki_text_start:wiki_text_end]

    wiki_output = open('users.txt', 'w')
    wiki_output.write(wiki_text)
    wiki_output.close()

def make_geojson():
    users = open('users.txt', 'r')
    output = open('output.geojson', 'w')

    geo_output = []

    for line in users:
        elements = line.split('"')

        coords = elements[0].strip(' ')
        coords = coords.split(',')
        latitude = float(coords[0])
        longitude = float(coords[1])
        name = elements[1].strip()
        comment = elements[2].strip()
        comment = comment[2:]

        point = Point((longitude, latitude))
        feature = Feature(geometry=point, properties={"Name": name})

        geo_output.append(feature)

    output.write(dumps(FeatureCollection(geo_output)))

    users.close()
    output.close()

if __name__ == "__main__":
    get_users()
    make_geojson()
<commit_msg>Added doctrings<commit_after>#!/usr/bin/env python3

from urllib.request import urlopen
from geojson import Feature, Point, FeatureCollection, dumps

def get_users():
    """This funtion parses users from the Arch wiki and writes it to users.txt"""

    wiki = urlopen("https://wiki.archlinux.org/index.php/ArchMap/List")

    wiki_source = wiki.read().decode()

    wiki_text_start = wiki_source.find('
', wiki_source.find('
') + 1) + 6
    wiki_text_end = wiki_source.find('
', wiki_source.find('
') + 1) - 1
    wiki_text = wiki_source[wiki_text_start:wiki_text_end]

    wiki_output = open('users.txt', 'w')
    wiki_output.write(wiki_text)
    wiki_output.close()

def make_geojson():
    """This function reads users.txt and outputs output.geojson"""

    users = open('users.txt', 'r')
    output = open('output.geojson', 'w')

    geo_output = []

    for line in users:
        elements = line.split('"')

        coords = elements[0].strip(' ')
        coords = coords.split(',')
        latitude = float(coords[0])
        longitude = float(coords[1])
        name = elements[1].strip()
        comment = elements[2].strip()
        comment = comment[2:]

        point = Point((longitude, latitude))
        feature = Feature(geometry=point, properties={"Name": name})

        geo_output.append(feature)

    output.write(dumps(FeatureCollection(geo_output)))

    users.close()
    output.close()

if __name__ == "__main__":
    get_users()
    make_geojson()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2016-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""\
One of the worst thing about Python is the `DB API 2.0`_
specification, which is unusable except for building frameworks. It
should have been a stepping stone for an usable DB API 3.0 that never
happened. So, instead of a good low level API, we had a proliferation
of Object Relational Mappers making our lives a lot harder. Fortunately,
there has always been good Pythonistas in the anti-ORM camp.

This module is heavily inspired by the dbapiext_ module by
Martin Blais, which is part of the antiorm_ package. The main (only)
difference is that I am using the question mark (?) for the placeholders
instead of the percent sign (%) to avoid confusions with other usages
of the %s, in particular in LIKE queries and in expressions like
`strftime('%s', time)` used in SQLite.

In less than 200 lines of code there is enough support
to build dynamic SQL queries and to make an ORM unneeded, since
we do not need database independence.

.. _DB API 2.0: https://www.python.org/dev/peps/pep-0249
.. _dbapiext: http://furius.ca/pubcode/pub/antiorm/lib/python/dbapiext.py.html
.. _antiorm: https://bitbucket.org/blais/antiorm

dbiapi tutorial
--------------------------

The only thing you must to know is the `Db` class, which is lazy wrapper
over a database connection. You instantiate it by passing a connection
function and its arguments:

>>> import sqlite3
>>> db = Db(sqlite3.connect, ':memory:')

Now you have an interface to your database, the `db` object. This object
is lazy, i.e. the connection is not yet instantiated, but it will be
when you will access its `.conn` attribute. This attribute is automatically
accessed when you call the interface to run a query, for instance to create
an empty table:

>>> curs = db('CREATE TABLE job ('
...     'id INTEGER PRIMARY KEY AUTOINCREMENT, value INTEGER)')

You can populate the table by using the `.insert` method:

>>> db.insert('job', ['value'], [(42,), (43,)]) # doctest: +ELLIPSIS
<sqlite3.Cursor object at ...>

Notice that this method returns a standard DB API 2.0 cursor and
you have access to all of its features: for instance here you could extract
the lastrowid.

Then you can run SELECT queries:

>>> rows = db('SELECT * FROM job')

The dbapi provides a `Row` class which is used to hold the results of
SELECT queries and is working as one would expect:

>>> rows
[<Row(id=1, value=42)>, <Row(id=2, value=43)>]
>>> tuple(rows[0])
(1, 42)
>>> rows[0].id
1
>>> rows[0].value
42
>>> rows[0]._fields
['id', 'value']

The queries can have different kind of `?` parameters:

- `?s` is for interpolated string parameters:

  >>> db('SELECT * FROM ?s', 'job')  # ?s is replaced by 'job'
  [<Row(id=1, value=42)>, <Row(id=2, value=43)>]

- `?x` is for escaped parameters (to avoid SQL injection):

  >>> db('SELECT * FROM job WHERE id=?x', 1)  # ?x is replaced by 1
  [<Row(id=1, value=42)>]

- `?s` and `?x` are for scalar parameters; `?S` and `?X` are for sequences:

  >>> db('INSERT INTO job (?S) VALUES (?X)', ['id', 'value'], (3, 44)) # doctest: +ELLIPSIS
  <sqlite3.Cursor object at ...>

You can see how the interpolation works by calling the `expand` method
that returns the interpolated template. In this case

>>> db.expand('INSERT INTO job (?S) VALUES (?X)', ['id', 'value'], [3, 44])
'INSERT INTO job (id, value) VALUES (?, ?)'

As you see, `?S` parameters work by replacing a list of strings with a comma
separated string, where `?X` parameters are replaced by a comma separated
sequence of question marks, i.e. the low level placeholder for SQLite.
The interpolation performs a regular search and replace,
so if you have a `?-` string in your template that must not be escaped,
you can run into issues. This is an error:

>>> match("SELECT * FROM job WHERE id=?x AND description='Lots of ?s'", 1)
Traceback (most recent call last):
   ...
ValueError: Incorrect number of ?-parameters in SELECT * FROM job WHERE id=?x AND description='Lots of ?s', expected 1

This is correct:

>>> match("SELECT * FROM job WHERE id=?x AND description=?x", 1, 'Lots of ?s')
('SELECT * FROM job WHERE id=? AND description=?', (1, 'Lots of ?s'))

There are three other `?` parameters:

- `?D` is for dictionaries and it is used mostly in UPDATE queries:

  >>> match('UPDATE mytable SET ?D WHERE id=?x', dict(value=33, other=5), 1)
  ('UPDATE mytable SET other=?, value=? WHERE id=?', (5, 33, 1))

- `?A` is for dictionaries and it is used in AND queries:

  >>> match('SELECT * FROM job WHERE ?A', dict(value=33, id=5))
  ('SELECT * FROM job WHERE id=? AND value=?', (5, 33))

- `?O` is for dictionaries and it is used in OR queries:

  >>> match('SELECT * FROM job WHERE ?O', dict(value=33, id=5))
  ('SELECT * FROM job WHERE id=? OR value=?', (5, 33))

The dictionary parameters are ordered per field name, just to make
the templates reproducible. `?A` and `?O` are smart enough to
treat specially `None` parameters, that are turned into `NULL`:

  >>> match('SELECT * FROM job WHERE ?A', dict(value=None, id=5))
  ('SELECT * FROM job WHERE id=? AND value IS NULL', (5,))

The `?` parameters are matched positionally; it is also possible to
pass to the `db` object a few keyword arguments to tune the standard
behavior. In particular, if you know that a query must return a
single row you can do the following:

>>> db('SELECT * FROM job WHERE id=?x', 1, one=True)
<Row(id=1, value=42)>

Without the `one=True` the query would have returned a list with a single
element. If you know that the query must return a scalar you can do the
following:

>>> db('SELECT value FROM job WHERE id=?x', 1, scalar=True)
42

If a query that should return a scalar returns something else, or if a
query that should return a row returns a different number of rows,
appropriate errors are raised:

>>> db('SELECT * FROM job WHERE id=?x', 1, scalar=True) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
   ...
TooManyColumns: 2, expected 1

>>> db('SELECT * FROM job', None, one=True) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
   ...
TooManyRows: 3, expected 1

If a row is expected but not found, a NotFound exception is raised:

>>> db('SELECT * FROM job WHERE id=?x', None, one=True) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
   ...
NotFound

"""
import re
import threading
import collections

class NotFound(Exception):
    """Raised when a scalar query has not output"""

class TooManyRows(Exception):
    """Raised when a scalar query produces more than one row"""

class TooManyColumns(Exception):
    """Raised when a scalar query has more than one column"""

class _Replacer(object):
    # helper class for the match function below
    rx = re.compile(r'\?S|\?X|\?D|\?A|\?O|\?s|\?x')
    ph = '?'

    def __init__(self, all_args):
        self.all_args = list(all_args)
        self.xargs = []
        self.sargs = []

    def __call__(self, mo):
        arg = self.all_args[0]  # can raise an IndexError
        del self.all_args[0]
        placeholder = mo.group()
        if placeholder == '?X':
            self.xargs.extend(arg)
            return ', '.join([self.ph] * len(arg))
        elif placeholder == '?S':
            self.sargs.extend(arg)
            return ', '.join(['{}'] * len(arg))
        elif placeholder == '?D':
            keys, values = zip(*sorted(arg.items()))
            self.sargs.extend(keys)
            self.xargs.extend(values)
            return ', '.join(['{}=' + self.ph] * len(arg))
        elif placeholder == '?A':
            return self.join(' AND ', arg) or '1'
        elif placeholder == '?O':
            return self.join(' OR ', arg) or '1'
        elif placeholder == '?x':
            self.xargs.append(arg)
            return self.ph
        elif placeholder == '?s':
            self.sargs.append(arg)
            return '{}'

    def join(self, sep, arg):
        ls = []
        for name in sorted(arg):
            self.sargs.append(name)
            value = arg[name]
            if value is None:
                ls.append('{} IS NULL')
            else:
                self.xargs.append(value)
                ls.append('{}=' + self.ph)
        return sep.join(ls)

    def match(self, m_templ):
        templ = self.rx.sub(self, m_templ)
        return templ.format(*self.sargs), tuple(self.xargs)

def match(m_templ, *m_args):
    """
    :param m_templ: a meta template string
    :param m_args: all arguments
    :returns: template, args

    Here is an example of usage:

    >>> match('SELECT * FROM job WHERE id=?x', 1)
    ('SELECT * FROM job WHERE id=?', (1,))
    """
    # strip commented lines
    m_templ = '\n'.join(line for line in m_templ.splitlines()
                        if not line.lstrip().startswith('--'))
    if not m_args:
        return m_templ, ()
    try:
        return _Replacer(m_args).match(m_templ)
    except IndexError:
        raise ValueError('Incorrect number of ?-parameters in %s, expected %s'
                         % (m_templ, len(m_args)))

class Db(object):
    """
    A wrapper over a DB API 2 connection. See the tutorial.
    """
    def __init__(self, connect, *args, **kw):
        self.connect = connect
        self.args = args
        self.kw = kw
        self.local = threading.local()

    @classmethod
    def expand(cls, m_templ, *m_args):
        """
        Performs partial interpolation of the template. Used for debugging.
        """
        return match(m_templ, *m_args)[0]

    @property
    def conn(self):
        try:
            return self.local.conn
        except AttributeError:
            self.local.conn = self.connect(*self.args, **self.kw)
            return self.local.conn

    @property
    def path(self):
        """Path to the underlying sqlite file"""
        return self.args[0]

    def __enter__(self):
        return self

    def __exit__(self, etype, exc, tb):
        if etype:
            self.conn.rollback()
        else:
            self.conn.commit()

    def __call__(self, m_templ, *m_args, **kw):
        cursor = self.conn.cursor()
        templ, args = match(m_templ, *m_args)
        if kw.get('debug'):
            print(templ, args)
        try:
            if args:
                cursor.execute(templ, args)
            else:
                cursor.execute(templ)
        except Exception as exc:
            raise exc.__class__('%s: %s %s' % (exc, templ, args))
        if templ.lstrip().lower().startswith(('select', 'pragma')):
            rows = cursor.fetchall()
            if kw.get('scalar'):  # scalar query
                if not rows:
                    raise NotFound
                elif len(rows) > 1:
                    raise TooManyRows('%s, expected 1' % len(rows))
                elif len(rows[0]) > 1:
                    raise TooManyColumns('%s, expected 1' % len(rows[0]))
                return rows[0][0]
            elif kw.get('one'):  # query returning a single row
                if not rows:
                    raise NotFound(args)
                elif len(rows) > 1:
                    raise TooManyRows('%s, expected 1' % len(rows))
            elif cursor.description is None:
                return cursor

            colnames = [r[0] for r in cursor.description]
            if kw.get('one'):
                return Row(colnames, rows[0])
            else:
                return Table(colnames, rows)
        else:
            return cursor

    def insert(self, table, columns, rows):
        """
        Insert several rows with executemany. Return a cursor.
        """
        cursor = self.conn.cursor()
        if len(rows):
            templ, _args = match('INSERT INTO ?s (?S) VALUES (?X)',
                                 table, columns, rows[0])
            cursor.executemany(templ, rows)
        return cursor

    def close(self):
        """
        Close the main thread connection and refresh the threadlocal object
        """
        self.conn.close()
        self.local = threading.local()

class Table(list):
    """Just a list of Rows with an attribute _fields"""
    def __init__(self, fields, rows):
        self._fields = fields
        for row in rows:
            self.append(Row(fields, row))

# we cannot use a namedtuple here because one would get a PicklingError:
# Can't pickle <class 'openquake.server.dbapi.Row'>: attribute lookup
# openquake.server.dbapi.Row failed
class Row(collections.Sequence):
    """
    A pickleable row, working both as a tuple and an object:

    >>> row = Row(['id', 'value'], (1, 2))
    >>> tuple(row)
    (1, 2)
    >>> assert row[0] == row.id and row[1] == row.value

    :param fields: a sequence of field names
    :param values: a sequence of values (one per field)
    """
    def __init__(self, fields, values):
        if len(values) != len(fields):
            raise ValueError('Got %d values, expected %d' %
                             (len(values), len(fields)))
        self._fields = fields
        self._values = values
        for f, v in zip(fields, values):
            setattr(self, f, v)

    def __getitem__(self, i):
        return self._values[i]

    def __len__(self):
        return len(self._values)

    def __repr__(self):
        items = ['%s=%s' % (f, getattr(self, f)) for f in self._fields]
        return '<Row(%s)>' % ', '.join(items)
<commit_msg>Small improvement on db(..., debug=True)<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2016-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""\
One of the worst thing about Python is the `DB API 2.0`_
specification, which is unusable except for building frameworks. It
should have been a stepping stone for an usable DB API 3.0 that never
happened. So, instead of a good low level API, we had a proliferation
of Object Relational Mappers making our lives a lot harder. Fortunately,
there has always been good Pythonistas in the anti-ORM camp.

This module is heavily inspired by the dbapiext_ module by
Martin Blais, which is part of the antiorm_ package. The main (only)
difference is that I am using the question mark (?) for the placeholders
instead of the percent sign (%) to avoid confusions with other usages
of the %s, in particular in LIKE queries and in expressions like
`strftime('%s', time)` used in SQLite.

In less than 200 lines of code there is enough support
to build dynamic SQL queries and to make an ORM unneeded, since
we do not need database independence.

.. _DB API 2.0: https://www.python.org/dev/peps/pep-0249
.. _dbapiext: http://furius.ca/pubcode/pub/antiorm/lib/python/dbapiext.py.html
.. _antiorm: https://bitbucket.org/blais/antiorm

dbiapi tutorial
--------------------------

The only thing you must to know is the `Db` class, which is lazy wrapper
over a database connection. You instantiate it by passing a connection
function and its arguments:

>>> import sqlite3
>>> db = Db(sqlite3.connect, ':memory:')

Now you have an interface to your database, the `db` object. This object
is lazy, i.e. the connection is not yet instantiated, but it will be
when you will access its `.conn` attribute. This attribute is automatically
accessed when you call the interface to run a query, for instance to create
an empty table:

>>> curs = db('CREATE TABLE job ('
...     'id INTEGER PRIMARY KEY AUTOINCREMENT, value INTEGER)')

You can populate the table by using the `.insert` method:

>>> db.insert('job', ['value'], [(42,), (43,)]) # doctest: +ELLIPSIS
<sqlite3.Cursor object at ...>

Notice that this method returns a standard DB API 2.0 cursor and
you have access to all of its features: for instance here you could extract
the lastrowid.

Then you can run SELECT queries:

>>> rows = db('SELECT * FROM job')

The dbapi provides a `Row` class which is used to hold the results of
SELECT queries and is working as one would expect:

>>> rows
[<Row(id=1, value=42)>, <Row(id=2, value=43)>]
>>> tuple(rows[0])
(1, 42)
>>> rows[0].id
1
>>> rows[0].value
42
>>> rows[0]._fields
['id', 'value']

The queries can have different kind of `?` parameters:

- `?s` is for interpolated string parameters:

  >>> db('SELECT * FROM ?s', 'job')  # ?s is replaced by 'job'
  [<Row(id=1, value=42)>, <Row(id=2, value=43)>]

- `?x` is for escaped parameters (to avoid SQL injection):

  >>> db('SELECT * FROM job WHERE id=?x', 1)  # ?x is replaced by 1
  [<Row(id=1, value=42)>]

- `?s` and `?x` are for scalar parameters; `?S` and `?X` are for sequences:

  >>> db('INSERT INTO job (?S) VALUES (?X)', ['id', 'value'], (3, 44)) # doctest: +ELLIPSIS
  <sqlite3.Cursor object at ...>

You can see how the interpolation works by calling the `expand` method
that returns the interpolated template (alternatively, there is a
`debug=True` flag when calling `db` that prints the same info). In this case

>>> db.expand('INSERT INTO job (?S) VALUES (?X)', ['id', 'value'], [3, 44])
'INSERT INTO job (id, value) VALUES (?, ?)'

As you see, `?S` parameters work by replacing a list of strings with a comma
separated string, where `?X` parameters are replaced by a comma separated
sequence of question marks, i.e. the low level placeholder for SQLite.
The interpolation performs a regular search and replace,
so if you have a `?-` string in your template that must not be escaped,
you can run into issues. This is an error:

>>> match("SELECT * FROM job WHERE id=?x AND description='Lots of ?s'", 1)
Traceback (most recent call last):
   ...
ValueError: Incorrect number of ?-parameters in SELECT * FROM job WHERE id=?x AND description='Lots of ?s', expected 1

This is correct:

>>> match("SELECT * FROM job WHERE id=?x AND description=?x", 1, 'Lots of ?s')
('SELECT * FROM job WHERE id=? AND description=?', (1, 'Lots of ?s'))

There are three other `?` parameters:

- `?D` is for dictionaries and it is used mostly in UPDATE queries:

  >>> match('UPDATE mytable SET ?D WHERE id=?x', dict(value=33, other=5), 1)
  ('UPDATE mytable SET other=?, value=? WHERE id=?', (5, 33, 1))

- `?A` is for dictionaries and it is used in AND queries:

  >>> match('SELECT * FROM job WHERE ?A', dict(value=33, id=5))
  ('SELECT * FROM job WHERE id=? AND value=?', (5, 33))

- `?O` is for dictionaries and it is used in OR queries:

  >>> match('SELECT * FROM job WHERE ?O', dict(value=33, id=5))
  ('SELECT * FROM job WHERE id=? OR value=?', (5, 33))

The dictionary parameters are ordered per field name, just to make
the templates reproducible. `?A` and `?O` are smart enough to
treat specially `None` parameters, that are turned into `NULL`:

  >>> match('SELECT * FROM job WHERE ?A', dict(value=None, id=5))
  ('SELECT * FROM job WHERE id=? AND value IS NULL', (5,))

The `?` parameters are matched positionally; it is also possible to
pass to the `db` object a few keyword arguments to tune the standard
behavior. In particular, if you know that a query must return a
single row you can do the following:

>>> db('SELECT * FROM job WHERE id=?x', 1, one=True)
<Row(id=1, value=42)>

Without the `one=True` the query would have returned a list with a single
element. If you know that the query must return a scalar you can do the
following:

>>> db('SELECT value FROM job WHERE id=?x', 1, scalar=True)
42

If a query that should return a scalar returns something else, or if a
query that should return a row returns a different number of rows,
appropriate errors are raised:

>>> db('SELECT * FROM job WHERE id=?x', 1, scalar=True) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
   ...
TooManyColumns: 2, expected 1

>>> db('SELECT * FROM job', None, one=True) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
   ...
TooManyRows: 3, expected 1

If a row is expected but not found, a NotFound exception is raised:

>>> db('SELECT * FROM job WHERE id=?x', None, one=True) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
   ...
NotFound

"""
import re
import threading
import collections

class NotFound(Exception):
    """Raised when a scalar query has not output"""

class TooManyRows(Exception):
    """Raised when a scalar query produces more than one row"""

class TooManyColumns(Exception):
    """Raised when a scalar query has more than one column"""

class _Replacer(object):
    # helper class for the match function below
    rx = re.compile(r'\?S|\?X|\?D|\?A|\?O|\?s|\?x')
    ph = '?'

    def __init__(self, all_args):
        self.all_args = list(all_args)
        self.xargs = []
        self.sargs = []

    def __call__(self, mo):
        arg = self.all_args[0]  # can raise an IndexError
        del self.all_args[0]
        placeholder = mo.group()
        if placeholder == '?X':
            self.xargs.extend(arg)
            return ', '.join([self.ph] * len(arg))
        elif placeholder == '?S':
            self.sargs.extend(arg)
            return ', '.join(['{}'] * len(arg))
        elif placeholder == '?D':
            keys, values = zip(*sorted(arg.items()))
            self.sargs.extend(keys)
            self.xargs.extend(values)
            return ', '.join(['{}=' + self.ph] * len(arg))
        elif placeholder == '?A':
            return self.join(' AND ', arg) or '1'
        elif placeholder == '?O':
            return self.join(' OR ', arg) or '1'
        elif placeholder == '?x':
            self.xargs.append(arg)
            return self.ph
        elif placeholder == '?s':
            self.sargs.append(arg)
            return '{}'

    def join(self, sep, arg):
        ls = []
        for name in sorted(arg):
            self.sargs.append(name)
            value = arg[name]
            if value is None:
                ls.append('{} IS NULL')
            else:
                self.xargs.append(value)
                ls.append('{}=' + self.ph)
        return sep.join(ls)

    def match(self, m_templ):
        templ = self.rx.sub(self, m_templ)
        return templ.format(*self.sargs), tuple(self.xargs)

def match(m_templ, *m_args):
    """
    :param m_templ: a meta template string
    :param m_args: all arguments
    :returns: template, args

    Here is an example of usage:

    >>> match('SELECT * FROM job WHERE id=?x', 1)
    ('SELECT * FROM job WHERE id=?', (1,))
    """
    # strip commented lines
    m_templ = '\n'.join(line for line in m_templ.splitlines()
                        if not line.lstrip().startswith('--'))
    if not m_args:
        return m_templ, ()
    try:
        return _Replacer(m_args).match(m_templ)
    except IndexError:
        raise ValueError('Incorrect number of ?-parameters in %s, expected %s'
                         % (m_templ, len(m_args)))

class Db(object):
    """
    A wrapper over a DB API 2 connection. See the tutorial.
    """
    def __init__(self, connect, *args, **kw):
        self.connect = connect
        self.args = args
        self.kw = kw
        self.local = threading.local()

    @classmethod
    def expand(cls, m_templ, *m_args):
        """
        Performs partial interpolation of the template. Used for debugging.
        """
        return match(m_templ, *m_args)[0]

    @property
    def conn(self):
        try:
            return self.local.conn
        except AttributeError:
            self.local.conn = self.connect(*self.args, **self.kw)
            return self.local.conn

    @property
    def path(self):
        """Path to the underlying sqlite file"""
        return self.args[0]

    def __enter__(self):
        return self

    def __exit__(self, etype, exc, tb):
        if etype:
            self.conn.rollback()
        else:
            self.conn.commit()

    def __call__(self, m_templ, *m_args, **kw):
        cursor = self.conn.cursor()
        templ, args = match(m_templ, *m_args)
        if kw.get('debug'):
            print(templ)
            print('args = %s' % repr(args))
        try:
            if args:
                cursor.execute(templ, args)
            else:
                cursor.execute(templ)
        except Exception as exc:
            raise exc.__class__('%s: %s %s' % (exc, templ, args))
        if templ.lstrip().lower().startswith(('select', 'pragma')):
            rows = cursor.fetchall()
            if kw.get('scalar'):  # scalar query
                if not rows:
                    raise NotFound
                elif len(rows) > 1:
                    raise TooManyRows('%s, expected 1' % len(rows))
                elif len(rows[0]) > 1:
                    raise TooManyColumns('%s, expected 1' % len(rows[0]))
                return rows[0][0]
            elif kw.get('one'):  # query returning a single row
                if not rows:
                    raise NotFound(args)
                elif len(rows) > 1:
                    raise TooManyRows('%s, expected 1' % len(rows))
            elif cursor.description is None:
                return cursor

            colnames = [r[0] for r in cursor.description]
            if kw.get('one'):
                return Row(colnames, rows[0])
            else:
                return Table(colnames, rows)
        else:
            return cursor

    def insert(self, table, columns, rows):
        """
        Insert several rows with executemany. Return a cursor.
        """
        cursor = self.conn.cursor()
        if len(rows):
            templ, _args = match('INSERT INTO ?s (?S) VALUES (?X)',
                                 table, columns, rows[0])
            cursor.executemany(templ, rows)
        return cursor

    def close(self):
        """
        Close the main thread connection and refresh the threadlocal object
        """
        self.conn.close()
        self.local = threading.local()

class Table(list):
    """Just a list of Rows with an attribute _fields"""
    def __init__(self, fields, rows):
        self._fields = fields
        for row in rows:
            self.append(Row(fields, row))

# we cannot use a namedtuple here because one would get a PicklingError:
# Can't pickle <class 'openquake.server.dbapi.Row'>: attribute lookup
# openquake.server.dbapi.Row failed
class Row(collections.Sequence):
    """
    A pickleable row, working both as a tuple and an object:

    >>> row = Row(['id', 'value'], (1, 2))
    >>> tuple(row)
    (1, 2)
    >>> assert row[0] == row.id and row[1] == row.value

    :param fields: a sequence of field names
    :param values: a sequence of values (one per field)
    """
    def __init__(self, fields, values):
        if len(values) != len(fields):
            raise ValueError('Got %d values, expected %d' %
                             (len(values), len(fields)))
        self._fields = fields
        self._values = values
        for f, v in zip(fields, values):
            setattr(self, f, v)

    def __getitem__(self, i):
        return self._values[i]

    def __len__(self):
        return len(self._values)

    def __repr__(self):
        items = ['%s=%s' % (f, getattr(self, f)) for f in self._fields]
        return '<Row(%s)>' % ', '.join(items)
<|endoftext|>


<commit_before>#!bin/python
import json
import lib_zpr
#import logging
#from logging.handlers import RotatingFileHandler

from flask import Flask, jsonify, make_response

app = Flask(__name__)

# app.logger.setLevel(logging.INFO)
# app.logger.disabled = False
# handler = logging.handlers.RotatingFileHandler(
#     '/var/log/zpr_flask.log',
#     'a',
#     maxBytes=1024 * 1024 * 100,
#     backupCount=20
#     )
# log = logging.getLogger('werkzeug')
# log.setLevel(logging.DEBUG)
# app.logger.addHandler(handler)

api_version = 'v1.0'
api_base = str('/zpr/{v}'.format(v=api_version))

@app.errorhandler(404)
def not_found(error):
    return make_response(jsonify({'error': 'Not found'}), 404)

# @app.route('/zpr/job')
# def ls_test():
#     return json.dumps(call('ls'))

# @app.route('{a}/job/rsync/<backup_host>'.format(a=api_base), methods=['GET'])
# def check_job(backup_host):
#     job = str(lib_zpr.check_zpr_rsync_job(backup_host))
#     return json.dumps(job)

@app.route('{a}/job/<backup_host>'.format(a=api_base), methods=['GET'])
def check_zpr_job(backup_host):
    lib_zpr.check_tsp_job(backup_host)
    return jsonify({'last result for {b}'.format(b=backup_host): lib_zpr.check_tsp_job_out[0]})

@app.route('{a}/job/<backup_host>/output'.format(a=api_base), methods=['GET'])
def check_zpr_job_summary(backup_host):
    lib_zpr.check_tsp_job(backup_host, show_changes=True)
#   job_out = {
#       'response': lib_zpr.check_tsp_job_out[0],
#       'name': backup_host
#   }
#   if lib_zpr.check_job_changes:
#       job_out['job_output'] = lib_zpr.check_job_changes[0]
#   job_checked = [job_out]
    return jsonify({'job_checked': lib_zpr.json_output})

    #return json.dumps(join_summary, indent=2)

if __name__ == '__main__':
#   formatter = logging.Formatter(\
#       "%(asctime)s - %(levelname)s - %(name)s: \t%(messages)s")
#   handler.setFormatter(formatter)
    app.run(debug=True, extra_files='/var/lib/zpr/api/lib_zpr.py')
<commit_msg>Add endpoint to check multiple jobs<commit_after>#!bin/python
import json
import lib_zpr
#import logging
#from logging.handlers import RotatingFileHandler

from flask import Flask, jsonify, make_response

app = Flask(__name__)

# app.logger.setLevel(logging.INFO)
# app.logger.disabled = False
# handler = logging.handlers.RotatingFileHandler(
#     '/var/log/zpr_flask.log',
#     'a',
#     maxBytes=1024 * 1024 * 100,
#     backupCount=20
#     )
# log = logging.getLogger('werkzeug')
# log.setLevel(logging.DEBUG)
# app.logger.addHandler(handler)

api_version = 'v1.0'
api_base = str('/zpr/{v}'.format(v=api_version))

@app.errorhandler(404)
def not_found(error):
    return make_response(jsonify({'error': 'Not found'}), 404)

# @app.route('/zpr/job')
# def ls_test():
#     return json.dumps(call('ls'))

# @app.route('{a}/job/rsync/<backup_host>'.format(a=api_base), methods=['GET'])
# def check_job(backup_host):
#     job = str(lib_zpr.check_zpr_rsync_job(backup_host))
#     return json.dumps(job)

@app.route('{a}/job/<backup_host>'.format(a=api_base), methods=['GET'])
def check_zpr_job(backup_host):
    lib_zpr.check_tsp_job(backup_host)
    return jsonify({'last result for {b}'.format(b=backup_host): lib_zpr.check_tsp_job_out[0]})

@app.route('{a}/job/<backup_host>/output'.format(a=api_base), methods=['GET'])
def check_zpr_job_summary(backup_host):
    lib_zpr.check_tsp_job(backup_host, show_changes=True)
    return jsonify({'job_checked': lib_zpr.json_output})

@app.route('{a}/job/<backup_host>/output/<int:count>'.format(a=api_base), methods=['GET'])
def check_zpr_job_summary_count(backup_host, count):
    lib_zpr.check_tsp_job(backup_host, count, show_changes=True)
    return jsonify({'job_checked': lib_zpr.json_output})

    #return json.dumps(join_summary, indent=2)

if __name__ == '__main__':
#   formatter = logging.Formatter(\
#       "%(asctime)s - %(levelname)s - %(name)s: \t%(messages)s")
#   handler.setFormatter(formatter)
    app.run(debug=True, extra_files='/var/lib/zpr/api/lib_zpr.py')
<|endoftext|>