<commit_before># -*- coding: utf-8 -*-

#-------------------------------------------------------------------------
# drawElements Quality Program utilities
# --------------------------------------
#
# Copyright 2015 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#-------------------------------------------------------------------------

import os
import re
import sys
import shlex
import subprocess
import multiprocessing
import string

try:
	import threading
except ImportError:
	import dummy_threading as threading

class NativeLib:
	def __init__ (self, apiVersion, abiVersion, prebuiltDir):
		self.apiVersion		= apiVersion
		self.abiVersion		= abiVersion
		self.prebuiltDir	= prebuiltDir

	def __str__ (self):
		return "(API: %s, ABI: %s)" % (self.apiVersion, self.abiVersion)

	def __repr__ (self):
		return "(API: %s, ABI: %s)" % (self.apiVersion, self.abiVersion)

def getPlatform ():
	if sys.platform.startswith('linux'):
		return 'linux'
	else:
		return sys.platform

def selectByOS (variants):
	platform = getPlatform()
	if platform in variants:
		return variants[platform]
	elif 'other' in variants:
		return variants['other']
	else:
		raise Exception("No configuration for '%s'" % platform)

def isExecutable (path):
	return os.path.isfile(path) and os.access(path, os.X_OK)

def which (binName):
	for path in os.environ['PATH'].split(os.pathsep):
		path = path.strip('"')
		fullPath = os.path.join(path, binName)
		if isExecutable(fullPath):
			return fullPath

	return None

def isBinaryInPath (binName):
	return which(binName) != None

def selectFirstExistingBinary (filenames):
	for filename in filenames:
		if filename != None and isExecutable(filename):
			return filename

	return None

def selectFirstExistingDir (paths):
	for path in paths:
		if path != None and os.path.isdir(path):
			return path

	return None

def die (msg):
	print msg
	exit(-1)

def shellquote(s):
	return '"%s"' % s.replace('\\', '\\\\').replace('"', '\"').replace('$', '\$').replace('`', '\`')

def execute (commandLine):
	args	= shlex.split(commandLine)
	retcode	= subprocess.call(args)
	if retcode != 0:
		raise Exception("Failed to execute '%s', got %d" % (commandLine, retcode))

def execArgs (args):
	# Make sure previous stdout prints have been written out.
	sys.stdout.flush()
	retcode	= subprocess.call(args)
	if retcode != 0:
		raise Exception("Failed to execute '%s', got %d" % (str(args), retcode))

def execArgsInDirectory (args, cwd, linePrefix=""):

	def readApplyPrefixAndPrint (source, prefix, sink):
		while True:
			line = source.readline()
			if len(line) == 0: # EOF
				break;
			sink.write(prefix + line)

	process = subprocess.Popen(args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	stdoutJob = threading.Thread(target=readApplyPrefixAndPrint, args=(process.stdout, linePrefix, sys.stdout))
	stderrJob = threading.Thread(target=readApplyPrefixAndPrint, args=(process.stderr, linePrefix, sys.stderr))
	stdoutJob.start()
	stderrJob.start()
	retcode = process.wait()
	if retcode != 0:
		raise Exception("Failed to execute '%s', got %d" % (str(args), retcode))

def serialApply(f, argsList):
	for args in argsList:
		f(*args)

def parallelApply(f, argsList):
	class ErrorCode:
		def __init__ (self):
			self.error = None;

	def applyAndCaptureError (func, args, errorCode):
		try:
			func(*args)
		except:
			errorCode.error = sys.exc_info()

	errorCode = ErrorCode()
	jobs = []
	for args in argsList:
		job = threading.Thread(target=applyAndCaptureError, args=(f, args, errorCode))
		job.start()
		jobs.append(job)

	for job in jobs:
		job.join()

	if errorCode.error:
		raise errorCode.error[0], errorCode.error[1], errorCode.error[2]

class Device:
	def __init__(self, serial, product, model, device):
		self.serial		= serial
		self.product	= product
		self.model		= model
		self.device		= device

	def __str__ (self):
		return "%s: {product: %s, model: %s, device: %s}" % (self.serial, self.product, self.model, self.device)

def getDevices (adb):
	proc = subprocess.Popen([adb, 'devices', '-l'], stdout=subprocess.PIPE)
	(stdout, stderr) = proc.communicate()

	if proc.returncode != 0:
		raise Exception("adb devices -l failed, got %d" % proc.returncode)

	ptrn = re.compile(r'^([a-zA-Z0-9:]+)\s+.*product:([^\s]+)\s+model:([^\s]+)\s+device:([^\s]+)')
	devices = []
	for line in stdout.splitlines()[1:]:
		if len(line.strip()) == 0:
			continue

		m = ptrn.match(line)
		if m == None:
			print "WARNING: Failed to parse device info '%s'" % line
			continue

		devices.append(Device(m.group(1), m.group(2), m.group(3), m.group(4)))

	return devices

def getWin32Generator ():
	if which("jom.exe") != None:
		return "NMake Makefiles JOM"
	else:
		return "NMake Makefiles"

def isNinjaSupported ():
	return which("ninja") != None

def getUnixGenerator ():
	if isNinjaSupported():
		return "Ninja"
	else:
		return "Unix Makefiles"

def getExtraBuildArgs (generator):
	if generator == "Unix Makefiles":
		return ["--", "-j%d" % multiprocessing.cpu_count()]
	else:
		return []

NDK_HOST_OS_NAMES = [
	"windows",
	"windows-x86_64",
	"darwin-x86",
	"darwin-x86_64",
	"linux-x86",
	"linux-x86_64"
]

def getNDKHostOsName (ndkPath):
	for name in NDK_HOST_OS_NAMES:
		if os.path.exists(os.path.join(ndkPath, "prebuilt", name)):
			return name

	raise Exception("Couldn't determine NDK host OS")

# deqp/android path
ANDROID_DIR				= os.path.realpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))

# Build configuration
NATIVE_LIBS				= [
		#		  API		ABI				prebuiltsDir
		NativeLib(13,		"armeabi-v7a",	'android-arm'),		# ARM v7a ABI
		NativeLib(13,		"x86",			'android-x86'),		# x86
		NativeLib(21,		"arm64-v8a",	'android-arm64'),	# ARM64 v8a ABI
		NativeLib(21,		"x86_64",		'android-x86_64'),	# x86_64
	]

ANDROID_JAVA_API		= "android-22"
NATIVE_LIB_NAME			= "libdeqp.so"

def selectNDKPath ():
	candidates =  [
		os.path.expanduser("~/android-ndk-r11"),
		"C:/android/android-ndk-r11",
		os.environ.get("ANDROID_NDK_PATH", None), # If not defined, return None
	]

	ndkPath = selectFirstExistingDir(candidates)

	if ndkPath == None:
		raise Exception("None of NDK directory candidates exist: %s. Check ANDROID_NDK_PATH in common.py" % candidates)

	return ndkPath

def noneSafePathJoin (*components):
	if None in components:
		return None
	return os.path.join(*components)

# NDK paths
ANDROID_NDK_PATH				= selectNDKPath()
ANDROID_NDK_HOST_OS				= getNDKHostOsName(ANDROID_NDK_PATH)
ANDROID_NDK_TOOLCHAIN_VERSION	= "r11" # Toolchain file is selected based on this

# Native code build settings
CMAKE_GENERATOR			= selectByOS({
		'win32':	getWin32Generator(),
		'other':	getUnixGenerator()
	})
EXTRA_BUILD_ARGS		= getExtraBuildArgs(CMAKE_GENERATOR)

# SDK paths
ANDROID_SDK_PATH		= selectFirstExistingDir([
		os.environ.get("ANDROID_SDK_PATH", None),
		os.path.expanduser("~/android-sdk-linux"),
		os.path.expanduser("~/android-sdk-mac_x86"),
		"C:/android/android-sdk-windows",
	])
ANDROID_BIN				= selectFirstExistingBinary([
		noneSafePathJoin(ANDROID_SDK_PATH, "tools", "android"),
		noneSafePathJoin(ANDROID_SDK_PATH, "tools", "android.bat"),
		which('android'),
	])
ADB_BIN					= selectFirstExistingBinary([
		which('adb'), # \note Prefer adb in path to avoid version issues on dev machines
		noneSafePathJoin(ANDROID_SDK_PATH, "platform-tools", "adb"),
		noneSafePathJoin(ANDROID_SDK_PATH, "platform-tools", "adb.exe"),
	])
ZIPALIGN_BIN			= selectFirstExistingBinary([
		noneSafePathJoin(ANDROID_SDK_PATH, "tools", "zipalign"),
		noneSafePathJoin(ANDROID_SDK_PATH, "tools", "zipalign.exe"),
		which('zipalign'),
	])
JARSIGNER_BIN			= which('jarsigner')

# Apache ant
ANT_BIN					= selectFirstExistingBinary([
		which('ant'),
		"C:/android/apache-ant-1.8.4/bin/ant.bat",
		"C:/android/apache-ant-1.9.2/bin/ant.bat",
		"C:/android/apache-ant-1.9.3/bin/ant.bat",
		"C:/android/apache-ant-1.9.4/bin/ant.bat",
	])

def makeNameValueTuple (name):
	return (name, str(eval(name)))

CONFIG_VAR_NAMES = [
		"ANDROID_DIR",
		"NATIVE_LIBS",
		"ANDROID_JAVA_API",
		"NATIVE_LIB_NAME",
		"ANDROID_NDK_PATH",
		"ANDROID_NDK_HOST_OS",
		"ANDROID_NDK_TOOLCHAIN_VERSION",
		"CMAKE_GENERATOR",
		"EXTRA_BUILD_ARGS",
		"ANDROID_SDK_PATH",
		"ANDROID_BIN",
		"ADB_BIN",
		"ZIPALIGN_BIN",
		"JARSIGNER_BIN",
		"ANT_BIN",
	]
CONFIG_STRINGS = [makeNameValueTuple(x) for x in CONFIG_VAR_NAMES]
<commit_msg>GetDevices: Also allow `\.` to be part of the host name am: dbf9715509 am: 133c610be6 am: cc0eaf79f0<commit_after># -*- coding: utf-8 -*-

#-------------------------------------------------------------------------
# drawElements Quality Program utilities
# --------------------------------------
#
# Copyright 2015 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#-------------------------------------------------------------------------

import os
import re
import sys
import shlex
import subprocess
import multiprocessing
import string

try:
	import threading
except ImportError:
	import dummy_threading as threading

class NativeLib:
	def __init__ (self, apiVersion, abiVersion, prebuiltDir):
		self.apiVersion		= apiVersion
		self.abiVersion		= abiVersion
		self.prebuiltDir	= prebuiltDir

	def __str__ (self):
		return "(API: %s, ABI: %s)" % (self.apiVersion, self.abiVersion)

	def __repr__ (self):
		return "(API: %s, ABI: %s)" % (self.apiVersion, self.abiVersion)

def getPlatform ():
	if sys.platform.startswith('linux'):
		return 'linux'
	else:
		return sys.platform

def selectByOS (variants):
	platform = getPlatform()
	if platform in variants:
		return variants[platform]
	elif 'other' in variants:
		return variants['other']
	else:
		raise Exception("No configuration for '%s'" % platform)

def isExecutable (path):
	return os.path.isfile(path) and os.access(path, os.X_OK)

def which (binName):
	for path in os.environ['PATH'].split(os.pathsep):
		path = path.strip('"')
		fullPath = os.path.join(path, binName)
		if isExecutable(fullPath):
			return fullPath

	return None

def isBinaryInPath (binName):
	return which(binName) != None

def selectFirstExistingBinary (filenames):
	for filename in filenames:
		if filename != None and isExecutable(filename):
			return filename

	return None

def selectFirstExistingDir (paths):
	for path in paths:
		if path != None and os.path.isdir(path):
			return path

	return None

def die (msg):
	print msg
	exit(-1)

def shellquote(s):
	return '"%s"' % s.replace('\\', '\\\\').replace('"', '\"').replace('$', '\$').replace('`', '\`')

def execute (commandLine):
	args	= shlex.split(commandLine)
	retcode	= subprocess.call(args)
	if retcode != 0:
		raise Exception("Failed to execute '%s', got %d" % (commandLine, retcode))

def execArgs (args):
	# Make sure previous stdout prints have been written out.
	sys.stdout.flush()
	retcode	= subprocess.call(args)
	if retcode != 0:
		raise Exception("Failed to execute '%s', got %d" % (str(args), retcode))

def execArgsInDirectory (args, cwd, linePrefix=""):

	def readApplyPrefixAndPrint (source, prefix, sink):
		while True:
			line = source.readline()
			if len(line) == 0: # EOF
				break;
			sink.write(prefix + line)

	process = subprocess.Popen(args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	stdoutJob = threading.Thread(target=readApplyPrefixAndPrint, args=(process.stdout, linePrefix, sys.stdout))
	stderrJob = threading.Thread(target=readApplyPrefixAndPrint, args=(process.stderr, linePrefix, sys.stderr))
	stdoutJob.start()
	stderrJob.start()
	retcode = process.wait()
	if retcode != 0:
		raise Exception("Failed to execute '%s', got %d" % (str(args), retcode))

def serialApply(f, argsList):
	for args in argsList:
		f(*args)

def parallelApply(f, argsList):
	class ErrorCode:
		def __init__ (self):
			self.error = None;

	def applyAndCaptureError (func, args, errorCode):
		try:
			func(*args)
		except:
			errorCode.error = sys.exc_info()

	errorCode = ErrorCode()
	jobs = []
	for args in argsList:
		job = threading.Thread(target=applyAndCaptureError, args=(f, args, errorCode))
		job.start()
		jobs.append(job)

	for job in jobs:
		job.join()

	if errorCode.error:
		raise errorCode.error[0], errorCode.error[1], errorCode.error[2]

class Device:
	def __init__(self, serial, product, model, device):
		self.serial		= serial
		self.product	= product
		self.model		= model
		self.device		= device

	def __str__ (self):
		return "%s: {product: %s, model: %s, device: %s}" % (self.serial, self.product, self.model, self.device)

def getDevices (adb):
	proc = subprocess.Popen([adb, 'devices', '-l'], stdout=subprocess.PIPE)
	(stdout, stderr) = proc.communicate()

	if proc.returncode != 0:
		raise Exception("adb devices -l failed, got %d" % proc.returncode)

	ptrn = re.compile(r'^([a-zA-Z0-9\.:]+)\s+.*product:([^\s]+)\s+model:([^\s]+)\s+device:([^\s]+)')
	devices = []
	for line in stdout.splitlines()[1:]:
		if len(line.strip()) == 0:
			continue

		m = ptrn.match(line)
		if m == None:
			print "WARNING: Failed to parse device info '%s'" % line
			continue

		devices.append(Device(m.group(1), m.group(2), m.group(3), m.group(4)))

	return devices

def getWin32Generator ():
	if which("jom.exe") != None:
		return "NMake Makefiles JOM"
	else:
		return "NMake Makefiles"

def isNinjaSupported ():
	return which("ninja") != None

def getUnixGenerator ():
	if isNinjaSupported():
		return "Ninja"
	else:
		return "Unix Makefiles"

def getExtraBuildArgs (generator):
	if generator == "Unix Makefiles":
		return ["--", "-j%d" % multiprocessing.cpu_count()]
	else:
		return []

NDK_HOST_OS_NAMES = [
	"windows",
	"windows-x86_64",
	"darwin-x86",
	"darwin-x86_64",
	"linux-x86",
	"linux-x86_64"
]

def getNDKHostOsName (ndkPath):
	for name in NDK_HOST_OS_NAMES:
		if os.path.exists(os.path.join(ndkPath, "prebuilt", name)):
			return name

	raise Exception("Couldn't determine NDK host OS")

# deqp/android path
ANDROID_DIR				= os.path.realpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))

# Build configuration
NATIVE_LIBS				= [
		#		  API		ABI				prebuiltsDir
		NativeLib(13,		"armeabi-v7a",	'android-arm'),		# ARM v7a ABI
		NativeLib(13,		"x86",			'android-x86'),		# x86
		NativeLib(21,		"arm64-v8a",	'android-arm64'),	# ARM64 v8a ABI
		NativeLib(21,		"x86_64",		'android-x86_64'),	# x86_64
	]

ANDROID_JAVA_API		= "android-22"
NATIVE_LIB_NAME			= "libdeqp.so"

def selectNDKPath ():
	candidates =  [
		os.path.expanduser("~/android-ndk-r11"),
		"C:/android/android-ndk-r11",
		os.environ.get("ANDROID_NDK_PATH", None), # If not defined, return None
	]

	ndkPath = selectFirstExistingDir(candidates)

	if ndkPath == None:
		raise Exception("None of NDK directory candidates exist: %s. Check ANDROID_NDK_PATH in common.py" % candidates)

	return ndkPath

def noneSafePathJoin (*components):
	if None in components:
		return None
	return os.path.join(*components)

# NDK paths
ANDROID_NDK_PATH				= selectNDKPath()
ANDROID_NDK_HOST_OS				= getNDKHostOsName(ANDROID_NDK_PATH)
ANDROID_NDK_TOOLCHAIN_VERSION	= "r11" # Toolchain file is selected based on this

# Native code build settings
CMAKE_GENERATOR			= selectByOS({
		'win32':	getWin32Generator(),
		'other':	getUnixGenerator()
	})
EXTRA_BUILD_ARGS		= getExtraBuildArgs(CMAKE_GENERATOR)

# SDK paths
ANDROID_SDK_PATH		= selectFirstExistingDir([
		os.environ.get("ANDROID_SDK_PATH", None),
		os.path.expanduser("~/android-sdk-linux"),
		os.path.expanduser("~/android-sdk-mac_x86"),
		"C:/android/android-sdk-windows",
	])
ANDROID_BIN				= selectFirstExistingBinary([
		noneSafePathJoin(ANDROID_SDK_PATH, "tools", "android"),
		noneSafePathJoin(ANDROID_SDK_PATH, "tools", "android.bat"),
		which('android'),
	])
ADB_BIN					= selectFirstExistingBinary([
		which('adb'), # \note Prefer adb in path to avoid version issues on dev machines
		noneSafePathJoin(ANDROID_SDK_PATH, "platform-tools", "adb"),
		noneSafePathJoin(ANDROID_SDK_PATH, "platform-tools", "adb.exe"),
	])
ZIPALIGN_BIN			= selectFirstExistingBinary([
		noneSafePathJoin(ANDROID_SDK_PATH, "tools", "zipalign"),
		noneSafePathJoin(ANDROID_SDK_PATH, "tools", "zipalign.exe"),
		which('zipalign'),
	])
JARSIGNER_BIN			= which('jarsigner')

# Apache ant
ANT_BIN					= selectFirstExistingBinary([
		which('ant'),
		"C:/android/apache-ant-1.8.4/bin/ant.bat",
		"C:/android/apache-ant-1.9.2/bin/ant.bat",
		"C:/android/apache-ant-1.9.3/bin/ant.bat",
		"C:/android/apache-ant-1.9.4/bin/ant.bat",
	])

def makeNameValueTuple (name):
	return (name, str(eval(name)))

CONFIG_VAR_NAMES = [
		"ANDROID_DIR",
		"NATIVE_LIBS",
		"ANDROID_JAVA_API",
		"NATIVE_LIB_NAME",
		"ANDROID_NDK_PATH",
		"ANDROID_NDK_HOST_OS",
		"ANDROID_NDK_TOOLCHAIN_VERSION",
		"CMAKE_GENERATOR",
		"EXTRA_BUILD_ARGS",
		"ANDROID_SDK_PATH",
		"ANDROID_BIN",
		"ADB_BIN",
		"ZIPALIGN_BIN",
		"JARSIGNER_BIN",
		"ANT_BIN",
	]
CONFIG_STRINGS = [makeNameValueTuple(x) for x in CONFIG_VAR_NAMES]
<|endoftext|>


<commit_before>from .cfg import CFG
from .vsa import VSA
from .oflow import OFlow
from .cdg import CDG
from .ddg import DDG
from .vfg import VFG
from .cgc import CGC
<commit_msg>Per @Dominik's request.<commit_after>from .cfg import CFG
from .vsa import VSA
from .oflow import OFlow
from .cdg import CDG
from .ddg import DDG
from .vfg import VFG
from .cgc import CGC
from .buffer_overlap import BufferOverlap
<|endoftext|>


<commit_before>"""This is the configuration file for pwm.
It usually resides ~/.config/pwmrc.py
"""

import pwm.commands as cmd
import pwm.widgets as widgets
from pwm.rules import Rule

class Values():
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

loglevel = "debug"

bar = Values(
    interval=1.0,
    font=Values(face="DejaVu Sans Mono", size=12),

    background="#222222",
    foreground="#ffffff",

    active_workspace_foreground="#ffffff",
    active_workspace_background="#285577",
    active_workspace_border="#4c7899",

    inactive_workspace_foreground="#888888",
    inactive_workspace_background="#222222",
    inactive_workspace_border="#333333",

    urgent_workspace_foreground="#ffffff",
    urgent_workspace_background="#900000",
    urgent_workspace_border="#2f343a",

    # Widgets are defined as a list of callables.
    # Every widget should return a tuple like:
    #     (color, text)
    # Where color is a hex value string such as "#ffffff".
    # If color is None the default foreground color will be used.
    #
    # The default widget functions create new callables with the passed
    # arguments. If you want to use your own widget, simply define a function
    # and add its name to the list. Note that your widget will not receive any
    # arguments when called.
    widgets=[
        widgets.volume(),
        widgets.separator(),
        widgets.disk("/"),
        widgets.separator(),
        widgets.load(),
        widgets.separator(),
        widgets.time(),
    ],

    # The color used for the separator widget.
    separator="#aaaaaa",
)

window = Values(
    border=2,
    focused="#1793D1",
    unfocused="#222222",
    urgent="#900000",

    # How fast should floating windows move
    move_speed=0.02)

workspaces = 10

# Keys are described as tuples.
# The first value should be a string describing the key.
# It should start with one or more modifiers following one key.
# Avaliable modifiers are:
#    Control, Shift, Mod1, Mod2, Mod3, Mod4, Mod5
# Whereas Mod4 is usually the Super/Windows key
#
# The second value is the function to execute.
keys = [
    ("Mod4-Shift-q", cmd.quit()),
    ("Mod4-Shift-r", cmd.restart()),
    ("Mod4-q", cmd.kill()),
    ("Mod4-Return", cmd.spawn("urxvt")),
    ("Mod4-p", cmd.menu()),
    ("Shift-Mod4-f", cmd.toggle_floating()),
    ("Mod4-f", cmd.toggle_fullscreen()),
    ("Mod4-space", cmd.toggle_focus_layer()),

    ("Mod4-h", cmd.focus("left")),
    ("Mod4-j", cmd.focus("below")),
    ("Mod4-k", cmd.focus("above")),
    ("Mod4-l", cmd.focus("right")),

    ("Shift-Mod4-h", cmd.move("left")),
    ("Shift-Mod4-j", cmd.move("down")),
    ("Shift-Mod4-k", cmd.move("up")),
    ("Shift-Mod4-l", cmd.move("right")),

    ("Control-Mod4-h", cmd.resize((-0.02, 0))),
    ("Control-Mod4-j", cmd.resize((0, 0.02))),
    ("Control-Mod4-k", cmd.resize((0, -0.02))),
    ("Control-Mod4-l", cmd.resize((0.02, 0))),

    ("XF86AudioRaiseVolume", cmd.spawn("amixer -q set Master 2dB+ unmute")),
    ("XF86AudioLowerVolume", cmd.spawn("amixer -q set Master 2dB- unmute")),
    ("XF86AudioMute", cmd.spawn("amixer -q set Master toggle")),

    ("XF86MonBrightnessUp", cmd.spawn("/usr/bin/xbacklight -inc 10")),
    ("XF86MonBrightnessDown", cmd.spawn("/usr/bin/xbacklight -dec 10")),
]

# Keys for every workspace.
# Note that workspace indices are zero-based.
for i in range(9):
    keys.append(("Mod4-{}".format(i+1), cmd.switch_workspace(i)))
    keys.append(("Shift-Mod4-{}".format(i+1), cmd.send_to_workspace(i)))
keys.append(("Mod4-0", cmd.switch_workspace(9)))
keys.append(("Shift-Mod4-0", cmd.send_to_workspace(9)))

# You can define rules for some windows.
# Each rule has a property and its value used to match a window.
# There are three possible properties: class, role and name.
# Use the xprop tool and look for WM_CLASS, WM_WINDOW_ROLE and _NET_WM_NAME
# (or WM_NAME), to get the correct values.
rules = [
    Rule("role", "preferences", floating=True),
    Rule("class", "truecrypt", floating=True),
    Rule("class", "vlc", floating=True)
]
<commit_msg>Update default config<commit_after>"""
This is the configuration file for pwm.
"""

import pwm.commands as cmd
import pwm.widgets as widgets
from pwm.rules import Rule

class Values():
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

loglevel = "debug"

bar = Values(
    interval=1.0,
    font=Values(face="DejaVu Sans Mono", size=12),

    background="#222222",
    foreground="#ffffff",

    active_workspace_foreground="#ffffff",
    active_workspace_background="#285577",
    active_workspace_border="#4c7899",

    inactive_workspace_foreground="#888888",
    inactive_workspace_background="#222222",
    inactive_workspace_border="#333333",

    urgent_workspace_foreground="#ffffff",
    urgent_workspace_background="#900000",
    urgent_workspace_border="#2f343a",

    # Widgets are defined as a list of callables.
    # Every widget should return a tuple like:
    #     (color, text)
    # Where color is a hex value string such as "#ffffff".
    # If color is None the default foreground color will be used.
    #
    # The default widget functions create new callables with the passed
    # arguments. If you want to use your own widget, simply define a function
    # and add its name to the list. Note that your widget will not receive any
    # arguments when called.
    widgets=[
        widgets.volume(),
        widgets.separator(),
        widgets.disk("/"),
        widgets.separator(),
        widgets.load(),
        widgets.separator(),
        widgets.time(),
    ],

    # The color used for the separator widget.
    separator="#aaaaaa",
)

window = Values(
    border=2,
    focused="#1793D1",
    unfocused="#222222",
    urgent="#900000",

    # How fast should floating windows move
    move_speed=0.02)

workspaces = 10

# Keys are described as tuples.
# The first value should be a string describing the key.
# It should start with one or more modifiers following one key.
# Avaliable modifiers are:
#    Control, Shift, Mod1, Mod2, Mod3, Mod4, Mod5
# Whereas Mod4 is usually the Super/Windows key
#
# The second value is the function to execute.
keys = [
    ("Mod4-Shift-q", cmd.quit()),
    ("Mod4-Shift-r", cmd.restart()),
    ("Mod4-q", cmd.kill()),
    ("Mod4-Return", cmd.spawn("urxvt")),
    ("Mod4-p", cmd.menu()),
    ("Shift-Mod4-f", cmd.toggle_floating()),
    ("Mod4-f", cmd.toggle_fullscreen()),
    ("Mod4-space", cmd.toggle_focus_layer()),

    ("Mod4-h", cmd.focus("left")),
    ("Mod4-j", cmd.focus("below")),
    ("Mod4-k", cmd.focus("above")),
    ("Mod4-l", cmd.focus("right")),

    ("Shift-Mod4-h", cmd.move("left")),
    ("Shift-Mod4-j", cmd.move("down")),
    ("Shift-Mod4-k", cmd.move("up")),
    ("Shift-Mod4-l", cmd.move("right")),

    ("Control-Mod4-h", cmd.resize((-0.02, 0))),
    ("Control-Mod4-j", cmd.resize((0, 0.02))),
    ("Control-Mod4-k", cmd.resize((0, -0.02))),
    ("Control-Mod4-l", cmd.resize((0.02, 0))),

    ("XF86AudioRaiseVolume", cmd.spawn("amixer -q set Master 2dB+ unmute")),
    ("XF86AudioLowerVolume", cmd.spawn("amixer -q set Master 2dB- unmute")),
    ("XF86AudioMute", cmd.spawn("amixer -q set Master toggle")),

    ("XF86MonBrightnessUp", cmd.spawn("xbacklight -inc 10")),
    ("XF86MonBrightnessDown", cmd.spawn("xbacklight -dec 10")),
]

# Keys for every workspace.
# Note that workspace indices are zero-based.
for i in range(9):
    keys.append(("Mod4-{}".format(i+1), cmd.switch_workspace(i)))
    keys.append(("Shift-Mod4-{}".format(i+1), cmd.send_to_workspace(i)))
keys.append(("Mod4-0", cmd.switch_workspace(9)))
keys.append(("Shift-Mod4-0", cmd.send_to_workspace(9)))

# You can define rules for some windows.
# Each rule has a property and its value used to match a window.
# There are three possible properties: class, role and name.
# Use the xprop tool and look for WM_CLASS, WM_WINDOW_ROLE and _NET_WM_NAME
# (or WM_NAME), to get the correct values.
rules = [
    Rule("role", "preferences", floating=True),
    Rule("class", "truecrypt", floating=True),
    Rule("class", "vlc", floating=True)
]
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import requests
import re as _re
import xml.etree.ElementTree as ET
import urllib.parse

def seek(chk_key, cur_epi):
    # Compose search keyword
    tepi = cur_epi+1
    chk_key = str(chk_key)
    chk_key = "%s %02d" % (chk_key, tepi)
    chk_key = urllib.parse.quote_plus(chk_key)

    # Get search result page
    url = "https://www.nyaa.se/?page=rss&term={}".format(chk_key)
    r = requests.get(url, timeout=2)
    r.encoding = "utf-8"

    tree = ET.fromstring(r.text)

    r = []
    for item in tree.iterfind("channel/item"):
        r.append({"title": item.find("title").text, "link": item.find('link').text})

    if len(r) == 0:
        return 0
    else:
        return r

if __name__ == '__main__':
    r = seek("All Out!! Ohys", 14)
    if r:
        for i in r:
            print(i['title'])
            print(i['link'])
    else:
        print('Not found')
<commit_msg>Update nyaase seeker<commit_after>#!/usr/bin/env python3
import requests
import xml.etree.ElementTree as ET
import urllib.parse

def seek(chk_key, cur_epi):
    # Compose search keyword
    tepi = cur_epi+1
    chk_key = str(chk_key)
    chk_key = "%s %02d" % (chk_key, tepi)
    chk_key = urllib.parse.quote_plus(chk_key)

    # Get search result page
    url = "https://nyaa.pantsu.cat/feed?c=_&s=0&limit=50\
&userID=0&q={}".format(chk_key)
    r = requests.get(url, timeout=2)
    r.encoding = "utf-8"

    tree = ET.fromstring(r.text)

    r = []
    for item in tree.iterfind("channel/item"):
        r.append({"title": item.find("title").text,
                  "link": item.find('guid').text})

    if len(r) == 0:
        return 0
    else:
        return r

if __name__ == '__main__':
    r = seek("All Out!! Ohys", 14)
    if r:
        for i in r:
            print(i['title'])
            print(i['link'])
    else:
        print('Not found')
<|endoftext|>


<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# for running shell commands
import subprocess
# for finding out the linux distro
import sys,os,platform
# for downloading rpm/ppc files
import urllib

# ctx is imported and used in operations
from cloudify import ctx

# put the operation decorator on any function that is a task
from cloudify.decorators import operation

# Import Cloudify exception
from cloudify.exceptions import NonRecoverableError

# Runs any Shell Command
def _run_shell_command(command):
  ctx.logger.info("Running shell command: {0}".format(command))
  try:
    run = subprocess.check_call(
      command)
  except subprocess.CalledProcessError:
  	ctx.logger.error("Unable to run shell command: {0}".format(command))
  	raise NonRecoverableError("Command failed: {0}".format(command))
  return run

# Gets the Distrobution
def _get_distro_version():
  info = platform.dist()
  distro = info[0]
  version = info[1]
  return distro,version

# Returns the Package Manager for the distrobution
def _get_package_manager():
  package_manager = "apt-get"
  distro,version = _get_distro_version()
  if distro == "Ubuntu":
    ctx.logger.info("{0} is the Linux distribution.".format(distro))
    package_manager = "apt-get"
  elif distro == "Centos":
    ctx.logger.info("{0} is the Linux distribution.".format(distro))
    package_manager = "yum"
  else:
    ctx.logger.error("Currently Operating System {0} is not supported".format(distro))
    exit(1)
  return package_manager

# Returns Install Arguments for a Package Manager
def _install_args(package_manager):
  if package_manager == "apt-get":
    quiet_output = "-qq"
    assume_yes = "-y"
  if package_manager == "yum":
    quiet_output = "-q"
    assume_yes = "-y"
  return quiet_output,assume_yes

# Upgrade the Package Manager
def _upgrade_package_manager(package_manager):
  q,y = _install_args(package_manager)
  ctx.logger.info("Upgrading {0}".format(package_manager))
  command = ["sudo",package_manager,"upgrade",y,q]
  _run_shell_command(command)

# Installs the EPEL Repo
def _install_epel_repo():
  wget_url = "http://dl.fedoraproject.org/pub/epel/5/x86_64/"
  distro,version = _get_distro_version()
  if version == "5*":
    rpm_version = "epel-release-5-4.noarch.rpm"
  elif version == "6*":
    rpm_version = "epel-release-6-8.noarch.rpm"
  elif version == "7*":
    rpm_version = "epel-release-7-1.noarch.rpm"
  else:
  	rpm_version = ""
  	ctx.logger.error("invalid version of CentOs: {0}".format(version))
  filename = os.path.join("/tmp",rpm_version)
  if not os.path.exists(filename):
    ctx.logger.info("Deleting Existing RPM file: {0}".format(filename))
    ctx.logger.info("Downloading EPEL RPM: {0}{1}".format(wget_url,rpm_version))
    urllib.urlretrieve(wget_url + rpm_version, filename)
  command = ["sudo","rpm","-Uvh","/tmp/" + rpm_version]
  _run_shell_command(command)

# Installs the PPA repos
def _install_ppa_repo():
  # Installs the Software Properties Common Dependency
  def install_dependency():
    command = ["sudo","apt-get","install","software-properties-common"]
    _run_shell_command(command)
  command = ["sudo","apt-add-repository","ppa:ansible/ansible"]
  _run_shell_command(command)

# Decides which repos to install
def _add_repo(package_manager):
  ctx.logger.info("Installing Additional Repositories to {0}".format(package_manager))
  if package_manager == "yum":
    _install_epel_repo()
  elif package_manager == "apt-get":
    _install_ppa_repo()

# Updates a Package Manager
def _update_package_manager(package_manager):
  command = ["sudo","apt-get","clean"]
  _run_shell_command(command)
  _add_repo(package_manager)
  ctx.logger.info("Updating {0}".format(package_manager))
  command = ["sudo",package_manager,"update"]
  _run_shell_command(command)

# validate the installation
def _validate_installation(package):
  ctx.logger.info("Validating {0}: ".format(package))
  command = [pakcage,"--version"]
  code = _run_shell_command(command)
  if code > 0:
    ctx.logger.info("Installation was unsuccessful")
  else:
    ctx.logger.info("Installation was successful")

# Installs a Package
def install_package(package):
  package_manager = _get_package_manager()
  ctx.logger.info("Installing {0}".format(package))
  _update_package_manager(package_manager)
  _upgrade_package_manager(package_manager)
  q,y = _install_args(package_manager)
  command = ["sudo",package_manager,"install",package,q,y]
  _run_shell_command(command)
  _validate_installation(package)

@operation
def install(**kwargs):
  if "package_name" in kwargs:
    package = package_name
  else:
    package = "ansible"
  install_package(package)
<commit_msg>added -f to upgrade apt-get<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# for running shell commands
import subprocess
# for finding out the linux distro
import sys,os,platform
# for downloading rpm/ppc files
import urllib

# ctx is imported and used in operations
from cloudify import ctx

# put the operation decorator on any function that is a task
from cloudify.decorators import operation

# Import Cloudify exception
from cloudify.exceptions import NonRecoverableError

# Runs any Shell Command
def _run_shell_command(command):
  ctx.logger.info("Running shell command: {0}".format(command))
  try:
    run = subprocess.check_call(
      command)
  except subprocess.CalledProcessError:
  	ctx.logger.error("Unable to run shell command: {0}".format(command))
  	raise NonRecoverableError("Command failed: {0}".format(command))
  return run

# Gets the Distrobution
def _get_distro_version():
  info = platform.dist()
  distro = info[0]
  version = info[1]
  return distro,version

# Returns the Package Manager for the distrobution
def _get_package_manager():
  package_manager = "apt-get"
  distro,version = _get_distro_version()
  if distro == "Ubuntu":
    ctx.logger.info("{0} is the Linux distribution.".format(distro))
    package_manager = "apt-get"
  elif distro == "Centos":
    ctx.logger.info("{0} is the Linux distribution.".format(distro))
    package_manager = "yum"
  else:
    ctx.logger.error("Currently Operating System {0} is not supported".format(distro))
    exit(1)
  return package_manager

# Returns Install Arguments for a Package Manager
def _install_args(package_manager):
  if package_manager == "apt-get":
    quiet_output = "-qq"
    assume_yes = "-y"
  if package_manager == "yum":
    quiet_output = "-q"
    assume_yes = "-y"
  return quiet_output,assume_yes

# Upgrade the Package Manager
def _upgrade_package_manager(package_manager):
  q,y = _install_args(package_manager)
  ctx.logger.info("Upgrading {0}".format(package_manager))
  command = ["sudo",package_manager,"upgrade",y,q]
  _run_shell_command(command)

# Installs the EPEL Repo
def _install_epel_repo():
  wget_url = "http://dl.fedoraproject.org/pub/epel/5/x86_64/"
  distro,version = _get_distro_version()
  if version == "5*":
    rpm_version = "epel-release-5-4.noarch.rpm"
  elif version == "6*":
    rpm_version = "epel-release-6-8.noarch.rpm"
  elif version == "7*":
    rpm_version = "epel-release-7-1.noarch.rpm"
  else:
  	rpm_version = ""
  	ctx.logger.error("invalid version of CentOs: {0}".format(version))
  filename = os.path.join("/tmp",rpm_version)
  if not os.path.exists(filename):
    ctx.logger.info("Deleting Existing RPM file: {0}".format(filename))
    ctx.logger.info("Downloading EPEL RPM: {0}{1}".format(wget_url,rpm_version))
    urllib.urlretrieve(wget_url + rpm_version, filename)
  command = ["sudo","rpm","-Uvh","/tmp/" + rpm_version]
  _run_shell_command(command)

# Installs the PPA repos
def _install_ppa_repo():
  # Installs the Software Properties Common Dependency
  def install_dependency():
    command = ["sudo","apt-get","install","software-properties-common"]
    _run_shell_command(command)
  command = ["sudo","apt-add-repository","ppa:ansible/ansible"]
  _run_shell_command(command)

# Decides which repos to install
def _add_repo(package_manager):
  ctx.logger.info("Installing Additional Repositories to {0}".format(package_manager))
  if package_manager == "yum":
    _install_epel_repo()
  elif package_manager == "apt-get":
    _install_ppa_repo()

# Updates a Package Manager
def _update_package_manager(package_manager):
  command = ["sudo","apt-get","clean"]
  _run_shell_command(command)
  _add_repo(package_manager)
  ctx.logger.info("Updating {0}".format(package_manager))
  command = ["sudo",package_manager,"update"]
  _run_shell_command(command)

# validate the installation
def _validate_installation(package):
  ctx.logger.info("Validating {0}: ".format(package))
  command = [pakcage,"--version"]
  code = _run_shell_command(command)
  if code > 0:
    ctx.logger.info("Installation was unsuccessful")
  else:
    ctx.logger.info("Installation was successful")

# Installs a Package
def install_package(package):
  package_manager = _get_package_manager()
  ctx.logger.info("Installing {0}".format(package))
  _update_package_manager(package_manager)
  _upgrade_package_manager(package_manager)
  q,y = _install_args(package_manager)
  command = ["sudo",package_manager,"install",package,q,y,"-f"]
  _run_shell_command(command)
  _validate_installation(package)

@operation
def install(**kwargs):
  if "package_name" in kwargs:
    package = package_name
  else:
    package = "ansible"
  install_package(package)
<|endoftext|>


<commit_before>"""Plan future DESI observations.
"""
from __future__ import print_function, division

import datetime
import os.path

import numpy as np

import astropy.table
import astropy.io.fits
import astropy.units as u

import desiutil.log

import desimodel.io

import desisurvey.utils

def load_design_hourangle(name='surveyinit.fits'):
    """Load design hour-angle assignments from disk.

    Reads column 'HA' from binary table HDU 'DESIGN'. This is the format
    saved by the ``surveyinit`` script, but any FITS file following the
    same convention can be used.

    Parameters
    ----------
    name : str
        Name of the FITS file to read. A relative path is assumed to
        refer to the output path specified in the configuration.

    Returns
    -------
    array
        1D array of design hour angles in degrees, with indexing
        that matches :class:`desisurvey.tiles.Tiles`.
    """
    config = desisurvey.config.Configuration()
    fullname = config.get_path(name)
    with astropy.io.fits.open(fullname, memmap=False) as hdus:
        HA = hdus['DESIGN'].data['HA'].copy()
    tiles = desisurvey.tiles.get_tiles()
    if HA.shape != (tiles.ntiles,):
        raise ValueError('Read unexpected HA shape.')
    return HA

def load_weather(start_date=None, stop_date=None, name='surveyinit.fits'):
    """Load dome-open fraction expected during each night of the survey.

    Reads Image HDU 'WEATHER'. This is the format saved by the
    ``surveyinit`` script, but any FITS file following the same
    convention can be used.

    Parameters
    ----------
    name : str
        Name of the FITS file to read. A relative path is assumed to
        refer to the output path specified in the configuration.
    start_date : date or None
        First night to include or use the first date of the survey. Must
        be convertible to a date using :func:`desisurvey.utils.get_date`.
    stop_date : date or None
        First night to include or use the last date of the survey. Must
        be convertible to a date using :func:`desisurvey.utils.get_date`.

    Returns
    -------
    array
        1D array of length equal to the span between stop_date and
        start_date. Values are between 0 (dome closed all night) and
        1 (dome open all night).
    """
    config = desisurvey.config.Configuration()
    if start_date is None:
        start_date = config.first_day()
    else:
        start_date = desisurvey.utils.get_date(start_date)
    if stop_date is None:
        stop_date = config.last_day()
    else:
        stop_date = desisurvey.utils.get_date(stop_date)
    if stop_date <= start_date:
        raise ValueError('Expected start_date < stop_date.')
    with astropy.io.fits.open(config.get_path(name), memmap=False) as hdus:
        weather = hdus['WEATHER'].data
        num_nights = len(weather)
        first = desisurvey.utils.get_date(hdus['WEATHER'].header['FIRST'])
        last = first + datetime.timedelta(num_nights)
    if start_date < first:
        raise ValueError('Weather not available before {}.'.format(first.isoformat()))
    num_nights = (stop_date - start_date).days
    if last < stop_date:
        raise ValueError('Weather not available after {}.',format(last.isoformat()))
    ilo, ihi = (start_date - first).days, (stop_date - first).days
    return weather[ilo:ihi]

class Planner(object):
    """Coordinate afternoon planning activities.

    Parameters
    ----------
    rules : object or None
        Object with an ``apply`` method that is used to implement survey strategy by updating
        tile priorities each afternoon.  When None, all tiles have equal priority.
    restore : str or None
        Restore internal state from the snapshot saved to this filename,
        or initialize a new planner when None. Use :meth:`save` to
        save a snapshot to be restored later. Filename is relative to
        the configured output path unless an absolute path is
        provided.
    """
    def __init__(self, rules=None, restore=None):
        self.log = desiutil.log.get_logger()
        self.rules = rules
        config = desisurvey.config.Configuration()
        self.fiberassign_cadence = config.fiber_assignment_cadence()
        if self.fiberassign_cadence not in ('daily', 'monthly'):
            raise ValueError('Invalid fiberassign_cadence: "{}".'.format(self.fiberassign_cadence))
        self.tiles = desisurvey.tiles.get_tiles()
        self.ephem = desisurvey.ephem.get_ephem()
        if restore is not None:
            # Restore the plan for a survey in progress.
            fullname = config.get_path(restore)
            if not os.path.exists(fullname):
                raise RuntimeError('Cannot restore planner from non-existent "{}".'.format(fullname))
            t = astropy.table.Table.read(fullname)
            if t.meta['CADENCE'] != self.fiberassign_cadence:
                raise ValueError('Fiberassign cadence mismatch.')
            first, last = t.meta['FIRST'], t.meta['LAST']
            self.first_night = desisurvey.utils.get_date(first) if first else None
            self.last_night = desisurvey.utils.get_date(last) if last else None
            self.tile_covered = t['COVERED'].data.copy()
            self.tile_countdown = t['COUNTDOWN'].data.copy()
            self.tile_available = t['AVAILABLE'].data.copy()
            self.tile_priority = t['PRIORITY'].data.copy()
            self.log.debug(
                'Restored plan with {} ({}) / {} tiles covered (available) from "{}".'
                .format(np.count_nonzero(self.tile_covered),
                        np.count_nonzero(self.tile_available), self.tiles.ntiles,
                        fullname))
        else:
            # Initialize the plan for a a new survey.
            self.first_night = self.last_night = None
            # Initialize per-tile arrays.
            self.tile_covered = np.full(self.tiles.ntiles, -1)
            self.tile_available = np.zeros(self.tiles.ntiles, bool)
            # Initailize the delay countdown for each tile.
            self.tile_countdown = self.tiles.fiberassign_delay.copy()
            # Initialize priorities.
            if self.rules is not None:
                none_completed = np.zeros(self.tiles.ntiles, bool)
                self.tile_priority = self.rules.apply(none_completed)
            else:
                self.tile_priority = np.ones(self.tiles.ntiles, float)
            # Mark tiles that are initially available.
            fiberassign_order = config.fiber_assignment_order
            for passnum in self.tiles.passes:
                if 'P{}'.format(passnum) not in fiberassign_order.keys:
                    # Mark tiles in this pass as initially available.
                    under = self.tiles.passnum == passnum
                    self.tile_covered[under] = 0
                    self.tile_available[under] = True
                    self.log.info('Pass {} available for initial observing.'.format(passnum))
        if not np.any(self.tile_priority > 0):
            raise RuntimeError('All tile priorities are all <= 0.')

    def save(self, name):
        """Save a snapshot of our current state that can be restored.

        The snapshot file size is about 400Kb.

        Parameters
        ----------
        name : str
            Name of FITS file where the snapshot will be saved. The file will
            be saved under our configuration's output path unless name is
            already an absolute path.  Pass the same name to the constructor's
            ``restore`` argument to restore this snapshot.
        """
        config = desisurvey.config.Configuration()
        fullname = config.get_path(name)
        t = astropy.table.Table(meta={
            'CADENCE': self.fiberassign_cadence,
            'FIRST': self.first_night.isoformat() if self.first_night else '',
            'LAST': self.last_night.isoformat() if self.last_night else '',
            })
        t['COVERED'] = self.tile_covered
        t['COUNTDOWN'] = self.tile_countdown
        t['AVAILABLE'] = self.tile_available
        t['PRIORITY'] = self.tile_priority
        t.write(fullname, overwrite=True)
        self.log.debug(
            'Saved plan with {} ({}) / {} tiles covered (available) to "{}".'
            .format(np.count_nonzero(self.tile_covered),
                    np.count_nonzero(self.tile_available), self.tiles.ntiles,
                    fullname))

    def fiberassign(self, night, completed):
        """Update fiber assignments.
        """
        # Calculate the number of elapsed nights in the survey.
        day_number = (night - self.first_night).days
        for passnum in self.tiles.passes:
            under = self.tiles.passnum == passnum
            over = self.tiles.tile_over[passnum]
            if not np.any(over):
                continue
            overlapping = self.tiles.overlapping[passnum]
            # Identify all tiles in this pass whose covering tiles are completed.
            covered = np.all(~overlapping | completed[over], axis=1)
            # Which tiles have been newly covered since the last call to fiberassign?
            new_covered = covered & (self.tile_covered[under] == -1)
            if np.any(new_covered):
                new_tiles = self.tiles.tileID[under][new_covered]
               # Record the day number when these tiles were first covered.
                new = under.copy()
                new[under] = new_covered
                self.tile_covered[new] = day_number
        # Identify tiles that have been covered but not yet had fiber assignment run.
        ready = ~self.tile_available & (self.tile_covered >= 0)
        # Run fiber assignment on ready tiles that have completed their countdown.
        run_now = ready & (self.tile_countdown == 0)
        self.tile_available[run_now] = True
        # Update delay countdown for the remaining ready tiles.
        delayed = ready & (self.tile_countdown > 0)
        self.tile_countdown[delayed] -= 1
        self.log.info('Fiber assigned {} tiles with {} delayed on {}.'
                      .format(np.count_nonzero(run_now), np.count_nonzero(delayed), night))

    def afternoon_plan(self, night, completed):
        self.log.debug('Starting afternoon planning for {} with {} / {} tiles completed.'
                       .format(night, np.count_nonzero(completed), self.tiles.ntiles))
        if self.first_night is None:
            # Remember the first night of the survey.
            self.first_night = night
        # Update fiber assignments this afternoon?
        if self.fiberassign_cadence == 'monthly':
            # Run fiber assignment on the afternoon before the full moon.
            dt = self.ephem.get_night(night)['nearest_full_moon']
            run_fiberassign = (dt > -0.5) and (dt <= 0.5)
            assert run_fiberassign == self.ephem.is_full_moon(night, num_nights=1)
        else:
            run_fiberassign = True
        if run_fiberassign:
            self.fiberassign(night, completed)
        # Update tile priorities.
        if self.rules is not None:
            self.tile_priority = self.rules.apply(completed)
        self.last_night = night
        return self.tile_available, self.tile_priority
<commit_msg>Save plan to binary table with EXTNAME=PLAN<commit_after>"""Plan future DESI observations.
"""
from __future__ import print_function, division

import datetime
import os.path

import numpy as np

import astropy.table
import astropy.io.fits
import astropy.units as u

import desiutil.log

import desimodel.io

import desisurvey.utils

def load_design_hourangle(name='surveyinit.fits'):
    """Load design hour-angle assignments from disk.

    Reads column 'HA' from binary table HDU 'DESIGN'. This is the format
    saved by the ``surveyinit`` script, but any FITS file following the
    same convention can be used.

    Parameters
    ----------
    name : str
        Name of the FITS file to read. A relative path is assumed to
        refer to the output path specified in the configuration.

    Returns
    -------
    array
        1D array of design hour angles in degrees, with indexing
        that matches :class:`desisurvey.tiles.Tiles`.
    """
    config = desisurvey.config.Configuration()
    fullname = config.get_path(name)
    with astropy.io.fits.open(fullname, memmap=False) as hdus:
        HA = hdus['DESIGN'].data['HA'].copy()
    tiles = desisurvey.tiles.get_tiles()
    if HA.shape != (tiles.ntiles,):
        raise ValueError('Read unexpected HA shape.')
    return HA

def load_weather(start_date=None, stop_date=None, name='surveyinit.fits'):
    """Load dome-open fraction expected during each night of the survey.

    Reads Image HDU 'WEATHER'. This is the format saved by the
    ``surveyinit`` script, but any FITS file following the same
    convention can be used.

    Parameters
    ----------
    name : str
        Name of the FITS file to read. A relative path is assumed to
        refer to the output path specified in the configuration.
    start_date : date or None
        First night to include or use the first date of the survey. Must
        be convertible to a date using :func:`desisurvey.utils.get_date`.
    stop_date : date or None
        First night to include or use the last date of the survey. Must
        be convertible to a date using :func:`desisurvey.utils.get_date`.

    Returns
    -------
    array
        1D array of length equal to the span between stop_date and
        start_date. Values are between 0 (dome closed all night) and
        1 (dome open all night).
    """
    config = desisurvey.config.Configuration()
    if start_date is None:
        start_date = config.first_day()
    else:
        start_date = desisurvey.utils.get_date(start_date)
    if stop_date is None:
        stop_date = config.last_day()
    else:
        stop_date = desisurvey.utils.get_date(stop_date)
    if stop_date <= start_date:
        raise ValueError('Expected start_date < stop_date.')
    with astropy.io.fits.open(config.get_path(name), memmap=False) as hdus:
        weather = hdus['WEATHER'].data
        num_nights = len(weather)
        first = desisurvey.utils.get_date(hdus['WEATHER'].header['FIRST'])
        last = first + datetime.timedelta(num_nights)
    if start_date < first:
        raise ValueError('Weather not available before {}.'.format(first.isoformat()))
    num_nights = (stop_date - start_date).days
    if last < stop_date:
        raise ValueError('Weather not available after {}.',format(last.isoformat()))
    ilo, ihi = (start_date - first).days, (stop_date - first).days
    return weather[ilo:ihi]

class Planner(object):
    """Coordinate afternoon planning activities.

    Parameters
    ----------
    rules : object or None
        Object with an ``apply`` method that is used to implement survey strategy by updating
        tile priorities each afternoon.  When None, all tiles have equal priority.
    restore : str or None
        Restore internal state from the snapshot saved to this filename,
        or initialize a new planner when None. Use :meth:`save` to
        save a snapshot to be restored later. Filename is relative to
        the configured output path unless an absolute path is
        provided.
    """
    def __init__(self, rules=None, restore=None):
        self.log = desiutil.log.get_logger()
        self.rules = rules
        config = desisurvey.config.Configuration()
        self.fiberassign_cadence = config.fiber_assignment_cadence()
        if self.fiberassign_cadence not in ('daily', 'monthly'):
            raise ValueError('Invalid fiberassign_cadence: "{}".'.format(self.fiberassign_cadence))
        self.tiles = desisurvey.tiles.get_tiles()
        self.ephem = desisurvey.ephem.get_ephem()
        if restore is not None:
            # Restore the plan for a survey in progress.
            fullname = config.get_path(restore)
            if not os.path.exists(fullname):
                raise RuntimeError('Cannot restore planner from non-existent "{}".'.format(fullname))
            t = astropy.table.Table.read(fullname, hdu='PLAN')
            if t.meta['CADENCE'] != self.fiberassign_cadence:
                raise ValueError('Fiberassign cadence mismatch.')
            first, last = t.meta['FIRST'], t.meta['LAST']
            self.first_night = desisurvey.utils.get_date(first) if first else None
            self.last_night = desisurvey.utils.get_date(last) if last else None
            self.tile_covered = t['COVERED'].data.copy()
            self.tile_countdown = t['COUNTDOWN'].data.copy()
            self.tile_available = t['AVAILABLE'].data.copy()
            self.tile_priority = t['PRIORITY'].data.copy()
            self.log.debug(
                'Restored plan with {} ({}) / {} tiles covered (available) from "{}".'
                .format(np.count_nonzero(self.tile_covered),
                        np.count_nonzero(self.tile_available), self.tiles.ntiles,
                        fullname))
        else:
            # Initialize the plan for a a new survey.
            self.first_night = self.last_night = None
            # Initialize per-tile arrays.
            self.tile_covered = np.full(self.tiles.ntiles, -1)
            self.tile_available = np.zeros(self.tiles.ntiles, bool)
            # Initailize the delay countdown for each tile.
            self.tile_countdown = self.tiles.fiberassign_delay.copy()
            # Initialize priorities.
            if self.rules is not None:
                none_completed = np.zeros(self.tiles.ntiles, bool)
                self.tile_priority = self.rules.apply(none_completed)
            else:
                self.tile_priority = np.ones(self.tiles.ntiles, float)
            # Mark tiles that are initially available.
            fiberassign_order = config.fiber_assignment_order
            for passnum in self.tiles.passes:
                if 'P{}'.format(passnum) not in fiberassign_order.keys:
                    # Mark tiles in this pass as initially available.
                    under = self.tiles.passnum == passnum
                    self.tile_covered[under] = 0
                    self.tile_available[under] = True
                    self.log.info('Pass {} available for initial observing.'.format(passnum))
        if not np.any(self.tile_priority > 0):
            raise RuntimeError('All tile priorities are all <= 0.')

    def save(self, name):
        """Save a snapshot of our current state that can be restored.

        The output file has a binary table (extname PLAN) with columns
        COVERED, COUNTDOWN, AVAILABLE and PRIORITY and header keywords
        CADENCE, FIRST, LAST. The saved file size is about 400Kb.

        Parameters
        ----------
        name : str
            Name of FITS file where the snapshot will be saved. The file will
            be saved under our configuration's output path unless name is
            already an absolute path.  Pass the same name to the constructor's
            ``restore`` argument to restore this snapshot.
        """
        config = desisurvey.config.Configuration()
        fullname = config.get_path(name)
        t = astropy.table.Table(meta={
            'CADENCE': self.fiberassign_cadence,
            'FIRST': self.first_night.isoformat() if self.first_night else '',
            'LAST': self.last_night.isoformat() if self.last_night else '',
            })
        t['COVERED'] = self.tile_covered
        t['COUNTDOWN'] = self.tile_countdown
        t['AVAILABLE'] = self.tile_available
        t['PRIORITY'] = self.tile_priority
        hdus = astropy.io.fits.HDUList()
        hdus.append(astropy.io.fits.PrimaryHDU())
        hdus.append(astropy.io.fits.table_to_hdu(t))
        hdus[-1].name = 'PLAN'
        hdus.writeto(fullname, overwrite=True)
        self.log.debug(
            'Saved plan with {} ({}) / {} tiles covered (available) to "{}".'
            .format(np.count_nonzero(self.tile_covered),
                    np.count_nonzero(self.tile_available), self.tiles.ntiles,
                    fullname))

    def fiberassign(self, night, completed):
        """Update fiber assignments.
        """
        # Calculate the number of elapsed nights in the survey.
        day_number = (night - self.first_night).days
        for passnum in self.tiles.passes:
            under = self.tiles.passnum == passnum
            over = self.tiles.tile_over[passnum]
            if not np.any(over):
                continue
            overlapping = self.tiles.overlapping[passnum]
            # Identify all tiles in this pass whose covering tiles are completed.
            covered = np.all(~overlapping | completed[over], axis=1)
            # Which tiles have been newly covered since the last call to fiberassign?
            new_covered = covered & (self.tile_covered[under] == -1)
            if np.any(new_covered):
                new_tiles = self.tiles.tileID[under][new_covered]
               # Record the day number when these tiles were first covered.
                new = under.copy()
                new[under] = new_covered
                self.tile_covered[new] = day_number
        # Identify tiles that have been covered but not yet had fiber assignment run.
        ready = ~self.tile_available & (self.tile_covered >= 0)
        # Run fiber assignment on ready tiles that have completed their countdown.
        run_now = ready & (self.tile_countdown == 0)
        self.tile_available[run_now] = True
        # Update delay countdown for the remaining ready tiles.
        delayed = ready & (self.tile_countdown > 0)
        self.tile_countdown[delayed] -= 1
        self.log.info('Fiber assigned {} tiles with {} delayed on {}.'
                      .format(np.count_nonzero(run_now), np.count_nonzero(delayed), night))

    def afternoon_plan(self, night, completed):
        self.log.debug('Starting afternoon planning for {} with {} / {} tiles completed.'
                       .format(night, np.count_nonzero(completed), self.tiles.ntiles))
        if self.first_night is None:
            # Remember the first night of the survey.
            self.first_night = night
        # Update fiber assignments this afternoon?
        if self.fiberassign_cadence == 'monthly':
            # Run fiber assignment on the afternoon before the full moon.
            dt = self.ephem.get_night(night)['nearest_full_moon']
            run_fiberassign = (dt > -0.5) and (dt <= 0.5)
            assert run_fiberassign == self.ephem.is_full_moon(night, num_nights=1)
        else:
            run_fiberassign = True
        if run_fiberassign:
            self.fiberassign(night, completed)
        # Update tile priorities.
        if self.rules is not None:
            self.tile_priority = self.rules.apply(completed)
        self.last_night = night
        return self.tile_available, self.tile_priority
<|endoftext|>


<commit_before>#!/usr/bin/python
# vim: set fileencoding=UTF-8 :
"""
Simple import script to load data from an API and push to csv
"""

import requests
import os #needed for Jenkisn environment variables
#import json
import pandas as pd
from pandas.io.json import json_normalize
import datetime

result = []
jarjestysluvat = []
oiva_maaraykset=[]
maaraykset=[]
lupa_data= []
loadtime=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
url = 'https://oiva.minedu.fi/api/export/jarjestysluvat'
try:
    api_key = os.environ['AUTH_API_KEY']
except KeyError:
    print("API-key missing")
try:
    api_user = os.environ['AUTH_API_USER']
except KeyError:
    print("API-user missing")

reqheaders = {'Content-Type': 'application/json'}

#reqheaders['Authorization'] = 'oiva '

def makerow_jarjestysluvat():
  return {
    "id": None,     # bigint	pk
    "edellinen_lupa_id": None,     # bigint,	optional
    "paatoskierros_id": None,     # bigint	not null
    "lupatila_id": None,     # bigint	not null
    "asiatyyppi_id": None,     # bigint	not null
    "diaarinumero": None,     # character varying(20)	not null
    "jarjestajaYtunnus": None,     # character varying(10)	not null
    "jarjestajaOid": None,     # character varying(255)	not null
    "alkupvm": None,     # date	not null
    "loppupvm": None,     # date, optional
    "meta.kirje": None,
    "meta.ministeri": None,
    "meta.ministeri_nimike": None,
    "meta.esittelija": None,
    "meta.esittelija_nimike": None,
    "meta.paatos_old_id": None,
    "meta.koulutustehtava_old_id": None,
    "maksu": None,     # boolean, optional
    "luoja": None,     # text, optional
    "luontipvm": None,     # timestamp without time zone, optional
    "paivittaja": None,     # text, optional
    "paivityspvm": None,     # timestamp without time zone, optional
    "source": None,
    "loadtime": None
    }
def makerow_maaraykset():
  return {
    "id": None,
    "lupaid": None,   # bigint not null
    "kohde.tunniste": None,
    "koodisto": None,   # character varying(255)
    "arvo": None,   # text, optional
    "koodiarvo": None,   # character varying(255), optional
    "maaraystyyppi.tunniste": None,
    #"meta": None,   # jsonb , optional
    "luoja": None,   # text, optional
    "luontipvm": None,   # timestamp without time zone, optional
    "paivittaja": None,   # text, optional
    "paivityspvm": None,   # timestamp without time zone, optional
    "source": None,
    "loadtime": None
  }
# URL Status check
def url_ok(url):
    r = requests.head(url,  auth=(api_user, api_key))
    return r.status_code == 200
# http status code
http_status_code = str(requests.head(url,  auth=(api_user, api_key)).status_code)

# Check if key 'x' is in dictionary 'y'
def key_check(x,y):
   if x in y:
      return y[x]
   else:
      return None

if url_ok(url):
   print("OIVA API was available with status code: " +http_status_code)
else:
   print("OIVA API was not accessible and returned status code:" + http_status_code)
   exit

# Loading the data into response  list (jarjestysluvat)
response = requests.get(url,  auth=(api_user, api_key)).json()

# select jarjestysluvat
for i in response:
    row = makerow_jarjestysluvat()
    row["id"] = i["maaraykset"][0]["lupaId"]
    row["edellinen_lupa_id"] =key_check("aatoskierros_id", i)
    row["lupatila_id"] = key_check("lupatila_id", i)
    row["asiatyyppi_id"] = key_check("asiatyyppi_id", i)
    row["diaarinumero"] = key_check("diaarinumero", i)
    row["jarjestajaYtunnus"] = key_check("jarjestajaYtunnus", i)
    row["jarjestajaOid"] = key_check("jarjestajaOid", i)
    row["alkupvm"] = key_check("alkupvm", i)
    row["loppupvm"] = key_check("loppupvm", i)
    row["meta.kirje"] = key_check("kirje", i["meta"])
    row["meta.ministeri"] = key_check("ministeri", i["meta"])
    row["meta.ministeri_nimike"] = key_check("ministeri_nimike", i["meta"])
    row["meta.esittelija"] = key_check("esittelija", i["meta"])
    row["meta.esittelija_nimike"] = key_check("esittelija_nimike", i["meta"])
    row["meta.paatos_old_id"] = key_check("paatos_old_id", i["meta"])
    row["meta.koulutustehtava_old_id"] = key_check("koulutustehtava_old_id", i["meta"])
    row["maksu"] = key_check("maksu", i)
    row["luoja"] = key_check("luoja", i)
    row["luontipvm"] = key_check("luontipvm", i)
    row["paivittaja"] = key_check("paivittaja", i)
    row["paivityspvm"] = key_check("paivityspvm", i)
    row["source"] = "arvo.csc.fi/api/vipunen/uraseuranta"
    row["loadtime"] = str(loadtime)

    lupa_data.append(row)
#   split maaraykset into own list from respone
    #maarays=i["maaraykset"]
    maaraykset.append(i["maaraykset"])

# select maaraykset
for k in maaraykset:
    for m in k:
        row = makerow_maaraykset()
        row["id"] = key_check("id", m)
        row["lupaId"] = key_check("lupaId", m)
        row["kohde.tunniste"] = key_check("tunniste", m["kohde"])
        row["koodisto"] = key_check("koodisto", m)
        row["koodiarvo"] = key_check("koodiarvo", m)
        row["arvo"] = key_check("arvo", m)
        row["maaraystyyppi.tunniste"] =  key_check("tunniste", m["maaraystyyppi"])
        #row["meta"] = m["meta"]	 if "meta" in m else  None
        row["luoja"] = key_check("luoja", m)
        row["luontipvm"] = key_check("luontipvm", m)
        row["paivittaja"] = key_check("paivittaja", m)
        row["paivityspvm"] = key_check("paivityspvm", m)
        row["source"] = "arvo.csc.fi/api/vipunen/uraseuranta"
        row["loadtime"] = str(loadtime)

        oiva_maaraykset.append(row)

jarjestysluvat = json_normalize(lupa_data)
oivamaaraykset = json_normalize(oiva_maaraykset)

## Export data to csv's

jarjestysluvat.to_csv(path_or_buf='d:/pdi_integrations/data/oiva/oiva_jarjestysluvat.csv', sep='|', na_rep='',
#data.to_csv(path_or_buf='C:/Documents/ARVO/uraseuranta.csv', sep='|', na_rep='',
               header=True, index=False, mode='w', encoding='utf-8',
               quotechar='"', line_terminator='\n', escapechar='$')
oivamaaraykset.to_csv(path_or_buf='d:/pdi_integrations/data/oiva/oiva_maaraykset.csv', sep='|', na_rep='',
#data.to_csv(path_or_buf='C:/Documents/ARVO/uraseuranta.csv', sep='|', na_rep='',
               header=True, index=False, mode='w', encoding='utf-8',
               quotechar='"', line_terminator='\n', escapechar='$')

<commit_msg>fix oiva etl<commit_after>#!/usr/bin/python
# vim: set fileencoding=UTF-8 :
"""
Simple import script to load data from an API and push to csv
"""

import requests
import os #needed for Jenkisn environment variables
#import json
import pandas as pd
from pandas.io.json import json_normalize
import datetime

result = []
jarjestysluvat = []
oiva_maaraykset=[]
maaraykset=[]
lupa_data= []
loadtime=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
url = 'https://oiva.minedu.fi/api/export/jarjestysluvat'
try:
    api_key = os.environ['AUTH_API_KEY']
except KeyError:
    print("API-key missing")
try:
    api_user = os.environ['AUTH_API_USER']
except KeyError:
    print("API-user missing")

reqheaders = {'Content-Type': 'application/json'}

#reqheaders['Authorization'] = 'oiva '

def makerow_jarjestysluvat():
  return {
    "id": None,     # bigint	pk
    "edellinen_lupa_id": None,     # bigint,	optional
    "paatoskierros_id": None,     # bigint	not null
    "lupatila_id": None,     # bigint	not null
    "asiatyyppi_id": None,     # bigint	not null
    "diaarinumero": None,     # character varying(20)	not null
    "jarjestajaYtunnus": None,     # character varying(10)	not null
    "jarjestajaOid": None,     # character varying(255)	not null
    "alkupvm": None,     # date	not null
    "loppupvm": None,     # date, optional
    "meta.kirje": None,
    "meta.ministeri": None,
    "meta.ministeri_nimike": None,
    "meta.esittelija": None,
    "meta.esittelija_nimike": None,
    "meta.paatos_old_id": None,
    "meta.koulutustehtava_old_id": None,
    "maksu": None,     # boolean, optional
    "luoja": None,     # text, optional
    "luontipvm": None,     # timestamp without time zone, optional
    "paivittaja": None,     # text, optional
    "paivityspvm": None,     # timestamp without time zone, optional
    "source": None,
    "loadtime": None
    }
def makerow_maaraykset():
  return {
    "id": None,
    "lupaid": None,   # bigint not null
    "kohde.tunniste": None,
    "koodisto": None,   # character varying(255)
    "arvo": None,   # text, optional
    "koodiarvo": None,   # character varying(255), optional
    "maaraystyyppi.tunniste": None,
    #"meta": None,   # jsonb , optional
    "luoja": None,   # text, optional
    "luontipvm": None,   # timestamp without time zone, optional
    "paivittaja": None,   # text, optional
    "paivityspvm": None,   # timestamp without time zone, optional
    "source": None,
    "loadtime": None
  }
# URL Status check
def url_ok(url):
    r = requests.head(url,  auth=(api_user, api_key))
    return r.status_code == 200
# http status code
http_status_code = str(requests.head(url,  auth=(api_user, api_key)).status_code)

# Check if key 'x' is in dictionary 'y'
def key_check(x,y):
   if x in y:
      return y[x]
   else:
      return None

if url_ok(url):
   print("OIVA API was available with status code: " +http_status_code)
else:
   print("OIVA API was not accessible and returned status code:" + http_status_code)
   exit

# Loading the data into response  list (jarjestysluvat)
response = requests.get(url,  auth=(api_user, api_key)).json()

# select jarjestysluvat
for i in response:
    row = makerow_jarjestysluvat()
    row["id"] = key_check("uuid",i)
    row["edellinen_lupa_id"] =key_check("paatoskierros_id", i)
    row["lupatila_id"] = key_check("lupatila_id", i)
    row["asiatyyppi_id"] = key_check("asiatyyppi_id", i)
    row["diaarinumero"] = key_check("diaarinumero", i)
    row["jarjestajaYtunnus"] = key_check("jarjestajaYtunnus", i)
    row["jarjestajaOid"] = key_check("jarjestajaOid", i)
    row["nimi"] = key_check("nimi", i)
    row["alkupvm"] = key_check("alkupvm", i)
    row["loppupvm"] = key_check("loppupvm", i)
    row["meta.kirje"] = key_check("kirje", i["meta"])
    row["meta.ministeri"] = key_check("ministeri", i["meta"])
    row["meta.ministeri_nimike"] = key_check("ministeri_nimike", i["meta"])
    row["meta.esittelija"] = key_check("esittelija", i["meta"])
    row["meta.esittelija_nimike"] = key_check("esittelija_nimike", i["meta"])
    row["meta.paatos_old_id"] = key_check("paatos_old_id", i["meta"])
    row["meta.koulutustehtava_old_id"] = key_check("koulutustehtava_old_id", i["meta"])
    row["maksu"] = key_check("maksu", i)
    row["luoja"] = key_check("luoja", i)
    row["luontipvm"] = key_check("luontipvm", i)
    row["paivittaja"] = key_check("paivittaja", i)
    row["paivityspvm"] = key_check("paivityspvm", i)
    row["source"] = "arvo.csc.fi/api/vipunen/uraseuranta"
    row["loadtime"] = str(loadtime)

    lupa_data.append(row)
#   split maaraykset into own list from respone
    #maarays=i["maaraykset"]
    maaraykset.append(i["maaraykset"])

# select maaraykset
for k in maaraykset:
    for m in k:
        row = makerow_maaraykset()
        row["id"] = key_check("uuid", m)
        row["lupaId"] = key_check("uuid", i)
        if "kohde" in m:
            row["kohde.tunniste"] = key_check("tunniste", m["kohde"])
        else:
            row["kohde.tunniste"] = None
        row["koodisto"] = key_check("koodisto", m)
        row["koodiarvo"] = key_check("koodiarvo", m)
        row["arvo"] = key_check("arvo", m)
        if "maaraystyyppi" in m:
            row["maaraystyyppi.tunniste"] =  key_check("tunniste", m["maaraystyyppi"])
        else:
            row["maaraystyyppi.tunniste"] = None
        #row["meta"] = m["meta"]	 if "meta" in m else  None
        row["luoja"] = key_check("luoja", m)
        row["luontipvm"] = key_check("luontipvm", m)
        row["paivittaja"] = key_check("paivittaja", m)
        row["paivityspvm"] = key_check("paivityspvm", m)
        row["source"] = "arvo.csc.fi/api/vipunen/uraseuranta"
        row["loadtime"] = str(loadtime)

        oiva_maaraykset.append(row)

jarjestysluvat = json_normalize(lupa_data)
oivamaaraykset = json_normalize(oiva_maaraykset)

## Export data to csv's

jarjestysluvat.to_csv(path_or_buf='d:/pdi_integrations/data/oiva/oiva_jarjestysluvat.csv', sep='|', na_rep='',
#data.to_csv(path_or_buf='C:/Documents/ARVO/uraseuranta.csv', sep='|', na_rep='',
               header=True, index=False, mode='w', encoding='utf-8',
               quotechar='"', line_terminator='\n', escapechar='$')
oivamaaraykset.to_csv(path_or_buf='d:/pdi_integrations/data/oiva/oiva_maaraykset.csv', sep='|', na_rep='',
#data.to_csv(path_or_buf='C:/Documents/ARVO/uraseuranta.csv', sep='|', na_rep='',
               header=True, index=False, mode='w', encoding='utf-8',
               quotechar='"', line_terminator='\n', escapechar='$')

<|endoftext|>


<commit_before>import core
from core import *

__all__ = core.__all__
<commit_msg><commit_after>import core
from core import *
import const

__all__ = core.__all__
__all__.append('const')
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

# @file activation_functions.py
# @brief
# @author QRS
# @blog qrsforever.github.io
# @version 1.0
# @date 2019-05-29 18:24:39

# run this file using jupyter-vim

#####################################################################################
# <codecell> header
#####################################################################################

import os
import numpy as np
import matplotlib.pyplot as plt

# %matplotlib inline # not work in jupyter-vim

#####################################################################################
# <codecell> global
#####################################################################################

plt.rcParams['font.sans-serif'] = 'SimHei'
plt.rcParams['axes.unicode_minus'] = False # 负号正常显示
plt.rcParams['figure.figsize'] = (16.0, 8.0)
plt.rcParams['text.usetex'] = True

xs = np.arange(-6, 6, 0.01)

## plot.show

def display(title, yaxis,
        func, ftxt = None, xpos=0, xytext=(0,0),
        dfunc=None, dftxt=None, dxpos=None, dxytext=None):

    plt.title(title, fontsize='xx-large', fontweight='bold', color='b')
    plt.ylim(yaxis)
    plt.locator_params('both', nbins=8) # 调整x,y坐标系的刻度划分
    plt.xticks(fontsize=14) # 设置坐标刻度字体
    plt.yticks(fontsize=14)
    plt.axhline(lw=1, c='black') # 轴线设置
    plt.axvline(lw=1, c='black')
    plt.grid(alpha=0.4, ls='-.')
    plt.box(on=None)
    plt.plot(xs, func(xs), c='r', lw=3)

    if ftxt:
        plt.annotate(ftxt, xy=(xpos, func(xpos)), xytext=xytext, fontsize=16,
                arrowprops=dict(width=1,headwidth=6,facecolor='black',shrink=0.05))
    if dfunc:
        plt.plot(xs, dfunc(xs), c='g', lw=3)

    if dftxt:
        plt.annotate(dftxt, xy=(dxpos, dfunc(dxpos)), xytext=dxytext, fontsize=16,
                arrowprops=dict(width=1,headwidth=6,facecolor='black',shrink=0.05))

    # jupyter-vim can not plot inline, so using eog
    plt.savefig('/tmp/af.png')
    os.system('eog /tmp/af.png')

#####################################################################################
# <codecell> binary step (二元阶跃)
#####################################################################################

step_func = np.vectorize(lambda x: 1 if x > 0 else 0, otypes=[np.float])

f_text  = r'$ f(x) = \bigg\{\begin{array}{ccc}'
f_text += r' 0 & for & x\leq 0 \\ '
f_text += r' 1 & for & x> 0 '
f_text += r'\end{array} $'

display('binary step', (-0.1, 1.6),
        step_func, f_text, 2, (2.2, 0.5))

#####################################################################################
# <codecell> piecewise linear (线性分段)
#####################################################################################

xmin = -3.5
xmax = 3.5
w = 1/(xmax - xmin)
b = xmax*w

piece_func = np.vectorize(lambda x: 1 if x > xmax else 0 if x < xmin else w*x+b,
        otypes=[np.float])
piece_dfunc = np.vectorize(lambda x: w, otypes=[np.float])

display('piecewise linear', (-0.1, 1.6),
        piece_func, -2, (-2.2, 0.5), 'aaaa',
        piece_dfunc, 2, (2.2, 0.5), 'bbbb')
<commit_msg>ML:learn:activation function<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

# @file activation_functions.py
# @brief
# @author QRS
# @blog qrsforever.github.io
# @version 1.0
# @date 2019-05-29 18:24:39

# run this file using jupyter-vim

#####################################################################################
# <codecell> header
#####################################################################################

import os
import numpy as np
import matplotlib.pyplot as plt

# %matplotlib inline # not work in jupyter-vim

#####################################################################################
# <codecell> global
#####################################################################################

plt.rcParams['font.sans-serif'] = 'SimHei'
plt.rcParams['axes.unicode_minus'] = False # 负号正常显示
plt.rcParams['figure.figsize'] = (16.0, 8.0)
plt.rcParams['text.usetex'] = True

xs = np.arange(-6, 6, 0.01)

##

def display(title, yaxis,
        func, ftxt = None, xpos=0, xytext=(0,0),
        dfunc=None, dftxt=None, dxpos=None, dxytext=None):

    plt.title(title, fontsize='xx-large', fontweight='bold', color='b')
    plt.ylim(yaxis)
    plt.locator_params('both', nbins=8) # 调整x,y坐标系的刻度划分
    plt.xticks(fontsize=14) # 设置坐标刻度字体
    plt.yticks(fontsize=14)
    plt.axhline(lw=1, c='black') # 轴线设置
    plt.axvline(lw=1, c='black')
    plt.grid(alpha=0.4, ls='-.')
    plt.box(on=None)
    plt.plot(xs, func(xs), c='r', lw=3)

    if ftxt:
        plt.annotate(ftxt, xy=(xpos, func(xpos)), xytext=xytext, fontsize=16,
                arrowprops=dict(width=1,headwidth=6,facecolor='black',shrink=0.05))
    if dfunc:
        plt.plot(xs, dfunc(xs), c='g', lw=3)

    if dftxt:
        plt.annotate(dftxt, xy=(dxpos, dfunc(dxpos)), xytext=dxytext, fontsize=16,
                arrowprops=dict(width=1,headwidth=6,facecolor='black',shrink=0.05))

    # jupyter-vim can not plot inline, so using eog
    plt.savefig('/tmp/af.png')
    os.system('eog /tmp/af.png')

#####################################################################################
# <codecell> Binary step (二元阶跃)
#####################################################################################

step_func = np.vectorize(lambda x: 1 if x > 0 else 0, otypes=[np.float])

f_text  = r'$ f(x) = \left\{\begin{array}{ccc}'
f_text += r' 0 & for & x\leq 0 \\ '
f_text += r' 1 & for & x> 0 '
f_text += r'\end{array}\right. $'

display('binary step', (-0.1, 1.6),
        step_func, f_text, 2, (2.2, 0.5))

#####################################################################################
# <codecell> Piecewise Linear (线性分段)
#####################################################################################

xmin = -3.5
xmax = 3.5
w = 1/(xmax - xmin)
b = xmax*w

piece_func = np.vectorize(lambda x: 1 if x > xmax else 0 if x < xmin else w*x+b,
        otypes=[np.float])
piece_dfunc = np.vectorize(lambda x: w, otypes=[np.float])

display('piecewise linear', (-0.1, 1.6),
        piece_func, -2, (-2.2, 0.5), 'aaaa',
        piece_dfunc, 2, (2.2, 0.5), 'bbbb')

#####################################################################################
# <codecell> Bipolar
#####################################################################################

bipolar_func = np.vectorize(lambda x: 1 if x > 0 else -1, otypes=[np.float])

display('bipolar', (-1.5, 1.5),
        bipolar_func, 'bipolar_func', 1, (2, 0.5))

#####################################################################################
# <codecell> Sigmoid
#####################################################################################

def sigmoid_func(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_dfunc(x):
    fx = sigmoid_func(x)
    return fx * (1 - fx)

display('sigmiod', (-0.2, 1.2),
        sigmoid_func, 'sigmoid_func', 1, (2, 0.5),
        sigmoid_dfunc, 'sigmoid_dfunc', -1, (-2, 0.5))

#####################################################################################
# <codecell> Bipolar Sigmoid
#####################################################################################

def bisigmoid_func(x):
    return (1 - np.exp(-x)) / (1 + np.exp(-x))

def bisigmoid_dfunc(x):
    return 2 * np.exp(x) / ((np.exp(x) + 1) ** 2)

display('bisigmiod', (-1.2, 1.2),
        bisigmoid_func, 'bisigmoid_func', 0.2, (1, -0.5),
        bisigmoid_dfunc, 'bisigmoid_dfunc', -5, (-4, 0.5))

#####################################################################################
# <codecell> Hyperbolic Tangent, TanH (双曲正切)
#####################################################################################

def tanh_func(x):
    return 2 / (1 + np.exp(-2 * x)) - 1

def tanh_dfunc(x):
    fx = tanh_func(x)
    return 1 - fx ** 2

display('hyperbolic tangent', (-1.2, 1.2),
        tanh_func, 'aaa', 0.2, (1, -0.5),
        tanh_dfunc, 'bbb', -1.0, (-2, 0.5))

#####################################################################################
# <codecell> Arctangent, Arctan
#####################################################################################

def arctan_func(x):
    return np.arctan(x)

def arctan_dfunc(x):
    return 1 / (1 + x ** 2)

display('arctangent', (-1.5, 1.5),
        arctan_func, 'aaa', 0.2, (1, -0.5),
        arctan_dfunc, 'bbb', -1.0, (-2, 0.5))

#####################################################################################
# <codecell> Rectified Linear Units, ReLU
#####################################################################################

relu_func = np.vectorize(lambda x: x if x > 0 else 0, otypes=[np.float])

relu_dfunc = np.vectorize(lambda x: 1 if x > 0 else 0, otypes=[np.float])

display('relu', (-0.5, 1.5),
        relu_func, 'aaa', 0.6, (1.8, 0.5),
        relu_dfunc, 'bbb', -2.0, (-3.2, 0.5))

#####################################################################################
# <codecell> Leaky Rectified Linear Units, Leaky ReLU
#####################################################################################

a = 0.1

lrelu_func = np.vectorize(lambda x: x if x > 0 else a*x, otypes=[np.float])

lrelu_dfunc = np.vectorize(lambda x: 1 if x > 0 else a, otypes=[np.float])

display('leaky relu', (-0.5, 1.5),
        lrelu_func, 'aaa', 0.6, (1.8, 0.5),
        lrelu_dfunc, 'bbb', -2.0, (-3.2, 0.5))

#####################################################################################
# <codecell> Exponential Linear Units, ELU
#####################################################################################

a = 0.5

elu_func = np.vectorize(lambda x: x if x > 0 else a*(np.exp(x)-1), otypes=[np.float])

elu_dfunc = np.vectorize(lambda x: 1 if x > 0 else a*np.exp(x), otypes=[np.float])

display('elu', (-0.5, 1.5),
        elu_func, 'aaa', 0.6, (1.8, 0.5),
        elu_dfunc, 'bbb', -2.0, (-3.2, 0.5))

#####################################################################################
# <codecell> SoftPlus
#####################################################################################

def softplus_func(x):
    return np.log(1+np.exp(x))

def softplus_dfunc(x):
    return 1/(1+np.exp(-x))

display('softplus', (-0.5, 1.5),
        softplus_func, 'aaa', 0.6, (1.8, 0.5),
        softplus_dfunc, 'bbb', -2.0, (-3.2, 0.5))
<|endoftext|>


<commit_before>import urlparse
import os, sys, re
import time
from flask import Flask, request, jsonify, abort, json, make_response
from msc_apps import *
import psycopg2, psycopg2.extras
from sqltools import *

tools_dir = os.environ.get('TOOLSDIR')
lib_path = os.path.abspath(tools_dir)
sys.path.append(lib_path)
data_dir_root = os.environ.get('DATADIR')

app = Flask(__name__)
app.debug = True

HISTORY_COUNT_CACHE = {}

@app.route('/categories', methods=['POST'])
def categories():
    categories_file = data_dir_root + "/www/categories.json"
    with open(categories_file, 'r') as f:
        try:
            categories = json.loads(f.read())
        except ValueError:
            print 'Error decoding JSON', categories_file.split('/')[-1][:-5]

    data = categories.keys()

    response = {
                'status' : 'OK',
                'categories' : data
                }

    return jsonify(response)

@app.route('/subcategories', methods=['POST'])
def subcategories():
    try:
        category = request.form['category']
    except KeyError:
        abort(make_response('No field \'category\' in request, request failed', 400))

    categories_file = data_dir_root + "/www/categories.json"
    with open(categories_file, 'r') as f:
        try:
            categories = json.loads(f.read())
        except ValueError:
            print 'Error decoding JSON', categories_file.split('/')[-1][:-5]

    try:
        data = categories[category]
    except KeyError:
        abort(make_response('Unexisting category, request failed', 400))

    response = {
                'status' : 'OK',
                'subcategories' : data
                }

    return jsonify(response)

@app.route('/list', methods=['POST'])
def list():
    query = ""
    try:
        value = int(re.sub(r'\D+', '', request.form['ecosystem']))
        valid_values = [1,2]
        if value not in valid_values:
            abort(make_response('Field \'ecosystem\' invalid value, request failed', 400))

        ecosystem = "Production" if value == 1 else "Test"
    except KeyError:
        abort(make_response('No field \'ecosystem\' in request, request failed', 400))
    except ValueError:
        abort(make_response('Field \'ecosystem\' invalid value, request failed', 400))
    try:
        issuer = re.sub(r'\D+', '', request.form['issuer_address']) #check alphanumeric
        query += " AND issuer='" + str(issuer) + "'"
    except KeyError:
        issuer = ""

    ROWS= dbSelect("select * from smartproperties where PropertyID > 2 AND ecosystem='%s'%s ORDER BY PropertyName,PropertyID", (ecosystem,query))
    data=[]
    for property in ROWS:
        data.append({"currencyId":property[1],"propertyName":property[6]}) #get the json representation

    response = {
                'status' : 'OK',
                'properties' : data
                }

    return jsonify(response)

@app.route('/listactivecrowdsales', methods=['POST'])
def listcrowdsales():
    try:
        value = int(re.sub(r'\D+', '', request.form['ecosystem']))
        valid_values = [1,2]
        if value not in valid_values:
            abort(make_response('Field \'ecosystem\' invalid value, request failed', 400))

        ecosystem = "Production" if value == 1 else "Test"
    except KeyError:
        abort(make_response('No field \'ecosystem\' in request, request failed', 400))
    except ValueError:
        abort(make_response('Field \'ecosystem\' invalid value, request failed', 400))

    ROWS= dbSelect("select PropertyData from smartproperties where PropertyData::json->>'fixedissuance'='false' AND PropertyData::json->>'active'='true' AND ecosystem='%s' ORDER BY PropertyName,PropertyID", [ecosystem])
    data=[row[0] for row in ROWS]

    response = {
                'status' : 'OK',
                'crowdsales' : data
                }

    return jsonify(response)

@app.route('/getdata/<int:property_id>')
def getdata(property_id):
    property=dbSelect("select PropertyData from smartproperties where PropertyID=%s",[property_id])[0]
    return jsonify(property[0])

@app.route('/gethistory/<int:property_id>', methods=["POST"])
def gethistory(property_id):
    try:
        start = int(request.form['start'])
    except KeyError:
        abort(make_response('No field \'start\' in request, request failed', 400))
    except ValueError:
        abort(make_response('Field \'start\' must be an integer, request failed', 400))

    try:
        count = int(request.form['count'])
    except KeyError:
        abort(make_response('No field \'count\' in request, request failed', 400))
    except ValueError:
        abort(make_response('Field \'count\' must be an integer, request failed', 400))

    transactions_query = "select txjson.txdata as data from propertyhistory ph, txjson where ph.txdbserialnum =txjson.txdbserialnum and ph.propertyid=%s order by ph.txdbserialnum LIMIT %s OFFSET %s;"
    total_query = "select count(*) as total from propertyhistory where propertyid =%s group by propertyid"

    try:
        cache=HISTORY_COUNT_CACHE[str(property_id)]
        if(time.time()-cache[1] > 6000000):
            total=dbSelect(totalquery,[property_id])[0][0]
            HISTORY_COUNT_CACHE[str(property_id)] = (total, time.time())
        else:
            total=cache[0]
    except KeyError:
        total=dbSelect(totalquery,[property_id])[0][0]
        HISTORY_COUNT_CACHE[str(property_id)] = (total, time.time())

    ROWS=dbSelect(transactions_query,[property_id])
    transactions=[row[0] for row in ROWS]

    response = {
                "total" : total,
                "transactions" : transactions
                }
    return jsonify(response)

@app.route('/info', methods=['POST'])
def info():
    try:
        property_ = request.form['property']
    except KeyError:
        abort(make_response('No field \'property\' in request, request failed', 400))

    try:
        property_ = json.loads(property_)
    except ValueError:
        abort(make_response('This endpoint only consumes valid JSON', 400))

    if type(property_) != type([]) or len(property_) == len([]):
        abort(make_response('Please provide data in a JSON array for processing, request failed', 400))

    for prop in property_:
        if type(prop) != type(0):
            abort(make_response('Array data must be of type int', 400))
        #property_ = map(( lambda prop_: re.sub('\D','', prop_ ) ), property_ )

    data = filterProperties(property_)

    response={
        'status': data[0],
        'data': data[1]
        }

    #DEBUG print response
    return jsonify(response)

# refactor this to be compatible with mastercored
def filterProperties( properties ):
    import glob

    addresses = glob.glob(data_dir_root + '/addr/*')
    addresses_data = []
    for prop in properties:
        for address_file in addresses:
            #print address[-5:]
            if address_file[-5:] == '.json':
                with open( address_file , 'r' ) as f:
                  try:
                    addr = json.loads(f.readline())

                    if str(prop) in addr:
                      addresses_data.append({ 'address': address_file.split('/')[-1][:-5], 'data': addr[str(prop)] })
                  except ValueError:
                    print 'Error decoding JSON', address_file.split('/')[-1][:-5]

    return ['OK',addresses_data]
<commit_msg>Remove quotes from query string<commit_after>import urlparse
import os, sys, re
import time
from flask import Flask, request, jsonify, abort, json, make_response
from msc_apps import *
import psycopg2, psycopg2.extras
from sqltools import *

tools_dir = os.environ.get('TOOLSDIR')
lib_path = os.path.abspath(tools_dir)
sys.path.append(lib_path)
data_dir_root = os.environ.get('DATADIR')

app = Flask(__name__)
app.debug = True

HISTORY_COUNT_CACHE = {}

@app.route('/categories', methods=['POST'])
def categories():
    categories_file = data_dir_root + "/www/categories.json"
    with open(categories_file, 'r') as f:
        try:
            categories = json.loads(f.read())
        except ValueError:
            print 'Error decoding JSON', categories_file.split('/')[-1][:-5]

    data = categories.keys()

    response = {
                'status' : 'OK',
                'categories' : data
                }

    return jsonify(response)

@app.route('/subcategories', methods=['POST'])
def subcategories():
    try:
        category = request.form['category']
    except KeyError:
        abort(make_response('No field \'category\' in request, request failed', 400))

    categories_file = data_dir_root + "/www/categories.json"
    with open(categories_file, 'r') as f:
        try:
            categories = json.loads(f.read())
        except ValueError:
            print 'Error decoding JSON', categories_file.split('/')[-1][:-5]

    try:
        data = categories[category]
    except KeyError:
        abort(make_response('Unexisting category, request failed', 400))

    response = {
                'status' : 'OK',
                'subcategories' : data
                }

    return jsonify(response)

@app.route('/list', methods=['POST'])
def list():
    query = ""
    try:
        value = int(re.sub(r'\D+', '', request.form['ecosystem']))
        valid_values = [1,2]
        if value not in valid_values:
            abort(make_response('Field \'ecosystem\' invalid value, request failed', 400))

        ecosystem = "Production" if value == 1 else "Test"
    except KeyError:
        abort(make_response('No field \'ecosystem\' in request, request failed', 400))
    except ValueError:
        abort(make_response('Field \'ecosystem\' invalid value, request failed', 400))
    try:
        issuer = re.sub(r'\D+', '', request.form['issuer_address']) #check alphanumeric
        query += " AND issuer='" + str(issuer) + "'"
    except KeyError:
        issuer = ""

    ROWS= dbSelect("select * from smartproperties where PropertyID > 2 AND ecosystem=%s%s ORDER BY PropertyName,PropertyID", (ecosystem,query))
    data=[]
    for property in ROWS:
        data.append({"currencyId":property[1],"propertyName":property[6]}) #get the json representation

    response = {
                'status' : 'OK',
                'properties' : data
                }

    return jsonify(response)

@app.route('/listactivecrowdsales', methods=['POST'])
def listcrowdsales():
    try:
        value = int(re.sub(r'\D+', '', request.form['ecosystem']))
        valid_values = [1,2]
        if value not in valid_values:
            abort(make_response('Field \'ecosystem\' invalid value, request failed', 400))

        ecosystem = "Production" if value == 1 else "Test"
    except KeyError:
        abort(make_response('No field \'ecosystem\' in request, request failed', 400))
    except ValueError:
        abort(make_response('Field \'ecosystem\' invalid value, request failed', 400))

    ROWS= dbSelect("select PropertyData from smartproperties where PropertyData::json->>'fixedissuance'='false' AND PropertyData::json->>'active'='true' AND ecosystem=%s ORDER BY PropertyName,PropertyID", [ecosystem])
    data=[row[0] for row in ROWS]

    response = {
                'status' : 'OK',
                'crowdsales' : data
                }

    return jsonify(response)

@app.route('/getdata/<int:property_id>')
def getdata(property_id):
    property=dbSelect("select PropertyData from smartproperties where PropertyID=%s",[property_id])[0]
    return jsonify(property[0])

@app.route('/gethistory/<int:property_id>', methods=["POST"])
def gethistory(property_id):
    try:
        start = int(request.form['start'])
    except KeyError:
        abort(make_response('No field \'start\' in request, request failed', 400))
    except ValueError:
        abort(make_response('Field \'start\' must be an integer, request failed', 400))

    try:
        count = int(request.form['count'])
    except KeyError:
        abort(make_response('No field \'count\' in request, request failed', 400))
    except ValueError:
        abort(make_response('Field \'count\' must be an integer, request failed', 400))

    transactions_query = "select txjson.txdata as data from propertyhistory ph, txjson where ph.txdbserialnum =txjson.txdbserialnum and ph.propertyid=%s order by ph.txdbserialnum LIMIT %s OFFSET %s;"
    total_query = "select count(*) as total from propertyhistory where propertyid =%s group by propertyid"

    try:
        cache=HISTORY_COUNT_CACHE[str(property_id)]
        if(time.time()-cache[1] > 6000000):
            total=dbSelect(totalquery,[property_id])[0][0]
            HISTORY_COUNT_CACHE[str(property_id)] = (total, time.time())
        else:
            total=cache[0]
    except KeyError:
        total=dbSelect(totalquery,[property_id])[0][0]
        HISTORY_COUNT_CACHE[str(property_id)] = (total, time.time())

    ROWS=dbSelect(transactions_query,[property_id])
    transactions=[row[0] for row in ROWS]

    response = {
                "total" : total,
                "transactions" : transactions
                }
    return jsonify(response)

@app.route('/info', methods=['POST'])
def info():
    try:
        property_ = request.form['property']
    except KeyError:
        abort(make_response('No field \'property\' in request, request failed', 400))

    try:
        property_ = json.loads(property_)
    except ValueError:
        abort(make_response('This endpoint only consumes valid JSON', 400))

    if type(property_) != type([]) or len(property_) == len([]):
        abort(make_response('Please provide data in a JSON array for processing, request failed', 400))

    for prop in property_:
        if type(prop) != type(0):
            abort(make_response('Array data must be of type int', 400))
        #property_ = map(( lambda prop_: re.sub('\D','', prop_ ) ), property_ )

    data = filterProperties(property_)

    response={
        'status': data[0],
        'data': data[1]
        }

    #DEBUG print response
    return jsonify(response)

# refactor this to be compatible with mastercored
def filterProperties( properties ):
    import glob

    addresses = glob.glob(data_dir_root + '/addr/*')
    addresses_data = []
    for prop in properties:
        for address_file in addresses:
            #print address[-5:]
            if address_file[-5:] == '.json':
                with open( address_file , 'r' ) as f:
                  try:
                    addr = json.loads(f.readline())

                    if str(prop) in addr:
                      addresses_data.append({ 'address': address_file.split('/')[-1][:-5], 'data': addr[str(prop)] })
                  except ValueError:
                    print 'Error decoding JSON', address_file.split('/')[-1][:-5]

    return ['OK',addresses_data]
<|endoftext|>