<commit_before># Log into Reddit and download the highest rated wallpaper
import argparse
import os
import praw
import requests
from urllib.parse import urlparse
import sys
import tempfile

def get_system_info():
    """
    Return os and desktop information

    Param: None
    Return: os_name - name of the current operating system
                    - linux, windows, osx, unknown
            desktop_env - name of the desktop environment
                        - gnome, unity, aqua, Explorer, Unknown
    """

    sys_type= sys.platform.lower()
    if sys_type == "linux":
        os_name = sys_type
        desktop_env = os.environ.get("DESKTOP_SESSION", "unknown").lower()
    elif sys_type == "win32":
        os_name = 'windows'
        desktop_env = "explorer"
    elif sys_type == "darwin":
        os_name = "osx"
        desktop_env = "aqua"
    else:
        os_name = "unknown"
        desktop_env = "unknown"

    return os_name, desktop_env

def parse_args():
    """
    Return command-line options passed in by the user

    Param: None
    Return: program_options - argparse object of options specified by the user
                            - as commandline arguments
    """
    parser = argparse.ArgumentParser(description="Wallpaper Downloader.",
            epilog="Downloads the most popular wallpaper of the day from /r/wallpapers")
    parser.add_argument('--save', dest='save_location', nargs='?', const='home',
            help="Save the downloaded wallpaper. If no save location is provided then ~/Pictures/Wallpapers will be used")

    return parser.parse_args()

def main():

    program_options = parse_args()

    os_name, desktop_env = get_system_info()

    if os_name == "linux":
        if desktop_env not in ["gnome", "unity"]:
            raise SystemExit("Only Gnome3 and Unity are currently supported.")

        # import gconf
        # not sure if python3 bindings exist
    elif os_name == "windows":
        # needed to run some win32.dlls
        import ctypes
    else:
        raise SystemExit("Only Linux with Gnome3 or Unity or Windows is currently supported.")

    if program_options.save_location is None:
        # user does not want to save the wallpaper
        wallpaper_dir = tempfile.gettempdir()
    elif program_options.save_location == 'home':
        try:
            wallpaper_dir = os.path.join(os.environ['HOME'], 'Pictures', 'Wallpapers')
        except KeyError:
            # if the user's home directory isn't set then fall back to tmp
            wallpaper_dir = tempfile.gettempdir()
    else:
        wallpaper_dir = program_options.save_location

    # validate that this location exists and we can read/write to it
    if not os.access(wallpaper_dir, os.R_OK | os.W_OK):
        raise SystemExit("Cannot write to {0}.".format(wallpaper_dir))

    # setup the useragent
    user_name = "openedground"
    user_agent = "{0}:wallpaper_downloader:v1 (by /u/{1})".format(os_name, user_name)
    subreddit_name = "wallpapers"

    r = praw.Reddit(user_agent = user_agent)
    print('Running')

    wp_downloaded = False
    subreddit = r.get_subreddit(subreddit_name)
    for submission in subreddit.get_hot(limit=10):
        if submission.is_self:
            # ignore self posts
            continue

        if submission.url is not None:
            # validate that this is a direct image link and not a link to webpage containing an image

            file_name = urlparse(submission.url).path
            if file_name[-1] == '/':
                continue

            file_ext = os.path.splitext(file_name)[1].lower()
            if file_ext not in ['.jpg', '.jpeg', '.png']:
                continue

            if file_name[0] == '/':
                file_name = file_name[1:]

            try:
                print("Downloading: " + submission.url)
                req = requests.get(submission.url, stream=True)
                with open(os.path.join(wallpaper_dir, file_name), 'wb') as f:
                    for chunk in req.iter_content(chunk_size=1024):
                        if chunk:
                            f.write(chunk)
                            f.flush()

                wp_downloaded = True
                break
            except PermissionError as ex:
                raise SystemExit("Could not save file to {0} - {1}".format(wallpaper_dir, ex))
            except Exception as ex:
                print("Error downloading - {0}... trying next image".format(ex))
                continue
            finally:
                # close the socket
                req.close()

    if wp_downloaded:
        if os_name == "linux":
            os.system("gsettings set org.gnome.desktop.background picture-uri \
                    file://{0}".format(os.path.join(wallpaper_dir, file_name)))
        elif sys_type == "windows":
            print("Not implemented for Windows yet")
        else:
            print("Not implemented yet.")

        print("All Done")

    else:
        raise SystemExit("Could not download Wallpaper")

    return

if __name__ == "__main__":
    try:
        main()
        sys.exit(0)
    except Exception as error:
        print(error)
        sys.exit(1)
<commit_msg>add check for hash of previously dowloaded image so we do not download the same image twice.<commit_after># Log into Reddit and download the highest rated wallpaper
import argparse
import hashlib
import os
import praw
import requests
from urllib.parse import urlparse
import shutil
import sys
import tempfile

# textfile containing md5 hashes of previously downloaded images  so we don't get duplicates
downloaded_images = os.path.join('resources', 'downloaded_images')
previous_downloads = {}

def get_system_info():
    """
    Return os and desktop information

    Param: None
    Return: os_name - name of the current operating system
                    - linux, windows, osx, unknown
            desktop_env - name of the desktop environment
                        - gnome, unity, aqua, Explorer, Unknown
    """

    sys_type= sys.platform.lower()
    if sys_type == "linux":
        os_name = sys_type
        desktop_env = os.environ.get("DESKTOP_SESSION", "unknown").lower()
    elif sys_type == "win32":
        os_name = 'windows'
        desktop_env = "explorer"
    elif sys_type == "darwin":
        os_name = "osx"
        desktop_env = "aqua"
    else:
        os_name = "unknown"
        desktop_env = "unknown"

    return os_name, desktop_env

def parse_args():
    """
    Return command-line options passed in by the user

    Param: None
    Return: program_options - argparse object of options specified by the user
                            - as commandline arguments
    """
    parser = argparse.ArgumentParser(description="Wallpaper Downloader.",
            epilog="Downloads the most popular wallpaper of the day from /r/wallpapers")
    parser.add_argument('--save', dest='save_location', nargs='?', const='home',
            help="Save the downloaded wallpaper. If no save location is provided then ~/Pictures/Wallpapers will be used")

    return parser.parse_args()

def validate_unique_image(image_path):
    """
    Verify that the image downloaded has been downloaded before or not.

    Param: image_path - relative or absolute path to the image
    Return: boolean - true if the image is unique
    """

    is_unique = None

    # get the md5 hash of the downloaded file
    m = hashlib.md5()

    try:
        with open(image_path, 'rb') as f:
            m.update(f.read())
            image_hash = m.hexdigest()

    except Exception as ex:
        raise SystemExit("Could not read image {0} - {1}".format(image_path, ex))

    if previous_downloads.get(image_hash, False):
        is_unique = False
    else:
        previous_downloads[image_hash] = True
        try:
            with open(downloaded_images, 'a') as f:
                f.write(image_hash + '\n')
        except OSError as ex:
            print("Could not updated list of downloaded images - {0}.".format(ex))

        is_unique = True

    return is_unique

def main():

    program_options = parse_args()

    os_name, desktop_env = get_system_info()

    if os_name == "linux":
        if desktop_env not in ["gnome", "unity"]:
            raise SystemExit("Only Gnome3 and Unity are currently supported.")

        # import gconf
        # not sure if python3 bindings exist
    elif os_name == "windows":
        # needed to run some win32.dlls
        import ctypes
    else:
        raise SystemExit("Only Linux with Gnome3 or Unity or Windows is currently supported.")

    # always save to the temporary directory first
    wallpaper_dir = None
    temp_dir = tempfile.gettempdir()
    if program_options.save_location is None:
        # user does not want to save the wallpaper
        None
    elif program_options.save_location == 'home':
        try:
            wallpaper_dir = os.path.join(os.environ['HOME'], 'Pictures', 'Wallpapers')
        except KeyError:
            # if the user's home directory isn't set then fall back to tmp
            None
    else:
        wallpaper_dir = program_options.save_location

    # validate that this location exists and we can read/write to it
    if not os.access(temp_dir, os.R_OK | os.W_OK):
        raise SystemExit("Cannot write to {0}.".format(temp_dir))
    if wallpaper_dir is not None:
        if not os.access(wallpaper_dir, os.R_OK | os.W_OK):
            raise SystemExit("Cannot write to {0}.".format(wallpaper_dir))

    # build dictionary of md5 hashes of previously downloaded images
    try:
        with open(downloaded_images, 'r') as f:
            for image_hash in f:
                # get rid of any newlines
                previous_downloads[image_hash.rstrip()] = True
    except IOError as ex:
        print("Could not build list of previously downloaded images - {0}".format(ex))

    # setup the useragent
    user_name = "openedground"
    user_agent = "{0}:wallpaper_downloader:v1 (by /u/{1})".format(os_name, user_name)
    subreddit_name = "wallpapers"

    r = praw.Reddit(user_agent = user_agent)
    print('Running')

    wp_downloaded = False
    subreddit = r.get_subreddit(subreddit_name)
    for submission in subreddit.get_hot(limit=10):
        if submission.is_self:
            # ignore self posts
            continue

        if submission.url is not None:
            # validate that this is a direct image link and not a link to webpage containing an image

            file_name = urlparse(submission.url).path
            if file_name[-1] == '/':
                continue

            file_ext = os.path.splitext(file_name)[1].lower()
            if file_ext not in ['.jpg', '.jpeg', '.png']:
                continue

            if file_name[0] == '/':
                file_name = file_name[1:]

            try:
                print("Downloading: " + submission.url)
                req = requests.get(submission.url, stream=True)
                with open(os.path.join(temp_dir, file_name), 'wb') as f:
                    for chunk in req.iter_content(chunk_size=1024):
                        if chunk:
                            f.write(chunk)
                            f.flush()

                unique_image =  validate_unique_image(os.path.join(temp_dir, file_name))
                if unique_image:
                    # this is the first time downloading the image
                    wp_downloaded = True
                    break
            except SystemExit:
                raise
            except PermissionError as ex:
                raise SystemExit("Could not save file to {0} - {1}".format(wallpaper_dir, ex))
            except Exception as ex:
                print("Error downloading - {0}... trying next image".format(ex))
                continue
            finally:
                # close the socket
                req.close()

    if wp_downloaded:

        if wallpaper_dir is not None:
            # first attempt to copy the wallpaper to the user's chosen directory. If this fails
            # we will still try to set the wallpaper
            try:
                shutil.copyfile(os.path.join(temp_dir, file_name), os.path.join(wallpaper_dir, file_name))
            except OSError as ex:
                print("Could not save wallpaper to {0} - {1}".format(os.path.join(wallpaper_dir, file_name), ex))

        if os_name == "linux":
            os.system("gsettings set org.gnome.desktop.background picture-uri \
                    file://{0}".format(os.path.join(temp_dir, file_name)))
        elif sys_type == "windows":
            print("Not implemented for Windows yet")
        else:
            print("Not implemented yet.")

        print("All Done")

    else:
        raise SystemExit("Could not download Wallpaper")

    return

if __name__ == "__main__":
    try:
        main()
        sys.exit(0)
    except Exception as error:
        print(error)
        sys.exit(1)
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
'''

__docformat__ = 'restructuredtext'
__version__ = '$Id: $'

import ctypes

from pyglet.app import windows, BaseEventLoop
from pyglet.window.carbon import carbon, types, constants

EventLoopTimerProc = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p)
kEventDurationForever = ctypes.c_double(constants.kEventDurationForever)

class CarbonEventLoop(BaseEventLoop):
    def run(self):
        self._setup()

        e = ctypes.c_void_p()
        event_dispatcher = carbon.GetEventDispatcherTarget()
        event_loop = carbon.GetMainEventLoop()
        event_queue = carbon.GetMainEventQueue()
        timer = ctypes.c_void_p()
        idle_event_proc = EventLoopTimerProc(self._timer_proc)
        carbon.InstallEventLoopTimer(event_loop,
                                     ctypes.c_double(0.1), #?
                                     kEventDurationForever,
                                     idle_event_proc,
                                     None,
                                     ctypes.byref(timer))

        self.dispatch_event('on_enter')

        while not self.has_exit:
            if carbon.ReceiveNextEvent(0, None, kEventDurationForever,
                                       True, ctypes.byref(e)) == 0:
                carbon.SendEventToEventTarget(e, event_dispatcher)
                carbon.ReleaseEvent(e)

            # Manual idle event
            if carbon.GetNumEventsInQueue(event_queue) == 0:
                self._timer_proc(timer, None)

        self.dispatch_event('on_exit')

    def _timer_proc(self, timer, data):
        for window in windows:
            # Check for live resizing
            if window._resizing is not None:
                old_width, old_height = window._resizing
                rect = types.Rect()
                carbon.GetWindowBounds(window._window,
                                       constants.kWindowContentRgn,
                                       ctypes.byref(rect))
                width = rect.right - rect.left
                height = rect.bottom - rect.top
                if width != old_width or height != old_height:
                    window._resizing = width, height
                    window.switch_to()
                    window.dispatch_event('on_resize', width, height)

        sleep_time = self.idle()

        if sleep_time is None:
            sleep_time = constants.kEventDurationForever
        carbon.SetEventLoopTimerNextFireTime(timer, ctypes.c_double(sleep_time))
<commit_msg>Carbon: Degenerate to polling if running behind the clock.<commit_after>#!/usr/bin/env python

'''
'''

__docformat__ = 'restructuredtext'
__version__ = '$Id: $'

import ctypes

from pyglet.app import windows, BaseEventLoop
from pyglet.window.carbon import carbon, types, constants

EventLoopTimerProc = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p)
kEventDurationForever = ctypes.c_double(constants.kEventDurationForever)

class CarbonEventLoop(BaseEventLoop):
    def run(self):
        self._setup()

        e = ctypes.c_void_p()
        event_dispatcher = carbon.GetEventDispatcherTarget()
        event_loop = carbon.GetMainEventLoop()
        event_queue = carbon.GetMainEventQueue()
        timer = ctypes.c_void_p()
        idle_event_proc = EventLoopTimerProc(self._timer_proc)
        carbon.InstallEventLoopTimer(event_loop,
                                     ctypes.c_double(0.1), #?
                                     kEventDurationForever,
                                     idle_event_proc,
                                     None,
                                     ctypes.byref(timer))

        self._sleep_time = None

        self.dispatch_event('on_enter')

        while not self.has_exit:
            duration = kEventDurationForever
            if self._sleep_time == 0.:
                duration = 0
            if carbon.ReceiveNextEvent(0, None, duration,
                                       True, ctypes.byref(e)) == 0:
                carbon.SendEventToEventTarget(e, event_dispatcher)
                carbon.ReleaseEvent(e)

            # Manual idle event
            if carbon.GetNumEventsInQueue(event_queue) == 0 or duration == 0:
                self._timer_proc(timer, None)

        self.dispatch_event('on_exit')

    def _timer_proc(self, timer, data):
        for window in windows:
            # Check for live resizing
            if window._resizing is not None:
                old_width, old_height = window._resizing
                rect = types.Rect()
                carbon.GetWindowBounds(window._window,
                                       constants.kWindowContentRgn,
                                       ctypes.byref(rect))
                width = rect.right - rect.left
                height = rect.bottom - rect.top
                if width != old_width or height != old_height:
                    window._resizing = width, height
                    window.switch_to()
                    window.dispatch_event('on_resize', width, height)

        self._sleep_time = sleep_time = self.idle()

        if sleep_time is None:
            sleep_time = constants.kEventDurationForever
        carbon.SetEventLoopTimerNextFireTime(timer, ctypes.c_double(sleep_time))
<|endoftext|>


<commit_before>"""
Functions for loading existing sequencing datasets into Synapse.

"""

# To do items:
#	change config to yaml
#	generalize tophat name/version id for other methods
#	unit tests

from synapseclient import Project, Folder, File, Activity, Evaluation
import subprocess, os

def readConfig(configFile):
	'''Reads external config file and stores entries in dictionary allArgDict organized by workflow step.'''

	allArgDict = dict()
	foldersToCreate = list()
	workflow = False
	stepNum = 0
	params = open(configFile, 'r')
	finalStep = 1
	for line in params:
		if line.startswith('# workflow'):
			workflow = True
			if stepNum > 0:
				allArgDict[stepNum] = workflowDict
			stepNum = line.strip().split()[3]
			workflowDict = dict()
			finalStep = finalStep + 1
			continue
		elif line.startswith('#') or line.startswith('\n'):
			continue
		fields = line.strip().split('\t')

		if workflow == True:
			if fields[0] == 'annotations':
				annotDict = dict()
				elements = fields[1].split(';')
				for item in elements:
					key, value = item.split(',')
					annotDict[key] = value
				workflowDict['annotations'] = annotDict
			elif fields[0] == 'folder':
				foldersToCreate.append(fields[1])
				workflowDict[fields[0]] = fields[1]
			else:
				workflowDict[fields[0]] = fields[1]
		else:
			try:
				allArgDict[fields[0]] = fields[1]
			except IndexError:
				continue
	allArgDict[stepNum] = workflowDict
	allArgDict['folderlist'] = foldersToCreate
	allArgDict['finalStep'] = finalStep
	return(allArgDict)

## test code to make sure dictionary is formed correctly
# for item in allArgDict:
# 	try:
# 		for k, v in allArgDict[item].iteritems():
# 			print '%s %s' % (k, v)
# 	except AttributeError:
# 		print '%s - not a dictionary' % item
# 		continue

def parseTophatCall(inBAMPath):
	'''For BAM files generated with TopHat, parses the cmd statement and tophat version from the sam header.'''

	cmd = ' '.join(['samtools view -H', inBAMPath])
	header = subprocess.check_output(cmd, shell=True)
	headerList = header.split('\n')
	for line in headerList:
		if line.startswith('@PG'):
			elements = line.split('\t')
			name = elements[1].split(':')[1]
			version = elements[2].split(':')[1]
#			print '%s %s' % (name, version)
			return(' '.join([name,version]), elements[3])

def str2bool(inString): # this function found on stackoverflow
	return inString.lower() in ("yes", "true", "t", "1")

def add_workflow_step_to_synapse(inFilePath, stepDict, step='1', store=False, software=None, parentid=None, syn=None, stepIDs=None, inFilename=None ):
	'''Uploads files with provenance and annotations to Synapse.'''
	if not inFilename:
		inFilename = os.path.basename(inFilePath.strip())
	if not software:
		software = stepDict['softwareName']
	if 'used' in stepDict:
		usedList = stepDict['used'].strip().split(',')
		if 'depends' in stepDict:
			usedList.append(stepIDs[stepDict['depends']])
	elif 'depends' in stepDict:
		usedList = stepIDs[stepDict['depends']]
	execList = stepDict['executed'].strip().split(';')

	act = Activity(name=stepDict['actName'],  description=stepDict['description'])
	act.used(usedList)
	for item in execList:
		splitItem = item.split(',')
		target = splitItem[0]
		version = 1
		if (len(splitItem) > 1):
			version = splitItem[1]
		act.executed(target=target, targetVersion=version)

	step_file = File(inFilePath, name=inFilename, description=stepDict['fileDescription'], parentId=parentid, synapseStore=str2bool(stepDict['store']))
	step_file = syn.store(step_file, activity=act, forceVersion=False)
	syn.setAnnotations(step_file, annotations=stepDict['annotations'])
	return(step_file.id)

def getExistingFolders(syn,pid):
	'''Returns dictionary of existing folders for the project. key = name, val = id.'''

	existingFolders = dict()
	project = syn.get(pid)
	results = syn.chunkedQuery(''.join(['SELECT id, name, concreteType FROM entity WHERE parentId=="', project.id, '"']))
	for result in results:
		if result['entity.concreteType'][0] == 'org.sagebionetworks.repo.model.Folder':
			existingFolders[result['entity.name']] = result['entity.id']
	return(existingFolders)

def setUpSynapseProject(foldersToCreate,syn,pid=None,pname=None):
	'''Creates Synapse project and necessary folders for the dataset.'''

	# Create a set of sub-folders expected for this project
	folderSchemaSet = set(foldersToCreate)

	# Get the project if it exists or create
	if pid == None:
		project = Project(pname)
		project = syn.store(project)
	else:
		project = syn.get(pid)
		print '%s' % project.name

		existingFolders = getExistingFolders(syn,project.id)
		if len(existingFolders) > 0:
			foldersToCreate = folderSchemaSet.difference(existingFolders.keys())

	# create the folders that don't exist
	for name in foldersToCreate:
		createFolder = Folder(name, parent=project.id)
		createFolder = syn.store(createFolder)
		existingFolders[name] = createFolder.id
	return(project, existingFolders)

<commit_msg>Changes to add_workflow: allows no annotations, specifies path for all files, deleted some unused arguments.<commit_after>"""
Functions for loading existing sequencing datasets into Synapse.

"""

# To do items:
#	change config to yaml
#	generalize tophat name/version id for other methods
#	unit tests

from synapseclient import Project, Folder, File, Activity, Evaluation
import subprocess, os

def readConfig(configFile):
	'''Reads external config file and stores entries in dictionary allArgDict organized by workflow step.'''

	allArgDict = dict()
	foldersToCreate = list()
	workflow = False
	stepNum = 0
	params = open(configFile, 'r')
	finalStep = 1
	for line in params:
		if line.startswith('# workflow'):
			workflow = True
			if stepNum > 0:
				allArgDict[stepNum] = workflowDict
			stepNum = line.strip().split()[3]
			workflowDict = dict()
			finalStep = finalStep + 1
			continue
		elif line.startswith('#') or line.startswith('\n'):
			continue
		fields = line.strip().split('\t')

		if workflow == True:
			if fields[0] == 'annotations':
				annotDict = dict()
				elements = fields[1].split(';')
				for item in elements:
					key, value = item.split(',')
					annotDict[key] = value
				workflowDict['annotations'] = annotDict
			elif fields[0] == 'folder':
				foldersToCreate.append(fields[1])
				workflowDict[fields[0]] = fields[1]
			else:
				workflowDict[fields[0]] = fields[1]
		else:
			try:
				allArgDict[fields[0]] = fields[1]
			except IndexError:
				continue
	allArgDict[stepNum] = workflowDict
	allArgDict['folderlist'] = foldersToCreate
	allArgDict['finalStep'] = finalStep
	return(allArgDict)

## test code to make sure dictionary is formed correctly
# for item in allArgDict:
# 	try:
# 		for k, v in allArgDict[item].iteritems():
# 			print '%s %s' % (k, v)
# 	except AttributeError:
# 		print '%s - not a dictionary' % item
# 		continue

def parseTophatCall(inBAMPath):
	'''For BAM files generated with TopHat, parses the cmd statement and tophat version from the sam header.'''

	cmd = ' '.join(['samtools view -H', inBAMPath])
	header = subprocess.check_output(cmd, shell=True)
	headerList = header.split('\n')
	for line in headerList:
		if line.startswith('@PG'):
			elements = line.split('\t')
			name = elements[1].split(':')[1]
			version = elements[2].split(':')[1]
#			print '%s %s' % (name, version)
			return(' '.join([name,version]), elements[3])

def str2bool(inString): # this function found on stackoverflow
	return inString.lower() in ("yes", "true", "t", "1")

def add_workflow_step_to_synapse(inFilePath, stepDict, step='1', software=None, parentid=None, syn=None, stepIDs=None, inFilename=None):
	'''Uploads files with provenance and annotations to Synapse.'''
	if not inFilename:
		inFilename = os.path.basename(inFilePath.strip())
	if not software:
		software = stepDict['softwareName']
	if 'used' in stepDict:
		usedList = stepDict['used'].strip().split(',')
		if 'depends' in stepDict:
			usedList.append(stepIDs[stepDict['depends']])
	elif 'depends' in stepDict:
		usedList = stepIDs[stepDict['depends']]
	execList = stepDict['executed'].strip().split(';')

	act = Activity(name=stepDict['actName'],  description=stepDict['description'])
	act.used(usedList)
	for item in execList:
		splitItem = item.split(',')
		target = splitItem[0]
		version = 1
		if (len(splitItem) > 1):
			version = splitItem[1]
		act.executed(target=target, targetVersion=version)

	step_file = File(path=inFilePath, name=inFilename, description=stepDict['fileDescription'], parentId=parentid, synapseStore=str2bool(stepDict['store']))
	step_file = syn.store(step_file, activity=act, forceVersion=False)
	if 'annotations' in stepDict:
		syn.setAnnotations(step_file, annotations=stepDict['annotations'])
	print 'new entity id %s' % step_file.id
	return(step_file.id)

def getExistingFolders(syn,pid):
	'''Returns dictionary of existing folders for the project. key = name, val = id.'''

	existingFolders = dict()
	project = syn.get(pid)
	results = syn.chunkedQuery(''.join(['SELECT id, name, concreteType FROM entity WHERE parentId=="', project.id, '"']))
	for result in results:
		if result['entity.concreteType'][0] == 'org.sagebionetworks.repo.model.Folder':
			existingFolders[result['entity.name']] = result['entity.id']
	return(existingFolders)

def setUpSynapseProject(foldersToCreate,syn,pid=None,pname=None):
	'''Creates Synapse project and necessary folders for the dataset.'''

	# Create a set of sub-folders expected for this project
	folderSchemaSet = set(foldersToCreate)

	# Get the project if it exists or create
	if pid == None:
		project = Project(pname)
		project = syn.store(project)
	else:
		project = syn.get(pid)
		print '%s' % project.name

		existingFolders = getExistingFolders(syn,project.id)
		if len(existingFolders) > 0:
			foldersToCreate = folderSchemaSet.difference(existingFolders.keys())

	# create the folders that don't exist
	for name in foldersToCreate:
		createFolder = Folder(name, parent=project.id)
		createFolder = syn.store(createFolder)
		existingFolders[name] = createFolder.id
	return(project, existingFolders)

<|endoftext|>


<commit_before>"Build up the sphinx autodoc file for the python code"

import os
import sys

pywin_folder = os.path.split(__file__)[0]
pywin_folder = os.path.split(pywin_folder)[0]

sys.path.append(pywin_folder)
pywin_folder = os.path.join(pywin_folder, "pywinauto")

excluded_dirs = ["unittests"]
excluded_files = [
    "_menux.py",
    "__init__.py",
    "win32defines.py",
    "win32structures.py",
    "win32functions.py"]

output_folder = "code"

try:
    os.mkdir(output_folder)
except WindowsError:
    pass

module_docs = []
for root, dirs, files in os.walk(pywin_folder):

    # Skip over directories we don't want to document
    for i, d in enumerate(dirs):
        if d in excluded_dirs:
            del dirs[i]

    py_files = [f for f in files if f.endswith(".py")]

    # skip over py files we don't want to document
    for filename in py_files:
        if filename in excluded_files:
            continue

        doc_source_filename = os.path.join(
            output_folder, filename + ".txt")
        if os.path.exists(doc_source_filename):
            continue

        print filename

        filepath =  os.path.join(root, filename)

        modulename = filepath.split("\\")
        modulename = modulename[modulename.index('pywinauto'):]

        # remove extension
        modulename[-1] = modulename[-1][:-3]

        modulename = ".".join(modulename)

        out = open(doc_source_filename, "w")

        out.write(modulename + "\n")
        out.write("-" * len(modulename) + "\n")
        out.write(" .. automodule:: %s\n"% modulename)
        out.write("    :members:\n")
        out.write("    :undoc-members:\n\n")
        #out.write("    :inherited-members:\n")
        #out.write(" .. autoattribute:: %s\n"% modulename)
        out.close()

        module_docs.append(doc_source_filename)

# This section needs to be updated - I should idealy parse the
# existing file to see if any new docs have been added, if not then
# I should just leave the file alone rathre than re-create.
#
#c = open(os.path.join(output_folder, "code.txt"), "w")
#c.write("Source Code\n")
#c.write("=" * 30 + "\n")
#
#c.write(".. toctree::\n")
#c.write("   :maxdepth: 3\n\n")
#for doc in module_docs:
#    c.write("   " + doc + "\n")
#
#c.close()
#        <commit_msg>use os.path.dirname() rather than os.path.split()[0]<commit_after>"Build up the sphinx autodoc file for the python code"

import os
import sys

docs_folder = os.path.dirname(__file__)
pywin_folder = os.path.dirname(docs_folder)

sys.path.append(pywin_folder)
pywin_folder = os.path.join(pywin_folder, "pywinauto")

excluded_dirs = ["unittests"]
excluded_files = [
    "_menux.py",
    "__init__.py",
    "win32defines.py",
    "win32structures.py",
    "win32functions.py"]

output_folder = "code"

try:
    os.mkdir(output_folder)
except WindowsError:
    pass

module_docs = []
for root, dirs, files in os.walk(pywin_folder):

    # Skip over directories we don't want to document
    for i, d in enumerate(dirs):
        if d in excluded_dirs:
            del dirs[i]

    py_files = [f for f in files if f.endswith(".py")]

    # skip over py files we don't want to document
    for filename in py_files:
        if filename in excluded_files:
            continue

        doc_source_filename = os.path.join(
            output_folder, filename + ".txt")
        if os.path.exists(doc_source_filename):
            continue

        print filename

        filepath =  os.path.join(root, filename)

        modulename = filepath.split("\\")
        modulename = modulename[modulename.index('pywinauto'):]

        # remove extension
        modulename[-1] = modulename[-1][:-3]

        modulename = ".".join(modulename)

        out = open(doc_source_filename, "w")

        out.write(modulename + "\n")
        out.write("-" * len(modulename) + "\n")
        out.write(" .. automodule:: %s\n"% modulename)
        out.write("    :members:\n")
        out.write("    :undoc-members:\n\n")
        #out.write("    :inherited-members:\n")
        #out.write(" .. autoattribute:: %s\n"% modulename)
        out.close()

        module_docs.append(doc_source_filename)

# This section needs to be updated - I should idealy parse the
# existing file to see if any new docs have been added, if not then
# I should just leave the file alone rathre than re-create.
#
#c = open(os.path.join(output_folder, "code.txt"), "w")
#c.write("Source Code\n")
#c.write("=" * 30 + "\n")
#
#c.write(".. toctree::\n")
#c.write("   :maxdepth: 3\n\n")
#for doc in module_docs:
#    c.write("   " + doc + "\n")
#
#c.close()
#        <|endoftext|>


<commit_before>#!/usr/bin/python3

"""

 - aggregate IPs to Class C?
 - Any usable libraries?
	- https://github.com/secynic/ipwhois
 - go through multiple services to not be rate limited too hard?

"""

from multiprocessing import Pool, TimeoutError
import fileinput
import csv
import sys
import config
import sqlite3
import re
import netaddr
import pprint
import db
from utils import *
import time
from abuseparsing import *
import traceback

import logging
logging.basicConfig(filename='debug.log',level=logging.DEBUG)

def addAddresses():

	ips  = []
	bads = 0
	for line in sys.stdin:
		for ip in extract_ips(line):
			ip = IP(ip)

			bad = not IsPublicIP(ip)

			if bad:
				if bads<15:
					sys.stderr.write("Bad IP: "+str(ip)+"\n")
				bads+=1
			else:
				ips.append(ip)

	print("IPs added "+str(db.addips(ips))+" Bad IPs: "+str(bads))

def continueProcessing():

	import whoiser
	whoiser.start()

	unprocessed_ips = db.get_unprocessed(random_order = True)
	queries_pending=[]

	# Status stuff
	req_sent = 0
	req_unsent = 0
	resp_recv = 0
	req_legacy = 0
	exhaust_print=False

	first=True
	while not whoiser.done() or len(queries_pending)>0 or first:
		if first:
			first=False

		# Responses from worker threads
		resp = whoiser.response(timeout=0.01)
		if resp:
			resp_recv+=1

			query_legacy = False

			(req,reply,*rest) = resp

			ip = IP(req['ip'])
			legacy_reply = req.get('legacy')

			sys.stdout.write(legacy_reply and '|' or "-")
			sys.stdout.flush()

			# We have a response
			if reply==True:
				# Push again

				sys.stdout.write(">")
				sys.stdout.flush()
				queries_pending.append(req)

			elif reply:

				quality = legacy_reply and 2 or 1
				(abuseemails,(startip,endip)) = legacy_reply and process_whois_abuse(reply) or process_rdap_abuse(reply)
				# one IP network is a possibility
				networkid = db.get_network(startip or ip, endip or ip, ip)
				db.add_network_abusemails(networkid,abuseemails,[quality]*len(abuseemails))
				db.ip_set_network(ip,networkid = networkid)

				if len(abuseemails)==0:
					query_legacy = True
					print("no emails found",req)

			# Worker failed (any reason)
			else:
				# We will try legacy query if rdap failed for any reason.
				# Only legacy query error will be stored, even if rdap succeeded but found no emails
				query_legacy = True
				exc = rest[0]
				store_error = "".join(traceback.format_exception(None, exc, exc.__traceback__))
				print("failquery",ip,legacy_reply and "(LEGACY)" or "",FormatShortError(exc),len(rest)>1 and rest[1])

				db.ip_set_network(ip,failurereason=store_error)

			if query_legacy and not legacy_reply:
				print("LEGACY QUERYING",ip)
				if reply:
					pprint.pprint(reply)
				req['legacy'] = True
				# Add to queries again
				queries_pending.append(req)

		# Get more queries if they do not exist in pending
		if len(queries_pending)<6:
			row = unprocessed_ips.fetchone()
			if row:
				(ip,) = row
				ip = IP(ip)
				networkid = db.network_from_ip(ip)

				#TODO: Loop if we didn't query stuff
				if networkid and db.network_has_mails_processed(networkid):
					db.ip_set_network(ip,networkid)
					req_unsent+=1
				else:
					queries_pending.append({"ip": ip })
				sys.stdout.write("_")
				sys.stdout.flush()
			else:
				if not exhaust_print:
					exhaust_print = True
					print("Exhausted queryable IPs")

		# Add queries from query queue
		try:
			request = queries_pending[0]
			if whoiser.request( request, timeout=0.01):
				queries_pending.pop(0)
				req_sent+=1
		except IndexError as e:
			pass

	print("Processing finished. Requests: {} Replies: {} instant, {} by whoiser".format(
									req_sent,	req_unsent,		resp_recv	))

def dumpAll(processedOnly=False):
	for r in db.dump_all():
		(ip,e1,e2,e3) = r
		if not processedOnly or e1 is not None:
			extra = e1 or "Unprocessed"
			ip = IP(ip)
			if ip.is_ipv4_mapped():
				ip = ip.ipv4()
			print( "%s%s" % ( str(ip), extra and (" %s")%(extra) or "" ) )

cmd=len(sys.argv)>1 and sys.argv[1]
if cmd=="add":
	addAddresses()
elif cmd=="process":
	continueProcessing()
elif cmd=="dump":
	dumpAll()
elif cmd=="process-failed":
	db.clear_failed_ips()
	continueProcessing()
elif cmd=="dump-processed":
	dumpAll(True)
else:
	print(
"""Usage:
	add				Add IPs from stdin
	process			Start processing IPs
	process-failed	Mark failed IPs to be reprocessed
	dump			Dump IPs with any results
	dump-processed	Dump IPs that have email results
""")

db.commit()<commit_msg>Add depth=10 back, otherwise parsing of deeper entities will not happen<commit_after>#!/usr/bin/python3

"""

 - aggregate IPs to Class C?
 - Any usable libraries?
	- https://github.com/secynic/ipwhois
 - go through multiple services to not be rate limited too hard?

"""

from multiprocessing import Pool, TimeoutError
import fileinput
import csv
import sys
import config
import sqlite3
import re
import netaddr
import pprint
import db
from utils import *
import time
from abuseparsing import *
import traceback

import logging
logging.basicConfig(filename='debug.log',level=logging.DEBUG)

def addAddresses():

	ips  = []
	bads = 0
	for line in sys.stdin:
		for ip in extract_ips(line):
			ip = IP(ip)

			bad = not IsPublicIP(ip)

			if bad:
				if bads<15:
					sys.stderr.write("Bad IP: "+str(ip)+"\n")
				bads+=1
			else:
				ips.append(ip)

	print("IPs added "+str(db.addips(ips))+" Bad IPs: "+str(bads))

def continueProcessing():

	import whoiser
	whoiser.start()

	unprocessed_ips = db.get_unprocessed(random_order = True)
	queries_pending=[]

	# Status stuff
	req_sent = 0
	req_unsent = 0
	resp_recv = 0
	req_legacy = 0
	exhaust_print=False

	first=True
	while not whoiser.done() or len(queries_pending)>0 or first:
		if first:
			first=False

		# Responses from worker threads
		resp = whoiser.response(timeout=0.01)
		if resp:
			resp_recv+=1

			query_legacy = False

			(req,reply,*rest) = resp

			ip = IP(req['ip'])
			legacy_reply = req.get('legacy')

			sys.stdout.write(legacy_reply and '^' or "-")
			sys.stdout.flush()

			# We have a response
			if reply==True:
				# Push again, try later, preferably much later, or by some other worker-proxy

				sys.stdout.write(">")
				sys.stdout.flush()
				queries_pending.append(req)

			elif reply:
				quality = legacy_reply and 2 or 1
				(abuseemails,(startip,endip)) = legacy_reply and process_whois_abuse(reply) or process_rdap_abuse(reply)
				#print(legacy_reply and "LEGACY" or "RDAP",abuseemails,startip,endip)
				# one IP network is a possibility
				networkid = db.get_network(startip or ip, endip or ip, ip)
				db.add_network_abusemails(networkid,abuseemails,[quality]*len(abuseemails))
				db.ip_set_network(ip,networkid = networkid)

				if len(abuseemails)==0:
					query_legacy = True
					if legacy_reply:
						print("no emails found",req)

			# Worker failed (any reason)
			else:
				# We will try legacy query if rdap failed for any reason.
				# Only legacy query error will be stored, even if rdap succeeded but found no emails
				query_legacy = True
				exc = rest[0]
				store_error = "".join(traceback.format_exception(None, exc, exc.__traceback__))
				print("failquery",ip,legacy_reply and "(LEGACY)" or "",FormatShortError(exc),len(rest)>1 and rest[1])

				db.ip_set_network(ip,failurereason=store_error)

			if query_legacy and not legacy_reply:

				sys.stdout.write("~")
				sys.stdout.flush()

				req['legacy'] = True
				# Add to queries again
				queries_pending.append(req)

		# Get more queries if they do not exist in pending
		if len(queries_pending)<6:
			row = unprocessed_ips.fetchone()
			if row:
				(ip,) = row
				ip = IP(ip)
				networkid = db.network_from_ip(ip)

				#TODO: Loop if we didn't query stuff
				if networkid and db.network_has_mails_processed(networkid):
					db.ip_set_network(ip,networkid)
					req_unsent+=1
				else:
					queries_pending.append({"ip": ip })
				sys.stdout.write("_")
				sys.stdout.flush()
			else:
				if not exhaust_print:
					exhaust_print = True
					print("Exhausted queryable IPs")

		# Add queries from query queue
		try:
			request = queries_pending[0]
			if whoiser.request( request, timeout=0.01):
				queries_pending.pop(0)
				req_sent+=1
		except IndexError as e:
			pass

	print("Processing finished. Requests: {} Replies: {} instant, {} by whoiser".format(
									req_sent,	req_unsent,		resp_recv	))

def dumpAll(processedOnly=False):
	for r in db.dump_all():
		(ip,networkid,failurereason,
			e1,e2,e3,q1,startip,endip,*rest) = r

		processed = networkid is not None

		ip = IP(ip)
		if ip.is_ipv4_mapped():
			ip = ip.ipv4()
		out = [ str(ip) ]
		if not processedOnly or processed:

			if not processed:
				out += ["UNPROCESSED"]
			else:
				if failurereason or networkid==-1:
					assert(failurereason)
					assert(networkid==-1)
					out += ["FAIL"]
				else:
					if not q1:
						pass
					elif q1==1:
						out += ["RDAP"]
					elif q1==2:
						out += ["WHOIS"]
					else:
						out += ["???"]

				#out += emaillist
				if e1:
					out += [e1]
				if e2:
					out += [e2]
				if e3:
					out += [e3]
				if not e1 and not e2 and not e3:
					out += ["NOMAILS"]

			print( " ".join(out) )

cmd=len(sys.argv)>1 and sys.argv[1]
if cmd=="add":
	addAddresses()
elif cmd=="process":
	continueProcessing()
elif cmd=="dump":
	dumpAll()
elif cmd=="process-failed":
	db.clear_failed_ips()
	continueProcessing()
elif cmd=="dump-processed":
	dumpAll(True)
else:
	print(
"""Usage:
	add				Add IPs from stdin
	process			Start processing IPs
	process-failed	Mark failed IPs to be reprocessed
	dump			Dump IPs with any results
	dump-processed	Dump IPs that have email results
""")

db.commit()<|endoftext|>


<commit_before>"""Data Preparation

Calculates and joins parameters required for pygypsy to a plot table

"""
#pylint: disable=no-member
from __future__ import division
import logging
import pandas as pd
from copy import deepcopy

from pygypsy.stand_density_factor import (
    estimate_sdf_aw,
    estimate_sdf_sb,
    estimate_sdf_sw,
    estimate_sdf_pl,
)
from pygypsy.density import (
    estimate_density_aw,
    estimate_density_sw,
    estimate_density_sb,
    estimate_density_pl,
)
from pygypsy.site_index import (
    get_site_indices_from_dominant_species,
    _estimate_site_index
)
from pygypsy.utils import (
    _get_gypsy_valid_species,
    _log_loop_progress,
    _generate_fplot_dict,
    _reclassify_and_sort_species,
)
from pygypsy.asaCompileAgeGivenSpSiHt import (
    computeTreeAge,
    ComputeGypsyTreeHeightGivenSiteIndexAndTotalAge,
    ComputeGypsySiteIndex,
)

LOGGER = logging.getLogger(__name__)

def _populate_species_dict_with_indices(species_dict, estimated_site_indices):
    """Fill the dictionary with estimated SIs

    Fill all the SIs to avoid IFs and loops. Some of them will not be
    used.

    :param species_dict: dictionary used to store params for all species
    :param estimated_site_indices: array of estimated site indices from
                                   get_species_site_indices function

    ..note: the dominant species in the estiamted_site_indices uses its
    observed site index

    """
    local_species_dict = deepcopy(species_dict)

    for species, site_index in estimated_site_indices.items():
        local_species_dict[species]['SI'] = site_index

    return local_species_dict

def populate_species_dict(partial_species_list,
                          sorted_species_abbrev_perc_tuples_list,
                          dominant_species=None, row=None,
                          dominant_species_current_age=None,
                          dominant_species_current_height=None):
    """Fill partial fplot

    Given fplot with proportion filled out, add age, top height, basal area,
    density

    :param partial_species_list:
    :param sorted_species_abbrev_perc_tuples_list:
    :param dominant_species:
    :param dominant_species_current_age:
    :param dominant_species_current_height:

    """
    species_dict = deepcopy(partial_species_list)
    outer_sorted_species_perc_list = sorted_species_abbrev_perc_tuples_list
    for species in outer_sorted_species_perc_list:
        # TODO: raise NoDominantSpeciesError and catch it where called instead of break
        # confirmed with julianno
        if species[1] == 0:
            break
        # TODO: this is always true on first pass of the loop
        elif species[0] == dominant_species[0]:
            species_dict[species[0]]['tage'] = dominant_species_current_age
            species_dict[species[0]]['topHeight'] = dominant_species_current_height
            species_dict[species[0]]['N'] = row['TPH'] * species[1] / 100
            species_dict[species[0]]['BA'] = row['BPH'] * species[1] / 100
            x_si = ComputeGypsySiteIndex(
                species[0],
                species_dict[species[0]]['topHeight'],
                0,
                species_dict[species[0]]['tage']
            )
            species_dict[species[0]]['bhage'] = x_si[0]
            # if, after re-arranging the proportions, dom species is another
            # one then we need to re-estimate everything  even for the new
            # dominant species
        else:
            si_sp = species_dict[species[0]]['SI']
            species_dict[species[0]]['PCT'] = species[1]
            species_dict[species[0]]['tage'] = computeTreeAge(
                species[0],
                treeHt=dominant_species_current_height,
                treeSi=si_sp,
                maxTreeAge=450
            )
            # density based on the proportion of the species
            species_dict[species[0]]['N'] = row['TPH'] * species[1]/100
            # Basal area from the species proportion as well
            species_dict[species[0]]['BA'] = row['BPH'] * species[1]/100
            # calling the ComputeGypsySiteIndex function, estimate bhage
            x_si = ComputeGypsySiteIndex(
                species[0], dominant_species_current_height, 0,
                species_dict[species[0]]['tage']
            )
            species_dict[species[0]]['bhage'] = x_si[0]
    return species_dict

def prep_standtable(data, minimum_age=25):
    '''Define site_index of all other species given the dominant species

    The site index is only defined for the dominant species in a plot. This
    function uses the dominant species and its site index to estimate the site
    indices for other species. It estimates other values and returns a data
    frame of all of the values for all plots input.

    estimates site_index Dom from inventory Age and HD
    this site_index is used to estimate the site index of the other species
    for example:
    - FD is dom species take HD and Age
    - assume FD is Sw generate site_index for Sw
    - get the SW site_index
    - calculate the SIs for other species from the conversion formulas

    :param data: input data frame

    '''
    plot_dict = {}
    n_rows = data.shape[0]
    for i, row in data.iterrows():
        _log_loop_progress(i, n_rows)

        # from the row, compile a list of species abbreviation and percents
        species_abbrev_percent_list = [
            (row['SP1'], row['PCT1']),
            (row['SP2'], row['PCT2']),
            (row['SP3'], row['PCT3']),
            (row['SP4'], row['PCT4']),
            (row['SP5'], row['PCT5'])
        ]

        # validate species percents add to 100
        check_prop = sum(zip(*species_abbrev_percent_list)[1])
        if check_prop != 100:
            raise ValueError('Species proportions not correct: %s' % check_prop)

        # pull some key vaiables off the row
        plot_id = row['id_l1']
        plot_dominant_species = row['SP1']
        dominant_species_current_age = row['AGE']
        dominant_species_current_height = row['HD']

        site_index = _estimate_site_index(
            plot_dominant_species,
            dominant_species_current_age,
            dominant_species_current_height
        )

        temp_dominant_species = _get_gypsy_valid_species(plot_dominant_species)
        gypsy_site_indices = get_site_indices_from_dominant_species(
            temp_dominant_species,
            site_index
        )
        empty_species_dict = _generate_fplot_dict()
        species_dict = _populate_species_dict_with_indices(empty_species_dict,
                                                           gypsy_site_indices)

        outer_sorted_species_perc_list, outer_species_perc_dict = \
            _reclassify_and_sort_species(species_abbrev_percent_list)

        species_dict['Aw']['PCT'] = outer_species_perc_dict['Aw']
        species_dict['Pl']['PCT'] = outer_species_perc_dict['Pl']
        species_dict['Sw']['PCT'] = outer_species_perc_dict['Sw']
        species_dict['Sb']['PCT'] = outer_species_perc_dict['Sb']

        dominant_species = outer_sorted_species_perc_list[0]

        species_dict = populate_species_dict(
            species_dict, outer_sorted_species_perc_list,
            dominant_species=dominant_species,
            row=row,
            dominant_species_current_age=dominant_species_current_age,
            dominant_species_current_height=dominant_species_current_height
        )

        density_aw = species_dict['Aw']['N']
        density_sw = species_dict['Sw']['N']
        density_pl = species_dict['Pl']['N']
        density_sb = species_dict['Sb']['N']

        tage_aw = species_dict['Aw']['tage']
        tage_sw = species_dict['Sw']['tage']
        tage_pl = species_dict['Pl']['tage']
        tage_sb = species_dict['Sb']['tage']

        ages = [tage_aw, tage_sw, tage_pl, tage_sb]
        if not any([age > minimum_age for age in ages]):
            msg = ('No speices have age greater than the minimum age %s;'
                   'skipping plot %s') % (minimum_age, plot_id)
            LOGGER.warning(msg)
            continue

        bhage_aw = species_dict['Aw']['bhage']
        bhage_sw = species_dict['Sw']['bhage']
        bhage_pl = species_dict['Pl']['bhage']
        bhage_sb = species_dict['Sb']['bhage']

        y2bh_aw = tage_aw - bhage_aw
        y2bh_sw = tage_sw - bhage_sw
        y2bh_pl = tage_pl - bhage_pl
        y2bh_sb = tage_sb - bhage_sb

        site_index_aw = species_dict['Aw']['SI']
        site_index_sw = species_dict['Sw']['SI']
        site_index_pl = species_dict['Pl']['SI']
        site_index_sb = species_dict['Sb']['SI']

        basal_area_aw = species_dict['Aw']['BA']
        basal_area_sb = species_dict['Sb']['BA']
        basal_area_sw = species_dict['Sw']['BA']
        basal_area_pl = species_dict['Pl']['BA']

        # Top height for all species
        top_height_aw = ComputeGypsyTreeHeightGivenSiteIndexAndTotalAge(
            'Aw', site_index_aw, tage_aw
        )
        top_height_sb = ComputeGypsyTreeHeightGivenSiteIndexAndTotalAge(
            'Sb', site_index_sb, tage_sb
        )
        top_height_pl = ComputeGypsyTreeHeightGivenSiteIndexAndTotalAge(
            'Pl', site_index_pl, tage_pl
        )
        top_height_sw = ComputeGypsyTreeHeightGivenSiteIndexAndTotalAge(
            'Sw', site_index_sw, tage_sw
        )

        # stand desnity factor for all species
        y_aw = estimate_sdf_aw('Aw', site_index_aw, bhage_aw, density_aw)
        sdf_aw0 = y_aw[1]
        density_bh_aw = y_aw[0]

        y_sb = estimate_sdf_sb('Sb', site_index_sb, tage_sb, density_sb)
        sdf_sb0 = y_sb[1]
        density_bh_sb = y_sb[0]

        y_sw = estimate_sdf_sw('Sw', site_index_sw, tage_sw, sdf_aw0, density_sw)
        sdf_sw0 = y_sw[1]
        density_bh_sw = y_sw[0]

        y_pl = estimate_sdf_pl(
            'Pl', site_index_pl, tage_pl,
            sdf_aw0, sdf_sw0, sdf_sb0, density_pl
        )
        sdf_pl0 = y_pl[1]
        density_bh_pl = y_pl[0]

        # estimating species densities at time zero
        initial_density_aw = estimate_density_aw(sdf_aw0, 0, site_index_aw)
        initial_density_sb = estimate_density_sb(sdf_sb0, 0, site_index_sb)
        initial_density_sw = estimate_density_sw(
            sdf_sw0, sdf_aw0, 0, site_index_sw
        )
        initial_density_pl = estimate_density_pl(
            sdf_aw0, sdf_sw0, sdf_sb0, sdf_pl0, 0, site_index_pl
        )

        plot_dict[plot_id] = {
            'SI_Aw': site_index_aw,
            'SI_Sw': site_index_sw,
            'SI_Pl': site_index_pl,
            'SI_Sb': site_index_sb,
            'N_Aw': density_aw,
            'N_Sw': density_sw,
            'N_Pl': density_pl,
            'N_Sb': density_sb,
            'y2bh_Aw': y2bh_aw,
            'y2bh_Sw': y2bh_sw,
            'y2bh_Pl': y2bh_pl,
            'y2bh_Sb': y2bh_sb,
            'tage_Aw': tage_aw,
            'tage_Sw': tage_sw,
            'tage_Pl': tage_pl,
            'tage_Sb': tage_sb,
            'BA_Aw': basal_area_aw,
            'BA_Sw': basal_area_sw,
            'BA_Pl': basal_area_pl,
            'BA_Sb': basal_area_sb,
            'SDF_Aw': sdf_aw0,
            'SDF_Sw': sdf_sw0,
            'SDF_Pl': sdf_pl0,
            'SDF_Sb': sdf_sb0,
            'N0_Aw': initial_density_aw,
            'N0_Sb': initial_density_sb,
            'N0_Sw': initial_density_sw,
            'N0_Pl': initial_density_pl,
            'StumpDOB_Aw': species_dict['Aw']['StumpDOB'],
            'StumpDOB_Sb': species_dict['Sb']['StumpDOB'],
            'StumpDOB_Sw': species_dict['Sw']['StumpDOB'],
            'StumpDOB_Pl': species_dict['Pl']['StumpDOB'],
            'StumpHeight_Aw': species_dict['Aw']['StumpHeight'],
            'StumpHeight_Sb': species_dict['Sb']['StumpHeight'],
            'Stumpheight_Sw': species_dict['Sw']['StumpHeight'],
            'StumpHeight_Pl': species_dict['Pl']['StumpHeight'],
            'TopDib_Aw': species_dict['Aw']['TopDib'],
            'TopDib_Sb': species_dict['Sb']['TopDib'],
            'TopDib_Sw': species_dict['Sw']['TopDib'],
            'TopDib_Pl': species_dict['Pl']['TopDib'],
            'topHeight_Aw': top_height_aw,
            'topHeight_Sw': top_height_sw,
            'topHeight_Sb': top_height_sb,
            'topHeight_Pl': top_height_pl
        }

    plot_df = pd.DataFrame(plot_dict)
    plot_df = plot_df.transpose()

    return plot_df
<commit_msg>Apply the minimum age check to dominant species<commit_after>"""Data Preparation

Calculates and joins parameters required for pygypsy to a plot table

"""
#pylint: disable=no-member
from __future__ import division
import logging
import pandas as pd
from copy import deepcopy

from pygypsy.stand_density_factor import (
    estimate_sdf_aw,
    estimate_sdf_sb,
    estimate_sdf_sw,
    estimate_sdf_pl,
)
from pygypsy.density import (
    estimate_density_aw,
    estimate_density_sw,
    estimate_density_sb,
    estimate_density_pl,
)
from pygypsy.site_index import (
    get_site_indices_from_dominant_species,
    _estimate_site_index
)
from pygypsy.utils import (
    _get_gypsy_valid_species,
    _log_loop_progress,
    _generate_fplot_dict,
    _reclassify_and_sort_species,
)
from pygypsy.asaCompileAgeGivenSpSiHt import (
    computeTreeAge,
    ComputeGypsyTreeHeightGivenSiteIndexAndTotalAge,
    ComputeGypsySiteIndex,
)

LOGGER = logging.getLogger(__name__)

def _populate_species_dict_with_indices(species_dict, estimated_site_indices):
    """Fill the dictionary with estimated SIs

    Fill all the SIs to avoid IFs and loops. Some of them will not be
    used.

    :param species_dict: dictionary used to store params for all species
    :param estimated_site_indices: array of estimated site indices from
                                   get_species_site_indices function

    ..note: the dominant species in the estiamted_site_indices uses its
    observed site index

    """
    local_species_dict = deepcopy(species_dict)

    for species, site_index in estimated_site_indices.items():
        local_species_dict[species]['SI'] = site_index

    return local_species_dict

def populate_species_dict(partial_species_list,
                          sorted_species_abbrev_perc_tuples_list,
                          dominant_species=None, row=None,
                          dominant_species_current_age=None,
                          dominant_species_current_height=None):
    """Fill partial fplot

    Given fplot with proportion filled out, add age, top height, basal area,
    density

    :param partial_species_list:
    :param sorted_species_abbrev_perc_tuples_list:
    :param dominant_species:
    :param dominant_species_current_age:
    :param dominant_species_current_height:

    """
    species_dict = deepcopy(partial_species_list)
    outer_sorted_species_perc_list = sorted_species_abbrev_perc_tuples_list
    for species in outer_sorted_species_perc_list:
        # TODO: raise NoDominantSpeciesError and catch it where called instead of break
        # confirmed with julianno
        if species[1] == 0:
            break
        # TODO: this is always true on first pass of the loop
        elif species[0] == dominant_species[0]:
            species_dict[species[0]]['tage'] = dominant_species_current_age
            species_dict[species[0]]['topHeight'] = dominant_species_current_height
            species_dict[species[0]]['N'] = row['TPH'] * species[1] / 100
            species_dict[species[0]]['BA'] = row['BPH'] * species[1] / 100
            x_si = ComputeGypsySiteIndex(
                species[0],
                species_dict[species[0]]['topHeight'],
                0,
                species_dict[species[0]]['tage']
            )
            species_dict[species[0]]['bhage'] = x_si[0]
            # if, after re-arranging the proportions, dom species is another
            # one then we need to re-estimate everything  even for the new
            # dominant species
        else:
            si_sp = species_dict[species[0]]['SI']
            species_dict[species[0]]['PCT'] = species[1]
            species_dict[species[0]]['tage'] = computeTreeAge(
                species[0],
                treeHt=dominant_species_current_height,
                treeSi=si_sp,
                maxTreeAge=450
            )
            # density based on the proportion of the species
            species_dict[species[0]]['N'] = row['TPH'] * species[1]/100
            # Basal area from the species proportion as well
            species_dict[species[0]]['BA'] = row['BPH'] * species[1]/100
            # calling the ComputeGypsySiteIndex function, estimate bhage
            x_si = ComputeGypsySiteIndex(
                species[0], dominant_species_current_height, 0,
                species_dict[species[0]]['tage']
            )
            species_dict[species[0]]['bhage'] = x_si[0]
    return species_dict

def prep_standtable(data, minimum_age=25):
    '''Define site_index of all other species given the dominant species

    The site index is only defined for the dominant species in a plot. This
    function uses the dominant species and its site index to estimate the site
    indices for other species. It estimates other values and returns a data
    frame of all of the values for all plots input.

    estimates site_index Dom from inventory Age and HD
    this site_index is used to estimate the site index of the other species
    for example:
    - FD is dom species take HD and Age
    - assume FD is Sw generate site_index for Sw
    - get the SW site_index
    - calculate the SIs for other species from the conversion formulas

    :param data: input data frame

    '''
    plot_dict = {}
    n_rows = data.shape[0]
    for i, row in data.iterrows():
        _log_loop_progress(i, n_rows)

        # from the row, compile a list of species abbreviation and percents
        species_abbrev_percent_list = [
            (row['SP1'], row['PCT1']),
            (row['SP2'], row['PCT2']),
            (row['SP3'], row['PCT3']),
            (row['SP4'], row['PCT4']),
            (row['SP5'], row['PCT5'])
        ]

        # validate species percents add to 100
        check_prop = sum(zip(*species_abbrev_percent_list)[1])
        if check_prop != 100:
            raise ValueError('Species proportions not correct: %s' % check_prop)

        # pull some key vaiables off the row
        plot_id = row['id_l1']
        plot_dominant_species = row['SP1']
        dominant_species_current_age = row['AGE']
        dominant_species_current_height = row['HD']

        if dominant_species_current_age < minimum_age:
            msg = ('Dominant species age is less than minimum age %s;'
                   'skipping plot %s') % (minimum_age, plot_id)
            LOGGER.warning(msg)
            continue

        site_index = _estimate_site_index(
            plot_dominant_species,
            dominant_species_current_age,
            dominant_species_current_height
        )

        temp_dominant_species = _get_gypsy_valid_species(plot_dominant_species)
        gypsy_site_indices = get_site_indices_from_dominant_species(
            temp_dominant_species,
            site_index
        )
        empty_species_dict = _generate_fplot_dict()
        species_dict = _populate_species_dict_with_indices(empty_species_dict,
                                                           gypsy_site_indices)

        outer_sorted_species_perc_list, outer_species_perc_dict = \
            _reclassify_and_sort_species(species_abbrev_percent_list)

        species_dict['Aw']['PCT'] = outer_species_perc_dict['Aw']
        species_dict['Pl']['PCT'] = outer_species_perc_dict['Pl']
        species_dict['Sw']['PCT'] = outer_species_perc_dict['Sw']
        species_dict['Sb']['PCT'] = outer_species_perc_dict['Sb']

        dominant_species = outer_sorted_species_perc_list[0]

        species_dict = populate_species_dict(
            species_dict, outer_sorted_species_perc_list,
            dominant_species=dominant_species,
            row=row,
            dominant_species_current_age=dominant_species_current_age,
            dominant_species_current_height=dominant_species_current_height
        )

        density_aw = species_dict['Aw']['N']
        density_sw = species_dict['Sw']['N']
        density_pl = species_dict['Pl']['N']
        density_sb = species_dict['Sb']['N']

        tage_aw = species_dict['Aw']['tage']
        tage_sw = species_dict['Sw']['tage']
        tage_pl = species_dict['Pl']['tage']
        tage_sb = species_dict['Sb']['tage']

        bhage_aw = species_dict['Aw']['bhage']
        bhage_sw = species_dict['Sw']['bhage']
        bhage_pl = species_dict['Pl']['bhage']
        bhage_sb = species_dict['Sb']['bhage']

        y2bh_aw = tage_aw - bhage_aw
        y2bh_sw = tage_sw - bhage_sw
        y2bh_pl = tage_pl - bhage_pl
        y2bh_sb = tage_sb - bhage_sb

        site_index_aw = species_dict['Aw']['SI']
        site_index_sw = species_dict['Sw']['SI']
        site_index_pl = species_dict['Pl']['SI']
        site_index_sb = species_dict['Sb']['SI']

        basal_area_aw = species_dict['Aw']['BA']
        basal_area_sb = species_dict['Sb']['BA']
        basal_area_sw = species_dict['Sw']['BA']
        basal_area_pl = species_dict['Pl']['BA']

        # Top height for all species
        top_height_aw = ComputeGypsyTreeHeightGivenSiteIndexAndTotalAge(
            'Aw', site_index_aw, tage_aw
        )
        top_height_sb = ComputeGypsyTreeHeightGivenSiteIndexAndTotalAge(
            'Sb', site_index_sb, tage_sb
        )
        top_height_pl = ComputeGypsyTreeHeightGivenSiteIndexAndTotalAge(
            'Pl', site_index_pl, tage_pl
        )
        top_height_sw = ComputeGypsyTreeHeightGivenSiteIndexAndTotalAge(
            'Sw', site_index_sw, tage_sw
        )

        # stand desnity factor for all species
        y_aw = estimate_sdf_aw('Aw', site_index_aw, bhage_aw, density_aw)
        sdf_aw0 = y_aw[1]
        density_bh_aw = y_aw[0]

        y_sb = estimate_sdf_sb('Sb', site_index_sb, tage_sb, density_sb)
        sdf_sb0 = y_sb[1]
        density_bh_sb = y_sb[0]

        y_sw = estimate_sdf_sw('Sw', site_index_sw, tage_sw, sdf_aw0, density_sw)
        sdf_sw0 = y_sw[1]
        density_bh_sw = y_sw[0]

        y_pl = estimate_sdf_pl(
            'Pl', site_index_pl, tage_pl,
            sdf_aw0, sdf_sw0, sdf_sb0, density_pl
        )
        sdf_pl0 = y_pl[1]
        density_bh_pl = y_pl[0]

        # estimating species densities at time zero
        initial_density_aw = estimate_density_aw(sdf_aw0, 0, site_index_aw)
        initial_density_sb = estimate_density_sb(sdf_sb0, 0, site_index_sb)
        initial_density_sw = estimate_density_sw(
            sdf_sw0, sdf_aw0, 0, site_index_sw
        )
        initial_density_pl = estimate_density_pl(
            sdf_aw0, sdf_sw0, sdf_sb0, sdf_pl0, 0, site_index_pl
        )

        plot_dict[plot_id] = {
            'SI_Aw': site_index_aw,
            'SI_Sw': site_index_sw,
            'SI_Pl': site_index_pl,
            'SI_Sb': site_index_sb,
            'N_Aw': density_aw,
            'N_Sw': density_sw,
            'N_Pl': density_pl,
            'N_Sb': density_sb,
            'y2bh_Aw': y2bh_aw,
            'y2bh_Sw': y2bh_sw,
            'y2bh_Pl': y2bh_pl,
            'y2bh_Sb': y2bh_sb,
            'tage_Aw': tage_aw,
            'tage_Sw': tage_sw,
            'tage_Pl': tage_pl,
            'tage_Sb': tage_sb,
            'BA_Aw': basal_area_aw,
            'BA_Sw': basal_area_sw,
            'BA_Pl': basal_area_pl,
            'BA_Sb': basal_area_sb,
            'SDF_Aw': sdf_aw0,
            'SDF_Sw': sdf_sw0,
            'SDF_Pl': sdf_pl0,
            'SDF_Sb': sdf_sb0,
            'N0_Aw': initial_density_aw,
            'N0_Sb': initial_density_sb,
            'N0_Sw': initial_density_sw,
            'N0_Pl': initial_density_pl,
            'StumpDOB_Aw': species_dict['Aw']['StumpDOB'],
            'StumpDOB_Sb': species_dict['Sb']['StumpDOB'],
            'StumpDOB_Sw': species_dict['Sw']['StumpDOB'],
            'StumpDOB_Pl': species_dict['Pl']['StumpDOB'],
            'StumpHeight_Aw': species_dict['Aw']['StumpHeight'],
            'StumpHeight_Sb': species_dict['Sb']['StumpHeight'],
            'Stumpheight_Sw': species_dict['Sw']['StumpHeight'],
            'StumpHeight_Pl': species_dict['Pl']['StumpHeight'],
            'TopDib_Aw': species_dict['Aw']['TopDib'],
            'TopDib_Sb': species_dict['Sb']['TopDib'],
            'TopDib_Sw': species_dict['Sw']['TopDib'],
            'TopDib_Pl': species_dict['Pl']['TopDib'],
            'topHeight_Aw': top_height_aw,
            'topHeight_Sw': top_height_sw,
            'topHeight_Sb': top_height_sb,
            'topHeight_Pl': top_height_pl
        }

    plot_df = pd.DataFrame(plot_dict)
    plot_df = plot_df.transpose()

    return plot_df
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2013, Vispy Development Team.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.

""" Helper for the vispy.gl._gl module.
"""

import sys
import vispy
from OpenGL import GL as _GL

def _make_unavailable_func(funcname):
    def cb(*args, **kwds):
       raise RuntimeError('OpenGL API call "%s" is not available.' % funcname)
    return cb

def get_gl_functions_from_pyopengl(NS, funcnames):
    """ Get GL functions from pyopengl.
    """
    show_warnings = vispy.config['show_warnings']

    for funcname in funcnames:
        # Get function
        try:
            func = getattr(_GL, funcname)
        except AttributeError:
            func = _make_unavailable_func(funcname)
            if True or show_warnings:
                print('warning: %s not available' % funcname )
        NS[funcname] = func

def fix(NS):
    """ Apply some fixes and patches.
    """

    # Fix glGetActiveAttrib, since if its just the ctypes function
    if (    ('glGetActiveAttrib' in NS) and
            hasattr(NS['glGetActiveAttrib'], 'restype') ):
        import ctypes
        def new_glGetActiveAttrib(program, index):
            # Prepare
            bufsize = 32
            length = ctypes.c_int()
            size = ctypes.c_int()
            type = ctypes.c_int()
            name = ctypes.create_string_buffer(bufsize)
            # Call
            _GL.glGetActiveAttrib(program, index,
                    bufsize, ctypes.byref(length), ctypes.byref(size),
                    ctypes.byref(type), name)
            # Return Python objects
            #return name.value.decode('utf-8'), size.value, type.value
            return name.value, size.value, type.value

        # Patch
        NS['glGetActiveAttrib'] = new_glGetActiveAttrib

    # Monkey-patch pyopengl to fix a bug in glBufferSubData
    if sys.version_info > (3,):
        _m = sys.modules[NS['glBufferSubData'].wrapperFunction.__module__]
        _m.long = int
<commit_msg>Take FBO stuff from OpenGL.GL.framebufferobjects. This should fix issue #17.<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2013, Vispy Development Team.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.

""" Helper for the vispy.gl._gl module.
"""

import sys
import vispy
from OpenGL import GL as _GL

def _make_unavailable_func(funcname):
    def cb(*args, **kwds):
       raise RuntimeError('OpenGL API call "%s" is not available.' % funcname)
    return cb

def get_gl_functions_from_pyopengl(NS, funcnames):
    """ Get GL functions from pyopengl.
    """
    show_warnings = vispy.config['show_warnings']
    import OpenGL.GL.framebufferobjects as FBO

    for funcname in funcnames:
        func = None
        # Get function from GL
        try:
            func = getattr(_GL, funcname)
        except AttributeError:
            # Get function from FBO
            try:
                func = getattr(FBO, funcname)
            except AttributeError:
                pass  # more options?

        # Set dummy function if we could not find it
        if func is None:
            func = _make_unavailable_func(funcname)
            if True or show_warnings:
                print('warning: %s not available' % funcname )
        # Set
        NS[funcname] = func

def fix(NS):
    """ Apply some fixes and patches.
    """

    # Fix glGetActiveAttrib, since if its just the ctypes function
    if (    ('glGetActiveAttrib' in NS) and
            hasattr(NS['glGetActiveAttrib'], 'restype') ):
        import ctypes
        def new_glGetActiveAttrib(program, index):
            # Prepare
            bufsize = 32
            length = ctypes.c_int()
            size = ctypes.c_int()
            type = ctypes.c_int()
            name = ctypes.create_string_buffer(bufsize)
            # Call
            _GL.glGetActiveAttrib(program, index,
                    bufsize, ctypes.byref(length), ctypes.byref(size),
                    ctypes.byref(type), name)
            # Return Python objects
            #return name.value.decode('utf-8'), size.value, type.value
            return name.value, size.value, type.value

        # Patch
        NS['glGetActiveAttrib'] = new_glGetActiveAttrib

    # Monkey-patch pyopengl to fix a bug in glBufferSubData
    if sys.version_info > (3,):
        _m = sys.modules[NS['glBufferSubData'].wrapperFunction.__module__]
        _m.long = int
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import pickle
import datetime

from pyhn.hnapi import HackerNewsAPI
from pyhn.config import Config

class CacheManager(object):

    def __init__(self, cache_path=None):
        self.cache_path = cache_path
        if cache_path is None:
            self.config = Config()
            self.cache_path = self.config.parser.get('settings', 'cache')

        self.cache_age = int(self.config.parser.get('settings', 'cache_age'))
        self.extra_page = int(self.config.parser.get('settings', 'extra_page'))
        self.api = HackerNewsAPI()

        if not os.path.exists(self.cache_path):
            self.refresh()

    def is_outdated(self, which="top"):
        if not os.path.exists(self.cache_path):
            return True

        try:
            cache = pickle.load(open(self.cache_path, 'rb'))
        except:
            cache = {}
        if not cache.get(which, False):
            return True

        cache_age = datetime.datetime.today() - cache[which]['date']
        if cache_age.seconds > self.cache_age * 60:
            return True
        else:
            return False

    def refresh(self, which="top"):
        if which == "top":
            stories = self.api.getTopStories(extra_page=self.extra_page)
        elif which == "newest":
            stories = self.api.getNewestStories(extra_page=self.extra_page)
        elif which == "best":
            stories = self.api.getBestStories(extra_page=self.extra_page)
        else:
            raise Exception('Bad value: top, newest and best stories')

        cache = {}
        if os.path.exists(self.cache_path):
            try:
                cache = pickle.load(open(self.cache_path, 'rb'))
            except:
                pass

        cache[which] = {'stories': stories, 'date': datetime.datetime.today()}
        pickle.dump(cache, open(self.cache_path, 'wb'))

    def get_stories(self, which="top"):
        cache = []
        if os.path.exists(self.cache_path):
            try:
                cache = pickle.load(open(self.cache_path, 'rb'))
            except:
                cache = {}

        if not cache.get(which, False):
            return []
        else:
            return cache[which]['stories']
<commit_msg>Style<commit_after># -*- coding: utf-8 -*-
import os
import pickle
import datetime

from pyhn.config import Config
from pyhn.hnapi import HackerNewsAPI

class CacheManager(object):

    def __init__(self, cache_path=None):
        self.cache_path = cache_path
        if cache_path is None:
            self.config = Config()
            self.cache_path = self.config.parser.get('settings', 'cache')

        self.cache_age = int(self.config.parser.get('settings', 'cache_age'))
        self.extra_page = int(self.config.parser.get('settings', 'extra_page'))
        self.api = HackerNewsAPI()

        if not os.path.exists(self.cache_path):
            self.refresh()

    def is_outdated(self, which="top"):
        if not os.path.exists(self.cache_path):
            return True

        try:
            cache = pickle.load(open(self.cache_path, 'rb'))
        except:
            cache = {}
        if not cache.get(which, False):
            return True

        cache_age = datetime.datetime.today() - cache[which]['date']
        if cache_age.seconds > self.cache_age * 60:
            return True
        else:
            return False

    def refresh(self, which="top"):
        if which == "top":
            stories = self.api.getTopStories(extra_page=self.extra_page)
        elif which == "newest":
            stories = self.api.getNewestStories(extra_page=self.extra_page)
        elif which == "best":
            stories = self.api.getBestStories(extra_page=self.extra_page)
        else:
            raise Exception('Bad value: top, newest and best stories')

        cache = {}
        if os.path.exists(self.cache_path):
            try:
                cache = pickle.load(open(self.cache_path, 'rb'))
            except:
                pass

        cache[which] = {'stories': stories, 'date': datetime.datetime.today()}
        pickle.dump(cache, open(self.cache_path, 'wb'))

    def get_stories(self, which="top"):
        cache = []
        if os.path.exists(self.cache_path):
            try:
                cache = pickle.load(open(self.cache_path, 'rb'))
            except:
                cache = {}

        if not cache.get(which, False):
            return []
        else:
            return cache[which]['stories']
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.conf import settings
from social_api.api import ApiAbstractBase, Singleton
from vkontakte import VKError as VkontakteError, API

__all__ = ['api_call', 'VkontakteError']

class VkontakteApi(ApiAbstractBase):
    __metaclass__ = Singleton

    provider = 'vkontakte'
    provider_social_auth = 'vk-oauth2'
    error_class = VkontakteError
    request_timeout = getattr(settings, 'VKONTAKTE_API_REQUEST_TIMEOUT', 1)

    def get_consistent_token(self):
        return getattr(settings, 'VKONTAKTE_API_ACCESS_TOKEN', None)

    def get_api(self, token):
        return API(token=token)

    def get_api_response(self, *args, **kwargs):
        return self.api.get(self.method, timeout=self.request_timeout, *args, **kwargs)

    def handle_error_code_5(self, e, *args, **kwargs):
        if self.user:
            raise e
        self.logger.info("Updating vkontakte access token, error %s, method: %s, recursion count: %d" %
                         (e, self.method, self.recursion_count))
        self.update_tokens()
        return self.repeat_call(*args, **kwargs)

    def handle_error_code_6(self, e, *args, **kwargs):
        # try access_token by another user
        self.logger.info("Vkontakte error 'Too many requests per second' on method: %s, recursion count: %d" % (
            self.method, self.recursion_count))
        self.used_access_tokens += [self.api.token]
        return self.repeat_call(*args, **kwargs)

    def handle_error_code_9(self, e, *args, **kwargs):
        self.logger.warning("Vkontakte flood control registered while executing method %s with params %s, \
            recursion count: %d" % (self.method, kwargs, self.recursion_count))
        self.used_access_tokens += [self.api.token]
        return self.sleep_repeat_call(*args, **kwargs)

    def handle_error_code_10(self, e, *args, **kwargs):
        self.logger.warning("Internal server error: Database problems, try later. Error registered while executing \
            method %s with params %s, recursion count: %d" % (self.method, kwargs, self.recursion_count))
        return self.sleep_repeat_call(*args, **kwargs)

    def handle_error_code_17(self, e, *args, **kwargs):
        # Validation required: please open redirect_uri in browser
        # TODO: cover with tests
        # TODO: remove dependancy from oauth_tokens
        from oauth_tokens.models import AccessToken
        self.logger.warning("Request error: %s. Error registered while executing \
            method %s with params %s, recursion count: %d" % (e, self.method, kwargs, self.recursion_count))

        user = AccessToken.objects.get(access_token=self.api.token).user_credentials
        auth_request = AccessToken.objects.get_token_for_user('vkontakte', user).auth_request
        auth_request.form_action_domain = 'https://m.vk.com'

        response = auth_request.session.get(e.redirect_uri)
        try:
            method, action, data = auth_request.get_form_data_from_content(response.content)
        except:
            raise Exception("There is no any form in response: %s" % response.content)
        data = {'code': auth_request.additional}
        response = getattr(auth_request.session, method)(url=action, headers=auth_request.headers, data=data)

        if 'success' not in response.url:
            raise Exception("Wrong response. Can not handle VK error 17. response: %s" % response.content)

        return self.sleep_repeat_call(*args, **kwargs)

    def handle_error_code_500(self, e, *args, **kwargs):
        # strange HTTP error appears sometimes
        return self.sleep_repeat_call(*args, **kwargs)

    def handle_error_code_501(self, e, *args, **kwargs):
        # strange HTTP error appears sometimes
        return self.sleep_repeat_call(*args, **kwargs)

    def handle_error_code_502(self, e, *args, **kwargs):
        # strange HTTP error appears sometimes
        return self.sleep_repeat_call(*args, **kwargs)

    def handle_error_code_504(self, e, *args, **kwargs):
        # strange HTTP error appears sometimes
        return self.sleep_repeat_call(*args, **kwargs)

def api_call(*args, **kwargs):
    api = VkontakteApi()
    return api.call(*args, **kwargs)
<commit_msg>updated handling errors<commit_after># -*- coding: utf-8 -*-
from django.conf import settings
from social_api.api import ApiAbstractBase, Singleton
from vkontakte import VKError as VkontakteError, API

__all__ = ['api_call', 'VkontakteError']

class VkontakteApi(ApiAbstractBase):
    __metaclass__ = Singleton

    provider = 'vkontakte'
    provider_social_auth = 'vk-oauth2'
    error_class = VkontakteError
    request_timeout = getattr(settings, 'VKONTAKTE_API_REQUEST_TIMEOUT', 1)

    def get_consistent_token(self):
        return getattr(settings, 'VKONTAKTE_API_ACCESS_TOKEN', None)

    def get_api(self, token):
        return API(token=token)

    def get_api_response(self, *args, **kwargs):
        return self.api.get(self.method, timeout=self.request_timeout, *args, **kwargs)

    def handle_error_code_5(self, e, *args, **kwargs):
        # code = 5, description = 'User authorization failed: invalid session.'
        self.used_access_tokens += [self.api.token]
        return self.repeat_call(*args, **kwargs)

    def handle_error_code_6(self, e, *args, **kwargs):
        # try access_token by another user
        self.logger.info("Vkontakte error 'Too many requests per second' on method: %s, recursion count: %d" % (
            self.method, self.recursion_count))
        self.used_access_tokens += [self.api.token]
        return self.repeat_call(*args, **kwargs)

    def handle_error_code_9(self, e, *args, **kwargs):
        self.logger.warning("Vkontakte flood control registered while executing method %s with params %s, \
            recursion count: %d" % (self.method, kwargs, self.recursion_count))
        self.used_access_tokens += [self.api.token]
        return self.sleep_repeat_call(*args, **kwargs)

    def handle_error_code_10(self, e, *args, **kwargs):
        self.logger.warning("Internal server error: Database problems, try later. Error registered while executing \
            method %s with params %s, recursion count: %d" % (self.method, kwargs, self.recursion_count))
        return self.sleep_repeat_call(*args, **kwargs)

    def handle_error_code_17(self, e, *args, **kwargs):
        # code = 17, description = 'Validation required: please open redirect_uri in browser'
        self.used_access_tokens += [self.api.token]
        return self.repeat_call(*args, **kwargs)
        # # TODO: cover with tests
        # # TODO: remove dependancy from oauth_tokens
        # from oauth_tokens.models import AccessToken
        # self.logger.warning("Request error: %s. Error registered while executing \
        #     method %s with params %s, recursion count: %d" % (e, self.method, kwargs, self.recursion_count))
        #
        # user = AccessToken.objects.get(access_token=self.api.token).user_credentials
        # auth_request = AccessToken.objects.get_token_for_user('vkontakte', user).auth_request
        # auth_request.form_action_domain = 'https://m.vk.com'
        #
        # response = auth_request.session.get(e.redirect_uri)
        # try:
        #     method, action, data = auth_request.get_form_data_from_content(response.content)
        # except:
        #     raise Exception("There is no any form in response: %s" % response.content)
        # data = {'code': auth_request.additional}
        # response = getattr(auth_request.session, method)(url=action, headers=auth_request.headers, data=data)
        #
        # if 'success' not in response.url:
        #     raise Exception("Wrong response. Can not handle VK error 17. response: %s" % response.content)
        #
        # return self.sleep_repeat_call(*args, **kwargs)

    def handle_error_code_500(self, e, *args, **kwargs):
        # strange HTTP error appears sometimes
        return self.sleep_repeat_call(*args, **kwargs)

    def handle_error_code_501(self, e, *args, **kwargs):
        # strange HTTP error appears sometimes
        return self.sleep_repeat_call(*args, **kwargs)

    def handle_error_code_502(self, e, *args, **kwargs):
        # strange HTTP error appears sometimes
        return self.sleep_repeat_call(*args, **kwargs)

    def handle_error_code_504(self, e, *args, **kwargs):
        # strange HTTP error appears sometimes
        return self.sleep_repeat_call(*args, **kwargs)

def api_call(*args, **kwargs):
    api = VkontakteApi()
    return api.call(*args, **kwargs)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    pylatex.arguments
    ~~~~~~~

    This module implements the class that deals with tables.

    :copyright: (c) 2014 by Jelte Fennema.
    :license: MIT, see License for more details.
"""
from .base_classes import BaseLaTeXClass

class Arguments(BaseLaTeXClass):
    """
    A class implementing LaTex arguments. It supports normal positional arguments, as well as key-value pairs. Arguments
    can be rendered optional within square brackets ``[]`` or required within braces ``{}``.
    ::
        >>> args = Arguments('a', 'b', 'c')
        >>> args.dumps()
        '{a,b,c}'
        >>> args.optional = True
        >>> args.dumps()
        '[a,b,c]'
        >>> args = Arguments('clip', width=50, height='25em', trim='1 2 3 4')
        >>> args.optional = True
        >>> args.dumps()
        '[clip,trim=1 2 3 4,width=50,height=25em]'

    :param optional: Specifies whether this arguments are optional or not
    :type optional: bool
    """

    optional = False

    def __init__(self, *args, **kwargs):
        self._positional_args = list(args)
        self._key_value_args = dict(kwargs)
        super().__init__(packages=None)

    def dumps(self):
        """
        Represents the arguments as a string in LaTeX syntax to be appended to a command.

        :return: The rendered arguments
        :rtype: str
        """
        args = []
        args.extend(self._positional_args)
        args.extend(['{k}={v}'.format(k=k, v=v) for k, v in self._key_value_args.items()])
        if len(args) <= 0:
            return ''
        if self.optional:
            string = '[{args}]'.format(args=','.join(args))
        else:
            string = '{{{args}}}'.format(args=','.join(args))
        return string
<commit_msg>Fixed incorrect format for required arguments.<commit_after># -*- coding: utf-8 -*-
"""
    pylatex.arguments
    ~~~~~~~

    This module implements the class that deals with arguments.

    :copyright: (c) 2014 by Jelte Fennema.
    :license: MIT, see License for more details.
"""
from .base_classes import BaseLaTeXClass

class Arguments(BaseLaTeXClass):
    """
    A class implementing LaTex arguments. It supports normal positional arguments, as well as key-value pairs. Arguments
    can be rendered optional within square brackets ``[]`` or required within braces ``{}``.
    ::
        >>> args = Arguments('a', 'b', 'c')
        >>> args.dumps()
        '{a}{b}{c}'
        >>> args.optional = True
        >>> args.dumps()
        '[a,b,c]'
        >>> args = Arguments('clip', width=50, height='25em', trim='1 2 3 4')
        >>> args.optional = True
        >>> args.dumps()
        '[clip,trim=1 2 3 4,width=50,height=25em]'

    :param optional: Specifies whether this arguments are optional or not
    :type optional: bool
    """

    optional = False

    def __init__(self, *args, **kwargs):
        self._positional_args = list(args)
        self._key_value_args = dict(kwargs)
        super().__init__(packages=None)

    def dumps(self):
        """
        Represents the arguments as a string in LaTeX syntax to be appended to a command.

        :return: The rendered arguments
        :rtype: str
        """
        args = []
        args.extend(self._positional_args)
        args.extend(['{k}={v}'.format(k=k, v=v) for k, v in self._key_value_args.items()])
        if len(args) <= 0:
            return ''
        if self.optional:
            string = '[{args}]'.format(args=','.join(args))
        else:
            string = '{{{args}}}'.format(args='}{'.join(args))
        return string
<|endoftext|>