<commit_before>#!/bin/python

# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# This is meant to be run from the root directory of the repo. It will
# look for the .regression.yml file and expect a regression_tests directory
# with results from test runs previously produced by run_tests.py

import os
import sys
import glob
import json

# TODO: different parsers could be run on output from different actions
# if we would use the xml output in stead of stdout/stderr
def style_output(o):
	ret = ''
	subtle = False
	for l in o.split('\n'):
		l = l.replace('<', '<')
		l = l.replace('>', '>')
		if 'TEST_CHECK' in l or 'TEST_EQUAL_ERROR' in l or l.startswith('EXIT STATUS: ') or \
			l.endswith(' second time limit exceeded') or l.startswith('signal: SIG'):
			ret += '%s\n' % l
		elif '**passed**' in l:
			ret += '%s\n' % l
		elif ': error: ' in l or ': fatal error: ' in l or ' : fatal error ' in l or \
			'failed to write output file' in l or ') : error C' in l or \
			' : error LNK' in l or ': undefined reference to ' in l:
			ret += '%s\n' % l
		elif ': warning: ' in l or ') : warning C' in l:
			ret += '%s\n' % l
		elif l == '====== END OUTPUT ======' and not subtle:
			ret += '%s\n' % l
			subtle = True
		else:
			ret += '%s\n' % l
	if subtle: ret += ''
	return ret

def modification_time(file):
	mtime = 0
	try:
		mtime = os.stat(file).st_mtime
	except: pass
	return mtime

def save_log_file(log_name, project_name, branch_name, test_name, timestamp, data):

	if not os.path.exists(os.path.split(log_name)[0]):
		os.mkdir(os.path.split(log_name)[0])

	try:
		# if the log file already exists, and it's newer than
		# the source, no need to re-parse it
		mtime = os.stat(log_name).st_mtime
		if mtime >= timestamp: return
	except: pass

	html = open(log_name, 'w+')
	print >>html, '''<title>%s - %s</title>

%s - %s
''' % (project_name, branch_name, project_name, branch_name)
	print >>html, '
%s

%s
' % \
		(test_name.encode('utf-8'), style_output(data).encode('utf-8'))

	print >>html, ''
	html.close()
	sys.stdout.write('.')
	sys.stdout.flush()

def parse_tests(rev_dir):

	# this contains mappings from platforms to
	# the next layer of dictionaries. The next
	# layer contains a mapping of toolsets to
	# dictionaries the next layer of dictionaries.
	# those dictionaries contain a mapping from
	# feature-sets to the next layer of dictionaries.
	# the next layer contains a mapping from
	# tests to information about those tests, such
	# as whether it passed and the output from the
	# command
	# example:

	# {
	#   darwin: {
	#     clang-4.2.1: {
	#       ipv6=off: {
	#         test_primitives: {
	#           output: ...
	#           status: 1
	#         }
	#       }
	#     }
	#   }
	# }

	platforms = {}

	tests = {}

	for f in glob.glob(os.path.join(rev_dir, '*.json')):
		platform_toolset = os.path.split(f)[1].split('.json')[0].split('#')
		try:
			j = json.loads(open(f, 'rb').read())
			timestamp = os.stat(f).st_mtime
		except:
			print '\nFAILED TO LOAD "%s"\n' %f
			continue

		platform = platform_toolset[0]
		toolset = platform_toolset[1]

		for cfg in j:
			test_name = cfg.split('|')[0]
			features = cfg.split('|')[1]

			if not features in tests:
				tests[features] = set()

			tests[features].add(test_name)

			if not platform in platforms:
				platforms[platform] = {}

			if not toolset in platforms[platform]:
				platforms[platform][toolset] = {}

			if not features in platforms[platform][toolset]:
				platforms[platform][toolset][features] = {}

			platforms[platform][toolset][features][test_name] = j[cfg]
			platforms[platform][toolset][features][test_name]['timestamp'] = timestamp

	return (platforms, tests)

# TODO: remove this dependency by encoding it in the output files
# this script should work from outside of the repo, just having
# access to the shared folder
project_name = 'libtorrent'

# maps branch name to latest rev
revs = {}

os.chdir('regression_tests')

for rev in os.listdir('.'):
	try:
		branch = rev.split('-')[0]
		if branch == 'logs': continue
		r = int(rev.split('-')[1])
		if not branch in revs:
			revs[branch] = r
		else:
			if r > revs[branch]:
				revs[branch] = r
	except:
		print 'ignoring %s' % rev

if revs == {}:
	print 'no test files found'
	sys.exit(1)

print 'latest versions'
for b in revs:
	print '%s\t%d' % (b, revs[b])

for branch_name in revs:

	latest_rev = revs[branch_name]

	html_file = '%s.html' % branch_name

	html = open(html_file, 'w+')

	print >>html, '''<title>regression tests, %s</title>

%s - %s
''' % (project_name, project_name, branch_name)

	print >>html, '
'

	for r in range(latest_rev, latest_rev - 20, -1):
		sys.stdout.write('.')
		sys.stdout.flush()

		print >>html, '
revision %d	' % r

		rev_dir = '%s-%d' % (branch_name, r)
		(platforms, tests) = parse_tests(rev_dir)

		for f in tests:
			print >>html, '%s	' % (len(tests[f]), len(tests[f])*9 - 5, f)
		print >>html, '
'

		for p in platforms:
			print >>html, '
%s	' % (len(platforms[p]), p)
			idx = 0
			for toolset in platforms[p]:
				if idx > 0: print >>html, '
'
				print >>html, '%s	' % toolset
				for f in platforms[p][toolset]:
					for t in platforms[p][toolset][f]:
						details = platforms[p][toolset][f][t]
						if details['status'] == 0: c = 'passed'
						else: c = 'failed'
						log_name = os.path.join('logs-%s-%d' % (branch_name, r), p + '~' + toolset + '~' + t + '~' + f.replace(' ', '.') + '.html')
						print >>html, '	' % (t, f, c, log_name)
						save_log_file(log_name, project_name, branch_name, '%s - %s' % (t, f), int(details['timestamp']), details['output'])

				print >>html, '
'
				idx += 1

	print >>html, '
'
	html.close()

	print ''

<commit_msg>improve parse_test_results<commit_after>#!/bin/python

# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# This is meant to be run from the root directory of the repo. It will
# look for the .regression.yml file and expect a regression_tests directory
# with results from test runs previously produced by run_tests.py

import os
import sys
import glob
import json

# TODO: different parsers could be run on output from different actions
# if we would use the xml output in stead of stdout/stderr
def style_output(o):
	ret = ''
	subtle = False
	for l in o.split('\n'):
		l = l.replace('<', '<')
		l = l.replace('>', '>')
		if 'TEST_CHECK' in l or 'TEST_EQUAL_ERROR' in l or l.startswith('EXIT STATUS: ') or \
			' second time limit exceeded' in l or l.startswith('signal: SIG'):
			ret += '%s\n' % l
		elif '**passed**' in l:
			ret += '%s\n' % l
		elif ': error: ' in l or ': fatal error: ' in l or ' : fatal error ' in l or \
			'failed to write output file' in l or ') : error C' in l or \
			' : error LNK' in l or ': undefined reference to ' in l:
			ret += '%s\n' % l
		elif ': warning: ' in l or ') : warning C' in l:
			ret += '%s\n' % l
		elif l == '====== END OUTPUT ======' and not subtle:
			ret += '%s\n' % l
			subtle = True
		else:
			ret += '%s\n' % l
	if subtle: ret += ''
	return ret

def modification_time(file):
	mtime = 0
	try:
		mtime = os.stat(file).st_mtime
	except: pass
	return mtime

def save_log_file(log_name, project_name, branch_name, test_name, timestamp, data):

	if not os.path.exists(os.path.split(log_name)[0]):
		os.mkdir(os.path.split(log_name)[0])

	try:
		# if the log file already exists, and it's newer than
		# the source, no need to re-parse it
		mtime = os.stat(log_name).st_mtime
		if mtime >= timestamp: return
	except: pass

	html = open(log_name, 'w+')
	print >>html, '''<title>%s - %s</title>

%s - %s
''' % (project_name, branch_name, project_name, branch_name)
	print >>html, '
%s

%s
' % \
		(test_name.encode('utf-8'), style_output(data).encode('utf-8'))

	print >>html, ''
	html.close()
	sys.stdout.write('.')
	sys.stdout.flush()

def parse_tests(rev_dir):

	# this contains mappings from platforms to
	# the next layer of dictionaries. The next
	# layer contains a mapping of toolsets to
	# dictionaries the next layer of dictionaries.
	# those dictionaries contain a mapping from
	# feature-sets to the next layer of dictionaries.
	# the next layer contains a mapping from
	# tests to information about those tests, such
	# as whether it passed and the output from the
	# command
	# example:

	# {
	#   darwin: {
	#     clang-4.2.1: {
	#       ipv6=off: {
	#         test_primitives: {
	#           output: ...
	#           status: 1
	#         }
	#       }
	#     }
	#   }
	# }

	platforms = {}

	tests = {}

	for f in glob.glob(os.path.join(rev_dir, '*.json')):
		platform_toolset = os.path.split(f)[1].split('.json')[0].split('#')
		try:
			j = json.loads(open(f, 'rb').read())
			timestamp = os.stat(f).st_mtime
		except:
			print '\nFAILED TO LOAD "%s"\n' %f
			continue

		platform = platform_toolset[0]
		toolset = platform_toolset[1]

		for cfg in j:
			test_name = cfg.split('|')[0]
			features = cfg.split('|')[1]

			if not features in tests:
				tests[features] = set()

			tests[features].add(test_name)

			if not platform in platforms:
				platforms[platform] = {}

			if not toolset in platforms[platform]:
				platforms[platform][toolset] = {}

			if not features in platforms[platform][toolset]:
				platforms[platform][toolset][features] = {}

			platforms[platform][toolset][features][test_name] = j[cfg]
			platforms[platform][toolset][features][test_name]['timestamp'] = timestamp

	return (platforms, tests)

# TODO: remove this dependency by encoding it in the output files
# this script should work from outside of the repo, just having
# access to the shared folder
project_name = 'libtorrent'

# maps branch name to latest rev
revs = {}

os.chdir('regression_tests')

for rev in os.listdir('.'):
	try:
		branch = rev.split('-')[0]
		if branch == 'logs': continue
		r = int(rev.split('-')[1])
		if not branch in revs:
			revs[branch] = r
		else:
			if r > revs[branch]:
				revs[branch] = r
	except:
		print 'ignoring %s' % rev

if revs == {}:
	print 'no test files found'
	sys.exit(1)

print 'latest versions'
for b in revs:
	print '%s\t%d' % (b, revs[b])

for branch_name in revs:

	latest_rev = revs[branch_name]

	html_file = '%s.html' % branch_name

	html = open(html_file, 'w+')

	print >>html, '''<title>regression tests, %s</title>

%s - %s
''' % (project_name, project_name, branch_name)

	print >>html, '
'

	num_printed_revs = 0;
	for r in range(latest_rev, latest_rev - 40, -1):
		sys.stdout.write('.')
		sys.stdout.flush()

		rev_dir = '%s-%d' % (branch_name, r)
		(platforms, tests) = parse_tests(rev_dir)

		if len(tests) + len(platforms) == 0: continue

		print >>html, '
revision %d	' % r

		for f in tests:
			print >>html, '%s	' % (len(tests[f]), len(tests[f])*9 - 5, f)
		print >>html, '
'

		for p in platforms:
			print >>html, '
%s	' % (len(platforms[p]), p)
			idx = 0
			for toolset in platforms[p]:
				if idx > 0: print >>html, '
'
				print >>html, '%s	' % toolset
				for f in platforms[p][toolset]:
					for t in platforms[p][toolset][f]:
						details = platforms[p][toolset][f][t]
						if details['status'] == 0: c = 'passed'
						else: c = 'failed'
						log_name = os.path.join('logs-%s-%d' % (branch_name, r), p + '~' + toolset + '~' + t + '~' + f.replace(' ', '.') + '.html')
						print >>html, '	' % (t, f, c, log_name)
						save_log_file(log_name, project_name, branch_name, '%s - %s' % (t, f), int(details['timestamp']), details['output'])

				print >>html, '
'
				idx += 1
		num_printed_revs += 1
		if num_printed_revs >= 20: break

	print >>html, '
'
	html.close()

	print ''

<|endoftext|>


<commit_before>#!/bin/env python
from selenium import webdriver
from selenium.common.exceptions import StaleElementReferenceException
from time import sleep
from getpass import getpass

if __name__ == '__main__':
    driver = webdriver.phantomjs.webdriver.WebDriver()
    driver.get('https://facebook.com')
    driver.find_element_by_id('email').send_keys(input('Email: '))
    driver.find_element_by_id('pass').send_keys(getpass())
    driver.find_element_by_id('loginbutton').click()
    driver.get('https://facebook.com/pokes/')
    assert "Forgot password?" not in driver.page_source

    c = 0
    e = 0
    while True:
        try:
            for i in driver.find_elements_by_link_text("Poke Back"):
                i.click()
                c += 1
                print("Clicked so far: " + str(c))
        except StaleElementReferenceException:
            e += 1
            if e == 10:
                print("Found exception, reloading page")
                driver.get('https://facebook.com/pokes/')
                e = 0
            else:
                print("Found exception, doing nothing")

        sleep(1)
<commit_msg>Add random delay<commit_after>#!/bin/env python
from selenium import webdriver
from selenium.common.exceptions import StaleElementReferenceException
from time import sleep
from getpass import getpass
from random import randint

if __name__ == '__main__':
    driver = webdriver.phantomjs.webdriver.WebDriver()
    driver.get('https://facebook.com')
    driver.find_element_by_id('email').send_keys(input('Email: '))
    driver.find_element_by_id('pass').send_keys(getpass())
    driver.find_element_by_id('loginbutton').click()
    driver.get('https://facebook.com/pokes/')
    assert "Forgot password?" not in driver.page_source

    c = 0
    e = 0
    while True:
        try:
            for i in driver.find_elements_by_link_text("Poke Back"):
                sleep(randint(3,300))
                i.click()
                c += 1
                print("Clicked so far: " + str(c))
        except StaleElementReferenceException:
            e += 1
            if e == 10:
                print("Found exception, reloading page")
                driver.get('https://facebook.com/pokes/')
                e = 0
            else:
                print("Found exception, doing nothing")

        sleep(1)
<|endoftext|>


<commit_before>#  Copyright 2017 Intel Corporation
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
"""I/O functionality."""

__all__ = [
    "load_boolean_mask",
    "load_images",
    "load_images_from_dir",
    "load_labels",
    "save_as_nifti_file",
]

from pathlib import Path
from typing import Callable, Iterable, List, Union

import logging
import nibabel as nib
import numpy as np

from nibabel.nifti1 import Nifti1Pair
from nibabel.spatialimages import SpatialImage

from .image import SingleConditionSpec

logger = logging.getLogger(__name__)

def load_images_from_dir(in_dir: Union[str, Path], suffix: str = "nii.gz",
                         ) -> Iterable[SpatialImage]:
    """Load images from directory.

    For efficiency, returns an iterator, not a sequence, so the results cannot
    be accessed by indexing.

    For every new iteration through the images, load_images_from_dir must be
    called again.

    Parameters
    ----------
    in_dir:
        Path to directory.
    suffix:
        Only load images with names that end like this.

    Yields
    ------
    SpatialImage
        Image.
    """
    if isinstance(in_dir, str):
        in_dir = Path(in_dir)
    files = sorted(in_dir.glob("*" + suffix))
    for f in files:
        logger.debug(
            'Starting to read file %s', f
        )
        yield nib.load(str(f))

def load_images(image_paths: Iterable[Union[str, Path]]
                ) -> Iterable[SpatialImage]:
    """Load images from paths.

    For efficiency, returns an iterator, not a sequence, so the results cannot
    be accessed by indexing.

    For every new iteration through the images, load_images must be called
    again.

    Parameters
    ----------
    image_paths:
        Paths to images.

    Yields
    ------
    SpatialImage
        Image.
    """
    for image_path in image_paths:
        if isinstance(image_path, Path):
            string_path = str(image_path)
        else:
            string_path = image_path
        logger.debug(
            'Starting to read file %s', string_path
        )
        yield nib.load(string_path)

def load_boolean_mask(path: Union[str, Path],
                      predicate: Callable[[np.ndarray], np.ndarray] = None
                      ) -> np.ndarray:
    """Load boolean nibabel.SpatialImage mask.

    Parameters
    ----------
    path
        Mask path.
    predicate
        Callable used to create boolean values, e.g. a threshold function
        ``lambda x: x > 50``.

    Returns
    -------
    np.ndarray
        Boolean array corresponding to mask.
    """
    if not isinstance(path, str):
        path = str(path)
    data = nib.load(path).get_data()
    if predicate is not None:
        mask = predicate(data)
    else:
        mask = data.astype(np.bool)
    return mask

def load_labels(path: Union[str, Path]) -> List[SingleConditionSpec]:
    """Load labels files.

    Parameters
    ----------
    path
        Path of labels file.

    Returns
    -------
    List[SingleConditionSpec]
        List of SingleConditionSpec stored in labels file.
    """
    condition_specs = np.load(str(path))
    return [c.view(SingleConditionSpec) for c in condition_specs]

def save_as_nifti_file(data: np.ndarray, affine: np.ndarray,
                       path: Union[str, Path]) -> None:
    """Create a Nifti file and save it.

    Parameters
    ----------
    data
        Brain data.
    affine
        Affine of the image, usually inherited from an existing image.
    path
        Output filename.
    """
    if not isinstance(path, str):
        path = str(path)
    img = Nifti1Pair(data, affine)
    nib.nifti1.save(img, path)
<commit_msg>docs: Add example in docstring of load_images_from_dir (#432)<commit_after>#  Copyright 2017 Intel Corporation
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
"""I/O functionality."""

__all__ = [
    "load_boolean_mask",
    "load_images",
    "load_images_from_dir",
    "load_labels",
    "save_as_nifti_file",
]

from pathlib import Path
from typing import Callable, Iterable, List, Union

import logging
import nibabel as nib
import numpy as np

from nibabel.nifti1 import Nifti1Pair
from nibabel.spatialimages import SpatialImage

from .image import SingleConditionSpec

logger = logging.getLogger(__name__)

def load_images_from_dir(in_dir: Union[str, Path], suffix: str = "nii.gz",
                         ) -> Iterable[SpatialImage]:
    """Load images from directory.

    For efficiency, returns an iterator, not a sequence, so the results cannot
    be accessed by indexing.

    For every new iteration through the images, load_images_from_dir must be
    called again.

    Parameters
    ----------
    in_dir:
        Path to directory.
    suffix:
        Only load images with names that end like this.

    Yields
    ------
    SpatialImage
        Image.

    Examples
    --------
    >>> images = list(io.load_images_from_dir("face_scene", "bet.nii.gz"))
    """
    if isinstance(in_dir, str):
        in_dir = Path(in_dir)
    files = sorted(in_dir.glob("*" + suffix))
    for f in files:
        logger.debug(
            'Starting to read file %s', f
        )
        yield nib.load(str(f))

def load_images(image_paths: Iterable[Union[str, Path]]
                ) -> Iterable[SpatialImage]:
    """Load images from paths.

    For efficiency, returns an iterator, not a sequence, so the results cannot
    be accessed by indexing.

    For every new iteration through the images, load_images must be called
    again.

    Parameters
    ----------
    image_paths:
        Paths to images.

    Yields
    ------
    SpatialImage
        Image.
    """
    for image_path in image_paths:
        if isinstance(image_path, Path):
            string_path = str(image_path)
        else:
            string_path = image_path
        logger.debug(
            'Starting to read file %s', string_path
        )
        yield nib.load(string_path)

def load_boolean_mask(path: Union[str, Path],
                      predicate: Callable[[np.ndarray], np.ndarray] = None
                      ) -> np.ndarray:
    """Load boolean nibabel.SpatialImage mask.

    Parameters
    ----------
    path
        Mask path.
    predicate
        Callable used to create boolean values, e.g. a threshold function
        ``lambda x: x > 50``.

    Returns
    -------
    np.ndarray
        Boolean array corresponding to mask.
    """
    if not isinstance(path, str):
        path = str(path)
    data = nib.load(path).get_data()
    if predicate is not None:
        mask = predicate(data)
    else:
        mask = data.astype(np.bool)
    return mask

def load_labels(path: Union[str, Path]) -> List[SingleConditionSpec]:
    """Load labels files.

    Parameters
    ----------
    path
        Path of labels file.

    Returns
    -------
    List[SingleConditionSpec]
        List of SingleConditionSpec stored in labels file.
    """
    condition_specs = np.load(str(path))
    return [c.view(SingleConditionSpec) for c in condition_specs]

def save_as_nifti_file(data: np.ndarray, affine: np.ndarray,
                       path: Union[str, Path]) -> None:
    """Create a Nifti file and save it.

    Parameters
    ----------
    data
        Brain data.
    affine
        Affine of the image, usually inherited from an existing image.
    path
        Output filename.
    """
    if not isinstance(path, str):
        path = str(path)
    img = Nifti1Pair(data, affine)
    nib.nifti1.save(img, path)
<|endoftext|>


<commit_before># Copyright (c) 2012-2013, Mark Peek <mark@peek.org>
# All rights reserved.
#
# See LICENSE file for full license.

from aws import Action

service_name = 'Amazon S3'
prefix = 's3'

class S3_ARN(ARN):
    def __init__(self, resource):
        sup = super(S3_ARN, self)
        sup.__init__('s3', '', '', resource)

AbortMultipartUpload = Action(prefix, 'AbortMultipartUpload')
CreateBucket = Action(prefix, 'CreateBucket')
DeleteBucket = Action(prefix, 'DeleteBucket')
DeleteBucketPolicy = Action(prefix, 'DeleteBucketPolicy')
DeleteBucketWebsite = Action(prefix, 'DeleteBucketWebsite')
DeleteObject = Action(prefix, 'DeleteObject')
DeleteObjectVersion = Action(prefix, 'DeleteObjectVersion')
GetBucketAcl = Action(prefix, 'GetBucketAcl')
GetBucketLocation = Action(prefix, 'GetBucketLocation')
GetBucketLogging = Action(prefix, 'GetBucketLogging')
GetBucketNotification = Action(prefix, 'GetBucketNotification')
GetBucketPolicy = Action(prefix, 'GetBucketPolicy')
GetBucketRequestPayment = Action(prefix, 'GetBucketRequestPayment')
GetBucketTagging = Action(prefix, 'GetBucketTagging')
GetBucketVersioning = Action(prefix, 'GetBucketVersioning')
GetBucketWebsite = Action(prefix, 'GetBucketWebsite')
GetCORSConfiguration = Action(prefix, 'GetCORSConfiguration')
GetLifecycleConfiguration = Action(prefix, 'GetLifecycleConfiguration')
GetObject = Action(prefix, 'GetObject')
GetObjectAcl = Action(prefix, 'GetObjectAcl')
GetObjectTorrent = Action(prefix, 'GetObjectTorrent')
GetObjectVersion = Action(prefix, 'GetObjectVersion')
GetObjectVersionAcl = Action(prefix, 'GetObjectVersionAcl')
GetObjectVersionTorrent = Action(prefix, 'GetObjectVersionTorrent')
ListAllMyBuckets = Action(prefix, 'ListAllMyBuckets')
ListBucket = Action(prefix, 'ListBucket')
ListBucketMultipartUploads = \
    Action(prefix, 'ListBucketMultipartUploads')
ListBucketVersions = Action(prefix, 'ListBucketVersions')
ListMultipartUploadParts = Action(prefix, 'ListMultipartUploadParts')
PutBucketAcl = Action(prefix, 'PutBucketAcl')
PutBucketLogging = Action(prefix, 'PutBucketLogging')
PutBucketNotification = Action(prefix, 'PutBucketNotification')
PutBucketPolicy = Action(prefix, 'PutBucketPolicy')
PutBucketRequestPayment = Action(prefix, 'PutBucketRequestPayment')
PutBucketTagging = Action(prefix, 'PutBucketTagging')
PutBucketVersioning = Action(prefix, 'PutBucketVersioning')
PutBucketWebsite = Action(prefix, 'PutBucketWebsite')
PutCORSConfiguration = Action(prefix, 'PutCORSConfiguration')
PutLifecycleConfiguration = Action(prefix, 'PutLifecycleConfiguration')
PutObject = Action(prefix, 'PutObject')
PutObjectAcl = Action(prefix, 'PutObjectAcl')
PutObjectVersionAcl = Action(prefix, 'PutObjectVersionAcl')
<commit_msg>Add an import for ARN<commit_after># Copyright (c) 2012-2013, Mark Peek <mark@peek.org>
# All rights reserved.
#
# See LICENSE file for full license.

from aws import Action, ARN

service_name = 'Amazon S3'
prefix = 's3'

class S3_ARN(ARN):
    def __init__(self, resource):
        sup = super(S3_ARN, self)
        sup.__init__('s3', '', '', resource)

AbortMultipartUpload = Action(prefix, 'AbortMultipartUpload')
CreateBucket = Action(prefix, 'CreateBucket')
DeleteBucket = Action(prefix, 'DeleteBucket')
DeleteBucketPolicy = Action(prefix, 'DeleteBucketPolicy')
DeleteBucketWebsite = Action(prefix, 'DeleteBucketWebsite')
DeleteObject = Action(prefix, 'DeleteObject')
DeleteObjectVersion = Action(prefix, 'DeleteObjectVersion')
GetBucketAcl = Action(prefix, 'GetBucketAcl')
GetBucketLocation = Action(prefix, 'GetBucketLocation')
GetBucketLogging = Action(prefix, 'GetBucketLogging')
GetBucketNotification = Action(prefix, 'GetBucketNotification')
GetBucketPolicy = Action(prefix, 'GetBucketPolicy')
GetBucketRequestPayment = Action(prefix, 'GetBucketRequestPayment')
GetBucketTagging = Action(prefix, 'GetBucketTagging')
GetBucketVersioning = Action(prefix, 'GetBucketVersioning')
GetBucketWebsite = Action(prefix, 'GetBucketWebsite')
GetCORSConfiguration = Action(prefix, 'GetCORSConfiguration')
GetLifecycleConfiguration = Action(prefix, 'GetLifecycleConfiguration')
GetObject = Action(prefix, 'GetObject')
GetObjectAcl = Action(prefix, 'GetObjectAcl')
GetObjectTorrent = Action(prefix, 'GetObjectTorrent')
GetObjectVersion = Action(prefix, 'GetObjectVersion')
GetObjectVersionAcl = Action(prefix, 'GetObjectVersionAcl')
GetObjectVersionTorrent = Action(prefix, 'GetObjectVersionTorrent')
ListAllMyBuckets = Action(prefix, 'ListAllMyBuckets')
ListBucket = Action(prefix, 'ListBucket')
ListBucketMultipartUploads = \
    Action(prefix, 'ListBucketMultipartUploads')
ListBucketVersions = Action(prefix, 'ListBucketVersions')
ListMultipartUploadParts = Action(prefix, 'ListMultipartUploadParts')
PutBucketAcl = Action(prefix, 'PutBucketAcl')
PutBucketLogging = Action(prefix, 'PutBucketLogging')
PutBucketNotification = Action(prefix, 'PutBucketNotification')
PutBucketPolicy = Action(prefix, 'PutBucketPolicy')
PutBucketRequestPayment = Action(prefix, 'PutBucketRequestPayment')
PutBucketTagging = Action(prefix, 'PutBucketTagging')
PutBucketVersioning = Action(prefix, 'PutBucketVersioning')
PutBucketWebsite = Action(prefix, 'PutBucketWebsite')
PutCORSConfiguration = Action(prefix, 'PutCORSConfiguration')
PutLifecycleConfiguration = Action(prefix, 'PutLifecycleConfiguration')
PutObject = Action(prefix, 'PutObject')
PutObjectAcl = Action(prefix, 'PutObjectAcl')
PutObjectVersionAcl = Action(prefix, 'PutObjectVersionAcl')
<|endoftext|>


<commit_before>if "bpy" in locals():
    import imp
    imp.reload(socket_api)
    imp.reload(_converters)
    imp.reload(processors)
    imp.reload(converter_thread)
    imp.reload(processor_thread)
else:
    import bpy
    from . import socket_api
    from . import converters as _converters
    from . import processors
    from . import converter_thread
    from . import processor_thread

import os
import socket
import struct
import subprocess
import sys
import time
import threading
import collections
import queue

import bpy
import mathutils

from OpenGL.GL import *

DEFAULT_WATCHLIST = [
    #"actions",
    #"armatures",
    "cameras",
    "images",
    "lamps",
    "materials",
    "meshes",
    "objects",
    "scenes",
    #"sounds",
    #"speakers",
    "textures",
    #"worlds",
]

ViewportTuple = collections.namedtuple('Viewport', ('height', 'width'))

# Currently need some things stored globally so they can be cleaned up after
# losing the RealTimeEngine reference
class G:
    done_event = None

def get_collection_name(collection):
    class_name = collection.rna_type.__class__.__name__
    clean_name = class_name.replace("BlendData", "").lower()
    return clean_name

class RealTimeEngine():
    bl_idname = 'RTE_FRAMEWORK'
    bl_label = "Real Time Engine Framework"

    def __del__(self):
        if G.done_event:
            G.done_event.set()

    def __init__(self, **kwargs):
        # Display image
        self.width = 1
        self.height = 1
        self.clock = time.perf_counter()

        G.done_event = threading.Event()

        self.queue_pre_convert = queue.Queue()
        self.queue_post_convert = queue.Queue()
        self.queue_update = queue.Queue()
        self.queue_image = queue.Queue()

        converter = kwargs.get('converter', _converters.BTFConverter())
        self.thread_converter = converter_thread.ConverterThread(converter,
            self.queue_pre_convert, self.queue_post_convert, G.done_event)
        self.thread_converter.start()

        processor = kwargs.get('processor', processors.DummyProcessor())
        self.thread_processor = processor_thread.ProcessorThread(processor,
            self.queue_post_convert, self.queue_update, self.queue_image,
            G.done_event)
        self.thread_processor.start()

        self.remove_delta = {}
        self.add_delta = {}
        self.update_delta = {}
        self.view_delta = {}

        watch_list = kwargs['watch_list'] if 'watch_list' in kwargs else DEFAULT_WATCHLIST
        self._watch_list = [getattr(bpy.data, i) for i in watch_list]

        self._tracking_sets = {}
        for collection in self._watch_list:
            collection_name = get_collection_name(collection)
            self._tracking_sets[collection_name] = set()

        self._old_vmat = None
        self._old_pmat = None
        self._old_viewport = None

        def main_loop(scene):
            try:
                new_time = time.perf_counter()
                dt = new_time - self.clock
                self.clock = new_time
                self.main_update(dt)
            except ReferenceError:
                bpy.app.handlers.scene_update_post.remove(main_loop)
                G.done_event.set()

        bpy.app.handlers.scene_update_post.append(main_loop)

        self.tex = glGenTextures(1)

    def view_update(self, context):
        """ Called when the scene is changed """

        for collection in self._watch_list:
            collection_name = get_collection_name(collection)
            collection_set = set(collection)
            tracking_set = self._tracking_sets[collection_name]

            # Check for new items
            add_set = collection_set - tracking_set
            self.add_delta[collection_name] = add_set
            tracking_set |= add_set

            # Check for removed items
            remove_set = tracking_set - collection_set
            self.remove_delta[collection_name] = remove_set
            tracking_set -= remove_set

            # Check for updates
            update_set = {item for item in collection if item.is_updated}
            self.update_delta[collection_name] = update_set

    def view_draw(self, context):
        """ Called when viewport settings change """
        region = context.region
        view = context.region_data

        vmat = view.view_matrix.copy()
        vmat_inv = vmat.inverted()
        pmat = view.perspective_matrix * vmat_inv

        viewport = [region.x, region.y, region.width, region.height]

        self.update_view(vmat, pmat, viewport)

        glPushAttrib(GL_ALL_ATTRIB_BITS)
        glActiveTexture(GL_TEXTURE0)
        glBindTexture(GL_TEXTURE_2D, self.tex)
        try:
            image_ref = self.queue_image.get_nowait()
            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, self.width, self.height, 0, GL_RGB,
                GL_UNSIGNED_BYTE, image_ref)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
            self.queue_image.task_done()
        except queue.Empty:
            pass

        glDisable(GL_DEPTH_TEST)
        glDisable(GL_CULL_FACE)
        glDisable(GL_STENCIL_TEST)
        glEnable(GL_TEXTURE_2D)

        glClearColor(0, 0, 1, 1)
        glClear(GL_COLOR_BUFFER_BIT)

        glMatrixMode(GL_MODELVIEW)
        glPushMatrix()
        glLoadIdentity()
        glMatrixMode(GL_PROJECTION)
        glPushMatrix()
        glLoadIdentity()

        glBegin(GL_QUADS)
        glColor3f(1.0, 1.0, 1.0)
        glTexCoord2f(0.0, 0.0)
        glVertex3i(-1, -1, 0)
        glTexCoord2f(1.0, 0.0)
        glVertex3i(1, -1, 0)
        glTexCoord2f(1.0, 1.0)
        glVertex3i(1, 1, 0)
        glTexCoord2f(0.0, 1.0)
        glVertex3i(-1, 1, 0)
        glEnd()

        glPopMatrix()
        glMatrixMode(GL_MODELVIEW)
        glPopMatrix()

        glPopAttrib()

    def update_view(self, view_matrix, projection_matrix, viewport):
        if view_matrix != self._old_vmat:
            self._old_vmat = view_matrix
            self.view_delta['view_matrix'] = view_matrix

        if projection_matrix != self._old_pmat:
            self._old_pmat = projection_matrix
            self.view_delta['projection_matrix'] = projection_matrix

        if viewport != self._old_viewport:
            self._old_viewport = viewport
            self.view_delta['viewport'] = ViewportTuple(width=viewport[2], height=viewport[3])

    def draw_callback(self):
        '''Forces a view_draw to occur'''
        self.tag_redraw()

    def main_update(self, dt):
        if self.add_delta or self.update_delta or self.view_delta:
            self.queue_pre_convert.put((
                self.add_delta.copy(),
                self.update_delta.copy(),
                self.remove_delta.copy(),
                self.view_delta.copy(),
            ))

        self.add_delta.clear()
        self.update_delta.clear()
        self.remove_delta.clear()
        self.view_delta.clear()

        self.queue_update.put(dt)

        self.draw_callback()

        # Useful debug information for checking if a queue is filling up
        # print('Approximate queue sizes:')
        # print('\tPre Convert:', self.queue_pre_convert.qsize())
        # print('\tPost Convert:', self.queue_post_convert.qsize())
        # print('\tUpdate:', self.queue_update.qsize())
        # print('\tImage:', self.queue_image.qsize())
<commit_msg>Start with a black texture for the viewport quad<commit_after>if "bpy" in locals():
    import imp
    imp.reload(socket_api)
    imp.reload(_converters)
    imp.reload(processors)
    imp.reload(converter_thread)
    imp.reload(processor_thread)
else:
    import bpy
    from . import socket_api
    from . import converters as _converters
    from . import processors
    from . import converter_thread
    from . import processor_thread

import os
import socket
import struct
import subprocess
import sys
import time
import threading
import collections
import queue

import bpy
import mathutils

from OpenGL.GL import *

DEFAULT_WATCHLIST = [
    #"actions",
    #"armatures",
    "cameras",
    "images",
    "lamps",
    "materials",
    "meshes",
    "objects",
    "scenes",
    #"sounds",
    #"speakers",
    "textures",
    #"worlds",
]

ViewportTuple = collections.namedtuple('Viewport', ('height', 'width'))

# Currently need some things stored globally so they can be cleaned up after
# losing the RealTimeEngine reference
class G:
    done_event = None

def get_collection_name(collection):
    class_name = collection.rna_type.__class__.__name__
    clean_name = class_name.replace("BlendData", "").lower()
    return clean_name

class RealTimeEngine():
    bl_idname = 'RTE_FRAMEWORK'
    bl_label = "Real Time Engine Framework"

    def __del__(self):
        if G.done_event:
            G.done_event.set()

    def __init__(self, **kwargs):
        # Display image
        self.width = 1
        self.height = 1
        self.clock = time.perf_counter()

        G.done_event = threading.Event()

        self.queue_pre_convert = queue.Queue()
        self.queue_post_convert = queue.Queue()
        self.queue_update = queue.Queue()
        self.queue_image = queue.Queue()

        converter = kwargs.get('converter', _converters.BTFConverter())
        self.thread_converter = converter_thread.ConverterThread(converter,
            self.queue_pre_convert, self.queue_post_convert, G.done_event)
        self.thread_converter.start()

        processor = kwargs.get('processor', processors.DummyProcessor())
        self.thread_processor = processor_thread.ProcessorThread(processor,
            self.queue_post_convert, self.queue_update, self.queue_image,
            G.done_event)
        self.thread_processor.start()

        self.remove_delta = {}
        self.add_delta = {}
        self.update_delta = {}
        self.view_delta = {}

        watch_list = kwargs['watch_list'] if 'watch_list' in kwargs else DEFAULT_WATCHLIST
        self._watch_list = [getattr(bpy.data, i) for i in watch_list]

        self._tracking_sets = {}
        for collection in self._watch_list:
            collection_name = get_collection_name(collection)
            self._tracking_sets[collection_name] = set()

        self._old_vmat = None
        self._old_pmat = None
        self._old_viewport = None

        def main_loop(scene):
            try:
                new_time = time.perf_counter()
                dt = new_time - self.clock
                self.clock = new_time
                self.main_update(dt)
            except ReferenceError:
                bpy.app.handlers.scene_update_post.remove(main_loop)
                G.done_event.set()

        bpy.app.handlers.scene_update_post.append(main_loop)

        self.tex = glGenTextures(1)
        glBindTexture(GL_TEXTURE_2D, self.tex)
        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, 1, 1, 0,
            GL_RGB, GL_UNSIGNED_BYTE, struct.pack('=BBB', 0, 0, 0))

    def view_update(self, context):
        """ Called when the scene is changed """

        for collection in self._watch_list:
            collection_name = get_collection_name(collection)
            collection_set = set(collection)
            tracking_set = self._tracking_sets[collection_name]

            # Check for new items
            add_set = collection_set - tracking_set
            self.add_delta[collection_name] = add_set
            tracking_set |= add_set

            # Check for removed items
            remove_set = tracking_set - collection_set
            self.remove_delta[collection_name] = remove_set
            tracking_set -= remove_set

            # Check for updates
            update_set = {item for item in collection if item.is_updated}
            self.update_delta[collection_name] = update_set

    def view_draw(self, context):
        """ Called when viewport settings change """
        region = context.region
        view = context.region_data

        vmat = view.view_matrix.copy()
        vmat_inv = vmat.inverted()
        pmat = view.perspective_matrix * vmat_inv

        viewport = [region.x, region.y, region.width, region.height]

        self.update_view(vmat, pmat, viewport)

        glPushAttrib(GL_ALL_ATTRIB_BITS)
        glActiveTexture(GL_TEXTURE0)
        glBindTexture(GL_TEXTURE_2D, self.tex)
        try:
            image_ref = self.queue_image.get_nowait()
            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, self.width, self.height, 0, GL_RGB,
                GL_UNSIGNED_BYTE, image_ref)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
            self.queue_image.task_done()
        except queue.Empty:
            pass

        glDisable(GL_DEPTH_TEST)
        glDisable(GL_CULL_FACE)
        glDisable(GL_STENCIL_TEST)
        glEnable(GL_TEXTURE_2D)

        glClearColor(0, 0, 1, 1)
        glClear(GL_COLOR_BUFFER_BIT)

        glMatrixMode(GL_MODELVIEW)
        glPushMatrix()
        glLoadIdentity()
        glMatrixMode(GL_PROJECTION)
        glPushMatrix()
        glLoadIdentity()

        glBegin(GL_QUADS)
        glColor3f(1.0, 1.0, 1.0)
        glTexCoord2f(0.0, 0.0)
        glVertex3i(-1, -1, 0)
        glTexCoord2f(1.0, 0.0)
        glVertex3i(1, -1, 0)
        glTexCoord2f(1.0, 1.0)
        glVertex3i(1, 1, 0)
        glTexCoord2f(0.0, 1.0)
        glVertex3i(-1, 1, 0)
        glEnd()

        glPopMatrix()
        glMatrixMode(GL_MODELVIEW)
        glPopMatrix()

        glPopAttrib()

    def update_view(self, view_matrix, projection_matrix, viewport):
        if view_matrix != self._old_vmat:
            self._old_vmat = view_matrix
            self.view_delta['view_matrix'] = view_matrix

        if projection_matrix != self._old_pmat:
            self._old_pmat = projection_matrix
            self.view_delta['projection_matrix'] = projection_matrix

        if viewport != self._old_viewport:
            self._old_viewport = viewport
            self.view_delta['viewport'] = ViewportTuple(width=viewport[2], height=viewport[3])

    def draw_callback(self):
        '''Forces a view_draw to occur'''
        self.tag_redraw()

    def main_update(self, dt):
        if self.add_delta or self.update_delta or self.view_delta:
            self.queue_pre_convert.put((
                self.add_delta.copy(),
                self.update_delta.copy(),
                self.remove_delta.copy(),
                self.view_delta.copy(),
            ))

        self.add_delta.clear()
        self.update_delta.clear()
        self.remove_delta.clear()
        self.view_delta.clear()

        self.queue_update.put(dt)

        self.draw_callback()

        # Useful debug information for checking if a queue is filling up
        # print('Approximate queue sizes:')
        # print('\tPre Convert:', self.queue_pre_convert.qsize())
        # print('\tPost Convert:', self.queue_post_convert.qsize())
        # print('\tUpdate:', self.queue_update.qsize())
        # print('\tImage:', self.queue_image.qsize())
<|endoftext|>


<commit_before>""" this module impls the logic for 'bru make', which builds the downloaded
   modules via gyp and local compiler toolchain
"""

import os
import re
import sys
import glob
import platform
import brulib.install

def cmd_make(config, verbose, targetPlatform="Native"):
    """ this command makes some educated guesses about which toolchain
        the user probably wants to run, then invokes gyp to create the
        makefiles for this toolchain and invokes the build. On Linux
        'bru make' is likely equivalent to:
           >gyp *gyp --depth=.
           >make
        On Windows it's likely equivalent to:
           >gyp --depth=. package.gyp -G msvs_version=2012
           >C:\Windows\Microsoft.NET\Framework\v4.0.30319\msbuild.exe package.sln
        The main purpose of 'bru make' is really to spit out these two
        command lines as a quick reminder for how to build via cmd line.
        param config contains 'Release' or 'Debug'
        param verbose 0 means not verbose, >= 1 means higher verbosity level
            (whatever that means in the underlying toolchain)
    """
    print("running 'bru make --config {} --targetPlatform {}'".format(config,targetPlatform))

    # first locate the single gyp in the cwd
    bru_file = brulib.install.get_single_bru_file('.')
    if bru_file == None:
        raise Exception("there's no *.bru file in current work dir, "
            'e.g. run "bru install googlemock" first to create one')
    gyp_file = bru_file[:-3] + 'gyp'
    if not os.path.exists(gyp_file):
        raise Exception(bru_file,'has no companion *.gyp file, '
            'e.g. recreate one via "bru install googlemock"')

    system = platform.system()
    if system == 'Windows':
    	if targetPlatform == 'Native':
    		cmd_make_win(gyp_file, config)
    	else:
    		raise Exception('targetPlatform {} not supported on platform {}'\
    		.format(targetPlatform, system))
    elif system == 'Linux':
    	if targetPlatform == 'Native':
    		cmd_make_linux(gyp_file, config, verbose)
    	else:
        	raise Exception('targetPlatform {} not supported on platform {}'\
        	.format(targetPlatform, system))
    elif system == 'Darwin':
    	if targetPlatform == 'iOS':
    		cmd_make_ios(gyp_file, config, verbose)
    	elif targetPlatform == 'Native':
    		cmd_make_macos(gyp_file, config, verbose)
    	else:
        	raise Exception('targetPlatform {} not supported on platform {}'\
            .format(targetPlatform, system))
    else:
        raise Exception('no idea how to invoke gyp & toolchain on platform {}'\
            .format(system))

def get_latest_msbuild_exe():
    """ return path to latest msbuild on Windows machine """
    env = os.environ
    windir = env['SystemRoot'] if 'SystemRoot' in env else env['windir']
    glob_expr = os.path.join(windir, 'Microsoft.NET', 'Framework',
        '**', 'msbuild.exe')
    msbuilds = glob.glob(glob_expr)
    return max(msbuilds)  # not alphanumeric, should be good enough tho

def get_latest_installed_msvs_version():
    """ e.g. return 110 (aka VC11 aka VS 2012)) if msvs 2012 is installed.
        If multiple vs versions are installed then pick latest.
        Return None if no installs are found
    """
    # whats a good way to detect the msvs version?
    # a) scan for install dirs like
    #    c:\Program Files (x86)\Microsoft Visual Studio 10.0
    # b) scan for env vars like VS110COMNTOOLS
    # Let's do (b) for now.
    # See also https://code.google.com/p/gyp/source/browse/trunk/pylib/gyp/MSVSVersion.py
    msvs_versions = []
    regex = re.compile('^VS([0-10]+)COMNTOOLS$')
    for key in os.environ:
        match = regex.match(key)
        if match != None:
            msvs_versions.append(int(match.group(1)))
    if len(msvs_versions) == 0:
        return None
    latest = max(msvs_versions) # e.g. 110
    if len(msvs_versions) > 1:
        print('detected installs of msvs {}, choosing latest {}'.format(
            msvs_versions, latest))
    return latest

def get_latest_installed_msvs_year():
    """ e.g. return 2012 (aka VC11) if msvs 2012 is installed. If multiple
        vs versions are installed then pick latest.
        Return None if no installs are found
    """
    latest = get_latest_installed_msvs_version()
    if latest == None:
        return None
    msvs_version2year = {
        80: 2005,
        90: 2008,
        100: 2010,
        110: 2012,
    }
    if not latest in msvs_version2year:
        print('not sure how to map VC{} to a VS year, defaulting to VS 2012'
            .format(latest))
        return 2012
    return msvs_version2year[latest]

def run_gyp(gyp_cmdline):
    print("running '{}'".format(gyp_cmdline))
    returncode = os.system(gyp_cmdline)
    if returncode != 0:
        raise Exception('error running gyp, did you install it?'
            ' Instructions at https://github.com/KjellSchubert/bru')

def cmd_make_win(gyp_filename, config):
    # TODO: locate msvs version via glob
    msvs_version = get_latest_installed_msvs_year()
    if msvs_version == None:
        print('WARNING: no msvs installation detected, did you install it? '
            'Defaulting to msvs 2012.')
        msvs_version = 2012
    gyp_cmdline = 'gyp --depth=. {} -G msvs_version={}'.format(
        gyp_filename, msvs_version)
    run_gyp(gyp_cmdline)
    # gyp should have created a *.sln file, verify that.
    # if it didnt that pass a msvc generator option to gyp in a more explicit
    # fashion (is -G msvs_version enough? need GYP_GENERATORS=msvs?).
    sln_filename = gyp_filename[:-3] + 'sln'
    if not os.path.exists(sln_filename):
        raise Exception('gyp unexpectedly did not generate a *.sln file, '
            'you may wanna invoke gyp manually to generate the expected '
            'make/sln/ninja files, e.g. set GYP_GENERATORS=msvs')

    # there are many ways to build the *.sln now, e.g. pass it to devenv
    # or alternatively to msbuild. Lets do msbuild here:
    # TODO locate msbuild via glob
    msbuild_exe = get_latest_msbuild_exe()
    if msbuild_exe == None:
        raise Exception('did not detect any installs of msbuild, these should'
            ' be part of .NET installations, please install msbuild or .NET')
    # TODO? add /toolsversion:<version> ?
    msbuild_cmdline = '{} {} /p:Configuration={} /verbosity:minimal'.format(
        msbuild_exe, sln_filename, config)
    print("running msvs via msbuild: '{}'".format(msbuild_cmdline))
    sys.stdout.flush() # otherwise msbuild log lines show up first
    returncode = os.system(msbuild_cmdline)
    if returncode != 0:
        raise Exception('msbuild failed with errors, returncode =', returncode)
    print('Build complete.')

def cmd_make_linux(gyp_filename, config, verbose):
    # Here we could check if ninja or some such is installed to generate ninja
    # project files. But for simplicity's sake let's just use whatever gyp
    # defaults to.

    # For some odd reason passing './package.gyp' as a param to gyp will
    # generate garbage, instead you gotta pass 'package.gyp'. Se let's
    # explicitly remove a leading ./
    dirname = os.path.dirname(gyp_filename)
    assert dirname == '.' or len(dirname) == 0
    gyp_filename = os.path.basename(gyp_filename)

    gyp_cmdline = 'gyp --depth=. {}'.format(gyp_filename)
    run_gyp(gyp_cmdline)
    if not os.path.exists('Makefile'):
        raise Exception('gyp did not generate ./Makefile, no idea how to '
            'build with your toolchain, please build manually')
    make_cmdline = 'make BUILDTYPE={} V={}'.format(config,
            '1' if verbose >= 1 else '')
    print("running '{}'".format(make_cmdline))
    returncode = os.system(make_cmdline)
    if returncode != 0:
        raise Exception('Build failed: make returned', returncode)
    print('Build complete.')

def cmd_make_macos(gyp_filename, config, verbose):
    # Here we could check if ninja or some such is installed to generate ninja
    # project files. But for simplicity's sake let's just use whatever gyp
    # defaults to.

    # For some odd reason passing './package.gyp' as a param to gyp will
    # generate garbage, instead you gotta pass 'package.gyp'. Se let's
    # explicitly remove a leading ./
    dirname = os.path.dirname(gyp_filename)
    assert dirname == '.' or len(dirname) == 0
    gyp_filename = os.path.basename(gyp_filename)
    gyp_cmdline = 'gyp --depth=. -f xcode {} --generator-output=./xcode-macos'.format(gyp_filename)
    run_gyp(gyp_cmdline)
    filepattern = './xcode-macos/*.xcodeproj'
    files = glob.glob(filepattern)
    print(filepattern)
    print(files)
    if len(files) == 0:
        raise Exception('gyp did not generate {}, no idea how to '
            'build with your toolchain, please build manually').format(filepattern)
    xcode_cmdline = 'xCodeBuild -alltargets -project {} -configuration {}'.format(files[0],config)
    print("running '{}'".format(xcode_cmdline))
    returncode = os.system(xcode_cmdline)
    if returncode != 0:
        raise Exception('Build failed: make returned', returncode)
    print('Build complete.')

def cmd_make_ios(gyp_filename, config, verbose):
    # Here we could check if ninja or some such is installed to generate ninja
    # project files. But for simplicity's sake let's just use whatever gyp
    # defaults to.

    # For some odd reason passing './package.gyp' as a param to gyp will
    # generate garbage, instead you gotta pass 'package.gyp'. Se let's
    # explicitly remove a leading ./
    dirname = os.path.dirname(gyp_filename)
    assert dirname == '.' or len(dirname) == 0
    gyp_filename = os.path.basename(gyp_filename)
    gyp_cmdline = 'gyp --depth=. -f xcode -DOS=iOS {} --generator-output=./xcode-ios'.format(gyp_filename)
    run_gyp(gyp_cmdline)

    filepattern = './xcode-ios/*.xcodeproj'
    files = glob.glob(filepattern)
    if len(files) == 0:
        raise Exception('gyp did not generate {}, no idea how to '
            'build with your toolchain, please build manually').format(filepattern)
    xcode_cmdline = 'xCodeBuild -alltargets -project {} -configuration {} -sdk iphonesimulator'.format(files[0],config)
    print("running '{}'".format(xcode_cmdline))
    returncode = os.system(xcode_cmdline)
    if returncode != 0:
        raise Exception('Build failed: make for device returned', returncode)
    xcode_cmdline = 'xCodeBuild -alltargets -project {} -configuration {} -sdk iphoneos'.format(files[0],config)
    print("running '{}'".format(xcode_cmdline))
    returncode = os.system(xcode_cmdline)
    if returncode != 0:
        raise Exception('Build failed: make for simulator returned', returncode)
    if os.path.exists("./lib"):
        mkdir_cmdline = 'mkdir lib/{}-Universial'.format(config)
        returncode = os.system(mkdir_cmdline)
        filepattern = './lib/{}-iphoneos/lib*.a'.format(config)
        for file in glob.glob(filepattern):
            name = os.path.basename(os.path.normpath(file))
            command = 'lipo -create lib/{}-iphoneos/{} lib/{}-iphonesimulator/{} -output lib/{}-Universial/{}'.format(config,name,config,name,config,name)
            print("running '{}'".format(command))
            returncode = os.system(command)
            if returncode != 0:
                raise Exception('Build failed: lipo returned', returncode)
    if os.path.exists("./bru_modules/lib"):
        mkdir_cmdline = 'mkdir bru_modules/lib/{}-Universial'.format(config)
        returncode = os.system(mkdir_cmdline)
        filepattern = './bru_modules/lib/{}-iphoneos/lib*.a'.format(config)
        for file in glob.glob(filepattern):
            name = os.path.basename(os.path.normpath(file))
            command = 'lipo -create bru_modules/lib/{}-iphoneos/{} bru_modules/lib/{}-iphonesimulator/{} -output bru_modules/lib/{}-Universial/{}'.format(config,name,config,name,config,name)
            print("running '{}'".format(command))
            returncode = os.system(command)
            if returncode != 0:
                raise Exception('Build failed: lipo returned', returncode)
    print('Build complete.')
<commit_msg>iOS: fixed bug where the wrong project file is compiled if multiple project files are generated<commit_after>""" this module impls the logic for 'bru make', which builds the downloaded
   modules via gyp and local compiler toolchain
"""

import os
import re
import sys
import glob
import platform
import brulib.install

def cmd_make(config, verbose, targetPlatform="Native"):
    """ this command makes some educated guesses about which toolchain
        the user probably wants to run, then invokes gyp to create the
        makefiles for this toolchain and invokes the build. On Linux
        'bru make' is likely equivalent to:
           >gyp *gyp --depth=.
           >make
        On Windows it's likely equivalent to:
           >gyp --depth=. package.gyp -G msvs_version=2012
           >C:\Windows\Microsoft.NET\Framework\v4.0.30319\msbuild.exe package.sln
        The main purpose of 'bru make' is really to spit out these two
        command lines as a quick reminder for how to build via cmd line.
        param config contains 'Release' or 'Debug'
        param verbose 0 means not verbose, >= 1 means higher verbosity level
            (whatever that means in the underlying toolchain)
    """
    print("running 'bru make --config {} --targetPlatform {}'".format(config,targetPlatform))

    # first locate the single gyp in the cwd
    bru_file = brulib.install.get_single_bru_file('.')
    if bru_file == None:
        raise Exception("there's no *.bru file in current work dir, "
            'e.g. run "bru install googlemock" first to create one')
    gyp_file = bru_file[:-3] + 'gyp'
    if not os.path.exists(gyp_file):
        raise Exception(bru_file,'has no companion *.gyp file, '
            'e.g. recreate one via "bru install googlemock"')

    system = platform.system()
    if system == 'Windows':
    	if targetPlatform == 'Native':
    		cmd_make_win(gyp_file, config)
    	else:
    		raise Exception('targetPlatform {} not supported on platform {}'\
    		.format(targetPlatform, system))
    elif system == 'Linux':
    	if targetPlatform == 'Native':
    		cmd_make_linux(gyp_file, config, verbose)
    	else:
        	raise Exception('targetPlatform {} not supported on platform {}'\
        	.format(targetPlatform, system))
    elif system == 'Darwin':
    	if targetPlatform == 'iOS':
    		cmd_make_ios(gyp_file, config, verbose)
    	elif targetPlatform == 'Native':
    		cmd_make_macos(gyp_file, config, verbose)
    	else:
        	raise Exception('targetPlatform {} not supported on platform {}'\
            .format(targetPlatform, system))
    else:
        raise Exception('no idea how to invoke gyp & toolchain on platform {}'\
            .format(system))

def get_latest_msbuild_exe():
    """ return path to latest msbuild on Windows machine """
    env = os.environ
    windir = env['SystemRoot'] if 'SystemRoot' in env else env['windir']
    glob_expr = os.path.join(windir, 'Microsoft.NET', 'Framework',
        '**', 'msbuild.exe')
    msbuilds = glob.glob(glob_expr)
    return max(msbuilds)  # not alphanumeric, should be good enough tho

def get_latest_installed_msvs_version():
    """ e.g. return 110 (aka VC11 aka VS 2012)) if msvs 2012 is installed.
        If multiple vs versions are installed then pick latest.
        Return None if no installs are found
    """
    # whats a good way to detect the msvs version?
    # a) scan for install dirs like
    #    c:\Program Files (x86)\Microsoft Visual Studio 10.0
    # b) scan for env vars like VS110COMNTOOLS
    # Let's do (b) for now.
    # See also https://code.google.com/p/gyp/source/browse/trunk/pylib/gyp/MSVSVersion.py
    msvs_versions = []
    regex = re.compile('^VS([0-10]+)COMNTOOLS$')
    for key in os.environ:
        match = regex.match(key)
        if match != None:
            msvs_versions.append(int(match.group(1)))
    if len(msvs_versions) == 0:
        return None
    latest = max(msvs_versions) # e.g. 110
    if len(msvs_versions) > 1:
        print('detected installs of msvs {}, choosing latest {}'.format(
            msvs_versions, latest))
    return latest

def get_latest_installed_msvs_year():
    """ e.g. return 2012 (aka VC11) if msvs 2012 is installed. If multiple
        vs versions are installed then pick latest.
        Return None if no installs are found
    """
    latest = get_latest_installed_msvs_version()
    if latest == None:
        return None
    msvs_version2year = {
        80: 2005,
        90: 2008,
        100: 2010,
        110: 2012,
    }
    if not latest in msvs_version2year:
        print('not sure how to map VC{} to a VS year, defaulting to VS 2012'
            .format(latest))
        return 2012
    return msvs_version2year[latest]

def run_gyp(gyp_cmdline):
    print("running '{}'".format(gyp_cmdline))
    returncode = os.system(gyp_cmdline)
    if returncode != 0:
        raise Exception('error running gyp, did you install it?'
            ' Instructions at https://github.com/KjellSchubert/bru')

def cmd_make_win(gyp_filename, config):
    # TODO: locate msvs version via glob
    msvs_version = get_latest_installed_msvs_year()
    if msvs_version == None:
        print('WARNING: no msvs installation detected, did you install it? '
            'Defaulting to msvs 2012.')
        msvs_version = 2012
    gyp_cmdline = 'gyp --depth=. {} -G msvs_version={}'.format(
        gyp_filename, msvs_version)
    run_gyp(gyp_cmdline)
    # gyp should have created a *.sln file, verify that.
    # if it didnt that pass a msvc generator option to gyp in a more explicit
    # fashion (is -G msvs_version enough? need GYP_GENERATORS=msvs?).
    sln_filename = gyp_filename[:-3] + 'sln'
    if not os.path.exists(sln_filename):
        raise Exception('gyp unexpectedly did not generate a *.sln file, '
            'you may wanna invoke gyp manually to generate the expected '
            'make/sln/ninja files, e.g. set GYP_GENERATORS=msvs')

    # there are many ways to build the *.sln now, e.g. pass it to devenv
    # or alternatively to msbuild. Lets do msbuild here:
    # TODO locate msbuild via glob
    msbuild_exe = get_latest_msbuild_exe()
    if msbuild_exe == None:
        raise Exception('did not detect any installs of msbuild, these should'
            ' be part of .NET installations, please install msbuild or .NET')
    # TODO? add /toolsversion:<version> ?
    msbuild_cmdline = '{} {} /p:Configuration={} /verbosity:minimal'.format(
        msbuild_exe, sln_filename, config)
    print("running msvs via msbuild: '{}'".format(msbuild_cmdline))
    sys.stdout.flush() # otherwise msbuild log lines show up first
    returncode = os.system(msbuild_cmdline)
    if returncode != 0:
        raise Exception('msbuild failed with errors, returncode =', returncode)
    print('Build complete.')

def cmd_make_linux(gyp_filename, config, verbose):
    # Here we could check if ninja or some such is installed to generate ninja
    # project files. But for simplicity's sake let's just use whatever gyp
    # defaults to.

    # For some odd reason passing './package.gyp' as a param to gyp will
    # generate garbage, instead you gotta pass 'package.gyp'. Se let's
    # explicitly remove a leading ./
    dirname = os.path.dirname(gyp_filename)
    assert dirname == '.' or len(dirname) == 0
    gyp_filename = os.path.basename(gyp_filename)

    gyp_cmdline = 'gyp --depth=. {}'.format(gyp_filename)
    run_gyp(gyp_cmdline)
    if not os.path.exists('Makefile'):
        raise Exception('gyp did not generate ./Makefile, no idea how to '
            'build with your toolchain, please build manually')
    make_cmdline = 'make BUILDTYPE={} V={}'.format(config,
            '1' if verbose >= 1 else '')
    print("running '{}'".format(make_cmdline))
    returncode = os.system(make_cmdline)
    if returncode != 0:
        raise Exception('Build failed: make returned', returncode)
    print('Build complete.')

def cmd_make_macos(gyp_filename, config, verbose):
    # Here we could check if ninja or some such is installed to generate ninja
    # project files. But for simplicity's sake let's just use whatever gyp
    # defaults to.

    # For some odd reason passing './package.gyp' as a param to gyp will
    # generate garbage, instead you gotta pass 'package.gyp'. Se let's
    # explicitly remove a leading ./
    dirname = os.path.dirname(gyp_filename)
    assert dirname == '.' or len(dirname) == 0
    gyp_filename = os.path.basename(gyp_filename)
    gyp_cmdline = 'gyp --depth=. -f xcode {} --generator-output=./xcode-macos'.format(gyp_filename)
    run_gyp(gyp_cmdline)
    xcodeprj = './xcode-macos/{}xcodeproj'.format(gyp_filename[:-3])
    filepattern = xcodeprj
    files = glob.glob(filepattern)
    print(filepattern)
    print(files)
    if len(files) == 0:
        raise Exception('gyp did not generate {}, no idea how to '
            'build with your toolchain, please build manually').format(filepattern)
    xcode_cmdline = 'xCodeBuild -alltargets -project {} -configuration {}'.format(xcodeprj,config)
    print("running '{}'".format(xcode_cmdline))
    returncode = os.system(xcode_cmdline)
    if returncode != 0:
        raise Exception('Build failed: make returned', returncode)
    print('Build complete.')

def cmd_make_ios(gyp_filename, config, verbose):
    # Here we could check if ninja or some such is installed to generate ninja
    # project files. But for simplicity's sake let's just use whatever gyp
    # defaults to.

    # For some odd reason passing './package.gyp' as a param to gyp will
    # generate garbage, instead you gotta pass 'package.gyp'. Se let's
    # explicitly remove a leading ./
    dirname = os.path.dirname(gyp_filename)
    assert dirname == '.' or len(dirname) == 0
    gyp_filename = os.path.basename(gyp_filename)
    gyp_cmdline = 'gyp --depth=. -f xcode -DOS=iOS {} --generator-output=./xcode-ios'.format(gyp_filename)
    run_gyp(gyp_cmdline)
    xcodeprj = './xcode-ios/{}xcodeproj'.format(gyp_filename[:-3])
    filepattern = xcodeprj
    files = glob.glob(filepattern)
    if len(files) == 0:
        raise Exception('gyp did not generate {}, no idea how to '
            'build with your toolchain, please build manually').format(filepattern)
    xcode_cmdline = 'xCodeBuild -alltargets -project {} -configuration {} -sdk iphonesimulator'.format(xcodeprj,config)
    print("running '{}'".format(xcode_cmdline))
    returncode = os.system(xcode_cmdline)
    if returncode != 0:
        raise Exception('Build failed: make for device returned', returncode)
    xcode_cmdline = 'xCodeBuild -alltargets -project {} -configuration {} -sdk iphoneos'.format(xcodeprj,config)
    print("running '{}'".format(xcode_cmdline))
    returncode = os.system(xcode_cmdline)
    if returncode != 0:
        raise Exception('Build failed: make for simulator returned', returncode)
    if os.path.exists("./lib"):
        mkdir_cmdline = 'mkdir lib/{}-Universial'.format(config)
        returncode = os.system(mkdir_cmdline)
        filepattern = './lib/{}-iphoneos/lib*.a'.format(config)
        for file in glob.glob(filepattern):
            name = os.path.basename(os.path.normpath(file))
            command = 'lipo -create lib/{}-iphoneos/{} lib/{}-iphonesimulator/{} -output lib/{}-Universial/{}'.format(config,name,config,name,config,name)
            print("running '{}'".format(command))
            returncode = os.system(command)
            if returncode != 0:
                raise Exception('Build failed: lipo returned', returncode)
    if os.path.exists("./bru_modules/lib"):
        mkdir_cmdline = 'mkdir bru_modules/lib/{}-Universial'.format(config)
        returncode = os.system(mkdir_cmdline)
        filepattern = './bru_modules/lib/{}-iphoneos/lib*.a'.format(config)
        for file in glob.glob(filepattern):
            name = os.path.basename(os.path.normpath(file))
            command = 'lipo -create bru_modules/lib/{}-iphoneos/{} bru_modules/lib/{}-iphonesimulator/{} -output bru_modules/lib/{}-Universial/{}'.format(config,name,config,name,config,name)
            print("running '{}'".format(command))
            returncode = os.system(command)
            if returncode != 0:
                raise Exception('Build failed: lipo returned', returncode)
    print('Build complete.')
<|endoftext|>


<commit_before>from . import core
from . import algos
from . import backtest

from .backtest import Backtest, run
from .core import Strategy, Algo, AlgoStack

import ffn
from ffn import utils, data, get, merge

__version__ = (0, 2, 1)
<commit_msg>bump to 0.2.2<commit_after>from . import core
from . import algos
from . import backtest

from .backtest import Backtest, run
from .core import Strategy, Algo, AlgoStack

import ffn
from ffn import utils, data, get, merge

__version__ = (0, 2, 2)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import lxml.html
import re

class Vastavirta(object):
    def __init__(self):
        self.data = None

        self.url = "http://vastavirta.net/fi/"
        self.name = "Vastavirta"
        self.city = "Tampere"
        self.country = "Finland"
        self.parseddata = None
        self.monthmap = {   \
                u"Tammi" : 1,  \
                u"Helmi" : 2,  \
                u"Maalis" : 3,  \
                u"Huhti" : 4,  \
                u"Touko" : 5,  \
                u"Kesä" : 6,  \
                u"Heinä" : 7,  \
                u"Elo" : 8,  \
                u"Syys" : 9,  \
                u"Loka" : 10, \
                u"Marras" : 11, \
                u"Jou" : 12 }

    def getVenueName(self):
        return self.name

    def getCity(self):
        return self.city

    def getCountry(self):
        return self.country

    def eventSQLentity(self):
        return { u"name" : self.name, \
                 u"city" : self.city, \
                 u"country" : self.country }

    def parsePrice(self, tag):
        parsedprice = tag.xpath('.//div[@class="event-details"]/*/text()')
        prices = " ".join(parsedprice)

        if prices:
            return u"%s" % (prices)
        else:
            return u"0"

    def parseDate(self, tag):
        day = tag.xpath('./*/div[@class="start-date"]/div[@class="event-day"]/text()')
        m = tag.xpath('./*/div[@class="start-date"]/div[@class="event-month"]/text()')
        year = tag.xpath('./*/div[@class="start-date"]/div[@class="event-year"]/text()')

        day = " ".join(day)
        month = " ".join(m).capitalize()
        year = " ".join(year)

        if day is "" or month is "" or year is "":
            return ""

        month = self.monthmap[month]
        return "%.4d-%.2d-%.2d" % (int(year), int(month), int(day))

    def parseEvent(self, event):
        date = self.parseDate(event)
        etitle = " ".join(event.xpath('./*/div[@class="event-title"]/*/text()'))
        prices = self.parsePrice(event)

        return { u"venue" : self.getVenueName(), \
                 u"date" : date,                 \
                 u"name" : etitle,               \
                 u"price" : prices }

    def parseEvents(self, data):
        doc = lxml.html.fromstring(data).getroottree().getroot()
        tags = doc.xpath('//div[@class="event-list"]/ul' \
                       + '[@class="event-list-view"]'    \
                       + '/li[@class="event "]')
        tmp = u""

        for event in reversed(tags):
            parsed = self.parseEvent(event)

            if parsed["date"] == "":
                tmp += ", " + parsed["name"]
            else:
                parsed ["name"] += tmp
                yield parsed
                tmp = u""

if __name__ == '__main__':
    import requests

    v = Vastavirta()
    r = requests.get(v.url)

    #with open("venues/vastavirta.html") as f:
    #    r = f.read()

    for event in v.parseEvents(r.content):
        for k, v in event.iteritems():
            print "%-10s: %s" % (k, v)
        print

<commit_msg>Joulu, not Jou.<commit_after># -*- coding: utf-8 -*-

import lxml.html
import re

class Vastavirta(object):
    def __init__(self):
        self.data = None

        self.url = "http://vastavirta.net/fi/"
        self.name = "Vastavirta"
        self.city = "Tampere"
        self.country = "Finland"
        self.parseddata = None
        self.monthmap = {   \
                u"Tammi" : 1,  \
                u"Helmi" : 2,  \
                u"Maalis" : 3,  \
                u"Huhti" : 4,  \
                u"Touko" : 5,  \
                u"Kesä" : 6,  \
                u"Heinä" : 7,  \
                u"Elo" : 8,  \
                u"Syys" : 9,  \
                u"Loka" : 10, \
                u"Marras" : 11, \
                u"Joulu" : 12 }

    def getVenueName(self):
        return self.name

    def getCity(self):
        return self.city

    def getCountry(self):
        return self.country

    def eventSQLentity(self):
        return { u"name" : self.name, \
                 u"city" : self.city, \
                 u"country" : self.country }

    def parsePrice(self, tag):
        parsedprice = tag.xpath('.//div[@class="event-details"]/*/text()')
        prices = " ".join(parsedprice)

        if prices:
            return u"%s" % (prices)
        else:
            return u"0"

    def parseDate(self, tag):
        day = tag.xpath('./*/div[@class="start-date"]/div[@class="event-day"]/text()')
        m = tag.xpath('./*/div[@class="start-date"]/div[@class="event-month"]/text()')
        year = tag.xpath('./*/div[@class="start-date"]/div[@class="event-year"]/text()')

        day = " ".join(day)
        month = " ".join(m).capitalize()
        year = " ".join(year)

        if day is "" or month is "" or year is "":
            return ""

        month = self.monthmap[month]
        return "%.4d-%.2d-%.2d" % (int(year), int(month), int(day))

    def parseEvent(self, event):
        date = self.parseDate(event)
        etitle = " ".join(event.xpath('./*/div[@class="event-title"]/*/text()'))
        prices = self.parsePrice(event)

        return { u"venue" : self.getVenueName(), \
                 u"date" : date,                 \
                 u"name" : etitle,               \
                 u"price" : prices }

    def parseEvents(self, data):
        doc = lxml.html.fromstring(data).getroottree().getroot()
        tags = doc.xpath('//div[@class="event-list"]/ul' \
                       + '[@class="event-list-view"]'    \
                       + '/li[@class="event "]')
        tmp = u""

        for event in reversed(tags):
            parsed = self.parseEvent(event)

            if parsed["date"] == "":
                tmp += ", " + parsed["name"]
            else:
                parsed ["name"] += tmp
                yield parsed
                tmp = u""

if __name__ == '__main__':
    import requests

    v = Vastavirta()
    r = requests.get(v.url)

    #with open("venues/vastavirta.html") as f:
    #    r = f.read()

    for event in v.parseEvents(r.content):
        for k, v in event.iteritems():
            print "%-10s: %s" % (k, v)
        print

<|endoftext|>


<commit_before># GYP file to build various tools.
#
# To build on Linux:
#  ./gyp_skia tools.gyp && make tools
#
# Building on other platforms not tested yet.
#
{
  'includes': [
    'apptype_console.gypi',
  ],
  'targets': [
    {
      # Build all executable targets defined below.
      'target_name': 'tools',
      'type': 'none',
      'dependencies': [
        'skdiff',
        'skhello',
        'skimage',
        'render_pictures',
        'bench_pictures',
        'pinspect',
      ],
    },
    {
      'target_name': 'skdiff',
      'type': 'executable',
      'sources': [
        '../tools/skdiff_main.cpp',
      ],
      'dependencies': [
        'core.gyp:core',
        'effects.gyp:effects',
        'images.gyp:images',
        'ports.gyp:ports',
        'utils.gyp:utils',
      ],
    },
    {
      'target_name': 'skhello',
      'type': 'executable',
      'sources': [
        '../tools/skhello.cpp',
      ],
      'dependencies': [
        'core.gyp:core',
        'effects.gyp:effects',
        'images.gyp:images',
        'ports.gyp:ports',
        'utils.gyp:utils',
      ],
    },
    {
      'target_name': 'skimage',
      'type': 'executable',
      'sources': [
        '../tools/skimage_main.cpp',
      ],
      'dependencies': [
        'core.gyp:core',
        'effects.gyp:effects',
        'images.gyp:images',
        'ports.gyp:ports',
        'utils.gyp:utils',
      ],
    },
    {
      'target_name': 'render_pictures',
      'type': 'executable',
      'sources': [
        '../tools/render_pictures_main.cpp',
      ],
      'include_dirs': [
        '../src/pipe/utils/',
      ],
      'dependencies': [
        'core.gyp:core',
        'ports.gyp:ports',
        'tools.gyp:picture_renderer',
        'tools.gyp:picture_utils',
      ],
    },
    {
      'target_name': 'bench_pictures',
      'type': 'executable',
      'sources': [
        '../bench/SkBenchLogger.h',
        '../bench/SkBenchLogger.cpp',
        '../bench/TimerData.h',
        '../bench/TimerData.cpp',
        '../tools/bench_pictures_main.cpp',
        '../tools/PictureBenchmark.cpp',
      ],
      'include_dirs': [
        '../bench',
      ],
      'dependencies': [
        'core.gyp:core',
        'effects.gyp:effects',
        'ports.gyp:ports',
        'tools.gyp:picture_utils',
        'tools.gyp:picture_renderer',
        'bench.gyp:bench_timer',
      ],
    },
    {
     'target_name': 'picture_renderer',
     'type': 'static_library',
     'sources': [
        '../tools/PictureRenderer.cpp',
        '../src/pipe/utils/SamplePipeControllers.h',
        '../src/pipe/utils/SamplePipeControllers.cpp',
     ],
     'include_dirs': [
       '../src/pipe/utils/',
       '../src/utils/',
     ],
     'dependencies': [
        'core.gyp:core',
        'images.gyp:images',
        'tools.gyp:picture_utils',
        'utils.gyp:utils',
     ],
     'export_dependent_settings': [
       'images.gyp:images',
     ],
     'conditions': [
       ['skia_gpu == 1', {
         'dependencies': [
           'gpu.gyp:gr',
           'gpu.gyp:skgr',
         ],
         'export_dependent_settings': [
            'gpu.gyp:gr',
         ],
       }],
    ],
    },
    {
      'target_name': 'picture_utils',
      'type': 'static_library',
      'sources': [
        '../tools/picture_utils.cpp',
      ],
      'dependencies': [
        'core.gyp:core',
      ],
    },
    {
      'target_name': 'pinspect',
      'type': 'executable',
      'sources': [
        '../tools/pinspect.cpp',
      ],
      'dependencies': [
        'core.gyp:core',
        'effects.gyp:effects',
        'images.gyp:images',
        'ports.gyp:ports',
        'utils.gyp:utils',
      ],
    },
  ],
}

# Local Variables:
# tab-width:2
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=2 shiftwidth=2:
<commit_msg>Fix Android build of render_pictures (Again)<commit_after># GYP file to build various tools.
#
# To build on Linux:
#  ./gyp_skia tools.gyp && make tools
#
# Building on other platforms not tested yet.
#
{
  'includes': [
    'apptype_console.gypi',
  ],
  'targets': [
    {
      # Build all executable targets defined below.
      'target_name': 'tools',
      'type': 'none',
      'dependencies': [
        'skdiff',
        'skhello',
        'skimage',
        'render_pictures',
        'bench_pictures',
        'pinspect',
      ],
    },
    {
      'target_name': 'skdiff',
      'type': 'executable',
      'sources': [
        '../tools/skdiff_main.cpp',
      ],
      'dependencies': [
        'core.gyp:core',
        'effects.gyp:effects',
        'images.gyp:images',
        'ports.gyp:ports',
        'utils.gyp:utils',
      ],
    },
    {
      'target_name': 'skhello',
      'type': 'executable',
      'sources': [
        '../tools/skhello.cpp',
      ],
      'dependencies': [
        'core.gyp:core',
        'effects.gyp:effects',
        'images.gyp:images',
        'ports.gyp:ports',
        'utils.gyp:utils',
      ],
    },
    {
      'target_name': 'skimage',
      'type': 'executable',
      'sources': [
        '../tools/skimage_main.cpp',
      ],
      'dependencies': [
        'core.gyp:core',
        'effects.gyp:effects',
        'images.gyp:images',
        'ports.gyp:ports',
        'utils.gyp:utils',
      ],
    },
    {
      'target_name': 'render_pictures',
      'type': 'executable',
      'sources': [
        '../tools/render_pictures_main.cpp',
      ],
      'include_dirs': [
        '../src/pipe/utils/',
      ],
      'dependencies': [
        'core.gyp:core',
        'ports.gyp:ports',
        'tools.gyp:picture_renderer',
        'tools.gyp:picture_utils',
      ],
    },
    {
      'target_name': 'bench_pictures',
      'type': 'executable',
      'sources': [
        '../bench/SkBenchLogger.h',
        '../bench/SkBenchLogger.cpp',
        '../bench/TimerData.h',
        '../bench/TimerData.cpp',
        '../tools/bench_pictures_main.cpp',
        '../tools/PictureBenchmark.cpp',
      ],
      'include_dirs': [
        '../bench',
      ],
      'dependencies': [
        'core.gyp:core',
        'effects.gyp:effects',
        'ports.gyp:ports',
        'tools.gyp:picture_utils',
        'tools.gyp:picture_renderer',
        'bench.gyp:bench_timer',
      ],
    },
    {
     'target_name': 'picture_renderer',
     'type': 'static_library',
     'sources': [
        '../tools/PictureRenderer.cpp',
        '../src/pipe/utils/SamplePipeControllers.h',
        '../src/pipe/utils/SamplePipeControllers.cpp',
     ],
     'include_dirs': [
       '../src/pipe/utils/',
       '../src/utils/',
     ],
     'dependencies': [
        'core.gyp:core',
        'effects.gyp:effects',
        'images.gyp:images',
        'tools.gyp:picture_utils',
        'utils.gyp:utils',
     ],
     'export_dependent_settings': [
       'images.gyp:images',
     ],
     'conditions': [
       ['skia_gpu == 1', {
         'dependencies': [
           'gpu.gyp:gr',
           'gpu.gyp:skgr',
         ],
         'export_dependent_settings': [
            'gpu.gyp:gr',
         ],
       }],
    ],
    },
    {
      'target_name': 'picture_utils',
      'type': 'static_library',
      'sources': [
        '../tools/picture_utils.cpp',
      ],
      'dependencies': [
        'core.gyp:core',
      ],
    },
    {
      'target_name': 'pinspect',
      'type': 'executable',
      'sources': [
        '../tools/pinspect.cpp',
      ],
      'dependencies': [
        'core.gyp:core',
        'effects.gyp:effects',
        'images.gyp:images',
        'ports.gyp:ports',
        'utils.gyp:utils',
      ],
    },
  ],
}

# Local Variables:
# tab-width:2
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=2 shiftwidth=2:
<|endoftext|>


<commit_before>from flask import flash, request, Markup, url_for
from flask.ext.login import current_user

from viaduct import application
from markdown import markdown

from resource import Resource

markdown_extensions = [
	'toc'
]

def flash_form_errors(form):
	for field, errors in form.errors.items():
		for error in errors:
			flash(error, 'error')

@application.template_filter('markdown')
def markdown_filter(data):
	return Markup(markdown(data, safe_mode='escape', enable_attributes=False,
		extensions=markdown_extensions))

@application.template_filter('safe_markdown')
def safe_markdown_filter(data):
	return Markup(markdown(data, extensions=markdown_extensions))

@application.template_filter('pages')
def pages_filter(data):
	content = '
'

	for i in range(len(data)):
		if i % 2 == 0:
			content += '
'

		if i == len(data) - 1 and i % 2 == 0:
			content += '
'
		else:
			content += '
'

		content += '
'

		if current_user.is_authenticated():
			content += ' View History'
			content += ' Edit Page'

		content += '
{0}
'.format(data[i].title)

		if data[i].filter_html:
			content += markdown(data[i].content, safe_mode='escape',
				enable_attributes=False, extensions=markdown_extensions)
		else:
			content += markdown(data[i].content, extensions=markdown_extensions)

		content += '

'

		if i == len(data) - 1 or i % 2 != 0:
			content += '
'

	content += '
'

	return Markup(content)

<commit_msg>Fixed the style.<commit_after>from flask import flash, request, Markup, url_for
from flask.ext.login import current_user

from viaduct import application
from markdown import markdown

from resource import Resource

markdown_extensions = [
	'toc'
]

def flash_form_errors(form):
	for field, errors in form.errors.items():
		for error in errors:
			flash(error, 'error')

@application.template_filter('markdown')
def markdown_filter(data):
	return Markup(markdown(data, safe_mode='escape', enable_attributes=False,
		extensions=markdown_extensions))

@application.template_filter('safe_markdown')
def safe_markdown_filter(data):
	return Markup(markdown(data, extensions=markdown_extensions))

@application.template_filter('pages')
def pages_filter(data):
	content = '
'

	for i in range(len(data)):
		if i % 2 == 0:
			content += '
'

		if i == len(data) - 1 and i % 2 == 0:
			content += '
'
		else:
			content += '
'

		content += '
'

		if current_user.is_authenticated():
			content += '
'
			content += ' View History'
			content += ' Edit Page'
			content += '
'

		content += '
{0}
'.format(data[i].title)

		if data[i].filter_html:
			content += markdown(data[i].content, safe_mode='escape',
				enable_attributes=False, extensions=markdown_extensions)
		else:
			content += markdown(data[i].content, extensions=markdown_extensions)

		content += '

'

		if i == len(data) - 1 or i % 2 != 0:
			content += '
'

	content += '
'

	return Markup(content)

<|endoftext|>


<commit_before>#!/usr/bin/env python3
from autobahn.asyncio.websocket import WebSocketServerProtocol, WebSocketServerFactory
import asyncio
import json
import random
import time
WORKERS_SERVER_IP = '0.0.0.0'
WORKERS_SERVER_PORT = 12121
COMMANDERS_SERVER_IP = '0.0.0.0'
COMMANDERS_SERVER_PORT = 21212

MAX_PONG_TIME = 3 # Seconds
PING_INTERVAL = 5 # Seconds

MAX_FAILURES = 3
job_queue = asyncio.Queue()
job_counter = 0
jobs = {}
worker_websockets = set()
commander_websockets = set()
worker_exists = asyncio.Condition()
def print_info():
	info_str = 'Workers: {}, Commanders: {}, Jobs: {}'.format(len(worker_websockets),len(commander_websockets),job_queue.qsize())  + ' ' * 20
	print(info_str,end='\r'*len(info_str))
class WorkerProtocol(WebSocketServerProtocol):
	def onConnect(self, request):
	    pass

	def onPong(self, payload):
		self.last_pong_time = int(time.time())

	async def onOpen(self):
		self.last_ping_time = None
		self.last_pong_time = None
		self.jobs = [] # Add a new attribute for storing job ids
		await worker_exists.acquire()
		worker_websockets.add(self)
		worker_exists.notify_all()
		worker_exists.release()
		print_info()
	async def onMessage(self, payload, isBinary):
		msg = json.loads(payload.decode('utf8'))
		job_id = msg["id"]
		jobs[job_id][1].result_available(job_id,msg["result"],False)
		del jobs[job_id]
		self.jobs.remove(job_id)
		print_info()

	async def cleanup(self):
		for j in self.jobs:
			jobs[j][3]+=1
			if jobs[j][3] > MAX_FAILURES:
				jobs[j][1].result_available(j,None,True)
				del jobs[j]
			else:
				await job_queue.put(j)
		del self.jobs[:]
		print_info()

	async def onClose(self, wasClean, code, reason):
		if self in worker_websockets:
			worker_websockets.remove(self)
		await self.cleanup()
		print_info()

class CommanderProtocol(WebSocketServerProtocol):
	def onConnect(self, request):
	    pass

	def onPong(self, payload):
		self.last_pong_time = int(time.time())

	async def onOpen(self):
		global job_counter
		self.last_ping_time = None
		self.last_pong_time = None
		self.buff = []
		self.buff_size = 1
		commander_websockets.add(self)
		websocket_queue = asyncio.Queue()
		print_info()

	def result_available(self,job_id,result,error):
		if error:
			try:
				self.sendMessage(json.dumps({"type":"result","results":None,"error":True}).encode('utf-8'),False)
			except:
				pass # Non of our business!
		else:
			self.buff.append(result)
			if len(self.buff) >= self.buff_size:
				self.flush()

	def flush(self):
		try:
			self.sendMessage(json.dumps({"type":"result","results":self.buff,"error":False}).encode('utf-8'),False)
			del self.buff[:]
		except:
			pass # Non of our business!

	def info(self):
		try:
			self.sendMessage(json.dumps({"type":"info","workersCount":len(worker_websockets)}).encode('utf-8'),False)
		except:
			pass # Non of our business!

	async def onMessage(self, payload, isBinary):
		global job_counter
		msg = json.loads(payload.decode('utf8'))
		if msg["type"] == "run":
			jobs[job_counter] = [msg["code"],self,msg["args"],0]
			await job_queue.put(job_counter)
			job_counter += 1
		elif msg["type"] == "for":
			for i in range(msg["start"],msg["end"]):
				jobs[job_counter] = [msg["code"],self,[i] + msg["extraArgs"],0]
				await job_queue.put(job_counter)
				job_counter += 1
		elif msg["type"] == "forEach":
			for args in msg["argsList"]:
				jobs[job_counter] = [msg["code"],self,args + msg["extraArgs"],0]
				await job_queue.put(job_counter)
				job_counter += 1
		elif msg["type"] == "flush":
			self.flush()
		elif msg["type"] == "info":
			self.info()
		elif msg["type"] == "set":
			if msg["property"] == "bufferSize":
				self.buff_size = msg["value"]
				self.flush()
		print_info()
	async def onClose(self, wasClean, code, reason):
		if self in commander_websockets:
			commander_websockets.remove(self)
		print_info()
async def balance_handler():
	while True:
		job = await job_queue.get()
		await worker_exists.acquire()
		await worker_exists.wait_for(lambda:len(worker_websockets) > 0)
		worker_exists.release()
		websocket = random.sample(worker_websockets, 1)[0]
		websocket.sendMessage(json.dumps({"id":job,"code":jobs[job][0],"args":jobs[job][2]}).encode('utf-8'),False)

		websocket.jobs.append(job)

async def watcher_handler():
	while True:
		for ws in worker_websockets:
			if len(ws.jobs) > 0:
				if ws.last_ping_time:
					if not ws.last_pong_time or ws.last_pong_time < ws.last_ping_time:
						elapsed = int(time.time()) - ws.last_ping_time
						if elapsed > MAX_PONG_TIME:
							ws.sendClose()
							await ws.cleanup()
					elif ws.last_pong_time:
						ws.last_ping_time = None
				else:
					ws.sendPing()
					ws.last_ping_time = int(time.time())
			else:
				ws.last_ping_time = None
		for ws in commander_websockets:
			if ws.last_ping_time:
				if not ws.last_pong_time or ws.last_pong_time < ws.last_ping_time:
					elapsed = int(time.time()) - ws.last_ping_time
					if elapsed > MAX_PONG_TIME:
						ws.sendClose()
				elif ws.last_pong_time:
					ws.last_ping_time = None
			else:
				ws.sendPing()
				ws.last_ping_time = int(time.time())
		await asyncio.sleep(PING_INTERVAL)

if __name__ == '__main__':
	import asyncio
	import ssl
	loop = asyncio.get_event_loop()

	ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
	ssl_ctx.load_cert_chain(certfile='/etc/letsencrypt/live/pooljs.ir/cert.pem',keyfile='/etc/letsencrypt/live/pooljs.ir/privkey.pem')
	#ssl_ctx = None

	commanderFactory = WebSocketServerFactory()
	commanderFactory.protocol = CommanderProtocol
	commanderCoro = loop.create_server(commanderFactory, '0.0.0.0', 21212,ssl=ssl_ctx)
	workerFactory = WebSocketServerFactory()
	workerFactory.protocol = WorkerProtocol
	workerCoro = loop.create_server(workerFactory, '0.0.0.0', 12121,ssl=ssl_ctx)
	print_info()
	all_tasks = asyncio.gather(commanderCoro,workerCoro,balance_handler(),watcher_handler())
	try:
		server = loop.run_until_complete(all_tasks)
	except KeyboardInterrupt:
		print()
		all_tasks.cancel()
		loop.run_forever()
		all_tasks.exception()
	finally:
		loop.close()
<commit_msg>also check for all workers in each iteration<commit_after>#!/usr/bin/env python3
from autobahn.asyncio.websocket import WebSocketServerProtocol, WebSocketServerFactory
import asyncio
import json
import random
import time
WORKERS_SERVER_IP = '0.0.0.0'
WORKERS_SERVER_PORT = 12121
COMMANDERS_SERVER_IP = '0.0.0.0'
COMMANDERS_SERVER_PORT = 21212

MAX_PONG_TIME = 3 # Seconds
PING_INTERVAL = 5 # Seconds

MAX_FAILURES = 3
job_queue = asyncio.Queue()
job_counter = 0
jobs = {}
worker_websockets = set()
commander_websockets = set()
worker_exists = asyncio.Condition()
def print_info():
	info_str = 'Workers: {}, Commanders: {}, Jobs: {}'.format(len(worker_websockets),len(commander_websockets),job_queue.qsize())  + ' ' * 20
	print(info_str,end='\r'*len(info_str))
class WorkerProtocol(WebSocketServerProtocol):
	def onConnect(self, request):
	    pass

	def onPong(self, payload):
		self.last_pong_time = int(time.time())

	async def onOpen(self):
		self.last_ping_time = None
		self.last_pong_time = None
		self.jobs = [] # Add a new attribute for storing job ids
		await worker_exists.acquire()
		worker_websockets.add(self)
		worker_exists.notify_all()
		worker_exists.release()
		print_info()
	async def onMessage(self, payload, isBinary):
		msg = json.loads(payload.decode('utf8'))
		job_id = msg["id"]
		jobs[job_id][1].result_available(job_id,msg["result"],False)
		del jobs[job_id]
		self.jobs.remove(job_id)
		print_info()

	async def cleanup(self):
		for j in self.jobs:
			jobs[j][3]+=1
			if jobs[j][3] > MAX_FAILURES:
				jobs[j][1].result_available(j,None,True)
				del jobs[j]
			else:
				await job_queue.put(j)
		del self.jobs[:]
		print_info()

	async def onClose(self, wasClean, code, reason):
		if self in worker_websockets:
			worker_websockets.remove(self)
		await self.cleanup()
		print_info()

class CommanderProtocol(WebSocketServerProtocol):
	def onConnect(self, request):
	    pass

	def onPong(self, payload):
		self.last_pong_time = int(time.time())

	async def onOpen(self):
		global job_counter
		self.last_ping_time = None
		self.last_pong_time = None
		self.buff = []
		self.buff_size = 1
		commander_websockets.add(self)
		websocket_queue = asyncio.Queue()
		print_info()

	def result_available(self,job_id,result,error):
		if error:
			try:
				self.sendMessage(json.dumps({"type":"result","results":None,"error":True}).encode('utf-8'),False)
			except:
				pass # Non of our business!
		else:
			self.buff.append(result)
			if len(self.buff) >= self.buff_size:
				self.flush()

	def flush(self):
		try:
			self.sendMessage(json.dumps({"type":"result","results":self.buff,"error":False}).encode('utf-8'),False)
			del self.buff[:]
		except:
			pass # Non of our business!

	def info(self):
		try:
			self.sendMessage(json.dumps({"type":"info","workersCount":len(worker_websockets)}).encode('utf-8'),False)
		except:
			pass # Non of our business!

	async def onMessage(self, payload, isBinary):
		global job_counter
		msg = json.loads(payload.decode('utf8'))
		if msg["type"] == "run":
			jobs[job_counter] = [msg["code"],self,msg["args"],0]
			await job_queue.put(job_counter)
			job_counter += 1
		elif msg["type"] == "for":
			for i in range(msg["start"],msg["end"]):
				jobs[job_counter] = [msg["code"],self,[i] + msg["extraArgs"],0]
				await job_queue.put(job_counter)
				job_counter += 1
		elif msg["type"] == "forEach":
			for args in msg["argsList"]:
				jobs[job_counter] = [msg["code"],self,args + msg["extraArgs"],0]
				await job_queue.put(job_counter)
				job_counter += 1
		elif msg["type"] == "flush":
			self.flush()
		elif msg["type"] == "info":
			self.info()
		elif msg["type"] == "set":
			if msg["property"] == "bufferSize":
				self.buff_size = msg["value"]
				self.flush()
		print_info()
	async def onClose(self, wasClean, code, reason):
		if self in commander_websockets:
			commander_websockets.remove(self)
		print_info()
async def balance_handler():
	while True:
		job = await job_queue.get()
		await worker_exists.acquire()
		await worker_exists.wait_for(lambda:len(worker_websockets) > 0)
		worker_exists.release()
		websocket = random.sample(worker_websockets, 1)[0]
		websocket.sendMessage(json.dumps({"id":job,"code":jobs[job][0],"args":jobs[job][2]}).encode('utf-8'),False)

		websocket.jobs.append(job)

async def close_unavailable_sockets(socks):
	for ws in socks:
		if ws.last_ping_time:
			if not ws.last_pong_time or ws.last_pong_time < ws.last_ping_time:
				elapsed = int(time.time()) - ws.last_ping_time
				if elapsed > MAX_PONG_TIME:
					ws.sendClose()
					await ws.cleanup()
			elif ws.last_pong_time:
				ws.last_ping_time = None
		else:
			ws.sendPing()
			ws.last_ping_time = int(time.time())

async def watcher_handler():
	while True:
		await close_unavailable_sockets(worker_websockets)
		await close_unavailable_sockets(commander_websockets)
		await asyncio.sleep(PING_INTERVAL)

if __name__ == '__main__':
	import asyncio
	import ssl
	loop = asyncio.get_event_loop()

	ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
	ssl_ctx.load_cert_chain(certfile='/etc/letsencrypt/live/pooljs.ir/cert.pem',keyfile='/etc/letsencrypt/live/pooljs.ir/privkey.pem')
	#ssl_ctx = None

	commanderFactory = WebSocketServerFactory()
	commanderFactory.protocol = CommanderProtocol
	commanderCoro = loop.create_server(commanderFactory, '0.0.0.0', 21212,ssl=ssl_ctx)
	workerFactory = WebSocketServerFactory()
	workerFactory.protocol = WorkerProtocol
	workerCoro = loop.create_server(workerFactory, '0.0.0.0', 12121,ssl=ssl_ctx)
	print_info()
	all_tasks = asyncio.gather(commanderCoro,workerCoro,balance_handler(),watcher_handler())
	try:
		server = loop.run_until_complete(all_tasks)
	except KeyboardInterrupt:
		print()
		all_tasks.cancel()
		loop.run_forever()
		all_tasks.exception()
	finally:
		loop.close()
<|endoftext|>