<commit_before># Copyright 2010-2011 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Base Test Case for all Unit Tests"""

import contextlib
import logging as std_logging
import os
import os.path
import sys
import traceback

import eventlet.timeout
import fixtures
import mock
from oslo.config import cfg
from oslo.messaging import conffixture as messaging_conffixture
import testtools

from neutron.common import config
from neutron.common import rpc as n_rpc
from neutron.tests import fake_notifier
from neutron.tests import post_mortem_debug

CONF = cfg.CONF
CONF.import_opt('state_path', 'neutron.common.config')
TRUE_STRING = ['True', '1']
LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s"

ROOTDIR = os.path.dirname(__file__)
ETCDIR = os.path.join(ROOTDIR, 'etc')

def etcdir(*p):
    return os.path.join(ETCDIR, *p)

def fake_use_fatal_exceptions(*args):
    return True

def fake_consume_in_threads(self):
    return []

class BaseTestCase(testtools.TestCase):

    @staticmethod
    def config_parse(conf=None, args=None):
        """Create the default configurations."""
        # neutron.conf.test includes rpc_backend which needs to be cleaned up
        if args is None:
            args = ['--config-file', etcdir('neutron.conf.test')]
        if conf is None:
            config.init(args=args)
        else:
            conf(args)

    def setUp(self):
        super(BaseTestCase, self).setUp()

        # Configure this first to ensure pm debugging support for setUp()
        debugger = os.environ.get('OS_POST_MORTEM_DEBUGGER')
        if debugger:
            self.addOnException(post_mortem_debug.get_exception_handler(
                debugger))

        if os.environ.get('OS_DEBUG') in TRUE_STRING:
            _level = std_logging.DEBUG
        else:
            _level = std_logging.INFO
        capture_logs = os.environ.get('OS_LOG_CAPTURE') in TRUE_STRING
        if not capture_logs:
            std_logging.basicConfig(format=LOG_FORMAT, level=_level)
        self.log_fixture = self.useFixture(
            fixtures.FakeLogger(
                format=LOG_FORMAT,
                level=_level,
                nuke_handlers=capture_logs,
            ))

        # suppress all but errors here
        self.useFixture(
            fixtures.FakeLogger(
                name='neutron.api.extensions',
                format=LOG_FORMAT,
                level=std_logging.ERROR,
                nuke_handlers=capture_logs,
            ))

        test_timeout = int(os.environ.get('OS_TEST_TIMEOUT', 0))
        if test_timeout == -1:
            test_timeout = 0
        if test_timeout > 0:
            self.useFixture(fixtures.Timeout(test_timeout, gentle=True))

        # If someone does use tempfile directly, ensure that it's cleaned up
        self.useFixture(fixtures.NestedTempfile())
        self.useFixture(fixtures.TempHomeDir())

        self.temp_dir = self.useFixture(fixtures.TempDir()).path
        cfg.CONF.set_override('state_path', self.temp_dir)

        self.addCleanup(mock.patch.stopall)
        self.addCleanup(CONF.reset)

        if os.environ.get('OS_STDOUT_CAPTURE') in TRUE_STRING:
            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
        if os.environ.get('OS_STDERR_CAPTURE') in TRUE_STRING:
            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
        self.useFixture(fixtures.MonkeyPatch(
            'neutron.common.exceptions.NeutronException.use_fatal_exceptions',
            fake_use_fatal_exceptions))

        self.setup_rpc_mocks()

        if sys.version_info < (2, 7) and getattr(self, 'fmt', '') == 'xml':
            raise self.skipException('XML Testing Skipped in Py26')

        self.setup_config()
        self.addOnException(self.check_for_systemexit)

    def setup_rpc_mocks(self):
        # don't actually start RPC listeners when testing
        self.useFixture(fixtures.MonkeyPatch(
            'neutron.common.rpc.Connection.consume_in_threads',
            fake_consume_in_threads))

        # immediately return RPC calls
        self.useFixture(fixtures.MonkeyPatch(
            'neutron.common.rpc.RpcProxy._RpcProxy__call_rpc_method',
            mock.MagicMock()))

        self.useFixture(fixtures.MonkeyPatch(
            'oslo.messaging.Notifier', fake_notifier.FakeNotifier))

        self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
        self.messaging_conf.transport_driver = 'fake'
        self.messaging_conf.response_timeout = 15
        self.useFixture(self.messaging_conf)

        self.addCleanup(n_rpc.clear_extra_exmods)
        n_rpc.add_extra_exmods('neutron.test')

        self.addCleanup(n_rpc.cleanup)
        n_rpc.init(CONF)

    def check_for_systemexit(self, exc_info):
        if isinstance(exc_info[1], SystemExit):
            self.fail("A SystemExit was raised during the test. %s"
                      % traceback.format_exception(*exc_info))

    def setup_config(self):
        """Tests that need a non-default config can override this method."""
        self.config_parse()

    def config(self, **kw):
        """Override some configuration values.

        The keyword arguments are the names of configuration options to
        override and their values.

        If a group argument is supplied, the overrides are applied to
        the specified configuration option group.

        All overrides are automatically cleared at the end of the current
        test by the fixtures cleanup process.
        """
        group = kw.pop('group', None)
        for k, v in kw.iteritems():
            CONF.set_override(k, v, group)

    @contextlib.contextmanager
    def assert_max_execution_time(self, max_execution_time=5):
        with eventlet.timeout.Timeout(max_execution_time, False):
            yield
            return
        self.fail('Execution of this test timed out')

    def assertOrderedEqual(self, expected, actual):
        expect_val = self.sort_dict_lists(expected)
        actual_val = self.sort_dict_lists(actual)
        self.assertEqual(expect_val, actual_val)

    def sort_dict_lists(self, dic):
        for key, value in dic.iteritems():
            if isinstance(value, list):
                dic[key] = sorted(value)
            elif isinstance(value, dict):
                dic[key] = self.sort_dict_lists(value)
        return dic

    def assertDictSupersetOf(self, expected_subset, actual_superset):
        """Checks that actual dict contains the expected dict.

        After checking that the arguments are of the right type, this checks
        that each item in expected_subset is in, and matches, what is in
        actual_superset. Separate tests are done, so that detailed info can
        be reported upon failure.
        """
        if not isinstance(expected_subset, dict):
            self.fail("expected_subset (%s) is not an instance of dict" %
                      type(expected_subset))
        if not isinstance(actual_superset, dict):
            self.fail("actual_superset (%s) is not an instance of dict" %
                      type(actual_superset))
        for k, v in expected_subset.items():
            self.assertIn(k, actual_superset)
            self.assertEqual(v, actual_superset[k],
                             "Key %(key)s expected: %(exp)r, actual %(act)r" %
                             {'key': k, 'exp': v, 'act': actual_superset[k]})
<commit_msg>Don't block on rpc calls in unit tests<commit_after># Copyright 2010-2011 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Base Test Case for all Unit Tests"""

import contextlib
import logging as std_logging
import os
import os.path
import sys
import traceback

import eventlet.timeout
import fixtures
import mock
from oslo.config import cfg
from oslo.messaging import conffixture as messaging_conffixture
import testtools

from neutron.common import config
from neutron.common import rpc as n_rpc
from neutron.tests import fake_notifier
from neutron.tests import post_mortem_debug

CONF = cfg.CONF
CONF.import_opt('state_path', 'neutron.common.config')
TRUE_STRING = ['True', '1']
LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s"

ROOTDIR = os.path.dirname(__file__)
ETCDIR = os.path.join(ROOTDIR, 'etc')

def etcdir(*p):
    return os.path.join(ETCDIR, *p)

def fake_use_fatal_exceptions(*args):
    return True

def fake_consume_in_threads(self):
    return []

class BaseTestCase(testtools.TestCase):

    @staticmethod
    def config_parse(conf=None, args=None):
        """Create the default configurations."""
        # neutron.conf.test includes rpc_backend which needs to be cleaned up
        if args is None:
            args = ['--config-file', etcdir('neutron.conf.test')]
        if conf is None:
            config.init(args=args)
        else:
            conf(args)

    def setUp(self):
        super(BaseTestCase, self).setUp()

        # Configure this first to ensure pm debugging support for setUp()
        debugger = os.environ.get('OS_POST_MORTEM_DEBUGGER')
        if debugger:
            self.addOnException(post_mortem_debug.get_exception_handler(
                debugger))

        if os.environ.get('OS_DEBUG') in TRUE_STRING:
            _level = std_logging.DEBUG
        else:
            _level = std_logging.INFO
        capture_logs = os.environ.get('OS_LOG_CAPTURE') in TRUE_STRING
        if not capture_logs:
            std_logging.basicConfig(format=LOG_FORMAT, level=_level)
        self.log_fixture = self.useFixture(
            fixtures.FakeLogger(
                format=LOG_FORMAT,
                level=_level,
                nuke_handlers=capture_logs,
            ))

        # suppress all but errors here
        self.useFixture(
            fixtures.FakeLogger(
                name='neutron.api.extensions',
                format=LOG_FORMAT,
                level=std_logging.ERROR,
                nuke_handlers=capture_logs,
            ))

        test_timeout = int(os.environ.get('OS_TEST_TIMEOUT', 0))
        if test_timeout == -1:
            test_timeout = 0
        if test_timeout > 0:
            self.useFixture(fixtures.Timeout(test_timeout, gentle=True))

        # If someone does use tempfile directly, ensure that it's cleaned up
        self.useFixture(fixtures.NestedTempfile())
        self.useFixture(fixtures.TempHomeDir())

        self.temp_dir = self.useFixture(fixtures.TempDir()).path
        cfg.CONF.set_override('state_path', self.temp_dir)

        self.addCleanup(mock.patch.stopall)
        self.addCleanup(CONF.reset)

        if os.environ.get('OS_STDOUT_CAPTURE') in TRUE_STRING:
            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
        if os.environ.get('OS_STDERR_CAPTURE') in TRUE_STRING:
            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
        self.useFixture(fixtures.MonkeyPatch(
            'neutron.common.exceptions.NeutronException.use_fatal_exceptions',
            fake_use_fatal_exceptions))

        self.setup_rpc_mocks()

        if sys.version_info < (2, 7) and getattr(self, 'fmt', '') == 'xml':
            raise self.skipException('XML Testing Skipped in Py26')

        self.setup_config()
        self.addOnException(self.check_for_systemexit)

    def setup_rpc_mocks(self):
        # don't actually start RPC listeners when testing
        self.useFixture(fixtures.MonkeyPatch(
            'neutron.common.rpc.Connection.consume_in_threads',
            fake_consume_in_threads))

        # immediately return RPC calls
        self.useFixture(fixtures.MonkeyPatch(
            'neutron.common.rpc.RpcProxy._RpcProxy__call_rpc_method',
            mock.MagicMock()))

        self.useFixture(fixtures.MonkeyPatch(
            'oslo.messaging.Notifier', fake_notifier.FakeNotifier))

        self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
        self.messaging_conf.transport_driver = 'fake'
        # NOTE(russellb) We want all calls to return immediately.
        self.messaging_conf.response_timeout = 0
        self.useFixture(self.messaging_conf)

        self.addCleanup(n_rpc.clear_extra_exmods)
        n_rpc.add_extra_exmods('neutron.test')

        self.addCleanup(n_rpc.cleanup)
        n_rpc.init(CONF)

    def check_for_systemexit(self, exc_info):
        if isinstance(exc_info[1], SystemExit):
            self.fail("A SystemExit was raised during the test. %s"
                      % traceback.format_exception(*exc_info))

    def setup_config(self):
        """Tests that need a non-default config can override this method."""
        self.config_parse()

    def config(self, **kw):
        """Override some configuration values.

        The keyword arguments are the names of configuration options to
        override and their values.

        If a group argument is supplied, the overrides are applied to
        the specified configuration option group.

        All overrides are automatically cleared at the end of the current
        test by the fixtures cleanup process.
        """
        group = kw.pop('group', None)
        for k, v in kw.iteritems():
            CONF.set_override(k, v, group)

    @contextlib.contextmanager
    def assert_max_execution_time(self, max_execution_time=5):
        with eventlet.timeout.Timeout(max_execution_time, False):
            yield
            return
        self.fail('Execution of this test timed out')

    def assertOrderedEqual(self, expected, actual):
        expect_val = self.sort_dict_lists(expected)
        actual_val = self.sort_dict_lists(actual)
        self.assertEqual(expect_val, actual_val)

    def sort_dict_lists(self, dic):
        for key, value in dic.iteritems():
            if isinstance(value, list):
                dic[key] = sorted(value)
            elif isinstance(value, dict):
                dic[key] = self.sort_dict_lists(value)
        return dic

    def assertDictSupersetOf(self, expected_subset, actual_superset):
        """Checks that actual dict contains the expected dict.

        After checking that the arguments are of the right type, this checks
        that each item in expected_subset is in, and matches, what is in
        actual_superset. Separate tests are done, so that detailed info can
        be reported upon failure.
        """
        if not isinstance(expected_subset, dict):
            self.fail("expected_subset (%s) is not an instance of dict" %
                      type(expected_subset))
        if not isinstance(actual_superset, dict):
            self.fail("actual_superset (%s) is not an instance of dict" %
                      type(actual_superset))
        for k, v in expected_subset.items():
            self.assertIn(k, actual_superset)
            self.assertEqual(v, actual_superset[k],
                             "Key %(key)s expected: %(exp)r, actual %(act)r" %
                             {'key': k, 'exp': v, 'act': actual_superset[k]})
<|endoftext|>


<commit_before>__author__ = 'Aaron Anderson'
__version__ = "1.0"
from nexradawsinterface import NexradAwsInterface<commit_msg>made import in __init__ relative as tests are still having problems in 3.6<commit_after>__author__ = 'Aaron Anderson'
__version__ = "1.0"
from .nexradawsinterface import NexradAwsInterface
<|endoftext|>


<commit_before>import sys
import argparse
import requests
import urllib
try:
	from urllib.request import urlopen
except ImportError:
	from urllib2 import urlopen
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from bs4 import BeautifulSoup
from .downloader import download_series, download_parallel
from .utils import FILE_EXTENSIONS, THREAT_EXTENSIONS, DEFAULTS

search_url = "https://www.google.com/search"

s = requests.Session()
# Max retries and back-off strategy so all requests to http:// sleep before retrying
retries = Retry(total=5,
				backoff_factor=0.1,
				status_forcelist=[ 500, 502, 503, 504 ])
s.mount('http://', HTTPAdapter(max_retries=retries))

def scrape(html):
	"""
	function to scrape file links from html response
	"""
	soup = BeautifulSoup(html, 'lxml')
	results = soup.findAll('h3', {'class': 'r'})
	links = []
	for result in results:
		link = result.a['href'][7:].split('&')[0]
		links.append(link)
	return links

def get_links(limit, params, headers):
	"""
	function to fetch links equal to limit

	every Google search result page has a start index.
	every page contains 10 search results.
	"""
	links = []
	for start_index in range(0, limit, 10):
		params['start'] = start_index
		resp = s.get(search_url, params = params, headers = headers)
		page_links = scrape(resp.content)
		links.extend(page_links)
	return links[:limit]

def get_url_nofollow(url):
	"""
	function to get return code of a url

	Credits: http://blog.jasonantman.com/2013/06/python-script-to-check-a-list-of-urls-for-return-code-and-final-return-code-if-redirected/
	"""
	try:
		response = urlopen(url)
		code = response.getcode()
		return code
	except urllib.error.HTTPError as e:
		return e.code
	except:
		return 0

def validate_links(links):
	"""
	function to validate urls based on http(s) prefix and return code
	"""
	valid_links = []
	for link in links:
		if link[:7] in "http://" or link[:8] in "https://":
			valid_links.append(link)

	# checking valid urls for return code
	urls = {}
	for link in valid_links:
		urls[link] = {'code': get_url_nofollow(link)}

	# printing valid urls with return code 200
	available_urls = []
	for url in urls:
		print("code: %d\turl: %s" % (urls[url]['code'], url))
		if urls[url]['code'] != 0:
			available_urls.append(url)

	return available_urls

def search(query, website, file_type = 'pdf', limit = 10):
	"""
	main function to search for links and return valid ones
	"""
	gquery = "filetype:{0} {1} site:{2}".format(file_type, query, website)
	params = {
		'q': gquery,
		'start': 0,
	}
	headers = {
		'User Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:53.0) \
		Gecko/20100101 Firefox/53.0'
	}
	links = get_links(limit, params, headers)
	valid_links = validate_links(links)
	return valid_links

def check_threats(**args):
	"""
	function to check input filetype against threat extensions list
	"""
	is_high_threat = False
	for val in THREAT_EXTENSIONS.values():
		if type(val) == list:
			for el in val:
				if args['file_type'] == el:
					is_high_threat = True
					break
		else:
			if args['file_type'] == val:
				is_high_threat = True
				break
	return is_high_threat

def validate_args(**args):
	"""
	function to check if input query is not None
	and set missing arguments to default value
	"""
	if not args['query']:
		print("\nMissing required query argument.")
		sys.exit()

	for key in DEFAULTS:
		if key not in args:
			args[key] = DEFAULTS[key]

	return args

def download_content(**args):
	"""
	main function to fetch links and download them
	"""
	args = validate_args(**args)

	if not args['directory']:
		args['directory'] = args['query'].replace(' ', '-')

	print("Downloading {0} {1} files on topic {2} from {3} and saving to directory: {4}"
		.format(args['limit'], args['file_type'], args['query'], args['website'], args['directory']))

	links = search(args['query'], args['website'], args['file_type'], args['limit'])

	if args['parallel']:
		download_parallel(links, args['directory'], args['min_file_size'], args['max_file_size'], args['no_redirects'])
	else:
		download_series(links, args['directory'], args['min_file_size'], args['max_file_size'], args['no_redirects'])

def show_filetypes(extensions):
	"""
	function to show valid file extensions
	"""
	for item in extensions.items():
		val = item[1]
		if type(item[1]) == list:
			val = ", ".join(str(x) for x in item[1])
		print("{0:4}: {1}".format(val, item[0]))

def main():
	parser = argparse.ArgumentParser(description = "Content Downloader",
									 epilog="Now download files on any topic in bulk!")

	# defining arguments for parser object
	parser.add_argument("query", type = str, default = None, nargs = '?',
						help = "Specify the query.")

	parser.add_argument("-f", "--file_type", type = str, default = 'pdf',
						help = "Specify the extension of files to download.")

	parser.add_argument("-l", "--limit", type = int, default = 10,
						help = "Limit the number of search results (in multiples of 10).")

	parser.add_argument("-d", "--directory", type = str, default = None,
						help = "Specify directory where files will be stored.")

	parser.add_argument("-p", "--parallel", action = 'store_true', default = False,
						help = "For parallel downloading.")

	parser.add_argument("-a", "--available", action='store_true',
						help = "Get list of all available filetypes.")

	parser.add_argument("-t", "--threats", action='store_true',
						help = "Get list of all common virus carrier filetypes.")

	parser.add_argument("-minfs", "--min-file-size", type = int, default = 0,
						help = "Specify minimum file size to download in Kilobytes (KB).")

	parser.add_argument("-maxfs", "--max-file-size", type = int, default = -1,
						help = "Specify maximum file size to download in Kilobytes (KB).")

	parser.add_argument("-nr", "--no-redirects", action = 'store_true', default = False,
						help = "Prevent download redirects.")

	parser.add_argument("-w", "--website", default = "google.com", type = str,
						help = "Specify a particular website to download content from.")

	args = parser.parse_args()
	args_dict = vars(args)

	if args.available:
		show_filetypes(FILE_EXTENSIONS)
		return

	if args.threats:
		show_filetypes(THREAT_EXTENSIONS)
		return

	high_threat = check_threats(**args_dict)

	if high_threat:
		def prompt(message, errormessage, isvalid, isexit):
			res = None
			while res is None:
				res = input(str(message)+': ')
				if isexit(res):
					sys.exit()
				if not isvalid(res):
					print(str(errormessage))
					res = None
			return res
		prompt(
			message = "WARNING: Downloading this file type may expose you to a heightened security risk.\nPress 'y' to proceed or 'n' to exit",
			errormessage= "Error: Invalid option provided.",
			isvalid = lambda x:True if x is 'y' else None,
			isexit = lambda x:True if x is 'n' else None
		)

	download_content(**args_dict)

if __name__ == "__main__":
	main()
<commit_msg>Fixed Github pdf download issue.<commit_after>import sys
import argparse
import requests
import urllib
try:
	from urllib.request import urlopen
except ImportError:
	from urllib2 import urlopen
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from bs4 import BeautifulSoup
from .downloader import download_series, download_parallel
from .utils import FILE_EXTENSIONS, THREAT_EXTENSIONS, DEFAULTS

search_url = "https://www.google.com/search"

s = requests.Session()
# Max retries and back-off strategy so all requests to http:// sleep before retrying
retries = Retry(total=5,
				backoff_factor=0.1,
				status_forcelist=[ 500, 502, 503, 504 ])
s.mount('http://', HTTPAdapter(max_retries=retries))

def scrape(html):
	"""
	function to scrape file links from html response
	"""
	soup = BeautifulSoup(html, 'lxml')
	results = soup.findAll('h3', {'class': 'r'})
	links = []
	for result in results:
		link = result.a['href'][7:].split('&')[0]
		links.append(link)
	return links

def get_links(limit, params, headers):
	"""
	function to fetch links equal to limit

	every Google search result page has a start index.
	every page contains 10 search results.
	"""
	links = []
	for start_index in range(0, limit, 10):
		params['start'] = start_index
		resp = s.get(search_url, params = params, headers = headers)
		page_links = scrape(resp.content)
		links.extend(page_links)
	return links[:limit]

def get_url_nofollow(url):
	"""
	function to get return code of a url

	Credits: http://blog.jasonantman.com/2013/06/python-script-to-check-a-list-of-urls-for-return-code-and-final-return-code-if-redirected/
	"""
	try:
		response = urlopen(url)
		code = response.getcode()
		return code
	except urllib.error.HTTPError as e:
		return e.code
	except:
		return 0

def validate_links(links, website):
	"""
	function to validate urls based on http(s) prefix and return code
	"""
	valid_links = []
	for link in links:
		if link[:7] in "http://" or link[:8] in "https://":
			valid_links.append(link)

	# checking valid urls for return code
	urls = {}
	for link in valid_links:
		urls[link] = {'code': get_url_nofollow(link)}
		if website == 'github.com' and '/blob/' in link:
			link = link.replace('/blob/', '/raw/')

	# printing valid urls with return code 200
	available_urls = []
	for url in urls:
		print("code: %d\turl: %s" % (urls[url]['code'], url))
		if urls[url]['code'] != 0:
			available_urls.append(url)

	return available_urls

def search(query, website, file_type = 'pdf', limit = 10):
	"""
	main function to search for links and return valid ones
	"""
	gquery = "filetype:{0} {1} site:{2}".format(file_type, query, website)
	params = {
		'q': gquery,
		'start': 0,
	}
	headers = {
		'User Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:53.0) \
		Gecko/20100101 Firefox/53.0'
	}
	links = get_links(limit, params, headers)
	valid_links = validate_links(links, website)
	return valid_links

def check_threats(**args):
	"""
	function to check input filetype against threat extensions list
	"""
	is_high_threat = False
	for val in THREAT_EXTENSIONS.values():
		if type(val) == list:
			for el in val:
				if args['file_type'] == el:
					is_high_threat = True
					break
		else:
			if args['file_type'] == val:
				is_high_threat = True
				break
	return is_high_threat

def validate_args(**args):
	"""
	function to check if input query is not None
	and set missing arguments to default value
	"""
	if not args['query']:
		print("\nMissing required query argument.")
		sys.exit()

	for key in DEFAULTS:
		if key not in args:
			args[key] = DEFAULTS[key]

	return args

def download_content(**args):
	"""
	main function to fetch links and download them
	"""
	args = validate_args(**args)

	if not args['directory']:
		args['directory'] = args['query'].replace(' ', '-')

	print("Downloading {0} {1} files on topic {2} from {3} and saving to directory: {4}"
		.format(args['limit'], args['file_type'], args['query'], args['website'], args['directory']))

	links = search(args['query'], args['website'], args['file_type'], args['limit'])

	if args['parallel']:
		download_parallel(links, args['directory'], args['min_file_size'], args['max_file_size'], args['no_redirects'])
	else:
		download_series(links, args['directory'], args['min_file_size'], args['max_file_size'], args['no_redirects'])

def show_filetypes(extensions):
	"""
	function to show valid file extensions
	"""
	for item in extensions.items():
		val = item[1]
		if type(item[1]) == list:
			val = ", ".join(str(x) for x in item[1])
		print("{0:4}: {1}".format(val, item[0]))

def main():
	parser = argparse.ArgumentParser(description = "Content Downloader",
									 epilog="Now download files on any topic in bulk!")

	# defining arguments for parser object
	parser.add_argument("query", type = str, default = None, nargs = '?',
						help = "Specify the query.")

	parser.add_argument("-f", "--file_type", type = str, default = 'pdf',
						help = "Specify the extension of files to download.")

	parser.add_argument("-l", "--limit", type = int, default = 10,
						help = "Limit the number of search results (in multiples of 10).")

	parser.add_argument("-d", "--directory", type = str, default = None,
						help = "Specify directory where files will be stored.")

	parser.add_argument("-p", "--parallel", action = 'store_true', default = False,
						help = "For parallel downloading.")

	parser.add_argument("-a", "--available", action='store_true',
						help = "Get list of all available filetypes.")

	parser.add_argument("-t", "--threats", action='store_true',
						help = "Get list of all common virus carrier filetypes.")

	parser.add_argument("-minfs", "--min-file-size", type = int, default = 0,
						help = "Specify minimum file size to download in Kilobytes (KB).")

	parser.add_argument("-maxfs", "--max-file-size", type = int, default = -1,
						help = "Specify maximum file size to download in Kilobytes (KB).")

	parser.add_argument("-nr", "--no-redirects", action = 'store_true', default = False,
						help = "Prevent download redirects.")

	parser.add_argument("-w", "--website", default = "google.com", type = str,
						help = "Specify a particular website to download content from.")

	args = parser.parse_args()
	args_dict = vars(args)

	if args.available:
		show_filetypes(FILE_EXTENSIONS)
		return

	if args.threats:
		show_filetypes(THREAT_EXTENSIONS)
		return

	high_threat = check_threats(**args_dict)

	if high_threat:
		def prompt(message, errormessage, isvalid, isexit):
			res = None
			while res is None:
				res = input(str(message)+': ')
				if isexit(res):
					sys.exit()
				if not isvalid(res):
					print(str(errormessage))
					res = None
			return res
		prompt(
			message = "WARNING: Downloading this file type may expose you to a heightened security risk.\nPress 'y' to proceed or 'n' to exit",
			errormessage= "Error: Invalid option provided.",
			isvalid = lambda x:True if x is 'y' else None,
			isexit = lambda x:True if x is 'n' else None
		)

	download_content(**args_dict)

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before># Copyright (c) 2015 Metaswitch Networks
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from subprocess import call, check_output, check_call, CalledProcessError
import socket
import logging
import logging.handlers
import os
import sys
import uuid

from netaddr import IPNetwork, IPAddress

from datastore import Endpoint, IF_PREFIX, VETH_NAME

_log = logging.getLogger(__name__)

HOSTNAME = socket.gethostname()

ROOT_NETNS = "1"
"""The pid of the root namespace.  On almost all systems, the init system is
pid 1
"""

PREFIX_LEN = {4: 32, 6: 128}
"""The IP address prefix length to assign, by IP version."""

PROC_ALIAS = "/proc_host"
"""The alias for /proc.  This is useful when the filesystem is containerized.
"""

def setup_logging(logfile):
    _log.setLevel(logging.DEBUG)
    formatter = logging.Formatter(
        '%(asctime)s [%(levelname)s] %(name)s %(lineno)d: %(message)s')
    handler = logging.StreamHandler(sys.stdout)
    handler.setLevel(logging.INFO)
    handler.setFormatter(formatter)
    _log.addHandler(handler)
    handler = logging.handlers.TimedRotatingFileHandler(logfile,
                                                        when='D',
                                                        backupCount=10)
    handler.setLevel(logging.DEBUG)
    handler.setFormatter(formatter)
    _log.addHandler(handler)

def remove_endpoint(ep_id):
    """
    Remove an endpoint.

    :param ep_id: The endpoint ID to remove
    :return: Nothing
    """
    iface = IF_PREFIX + ep_id[:11]
    call("ip link delete %s" % iface, shell=True)

def add_ip_to_interface(container_pid, ip, interface_name,
                        proc_alias=PROC_ALIAS):
    """
    Add an IP to an interface in a container.

    :param container_pid: The PID of the namespace to operate in.
    :param ip: The IPAddress to add.
    :param interface_name: The interface to add the address to.
    :param proc_alias: The location of the /proc filesystem on the host.
    :return: None. raises CalledProcessError on error.
    """
    with NamedNamespace(container_pid, proc=proc_alias) as ns:
        ns.check_call("ip -%(version)s addr add %(addr)s/%(len)s "
                      "dev %(device)s" %
                        {"version": ip.version,
                         "len": PREFIX_LEN[ip.version],
                         "addr": ip,
                         "device": interface_name},
                      shell=True)

def remove_ip_from_interface(container_pid, ip, interface_name,
                             proc_alias=PROC_ALIAS):
    """
    Remove an IP from an interface in a container.

    :param container_pid: The PID of the namespace to operate in.
    :param ip: The IPAddress to remove.
    :param interface_name: The interface to remove the address from.
    :param proc_alias: The location of the /proc filesystem on the host.
    :return: None. raises CalledProcessError on error.
    """
    with NamedNamespace(container_pid, proc=proc_alias) as ns:
        ns.check_call("ip -%(version)s addr del %(addr)s/%(len)s "
                      "dev %(device)s" %
                        {"version": ip.version,
                         "len": PREFIX_LEN[ip.version],
                         "addr": ip,
                         "device": interface_name},
                      shell=True)

def set_up_endpoint(ip, cpid, next_hop_ips,
                    veth_name=VETH_NAME,
                    proc_alias=PROC_ALIAS,
                    ep_id=None,
                    mac=None):
    """
    Set up an endpoint (veth) in the network namespace identified by the PID.

    :param ip: The IP address to assign to the endpoint (veth) as Netaddr
    IPAddress.
    :param cpid: The PID of a process currently running in the namespace.
    :param next_hop_ips: Dict of {version: IPAddress} for the next hops of the
    default routes namespace, as opposed to the root namespace.  If so, this
    method also moves the other end of the veth into the root namespace.
    :param veth_name: The name of the interface inside the container namespace,
    e.g. eth1
    :param proc_alias: The location of the /proc filesystem on the host.
    :param ep_id: The endpoint ID to use.  Set to None if this is a new
    endpoint, or set to the existing endpoint ID that is being re-added.
    :param mac: The interface MAC to use.  Set to None to auto assign a MAC.
    :return: An Endpoint describing the veth just created.
    """
    assert isinstance(ip, IPAddress)

    # Generate a new endpoint ID if required.
    ep_id = ep_id or uuid.uuid1().hex

    iface = IF_PREFIX + ep_id[:11]
    iface_tmp = "tmp" + ep_id[:11]

    # Provision the networking.  We create a temporary link from the proc
    # alias to the /var/run/netns to provide a named namespace.  If we don't
    # do this, when run from the calico-node container the PID of the
    # container process is not recognised by `ip link set <if> netns <pid>`
    # command because that uses /proc rather than the proc alias to
    # dereference the PID.
    with NamedNamespace(cpid, proc=proc_alias) as ns:
        # Create the veth pair and move one end into container:
        check_call("ip link add %s type veth peer name %s" %
                     (iface, iface_tmp),
                   shell=True)
        check_call("ip link set %s up" % iface, shell=True)
        check_call("ip link set %s netns %s" % (iface_tmp, ns.name),
                   shell=True)
        _log.debug(check_output("ip link", shell=True))

    with Namespace(cpid, 'net', proc=proc_alias):
        if mac:
            ns.check_call("ip link set dev %s name %s address %s" %
                            (iface_tmp, veth_name, str(mac)),
                          shell=True)
        else:
            ns.check_call("ip link set dev %s name %s" %
                            (iface_tmp, veth_name),
                          shell=True)
        ns.check_call("ip link set %s up" % veth_name, shell=True)

    # Add an IP address.
    add_ip_to_interface(cpid, ip, veth_name, proc_alias=proc_alias)

    with NamedNamespace(cpid, proc=proc_alias) as ns:
        # Connected route to next hop & default route.
        next_hop = next_hop_ips[ip.version]
        ns.check_call("ip -%(version)s route replace"
                      " %(next_hop)s dev %(device)s" %
                      {"version": ip.version,
                       "device": veth_name,
                       "next_hop": next_hop},
                      shell=True)
        ns.check_call("ip -%(version)s route replace"
                      " default via %(next_hop)s dev %(device)s" %
                      {"version": ip.version,
                       "device": veth_name,
                       "next_hop": next_hop},
                      shell=True)

        # Get the MAC address.
        mac = ns.check_output(
                "ip link show %s | grep ether | awk '{print $2}'" %
                (veth_name), shell=True).strip()

    # Return an Endpoint.
    network = IPNetwork(IPAddress(ip))
    ep = Endpoint(ep_id=ep_id, state="active", mac=mac, if_name=veth_name)
    if network.version == 4:
        ep.ipv4_nets.add(network)
        ep.ipv4_gateway = next_hop
    else:
        ep.ipv6_nets.add(network)
        ep.ipv6_gateway = next_hop
    return ep

def reinstate_endpoint(cpid, old_endpoint, next_hop_ips,
                       proc_alias=PROC_ALIAS):
    """
    Re-instate and endpoint that has been removed.
    :param cpid: The PID of the namespace to operate in.
    :param old_endpoint: The old endpoint that is being re-instated.
    :param next_hop_ips: Dict of {version: IPAddress} for the next hops of the
    default routes namespace.
    :param proc_alias: The location of the /proc filesystem on the host.
    :return: A new Endpoint replacing the old one.
    """
    nets = old_endpoint.ipv4_nets | old_endpoint.ipv6_nets
    if_name = old_endpoint.if_name
    net = nets.pop()
    new_endpoint = set_up_endpoint(ip=net.ip,
                                   cpid=cpid,
                                   next_hop_ips=next_hop_ips,
                                   veth_name=if_name,
                                   proc_alias=proc_alias,
                                   ep_id=old_endpoint.ep_id,
                                   mac=old_endpoint.mac)
    for net in nets:
        add_ip_to_interface(cpid, net.ip, if_name, proc_alias=proc_alias)

    # Copy across the IP and profile data from the old endpoint since this is
    # unchanged.
    new_endpoint.ipv4_nets = set(old_endpoint.ipv4_nets)
    new_endpoint.ipv6_nets = set(old_endpoint.ipv6_nets)
    new_endpoint.profile_id = old_endpoint.profile_id

    return new_endpoint

class NamedNamespace(object):
    """
    Create a named namespace to allow commands to be run within the namespace
    in both the calico-node and within the root namespace.
    """
    def __init__(self, cpid, proc=PROC_ALIAS):
        self.name = uuid.uuid1().hex
        self.pid_dir = "%s/%s/ns/net" % (proc, cpid)
        self.nsn_dir = "/var/run/netns/%s" % self.name

    def __enter__(self):
        """
        Add the appropriate configuration to name the namespace.  This links
        the PID to the namespace name.
        """
        _log.debug("Creating link between ns name and PID")
        try:
            os.makedirs("/var/run/netns")
        except os.error:
            _log.info("Unable to create /var/run/netns dir")
        os.symlink(self.pid_dir, self.nsn_dir)
        return self

    def __exit__(self, _type, _value, _traceback):
        try:
            os.unlink(self.nsn_dir)
        except BaseException:
            _log.exception("Failed to remove link: %s", self.nsn_dir)
        return False

    def check_call(self, command, shell=False):
        """
        Run a command within the named namespace.
        :param command: The command to run.
        :param shell: Whether this is a shell command.
        """
        _log.debug("Run command: %s", command)
        check_call("ip netns exec %s %s" % (self.name, command), shell=shell)

    def check_output(self, command, shell=False):
        """
        Run a command within the named namespace.
        :param command: The command to run.
        :param shell: Whether this is a shell command.
        """
        _log.debug("Run command: %s", command)
        return check_output("ip netns exec %s %s" % (self.name, command),
                            shell=shell)
<commit_msg>Fix merge error<commit_after># Copyright (c) 2015 Metaswitch Networks
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from subprocess import call, check_output, check_call, CalledProcessError
import socket
import logging
import logging.handlers
import os
import sys
import uuid

from netaddr import IPNetwork, IPAddress

from datastore import Endpoint, IF_PREFIX, VETH_NAME

_log = logging.getLogger(__name__)

HOSTNAME = socket.gethostname()

ROOT_NETNS = "1"
"""The pid of the root namespace.  On almost all systems, the init system is
pid 1
"""

PREFIX_LEN = {4: 32, 6: 128}
"""The IP address prefix length to assign, by IP version."""

PROC_ALIAS = "/proc_host"
"""The alias for /proc.  This is useful when the filesystem is containerized.
"""

def setup_logging(logfile):
    _log.setLevel(logging.DEBUG)
    formatter = logging.Formatter(
        '%(asctime)s [%(levelname)s] %(name)s %(lineno)d: %(message)s')
    handler = logging.StreamHandler(sys.stdout)
    handler.setLevel(logging.INFO)
    handler.setFormatter(formatter)
    _log.addHandler(handler)
    handler = logging.handlers.TimedRotatingFileHandler(logfile,
                                                        when='D',
                                                        backupCount=10)
    handler.setLevel(logging.DEBUG)
    handler.setFormatter(formatter)
    _log.addHandler(handler)

def remove_endpoint(ep_id):
    """
    Remove an endpoint.

    :param ep_id: The endpoint ID to remove
    :return: Nothing
    """
    iface = IF_PREFIX + ep_id[:11]
    call("ip link delete %s" % iface, shell=True)

def add_ip_to_interface(container_pid, ip, interface_name,
                        proc_alias=PROC_ALIAS):
    """
    Add an IP to an interface in a container.

    :param container_pid: The PID of the namespace to operate in.
    :param ip: The IPAddress to add.
    :param interface_name: The interface to add the address to.
    :param proc_alias: The location of the /proc filesystem on the host.
    :return: None. raises CalledProcessError on error.
    """
    with NamedNamespace(container_pid, proc=proc_alias) as ns:
        ns.check_call("ip -%(version)s addr add %(addr)s/%(len)s "
                      "dev %(device)s" %
                        {"version": ip.version,
                         "len": PREFIX_LEN[ip.version],
                         "addr": ip,
                         "device": interface_name},
                      shell=True)

def remove_ip_from_interface(container_pid, ip, interface_name,
                             proc_alias=PROC_ALIAS):
    """
    Remove an IP from an interface in a container.

    :param container_pid: The PID of the namespace to operate in.
    :param ip: The IPAddress to remove.
    :param interface_name: The interface to remove the address from.
    :param proc_alias: The location of the /proc filesystem on the host.
    :return: None. raises CalledProcessError on error.
    """
    with NamedNamespace(container_pid, proc=proc_alias) as ns:
        ns.check_call("ip -%(version)s addr del %(addr)s/%(len)s "
                      "dev %(device)s" %
                        {"version": ip.version,
                         "len": PREFIX_LEN[ip.version],
                         "addr": ip,
                         "device": interface_name},
                      shell=True)

def set_up_endpoint(ip, cpid, next_hop_ips,
                    veth_name=VETH_NAME,
                    proc_alias=PROC_ALIAS,
                    ep_id=None,
                    mac=None):
    """
    Set up an endpoint (veth) in the network namespace identified by the PID.

    :param ip: The IP address to assign to the endpoint (veth) as Netaddr
    IPAddress.
    :param cpid: The PID of a process currently running in the namespace.
    :param next_hop_ips: Dict of {version: IPAddress} for the next hops of the
    default routes namespace, as opposed to the root namespace.  If so, this
    method also moves the other end of the veth into the root namespace.
    :param veth_name: The name of the interface inside the container namespace,
    e.g. eth1
    :param proc_alias: The location of the /proc filesystem on the host.
    :param ep_id: The endpoint ID to use.  Set to None if this is a new
    endpoint, or set to the existing endpoint ID that is being re-added.
    :param mac: The interface MAC to use.  Set to None to auto assign a MAC.
    :return: An Endpoint describing the veth just created.
    """
    assert isinstance(ip, IPAddress)

    # Generate a new endpoint ID if required.
    ep_id = ep_id or uuid.uuid1().hex

    iface = IF_PREFIX + ep_id[:11]
    iface_tmp = "tmp" + ep_id[:11]

    # Provision the networking.  We create a temporary link from the proc
    # alias to the /var/run/netns to provide a named namespace.  If we don't
    # do this, when run from the calico-node container the PID of the
    # container process is not recognised by `ip link set <if> netns <pid>`
    # command because that uses /proc rather than the proc alias to
    # dereference the PID.
    with NamedNamespace(cpid, proc=proc_alias) as ns:
        # Create the veth pair and move one end into container:
        check_call("ip link add %s type veth peer name %s" %
                     (iface, iface_tmp),
                   shell=True)
        check_call("ip link set %s up" % iface, shell=True)
        check_call("ip link set %s netns %s" % (iface_tmp, ns.name),
                   shell=True)
        _log.debug(check_output("ip link", shell=True))

        if mac:
            ns.check_call("ip link set dev %s name %s address %s" %
                            (iface_tmp, veth_name, str(mac)),
                          shell=True)
        else:
            ns.check_call("ip link set dev %s name %s" %
                            (iface_tmp, veth_name),
                          shell=True)
        ns.check_call("ip link set %s up" % veth_name, shell=True)

    # Add an IP address.
    add_ip_to_interface(cpid, ip, veth_name, proc_alias=proc_alias)

    with NamedNamespace(cpid, proc=proc_alias) as ns:
        # Connected route to next hop & default route.
        next_hop = next_hop_ips[ip.version]
        ns.check_call("ip -%(version)s route replace"
                      " %(next_hop)s dev %(device)s" %
                      {"version": ip.version,
                       "device": veth_name,
                       "next_hop": next_hop},
                      shell=True)
        ns.check_call("ip -%(version)s route replace"
                      " default via %(next_hop)s dev %(device)s" %
                      {"version": ip.version,
                       "device": veth_name,
                       "next_hop": next_hop},
                      shell=True)

        # Get the MAC address.
        mac = ns.check_output(
                "ip link show %s | grep ether | awk '{print $2}'" %
                (veth_name), shell=True).strip()

    # Return an Endpoint.
    network = IPNetwork(IPAddress(ip))
    ep = Endpoint(ep_id=ep_id, state="active", mac=mac, if_name=veth_name)
    if network.version == 4:
        ep.ipv4_nets.add(network)
        ep.ipv4_gateway = next_hop
    else:
        ep.ipv6_nets.add(network)
        ep.ipv6_gateway = next_hop
    return ep

def reinstate_endpoint(cpid, old_endpoint, next_hop_ips,
                       proc_alias=PROC_ALIAS):
    """
    Re-instate and endpoint that has been removed.
    :param cpid: The PID of the namespace to operate in.
    :param old_endpoint: The old endpoint that is being re-instated.
    :param next_hop_ips: Dict of {version: IPAddress} for the next hops of the
    default routes namespace.
    :param proc_alias: The location of the /proc filesystem on the host.
    :return: A new Endpoint replacing the old one.
    """
    nets = old_endpoint.ipv4_nets | old_endpoint.ipv6_nets
    if_name = old_endpoint.if_name
    net = nets.pop()
    new_endpoint = set_up_endpoint(ip=net.ip,
                                   cpid=cpid,
                                   next_hop_ips=next_hop_ips,
                                   veth_name=if_name,
                                   proc_alias=proc_alias,
                                   ep_id=old_endpoint.ep_id,
                                   mac=old_endpoint.mac)
    for net in nets:
        add_ip_to_interface(cpid, net.ip, if_name, proc_alias=proc_alias)

    # Copy across the IP and profile data from the old endpoint since this is
    # unchanged.
    new_endpoint.ipv4_nets = set(old_endpoint.ipv4_nets)
    new_endpoint.ipv6_nets = set(old_endpoint.ipv6_nets)
    new_endpoint.profile_id = old_endpoint.profile_id

    return new_endpoint

class NamedNamespace(object):
    """
    Create a named namespace to allow commands to be run within the namespace
    in both the calico-node and within the root namespace.
    """
    def __init__(self, cpid, proc=PROC_ALIAS):
        self.name = uuid.uuid1().hex
        self.pid_dir = "%s/%s/ns/net" % (proc, cpid)
        self.nsn_dir = "/var/run/netns/%s" % self.name

    def __enter__(self):
        """
        Add the appropriate configuration to name the namespace.  This links
        the PID to the namespace name.
        """
        _log.debug("Creating link between ns name and PID")
        try:
            os.makedirs("/var/run/netns")
        except os.error:
            _log.info("Unable to create /var/run/netns dir")
        os.symlink(self.pid_dir, self.nsn_dir)
        return self

    def __exit__(self, _type, _value, _traceback):
        try:
            os.unlink(self.nsn_dir)
        except BaseException:
            _log.exception("Failed to remove link: %s", self.nsn_dir)
        return False

    def check_call(self, command, shell=False):
        """
        Run a command within the named namespace.
        :param command: The command to run.
        :param shell: Whether this is a shell command.
        """
        _log.debug("Run command: %s", command)
        check_call("ip netns exec %s %s" % (self.name, command), shell=shell)

    def check_output(self, command, shell=False):
        """
        Run a command within the named namespace.
        :param command: The command to run.
        :param shell: Whether this is a shell command.
        """
        _log.debug("Run command: %s", command)
        return check_output("ip netns exec %s %s" % (self.name, command),
                            shell=shell)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Render curved 4d polychoron examples
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright (c) 2018 by Zhao Liang.
"""
import os
import subprocess
from fractions import Fraction
from polytopes.models import Polychora
from polytopes.povray import pov_vector, pov_index_array1d

IMAGE_DIR = "polychora_frames"        # directory to save the frames
POV_EXE = "povray"                    # POV-Ray exe binary
SCENE_FILE = "polychora_curved.pov"   # the main scene file
IMAGE_SIZE = 600                      # image size in pixels
FRAMES = 1                            # number of frames
IMAGE_QUALITY_LEVEL = 11              # between 0-11
SUPER_SAMPLING_LEVEL = 5              # between 1-9
ANTIALIASING_LEVEL = 0.001            # lower for better quality
DATAFILE_NAME = "polychora-data.inc"  # export data to this file

data_file = os.path.join(os.getcwd(), "povray", DATAFILE_NAME)

if not os.path.exists(IMAGE_DIR):
    os.makedirs(IMAGE_DIR)

POV_COMMAND = " cd povray && " + \
              " {} +I{}".format(POV_EXE, SCENE_FILE) + \
              " +W{} +H{}".format(IMAGE_SIZE, IMAGE_SIZE) + \
              " +Q{}".format(IMAGE_QUALITY_LEVEL) + \
              " +A{}".format(ANTIALIASING_LEVEL) + \
              " +R{}".format(SUPER_SAMPLING_LEVEL) + \
              " +KFI0" + \
              " +KFF{}".format(FRAMES - 1) + \
              " -V" + \
              " +O../{}/".format(IMAGE_DIR) + "{}"

POV_TEMPLATE = """
#declare bg_color = {};
#declare vertex_size = {};
#declare edge_size = {};
#declare camera_loc = {};
#declare obj_rotation = {};
#declare size_func = {};
#declare face_max= {};
#declare face_min = {};
#declare face_index = {};
#declare use_area_light = {};

// this macro is used for adjusting the size of edges
// according to their positions in the space.
#macro get_size(q)
  #local len = vlength(q);
  #if (size_func = 0)
    #local len = (1.0 + len * len) / 4;
  #else #if (size_func = 1)
    #local len = 2.0 * log(2.0 + len * len);
  #else
    #local len = 2.0 * log(1.13 + len * len);
  #end
  #end
  len
#end

#macro choose_face(i, face_size)
  #local chosen = false;
  #for (ind, 0, dimension_size(face_index, 1) - 1)
    #if (i = face_index[ind])
      #if (face_size > face_min & face_size < face_max)
        #local chosen = true;
      #end
    #end
  #end
  chosen
#end

#declare vertices = {};

#declare edges = {};

#declare faces = {};
"""

def draw(coxeter_diagram,
         trunc_type,
         extra_relations=(),
         description="polychora",
         bg_color="SkyBlue",
         camera_loc=(0, 0, 30),
         rotation=(0, 0, 0),
         vertex_size=0.04,
         edge_size=0.02,
         size_func=0,
         face_index=(0,),
         face_max=3.0,
         face_min=0.5,
         use_area_light=False):
    """
    Export data to povray .inc file and call the rendering process.

    :param camera_loc: location of the camera.

    :param size_func: choose which sizing funcion to use, currently only 0, 1, 2.

    :param face_index: a list controls which types of faces are shown.

    :param face_max: only faces smaller than this threshold are shown.

    :param face_min: only faces larger than this threshold are shown.
    """
    P = Polychora(coxeter_diagram, trunc_type, extra_relations)
    P.build_geometry()
    vert_data, edge_data, face_data = P.get_povray_data()
    with open(data_file, "w") as f:
        f.write(POV_TEMPLATE.format(
            bg_color,
            vertex_size,
            edge_size,
            pov_vector(camera_loc),
            pov_vector(rotation),
            size_func,
            face_max,
            face_min,
            pov_index_array1d(face_index),
            int(use_area_light),
            vert_data,
            edge_data,
            face_data))

    print("rendering {}: {} vertices, {} edges, {} faces".format(
        description,
        P.num_vertices,
        P.num_edges,
        P.num_faces))

    process = subprocess.Popen(
        POV_COMMAND.format(description),
        shell=True,
        stderr=subprocess.PIPE,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE)

    _, err = process.communicate()
    if process.returncode:
        print(type(err), err)
        raise IOError("POVRay error: " + err.decode("ascii"))

def main():
    draw((3, 2, 2, 3, 2, 3), (1, 0, 0, 0), description="5-cell",
         camera_loc=(0, 0, 120), vertex_size=0.08, rotation=(-30, 60, 0),
         edge_size=0.04, size_func=1)

    draw((4, 2, 2, 3, 2, 3), (1, 0, 0, 0), description="4d-cube",
         camera_loc=(0, 0, 130), vertex_size=0.06, rotation=(60, 0, 0),
         edge_size=0.03, size_func=1, face_min=0.2, face_max=0.8)

    draw((3, 2, 2, 3, 2, 4), (1, 0, 0, 0), description="16-cell",
         camera_loc=(0, 0, 160), vertex_size=0.08,
         edge_size=0.03, size_func=2, face_min=1.0, face_max=1.2)

    draw((3, 2, 2, 4, 2, 3), (1, 0, 0, 0), description="24-cell",
         camera_loc=(0, 0, 200), vertex_size=0.06,
         edge_size=0.04, size_func=2, face_min=0.2, face_max=0.8)

    draw((5, 2, 2, 3, 2, 3), (1, 0, 0, 0), description="120-cell",
         camera_loc=(0, 0, 400), vertex_size=0.05,
         edge_size=0.025, size_func=0, face_min=3.0, face_max=100.0)

    draw((3, 2, 2, 3, 2, 5), (1, 0, 0, 0), description="600-cell",
         bg_color="White", camera_loc=(0, 0, 500), vertex_size=0.12,
         edge_size=0.04, size_func=2, face_max=4.0, face_min=3.0)

    draw((3, 2, 2, 3, 2, 4), (1, 0, 0, 1), description="runcinated-16-cell",
         bg_color="White", camera_loc=(0, 0, 450), vertex_size=0.1,
         face_index=(0, 1, 2, 3), edge_size=0.04, size_func=1,
         face_min=0, face_max=3)

    draw((5, 2, 2, 3, 2, 3), (1, 0, 0, 1), description="runcinated-120-cell",
         camera_loc=(0, 0, 360), vertex_size=0.028, edge_size=0.014, face_min=20)

    # this is the settings I used to render the movie at
    # http://pywonderland.com/polytopes/rectified-grand-stellated-120-cell.mp4
    # (the parameters are not exactly the same but very similar)
    # take quite a while to render.
    draw((Fraction(5, 2), 2, 2, 5, 2, Fraction(5, 2)), (0, 1, 0, 0),
         extra_relations=((0, 1, 2, 1)*3, (1, 2, 3, 2)*3),
         description="rectified-grand-stellated-120-cell",
         size_func=1, vertex_size=0.06, edge_size=0.03,
         use_area_light=1, face_max=0.0, camera_loc=(0, 0, 400))

if __name__ == "__main__":
    main()
<commit_msg>fix broken link in comment<commit_after># -*- coding: utf-8 -*-
"""
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Render curved 4d polychoron examples
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright (c) 2018 by Zhao Liang.
"""
import os
import subprocess
from fractions import Fraction
from polytopes.models import Polychora
from polytopes.povray import pov_vector, pov_index_array1d

IMAGE_DIR = "polychora_frames"        # directory to save the frames
POV_EXE = "povray"                    # POV-Ray exe binary
SCENE_FILE = "polychora_curved.pov"   # the main scene file
IMAGE_SIZE = 600                      # image size in pixels
FRAMES = 1                            # number of frames
IMAGE_QUALITY_LEVEL = 11              # between 0-11
SUPER_SAMPLING_LEVEL = 5              # between 1-9
ANTIALIASING_LEVEL = 0.001            # lower for better quality
DATAFILE_NAME = "polychora-data.inc"  # export data to this file

data_file = os.path.join(os.getcwd(), "povray", DATAFILE_NAME)

if not os.path.exists(IMAGE_DIR):
    os.makedirs(IMAGE_DIR)

POV_COMMAND = " cd povray && " + \
              " {} +I{}".format(POV_EXE, SCENE_FILE) + \
              " +W{} +H{}".format(IMAGE_SIZE, IMAGE_SIZE) + \
              " +Q{}".format(IMAGE_QUALITY_LEVEL) + \
              " +A{}".format(ANTIALIASING_LEVEL) + \
              " +R{}".format(SUPER_SAMPLING_LEVEL) + \
              " +KFI0" + \
              " +KFF{}".format(FRAMES - 1) + \
              " -V" + \
              " +O../{}/".format(IMAGE_DIR) + "{}"

POV_TEMPLATE = """
#declare bg_color = {};
#declare vertex_size = {};
#declare edge_size = {};
#declare camera_loc = {};
#declare obj_rotation = {};
#declare size_func = {};
#declare face_max= {};
#declare face_min = {};
#declare face_index = {};
#declare use_area_light = {};

// this macro is used for adjusting the size of edges
// according to their positions in the space.
#macro get_size(q)
  #local len = vlength(q);
  #if (size_func = 0)
    #local len = (1.0 + len * len) / 4;
  #else #if (size_func = 1)
    #local len = 2.0 * log(2.0 + len * len);
  #else
    #local len = 2.0 * log(1.13 + len * len);
  #end
  #end
  len
#end

#macro choose_face(i, face_size)
  #local chosen = false;
  #for (ind, 0, dimension_size(face_index, 1) - 1)
    #if (i = face_index[ind])
      #if (face_size > face_min & face_size < face_max)
        #local chosen = true;
      #end
    #end
  #end
  chosen
#end

#declare vertices = {};

#declare edges = {};

#declare faces = {};
"""

def draw(coxeter_diagram,
         trunc_type,
         extra_relations=(),
         description="polychora",
         bg_color="SkyBlue",
         camera_loc=(0, 0, 30),
         rotation=(0, 0, 0),
         vertex_size=0.04,
         edge_size=0.02,
         size_func=0,
         face_index=(0,),
         face_max=3.0,
         face_min=0.5,
         use_area_light=False):
    """
    Export data to povray .inc file and call the rendering process.

    :param camera_loc: location of the camera.

    :param size_func: choose which sizing funcion to use, currently only 0, 1, 2.

    :param face_index: a list controls which types of faces are shown.

    :param face_max: only faces smaller than this threshold are shown.

    :param face_min: only faces larger than this threshold are shown.
    """
    P = Polychora(coxeter_diagram, trunc_type, extra_relations)
    P.build_geometry()
    vert_data, edge_data, face_data = P.get_povray_data()
    with open(data_file, "w") as f:
        f.write(POV_TEMPLATE.format(
            bg_color,
            vertex_size,
            edge_size,
            pov_vector(camera_loc),
            pov_vector(rotation),
            size_func,
            face_max,
            face_min,
            pov_index_array1d(face_index),
            int(use_area_light),
            vert_data,
            edge_data,
            face_data))

    print("rendering {}: {} vertices, {} edges, {} faces".format(
        description,
        P.num_vertices,
        P.num_edges,
        P.num_faces))

    process = subprocess.Popen(
        POV_COMMAND.format(description),
        shell=True,
        stderr=subprocess.PIPE,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE)

    _, err = process.communicate()
    if process.returncode:
        print(type(err), err)
        raise IOError("POVRay error: " + err.decode("ascii"))

def main():
    draw((3, 2, 2, 3, 2, 3), (1, 0, 0, 0), description="5-cell",
         camera_loc=(0, 0, 120), vertex_size=0.08, rotation=(-30, 60, 0),
         edge_size=0.04, size_func=1)

    draw((4, 2, 2, 3, 2, 3), (1, 0, 0, 0), description="4d-cube",
         camera_loc=(0, 0, 130), vertex_size=0.06, rotation=(60, 0, 0),
         edge_size=0.03, size_func=1, face_min=0.2, face_max=0.8)

    draw((3, 2, 2, 3, 2, 4), (1, 0, 0, 0), description="16-cell",
         camera_loc=(0, 0, 160), vertex_size=0.08,
         edge_size=0.03, size_func=2, face_min=1.0, face_max=1.2)

    draw((3, 2, 2, 4, 2, 3), (1, 0, 0, 0), description="24-cell",
         camera_loc=(0, 0, 200), vertex_size=0.06,
         edge_size=0.04, size_func=2, face_min=0.2, face_max=0.8)

    draw((5, 2, 2, 3, 2, 3), (1, 0, 0, 0), description="120-cell",
         camera_loc=(0, 0, 400), vertex_size=0.05,
         edge_size=0.025, size_func=0, face_min=3.0, face_max=100.0)

    draw((3, 2, 2, 3, 2, 5), (1, 0, 0, 0), description="600-cell",
         bg_color="White", camera_loc=(0, 0, 500), vertex_size=0.12,
         edge_size=0.04, size_func=2, face_max=4.0, face_min=3.0)

    draw((3, 2, 2, 3, 2, 4), (1, 0, 0, 1), description="runcinated-16-cell",
         bg_color="White", camera_loc=(0, 0, 450), vertex_size=0.1,
         face_index=(0, 1, 2, 3), edge_size=0.04, size_func=1,
         face_min=0, face_max=3)

    draw((5, 2, 2, 3, 2, 3), (1, 0, 0, 1), description="runcinated-120-cell",
         camera_loc=(0, 0, 360), vertex_size=0.028, edge_size=0.014, face_min=20)

    # this is the settings I used to render the movie at
    # http://pywonderland.com/images/cn/polytopes/rectified-grand-stellated-120-cell.mp4
    # (the parameters are not exactly the same but very similar)
    # take quite a while to render.
    draw((Fraction(5, 2), 2, 2, 5, 2, Fraction(5, 2)), (0, 1, 0, 0),
         extra_relations=((0, 1, 2, 1)*3, (1, 2, 3, 2)*3),
         description="rectified-grand-stellated-120-cell",
         size_func=1, vertex_size=0.06, edge_size=0.03,
         use_area_light=1, face_max=0.0, camera_loc=(0, 0, 400))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Render curved 4d polychoron examples in POV-Ray
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright (c) 2018 by Zhao Liang.
"""
import os
import subprocess
from fractions import Fraction

from polytopes.models import Polychora
from polytopes.povray import pov_index_array1d, pov_vector

IMAGE_DIR = "polychora_frames"  # directory to save the frames
POV_EXE = "povray"  # POV-Ray exe binary
SCENE_FILE = "polychora_curved.pov"  # the main scene file
IMAGE_SIZE = 600  # image size in pixels
FRAMES = 1  # number of frames
IMAGE_QUALITY_LEVEL = 11  # between 0-11
SUPER_SAMPLING_LEVEL = 5  # between 1-9
ANTIALIASING_LEVEL = 0.001  # lower for better quality
DATAFILE_NAME = "polychora-data.inc"  # export data to this file

data_file = os.path.join(os.getcwd(), "povray", DATAFILE_NAME)

if not os.path.exists(IMAGE_DIR):
    os.makedirs(IMAGE_DIR)

POV_COMMAND = (
    " cd povray && "
    + " {} +I{}".format(POV_EXE, SCENE_FILE)
    + " +W{} +H{}".format(IMAGE_SIZE, IMAGE_SIZE)
    + " +Q{}".format(IMAGE_QUALITY_LEVEL)
    + " +A{}".format(ANTIALIASING_LEVEL)
    + " +R{}".format(SUPER_SAMPLING_LEVEL)
    + " +KFI0"
    + " +KFF{}".format(FRAMES - 1)
    + " -V"
    + " +O../{}/".format(IMAGE_DIR)
    + "{}"
)

POV_TEMPLATE = """
#declare bg_color = {};
#declare vertex_size = {};
#declare edge_size = {};
#declare camera_loc = {};
#declare obj_rotation = {};
#declare size_func = {};
#declare face_max= {};
#declare face_min = {};
#declare face_index = {};
#declare use_area_light = {};

// this macro is used for adjusting the size of edges
// according to their positions in the space.
#macro get_size(q)
  #local len = vlength(q);
  #if (size_func = 0)
    #local len = (1.0 + len * len) / 4;
  #else #if (size_func = 1)
    #local len = 2.0 * log(2.0 + len * len);
  #else
    #local len = 2.0 * log(1.13 + len * len);
  #end
  #end
  len
#end

#macro choose_face(i, face_size)
  #local chosen = false;
  #for (ind, 0, dimension_size(face_index, 1) - 1)
    #if (i = face_index[ind])
      #if (face_size > face_min & face_size < face_max)
        #local chosen = true;
      #end
    #end
  #end
  chosen
#end

#declare vertices = {};

#declare edges = {};

#declare faces = {};
"""

def draw(
    coxeter_diagram,
    trunc_type,
    extra_relations=(),
    description="polychora",
    bg_color="SkyBlue",
    camera_loc=(0, 0, 30),
    rotation=(0, 0, 0),
    vertex_size=0.04,
    edge_size=0.02,
    size_func=0,
    face_index=(0,),
    face_max=3.0,
    face_min=0.5,
    use_area_light=False,
):
    """
    Export data to povray .inc file and call the rendering process.

    :param camera_loc:
        location of the camera.

    :param size_func:
        choose which sizing funcion to use, currently only 0, 1, 2.

    :param face_index:
        a list controls which types of faces are shown.

    :param face_max:
        only faces smaller than this threshold are shown.

    :param face_min:
        only faces larger than this threshold are shown.
    """
    P = Polychora(coxeter_diagram, trunc_type, extra_relations)
    P.build_geometry()
    vert_data, edge_data, face_data = P.get_povray_data()
    with open(data_file, "w") as f:
        f.write(
            POV_TEMPLATE.format(
                bg_color,
                vertex_size,
                edge_size,
                pov_vector(camera_loc),
                pov_vector(rotation),
                size_func,
                face_max,
                face_min,
                pov_index_array1d(face_index),
                int(use_area_light),
                vert_data,
                edge_data,
                face_data,
            )
        )

    print(
        "rendering {}: {} vertices, {} edges, {} faces".format(
            description, P.num_vertices, P.num_edges, P.num_faces
        )
    )

    process = subprocess.Popen(
        POV_COMMAND.format(description),
        shell=True,
        stderr=subprocess.PIPE,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
    )

    _, err = process.communicate()
    if process.returncode:
        print(type(err), err)
        raise IOError("POVRay error: " + err.decode("ascii"))

def main():
    draw(
        (3, 2, 2, 3, 2, 3),
        (1, 0, 0, 0),
        description="5-cell",
        camera_loc=(0, 0, 120),
        vertex_size=0.08,
        rotation=(-30, 60, 0),
        edge_size=0.04,
        size_func=1,
    )

    draw(
        (4, 2, 2, 3, 2, 3),
        (1, 0, 0, 0),
        description="4d-cube",
        camera_loc=(0, 0, 130),
        vertex_size=0.06,
        rotation=(60, 0, 0),
        edge_size=0.03,
        size_func=1,
        face_min=0.2,
        face_max=0.8,
    )

    draw(
        (3, 2, 2, 3, 2, 4),
        (1, 0, 0, 0),
        description="16-cell",
        camera_loc=(0, 0, 160),
        vertex_size=0.08,
        edge_size=0.03,
        size_func=2,
        face_min=1.0,
        face_max=1.2,
    )

    draw(
        (3, 2, 2, 4, 2, 3),
        (1, 0, 0, 0),
        description="24-cell",
        camera_loc=(0, 0, 200),
        vertex_size=0.06,
        edge_size=0.04,
        size_func=2,
        face_min=0.2,
        face_max=0.8,
    )

    draw(
        (5, 2, 2, 3, 2, 3),
        (1, 0, 0, 0),
        description="120-cell",
        camera_loc=(0, 0, 400),
        vertex_size=0.05,
        edge_size=0.025,
        size_func=0,
        face_min=3.0,
        face_max=100.0,
    )

    draw(
        (3, 2, 2, 3, 2, 5),
        (1, 0, 0, 0),
        description="600-cell",
        bg_color="White",
        camera_loc=(0, 0, 500),
        vertex_size=0.12,
        edge_size=0.04,
        size_func=2,
        face_max=4.0,
        face_min=3.0,
    )

    draw(
        (3, 2, 2, 3, 2, 4),
        (1, 0, 0, 1),
        description="runcinated-16-cell",
        bg_color="White",
        camera_loc=(0, 0, 450),
        vertex_size=0.1,
        face_index=(0, 1, 2, 3),
        edge_size=0.04,
        size_func=1,
        face_min=0,
        face_max=3,
    )

    draw(
        (5, 2, 2, 3, 2, 3),
        (1, 0, 0, 1),
        description="runcinated-120-cell",
        camera_loc=(0, 0, 360),
        vertex_size=0.028,
        edge_size=0.014,
        face_min=20,
    )

    # this is the settings I used to render the movie at
    # http://pywonderland.com/images/cn/polytopes/rectified-grand-stellated-120-cell.mp4
    # (the parameters are not exactly the same but very similar)
    # take quite a while to render.
    draw(
        (Fraction(5, 2), 2, 2, 5, 2, Fraction(5, 2)),
        (0, 1, 0, 0),
        extra_relations=((0, 1, 2, 1) * 3, (1, 2, 3, 2) * 3),
        description="rectified-grand-stellated-120-cell",
        size_func=1,
        vertex_size=0.06,
        edge_size=0.03,
        use_area_light=1,
        face_max=0.0,
        camera_loc=(0, 0, 400),
    )

    draw(
        (Fraction(5, 2), 2, 2, 5, 2, Fraction(5, 2)),
        (0, 1, 1, 0),
        extra_relations=((0, 1, 2, 1) * 3, (1, 2, 3, 2) * 3),
        description="runcinated-grand-stellated-120-cell",
        size_func=1,
        vertex_size=0.06,
        edge_size=0.03,
        use_area_light=1,
        face_min=0.3,
        face_max=1,
        camera_loc=(0, 0, 380),
    )

if __name__ == "__main__":
    main()
<commit_msg>Fix broken link in the comment<commit_after>"""
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Render curved 4d polychoron examples in POV-Ray
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright (c) 2018 by Zhao Liang.
"""
import os
import subprocess
from fractions import Fraction

from polytopes.models import Polychora
from polytopes.povray import pov_index_array1d, pov_vector

IMAGE_DIR = "polychora_frames"  # directory to save the frames
POV_EXE = "povray"  # POV-Ray exe binary
SCENE_FILE = "polychora_curved.pov"  # the main scene file
IMAGE_SIZE = 600  # image size in pixels
FRAMES = 1  # number of frames
IMAGE_QUALITY_LEVEL = 11  # between 0-11
SUPER_SAMPLING_LEVEL = 5  # between 1-9
ANTIALIASING_LEVEL = 0.001  # lower for better quality
DATAFILE_NAME = "polychora-data.inc"  # export data to this file

data_file = os.path.join(os.getcwd(), "povray", DATAFILE_NAME)

if not os.path.exists(IMAGE_DIR):
    os.makedirs(IMAGE_DIR)

POV_COMMAND = (
    " cd povray && "
    + " {} +I{}".format(POV_EXE, SCENE_FILE)
    + " +W{} +H{}".format(IMAGE_SIZE, IMAGE_SIZE)
    + " +Q{}".format(IMAGE_QUALITY_LEVEL)
    + " +A{}".format(ANTIALIASING_LEVEL)
    + " +R{}".format(SUPER_SAMPLING_LEVEL)
    + " +KFI0"
    + " +KFF{}".format(FRAMES - 1)
    + " -V"
    + " +O../{}/".format(IMAGE_DIR)
    + "{}"
)

POV_TEMPLATE = """
#declare bg_color = {};
#declare vertex_size = {};
#declare edge_size = {};
#declare camera_loc = {};
#declare obj_rotation = {};
#declare size_func = {};
#declare face_max= {};
#declare face_min = {};
#declare face_index = {};
#declare use_area_light = {};

// this macro is used for adjusting the size of edges
// according to their positions in the space.
#macro get_size(q)
  #local len = vlength(q);
  #if (size_func = 0)
    #local len = (1.0 + len * len) / 4;
  #else #if (size_func = 1)
    #local len = 2.0 * log(2.0 + len * len);
  #else
    #local len = 2.0 * log(1.13 + len * len);
  #end
  #end
  len
#end

#macro choose_face(i, face_size)
  #local chosen = false;
  #for (ind, 0, dimension_size(face_index, 1) - 1)
    #if (i = face_index[ind])
      #if (face_size > face_min & face_size < face_max)
        #local chosen = true;
      #end
    #end
  #end
  chosen
#end

#declare vertices = {};

#declare edges = {};

#declare faces = {};
"""

def draw(
    coxeter_diagram,
    trunc_type,
    extra_relations=(),
    description="polychora",
    bg_color="SkyBlue",
    camera_loc=(0, 0, 30),
    rotation=(0, 0, 0),
    vertex_size=0.04,
    edge_size=0.02,
    size_func=0,
    face_index=(0,),
    face_max=3.0,
    face_min=0.5,
    use_area_light=False,
):
    """
    Export data to povray .inc file and call the rendering process.

    :param camera_loc:
        location of the camera.

    :param size_func:
        choose which sizing funcion to use, currently only 0, 1, 2.

    :param face_index:
        a list controls which types of faces are shown.

    :param face_max:
        only faces smaller than this threshold are shown.

    :param face_min:
        only faces larger than this threshold are shown.
    """
    P = Polychora(coxeter_diagram, trunc_type, extra_relations)
    P.build_geometry()
    vert_data, edge_data, face_data = P.get_povray_data()
    with open(data_file, "w") as f:
        f.write(
            POV_TEMPLATE.format(
                bg_color,
                vertex_size,
                edge_size,
                pov_vector(camera_loc),
                pov_vector(rotation),
                size_func,
                face_max,
                face_min,
                pov_index_array1d(face_index),
                int(use_area_light),
                vert_data,
                edge_data,
                face_data,
            )
        )

    print(
        "rendering {}: {} vertices, {} edges, {} faces".format(
            description, P.num_vertices, P.num_edges, P.num_faces
        )
    )

    process = subprocess.Popen(
        POV_COMMAND.format(description),
        shell=True,
        stderr=subprocess.PIPE,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
    )

    _, err = process.communicate()
    if process.returncode:
        print(type(err), err)
        raise IOError("POVRay error: " + err.decode("ascii"))

def main():
    draw(
        (3, 2, 2, 3, 2, 3),
        (1, 0, 0, 0),
        description="5-cell",
        camera_loc=(0, 0, 120),
        vertex_size=0.08,
        rotation=(-30, 60, 0),
        edge_size=0.04,
        size_func=1,
    )

    draw(
        (4, 2, 2, 3, 2, 3),
        (1, 0, 0, 0),
        description="4d-cube",
        camera_loc=(0, 0, 130),
        vertex_size=0.06,
        rotation=(60, 0, 0),
        edge_size=0.03,
        size_func=1,
        face_min=0.2,
        face_max=0.8,
    )

    draw(
        (3, 2, 2, 3, 2, 4),
        (1, 0, 0, 0),
        description="16-cell",
        camera_loc=(0, 0, 160),
        vertex_size=0.08,
        edge_size=0.03,
        size_func=2,
        face_min=1.0,
        face_max=1.2,
    )

    draw(
        (3, 2, 2, 4, 2, 3),
        (1, 0, 0, 0),
        description="24-cell",
        camera_loc=(0, 0, 200),
        vertex_size=0.06,
        edge_size=0.04,
        size_func=2,
        face_min=0.2,
        face_max=0.8,
    )

    draw(
        (5, 2, 2, 3, 2, 3),
        (1, 0, 0, 0),
        description="120-cell",
        camera_loc=(0, 0, 400),
        vertex_size=0.05,
        edge_size=0.025,
        size_func=0,
        face_min=3.0,
        face_max=100.0,
    )

    draw(
        (3, 2, 2, 3, 2, 5),
        (1, 0, 0, 0),
        description="600-cell",
        bg_color="White",
        camera_loc=(0, 0, 500),
        vertex_size=0.12,
        edge_size=0.04,
        size_func=2,
        face_max=4.0,
        face_min=3.0,
    )

    draw(
        (3, 2, 2, 3, 2, 4),
        (1, 0, 0, 1),
        description="runcinated-16-cell",
        bg_color="White",
        camera_loc=(0, 0, 450),
        vertex_size=0.1,
        face_index=(0, 1, 2, 3),
        edge_size=0.04,
        size_func=1,
        face_min=0,
        face_max=3,
    )

    draw(
        (5, 2, 2, 3, 2, 3),
        (1, 0, 0, 1),
        description="runcinated-120-cell",
        camera_loc=(0, 0, 360),
        vertex_size=0.028,
        edge_size=0.014,
        face_min=20,
    )

    # this is the settings I used to render the movie at
    # http://pywonderland.com/images/polytopes/rectified-grand-stellated-120-cell.mp4
    # (the parameters are not exactly the same but very similar)
    # take quite a while to render.
    draw(
        (Fraction(5, 2), 2, 2, 5, 2, Fraction(5, 2)),
        (0, 1, 0, 0),
        extra_relations=((0, 1, 2, 1) * 3, (1, 2, 3, 2) * 3),
        description="rectified-grand-stellated-120-cell",
        size_func=1,
        vertex_size=0.06,
        edge_size=0.03,
        use_area_light=1,
        face_max=0.0,
        camera_loc=(0, 0, 400),
    )

    draw(
        (Fraction(5, 2), 2, 2, 5, 2, Fraction(5, 2)),
        (0, 1, 1, 0),
        extra_relations=((0, 1, 2, 1) * 3, (1, 2, 3, 2) * 3),
        description="runcinated-grand-stellated-120-cell",
        size_func=1,
        vertex_size=0.06,
        edge_size=0.03,
        use_area_light=1,
        face_min=0.3,
        face_max=1,
        camera_loc=(0, 0, 380),
    )

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
@file
@brief Various functions to process text
"""

import os

def replace_comma_by_point(file):
    """
    replaces all commas by point in a file (do that inplace)

    @param      file        file to process
    """
    with open(file, "r") as f:
        text = f.read()
    text = text.replace(",", ".")
    with open(file, "w") as f:
        f.write(text)

def file_head(filename, nbline=10, encoding="utf8"):
    """
    extracts the first nbline of a file (assuming it is text file)

    @param      filename        filename
    @param      nbline          number of lines
    @param      encoding        encoding
    @return                     list of lines

    .. versionadded:: 1.1
    """
    if not os.path.exists(filename):
        raise FileNotFoundError(filename)
    if not os.path.isfile(filename):
        raise FileNotFoundError("{0} is not a file".format(filename))
    rows = []
    with open(filename, "r", encoding=encoding) as f:
        for line in f:
            rows.append(line)
            if len(rows) >= nbline:
                break
    return rows

def file_tail(filename, nbline=10, encoding="utf8", threshold=2 ** 14):
    """
    extracts the first nbline of a file (assuming it is text file)

    @param      filename        filename
    @param      nbline          number of lines
    @param      encoding        encoding
    @param      threshold       if the file size is above, it will not read the beginning
    @return                     list of lines

    The line marked as *A* has an issue because the cursor
    could fall on a character (= byte) in the middle of a character
    if the file is encoded in utf-8 character.
    The next line fails. That's why we try again
    by moving the cursor by one character (see line B).

    The first returned line may be incomplete.

    .. versionadded:: 1.1
    """
    if not os.path.exists(filename):
        raise FileNotFoundError(filename)
    if not os.path.isfile(filename):
        raise FileNotFoundError("{0} is not a file".format(filename))

    size = os.stat(filename).st_size
    if size < threshold:
        with open(filename, "r", encoding=encoding) as f:
            rows = f.readlines()
        return rows[-nbline:] if len(rows) > nbline else rows
    else:
        with open(filename, "r", encoding=encoding) as f:
            f.seek(size - threshold)  # line A
            try:
                content = f.read()
            except UnicodeDecodeError:
                f.seek(size - threshold - 1)  # line B
                content = f.read()

        rows = content.split("\n")
        return rows[-nbline:] if len(rows) > nbline else rows
<commit_msg>fix a bug in file_tail, make it similar to file_head<commit_after>"""
@file
@brief Various functions to process text
"""

import os

def replace_comma_by_point(file):
    """
    replaces all commas by point in a file (do that inplace)

    @param      file        file to process
    """
    with open(file, "r") as f:
        text = f.read()
    text = text.replace(",", ".")
    with open(file, "w") as f:
        f.write(text)

def file_head(filename, nbline=10, encoding="utf8"):
    """
    extracts the first nbline of a file (assuming it is text file)

    @param      filename        filename
    @param      nbline          number of lines
    @param      encoding        encoding
    @return                     list of lines

    .. versionadded:: 1.1
    """
    if not os.path.exists(filename):
        raise FileNotFoundError(filename)
    if not os.path.isfile(filename):
        raise FileNotFoundError("{0} is not a file".format(filename))
    rows = []
    with open(filename, "r", encoding=encoding) as f:
        for line in f:
            rows.append(line)
            if len(rows) >= nbline:
                break
    return rows

def file_tail(filename, nbline=10, encoding="utf8", threshold=2 ** 14):
    """
    extracts the first nbline of a file (assuming it is text file)

    @param      filename        filename
    @param      nbline          number of lines
    @param      encoding        encoding
    @param      threshold       if the file size is above, it will not read the beginning
    @return                     list of lines

    The line marked as *A* has an issue because the cursor
    could fall on a character (= byte) in the middle of a character
    if the file is encoded in utf-8 character.
    The next line fails. That's why we try again
    by moving the cursor by one character (see line B).

    The first returned line may be incomplete.

    .. versionadded:: 1.1
    """
    if not os.path.exists(filename):
        raise FileNotFoundError(filename)
    if not os.path.isfile(filename):
        raise FileNotFoundError("{0} is not a file".format(filename))

    size = os.stat(filename).st_size
    if size < threshold:
        with open(filename, "r", encoding=encoding) as f:
            rows = f.readlines()
        return rows[-nbline:] if len(rows) > nbline else rows
    else:
        with open(filename, "r", encoding=encoding) as f:
            f.seek(size - threshold)  # line A
            try:
                content = f.read()
            except UnicodeDecodeError:
                f.seek(size - threshold - 1)  # line B
                content = f.read()

        rows = content.split("\n")
        res = rows[-nbline:] if len(rows) > nbline else rows
        return [_ + "\n" for _ in res]
<|endoftext|>


<commit_before>import os

from epucontrol.api.exceptions import *
import epucontrol.main.ec_args as ec_args
from epucontrol.main import ACTIONS

import child

class DefaultRunlogs:

    def __init__(self, params, common):
        self.p = params
        self.c = common
        self.validated = False
        self.thisrundir = None

        # temporary assumption that this is same on every VM, see events.conf
        self.allvmslogdir = None

    def validate(self):

        action = self.p.get_arg_or_none(ec_args.ACTION)
        if action not in [ACTIONS.CREATE, ACTIONS.LOGFETCH, ACTIONS.FETCH_KILL,
                          ACTIONS.FIND_WORKERS, ACTIONS.KILLRUN]:
            if self.c.trace:
                self.c.log.debug("validation for runlogs module complete, '%s' is not a relevant action" % action)
            return

        run_name = self.p.get_arg_or_none(ec_args.NAME)

        runlogdir = self.p.get_conf_or_none("events", "runlogdir")
        if not runlogdir:
            raise InvalidConfig("There is no runlogdir configuration")

        if not os.path.isabs(runlogdir):
            runlogdir = self.c.resolve_var_dir(runlogdir)

        if not os.path.exists(runlogdir):
            raise InvalidConfig("The runlogdir does not exist: %s" % runlogdir)

        self.thisrundir = os.path.join(runlogdir, run_name)
        if not os.path.exists(self.thisrundir):
            os.mkdir(self.thisrundir)
            self.c.log.debug("Created a new directory for the logfiles generated on nodes in this run: %s" % self.thisrundir)
        else:
            self.c.log.debug("Directory of logfiles generated on nodes in this run: %s" % self.thisrundir)

        self.allvmslogdir = self.p.get_conf_or_none("events", "vmlogdir")
        if not self.allvmslogdir:
            raise InvalidConfig("There is no events:vmlogdir configuration")

        self.validated = True

    def new_vm(self, newvm):
        """Make the module aware of a new VM.
        It also will annotate the VM object.  It can handle a VM that has
        been through this process before, so no need to check for it.
        """

        if not self.validated:
            raise ProgrammingError("operation called without necessary validation")

        if not newvm.instanceid:
            raise ProgrammingError("Cannot determine VM instance ID")

        thisvm_runlog_dir = os.path.join(self.thisrundir, newvm.instanceid)

        if newvm.runlogdir:
            if newvm.runlogdir != thisvm_runlog_dir:
                self.c.log.warn("The runlog directory for the VM was recorded but it is not what we would expect (%s != %s)" % (newvm.runlogdir, thisvm_runlog_dir))
        elif not os.path.exists(thisvm_runlog_dir):
            os.mkdir(thisvm_runlog_dir)
            newvm.runlogdir = thisvm_runlog_dir
            self.c.log.debug("created runlog directory for this instance: %s" % thisvm_runlog_dir)
        else:
            newvm.runlogdir = thisvm_runlog_dir

        if not os.path.exists(newvm.runlogdir):
            raise IncompatibleEnvironment("Could not find the runlog directory: %s" % newvm.runlogdir)

        newvm.vmlogdir = self.allvmslogdir

    def fetch_logs(self, vm, m):

        if not self.validated:
            raise ProgrammingError("operation called without necessary validation")

        if not vm.hostname:
            self.c.log.warn("Cannot retrieve logs for '%s', hostname is unknown" % vm.instanceid)
            return

        scpcmd = m.iaas.scp_cmd(vm.hostname)

        # last arg is "user@host:", we need to enhance this with the path
        scpcmd[-1] = scpcmd[-1] + vm.vmlogdir

        # and then the glob
        scpcmd[-1] = scpcmd[-1] + "/*log"

        # transfer destination
        scpcmd.append(vm.runlogdir)

        self._run_one_cmd(scpcmd)

    # TODO: this is a copied snippet from iaas.py
    def _run_one_cmd(self, args):
        cmd = ' '.join(args)
        self.c.log.debug("command = '%s'" % cmd)

        timeout = 30.0 # seconds
        (killed, retcode, stdout, stderr) = \
            child.child(cmd, timeout=timeout)

        if killed:
            self.c.log.error("TIMED OUT: '%s'" % cmd)
            return False

        if retcode == 0:
            self.c.log.debug("command succeeded: '%s'" % cmd)
            return True
        else:
            errmsg = "problem running command, "
            if retcode < 0:
                errmsg += "killed by signal:"
            if retcode > 0:
                errmsg += "exited non-zero:"
            errmsg += "'%s' ::: return code" % cmd
            errmsg += ": %d ::: error:\n%s\noutput:\n%s" % (retcode, stdout, stderr)

            # these commands will commonly fail
            if self.c.trace:
                self.c.log.debug(errmsg)
            return False
<commit_msg>get around log rotation issues, where there is 'log.1'<commit_after>import os

from epucontrol.api.exceptions import *
import epucontrol.main.ec_args as ec_args
from epucontrol.main import ACTIONS

import child

class DefaultRunlogs:

    def __init__(self, params, common):
        self.p = params
        self.c = common
        self.validated = False
        self.thisrundir = None

        # temporary assumption that this is same on every VM, see events.conf
        self.allvmslogdir = None

    def validate(self):

        action = self.p.get_arg_or_none(ec_args.ACTION)
        if action not in [ACTIONS.CREATE, ACTIONS.LOGFETCH, ACTIONS.FETCH_KILL,
                          ACTIONS.FIND_WORKERS, ACTIONS.KILLRUN]:
            if self.c.trace:
                self.c.log.debug("validation for runlogs module complete, '%s' is not a relevant action" % action)
            return

        run_name = self.p.get_arg_or_none(ec_args.NAME)

        runlogdir = self.p.get_conf_or_none("events", "runlogdir")
        if not runlogdir:
            raise InvalidConfig("There is no runlogdir configuration")

        if not os.path.isabs(runlogdir):
            runlogdir = self.c.resolve_var_dir(runlogdir)

        if not os.path.exists(runlogdir):
            raise InvalidConfig("The runlogdir does not exist: %s" % runlogdir)

        self.thisrundir = os.path.join(runlogdir, run_name)
        if not os.path.exists(self.thisrundir):
            os.mkdir(self.thisrundir)
            self.c.log.debug("Created a new directory for the logfiles generated on nodes in this run: %s" % self.thisrundir)
        else:
            self.c.log.debug("Directory of logfiles generated on nodes in this run: %s" % self.thisrundir)

        self.allvmslogdir = self.p.get_conf_or_none("events", "vmlogdir")
        if not self.allvmslogdir:
            raise InvalidConfig("There is no events:vmlogdir configuration")

        self.validated = True

    def new_vm(self, newvm):
        """Make the module aware of a new VM.
        It also will annotate the VM object.  It can handle a VM that has
        been through this process before, so no need to check for it.
        """

        if not self.validated:
            raise ProgrammingError("operation called without necessary validation")

        if not newvm.instanceid:
            raise ProgrammingError("Cannot determine VM instance ID")

        thisvm_runlog_dir = os.path.join(self.thisrundir, newvm.instanceid)

        if newvm.runlogdir:
            if newvm.runlogdir != thisvm_runlog_dir:
                self.c.log.warn("The runlog directory for the VM was recorded but it is not what we would expect (%s != %s)" % (newvm.runlogdir, thisvm_runlog_dir))
        elif not os.path.exists(thisvm_runlog_dir):
            os.mkdir(thisvm_runlog_dir)
            newvm.runlogdir = thisvm_runlog_dir
            self.c.log.debug("created runlog directory for this instance: %s" % thisvm_runlog_dir)
        else:
            newvm.runlogdir = thisvm_runlog_dir

        if not os.path.exists(newvm.runlogdir):
            raise IncompatibleEnvironment("Could not find the runlog directory: %s" % newvm.runlogdir)

        newvm.vmlogdir = self.allvmslogdir

    def fetch_logs(self, vm, m):

        if not self.validated:
            raise ProgrammingError("operation called without necessary validation")

        if not vm.hostname:
            self.c.log.warn("Cannot retrieve logs for '%s', hostname is unknown" % vm.instanceid)
            return

        scpcmd = m.iaas.scp_cmd(vm.hostname)

        # last arg is "user@host:", we need to enhance this with the path
        scpcmd[-1] = scpcmd[-1] + vm.vmlogdir

        # and then the glob
        scpcmd[-1] = scpcmd[-1] + "/*log*"

        # transfer destination
        scpcmd.append(vm.runlogdir)

        self._run_one_cmd(scpcmd)

    # TODO: this is a copied snippet from iaas.py
    def _run_one_cmd(self, args):
        cmd = ' '.join(args)
        self.c.log.debug("command = '%s'" % cmd)

        timeout = 30.0 # seconds
        (killed, retcode, stdout, stderr) = \
            child.child(cmd, timeout=timeout)

        if killed:
            self.c.log.error("TIMED OUT: '%s'" % cmd)
            return False

        if retcode == 0:
            self.c.log.debug("command succeeded: '%s'" % cmd)
            return True
        else:
            errmsg = "problem running command, "
            if retcode < 0:
                errmsg += "killed by signal:"
            if retcode > 0:
                errmsg += "exited non-zero:"
            errmsg += "'%s' ::: return code" % cmd
            errmsg += ": %d ::: error:\n%s\noutput:\n%s" % (retcode, stdout, stderr)

            # these commands will commonly fail
            if self.c.trace:
                self.c.log.debug(errmsg)
            return False
<|endoftext|>