<commit_before>import sys
import os, os.path
from tornado.concurrent import Future
from tornado.ioloop import IOLoop
from tornado import gen
from subprocess import Popen, PIPE
import logging
import tempfile
from .eggstorage import FilesystemEggStorage
import shutil
from scrapydd.exceptions import ProcessFailed, InvalidProjectEgg
import json
from zipfile import ZipFile
from w3lib.url import path_to_file_uri

logger = logging.getLogger(__name__)

class ProjectWorkspace(object):
    pip = None
    python = None
    processes = None
    project_workspace_dir = None
    project_check = None
    temp_dir = None

    def __init__(self, project_name, base_workdir=None):
        if not base_workdir:
            base_workdir = tempfile.mkdtemp(prefix='scrapydd-tmp')
        project_workspace_dir = os.path.abspath(os.path.join(base_workdir, 'workspace', project_name))
        self.project_workspace_dir = project_workspace_dir
        self.project_name = project_name
        self.egg_storage = FilesystemEggStorage({'eggs_dir':os.path.join(project_workspace_dir, 'eggs')})
        self.processes = []
        if sys.platform.startswith('linux'):
            self.pip = os.path.join(project_workspace_dir, 'bin', 'pip')
            self.python = os.path.join(project_workspace_dir, 'bin', 'python')
        elif sys.platform.startswith('win'):
            self.pip = os.path.join(project_workspace_dir, 'Scripts', 'pip.exe')
            self.python = os.path.join(project_workspace_dir, 'Scripts', 'python.exe')
        else:
            raise NotImplementedError('Unsupported system %s' % sys.platform)

    def init(self):
        '''
        init project isolated workspace,
        :return: future
        '''
        future = Future()
        if os.path.exists(self.pip) and os.path.exists(self.python):
            future.set_result(self)
            return future

        logger.info('start creating virtualenv.')
        try:
            process = Popen(['virtualenv', '--system-site-packages', self.project_workspace_dir], stdout=PIPE, stderr=PIPE)
            self.processes.append(process)
        except Exception as e:
            future.set_exception(e)
            return future

        def done(process):
            self.processes.remove(process)
            retcode = process.returncode
            if retcode is not None:
                if retcode == 0:
                    future.set_result(self)
                else:
                    std_output = process.stdout.read()
                    err_output = process.stderr.read()
                    future.set_exception(ProcessFailed('Error when init workspace virtualenv ', std_output=std_output, err_output=err_output))

        wait_process(process, done)
        return future

    def find_project_requirements(self, project=None, egg_storage=None, eggf=None):
        project = self.project_name
        if eggf is None:
            if egg_storage is None:
                egg_storage = self.egg_storage
            version, eggf = egg_storage.get(project)
        with ZipFile(eggf) as egg_zip_file:
            try:
                requires_fileinfo = egg_zip_file.getinfo('EGG-INFO/requires.txt')
                return egg_zip_file.read(requires_fileinfo).split()
            except KeyError:
                return []

    @gen.coroutine
    def install_requirements(self):
        requirements = self.find_project_requirements(self.project_name)
        requirements += ['scrapydd']
        if requirements:
            yield self.pip_install(requirements)

        raise gen.Return()

    def pip_install(self, requirements):
        logger.debug('installing requirements: %s' % requirements)
        future = Future()
        try:
            process = Popen([self.pip, 'install'] + requirements, stdout=PIPE, stderr=PIPE)
            self.processes.append(process)
        except Exception as e:
            future.set_exception(e)
            return future

        def done(process):
            self.processes.remove(process)
            retcode = process.returncode
            if retcode is not None:
                if retcode == 0:
                    future.set_result(self)
                else:
                    std_out = process.stdout.read()
                    err_out = process.stderr.read()
                    future.set_exception(ProcessFailed(std_output=std_out, err_output=err_out))

        wait_process(process, done)
        return future

    def spider_list(self):
        future = Future()
        cwd = self.project_workspace_dir
        try:
            env = os.environ.copy()
            env['SCRAPY_PROJECT'] = self.project_name
            process = Popen([self.python, '-m', 'scrapydd.utils.runner', 'list'], env = env, cwd=cwd, stdout = PIPE, stderr= PIPE)
        except Exception as e:
            logger.error(e)
            future.set_exception(e)
            return future

        def check_process():
            logger.debug('poll')
            retcode = process.poll()
            if retcode is not None:
                if retcode == 0:
                    future.set_result(process.stdout.read().splitlines())
                else:
                    #future.set_exception(ProcessFailed(std_output=process.stdout.read(), err_output=process.stderr.read()))
                    future.set_exception(InvalidProjectEgg(detail=process.stderr.read()))
                return
            IOLoop.current().call_later(1, check_process)

        check_process()
        return future

    def run_spider(self, spider, spider_parameters=None, f_output=None, project=None):
        ret_future = Future()
        items_file = os.path.join(self.project_workspace_dir, 'items.jl')
        runner = 'scrapydd.utils.runner'
        pargs = [self.python, '-m', runner, 'crawl', spider]
        if project:
            spider_parameters['BOT_NAME'] = project
        if spider_parameters:
            for spider_parameter_key, spider_parameter_value in spider_parameters.items():
                pargs += [
                            '-s',
                            '%s=%s' % (spider_parameter_key, spider_parameter_value)
                          ]

        env = os.environ.copy()
        env['SCRAPY_PROJECT'] = str(self.project_name)
        #env['SCRAPY_JOB'] = str(self.task.id)
        env['SCRAPY_FEED_URI'] = str(path_to_file_uri(items_file))

        p = Popen(pargs, env = env, stdout = f_output, cwd = self.project_workspace_dir, stderr = f_output)
        self.processes.append(p)

        def done(process):
            self.processes.remove(process)
            if process.returncode:
                return ret_future.set_exception(ProcessFailed())

            return ret_future.set_result(items_file)

        wait_process(p, done)
        return ret_future

    def find_project_settings(self, project):
        cwd = self.project_workspace_dir
        future = Future()
        try:
            env = os.environ.copy()
            env['SCRAPY_PROJECT'] = project
            process = Popen([self.python, '-m', 'scrapydd.utils.extract_settings', project], env = env, cwd=cwd, stdout = PIPE, stderr= PIPE)
            self.processes.append(process)
        except Exception as e:
            logger.error(e)
            future.set_exception(e)
            return future

        def done(process):
            retcode = process.poll()
            self.processes.remove(process)
            if retcode == 0:
                proc_output = process.stdout.read()
                logger.debug('find_project_settings output:')
                logger.debug(proc_output)
                return future.set_result(json.loads(proc_output))
            else:
                # future.set_exception(ProcessFailed(std_output=process.stdout.read(), err_output=process.stderr.read()))
                return future.set_exception(InvalidProjectEgg(detail=process.stderr.read()))

        wait_process(process, done)
        return future

    def put_egg(self, eggfile, version):
        eggfile.seek(0)
        self.egg_storage.put(eggfile=eggfile, project=self.project_name, version=version)

    def get_egg(self, version=None):
        return self.egg_storage.get(self.project_name, version=version)

    def delete_egg(self, project, version=None):
        logger.info('deleting project eggs')
        return self.egg_storage.delete(project, version)

    def list_versions(self, project):
        return self.egg_storage.list(project)

    @gen.coroutine
    def kill_process(self):
        logger.info('killing process')
        for process in self.processes:
            if process.poll() is not None:
                continue
            try:
                process.terminate()
            except OSError as e:
                logger.error('Caught OSError when try to terminate subprocess: %s' % e)

            gen.sleep(10)

            if process.poll() is not None:
                continue
            try:
                process.kill()
            except OSError as e:
                logger.error('Caught OSError when try to kill subprocess: %s' % e)

    def __del__(self):
        if self.project_workspace_dir and os.path.exists(self.project_workspace_dir):
            shutil.rmtree(self.project_workspace_dir)

def wait_process(process, callback):
    retcode = process.poll()
    if retcode is not None:
        return callback(process)

    IOLoop.current().call_later(1, wait_process, process, callback)<commit_msg>fix travis fail issue, cannot load the current version scrapydd in virtualenv running enviroment<commit_after>import sys
import os, os.path
from tornado.concurrent import Future
from tornado.ioloop import IOLoop
from tornado import gen
from subprocess import Popen, PIPE
import logging
import tempfile
from .eggstorage import FilesystemEggStorage
import shutil
from scrapydd.exceptions import ProcessFailed, InvalidProjectEgg
import json
from zipfile import ZipFile
from w3lib.url import path_to_file_uri

logger = logging.getLogger(__name__)

class ProjectWorkspace(object):
    pip = None
    python = None
    processes = None
    project_workspace_dir = None
    project_check = None
    temp_dir = None

    def __init__(self, project_name, base_workdir=None):
        if not base_workdir:
            base_workdir = tempfile.mkdtemp(prefix='scrapydd-tmp')
        project_workspace_dir = os.path.abspath(os.path.join(base_workdir, 'workspace', project_name))
        self.project_workspace_dir = project_workspace_dir
        self.project_name = project_name
        self.egg_storage = FilesystemEggStorage({'eggs_dir':os.path.join(project_workspace_dir, 'eggs')})
        self.processes = []
        if sys.platform.startswith('linux'):
            self.pip = os.path.join(project_workspace_dir, 'bin', 'pip')
            self.python = os.path.join(project_workspace_dir, 'bin', 'python')
        elif sys.platform.startswith('win'):
            self.pip = os.path.join(project_workspace_dir, 'Scripts', 'pip.exe')
            self.python = os.path.join(project_workspace_dir, 'Scripts', 'python.exe')
        else:
            raise NotImplementedError('Unsupported system %s' % sys.platform)

    def init(self):
        '''
        init project isolated workspace,
        :return: future
        '''
        future = Future()
        if os.path.exists(self.pip) and os.path.exists(self.python):
            future.set_result(self)
            return future

        logger.info('start creating virtualenv.')
        try:
            process = Popen(['virtualenv', '--system-site-packages', self.project_workspace_dir], stdout=PIPE, stderr=PIPE)
            self.processes.append(process)
        except Exception as e:
            future.set_exception(e)
            return future

        def done(process):
            self.processes.remove(process)
            retcode = process.returncode
            if retcode is not None:
                if retcode == 0:
                    future.set_result(self)
                else:
                    std_output = process.stdout.read()
                    err_output = process.stderr.read()
                    future.set_exception(ProcessFailed('Error when init workspace virtualenv ', std_output=std_output, err_output=err_output))

        wait_process(process, done)
        return future

    def find_project_requirements(self, project=None, egg_storage=None, eggf=None):
        project = self.project_name
        if eggf is None:
            if egg_storage is None:
                egg_storage = self.egg_storage
            version, eggf = egg_storage.get(project)
        with ZipFile(eggf) as egg_zip_file:
            try:
                requires_fileinfo = egg_zip_file.getinfo('EGG-INFO/requires.txt')
                return egg_zip_file.read(requires_fileinfo).split()
            except KeyError:
                return []

    @gen.coroutine
    def install_requirements(self):
        requirements = self.find_project_requirements(self.project_name)
        requirements += ['scrapydd', 'scrapyd']
        if requirements:
            yield self.pip_install(requirements)

        raise gen.Return()

    def pip_install(self, requirements):
        logger.debug('installing requirements: %s' % requirements)
        future = Future()
        try:
            process = Popen([self.pip, 'install'] + requirements, stdout=PIPE, stderr=PIPE)
            self.processes.append(process)
        except Exception as e:
            future.set_exception(e)
            return future

        def done(process):
            self.processes.remove(process)
            retcode = process.returncode
            if retcode is not None:
                if retcode == 0:
                    future.set_result(self)
                else:
                    std_out = process.stdout.read()
                    err_out = process.stderr.read()
                    future.set_exception(ProcessFailed(std_output=std_out, err_output=err_out))

        wait_process(process, done)
        return future

    def spider_list(self):
        future = Future()
        cwd = self.project_workspace_dir
        try:
            env = os.environ.copy()
            env['SCRAPY_PROJECT'] = self.project_name
            process = Popen([self.python, '-m', 'scrapyd.runner', 'list'], env = env, cwd=cwd, stdout = PIPE, stderr= PIPE)
        except Exception as e:
            logger.error(e)
            future.set_exception(e)
            return future

        def check_process():
            logger.debug('poll')
            retcode = process.poll()
            if retcode is not None:
                if retcode == 0:
                    future.set_result(process.stdout.read().splitlines())
                else:
                    #future.set_exception(ProcessFailed(std_output=process.stdout.read(), err_output=process.stderr.read()))
                    future.set_exception(InvalidProjectEgg(detail=process.stderr.read()))
                return
            IOLoop.current().call_later(1, check_process)

        check_process()
        return future

    def run_spider(self, spider, spider_parameters=None, f_output=None, project=None):
        ret_future = Future()
        items_file = os.path.join(self.project_workspace_dir, 'items.jl')
        runner = 'scrapyd.runner'
        pargs = [self.python, '-m', runner, 'crawl', spider]
        if project:
            spider_parameters['BOT_NAME'] = project
        if spider_parameters:
            for spider_parameter_key, spider_parameter_value in spider_parameters.items():
                pargs += [
                            '-s',
                            '%s=%s' % (spider_parameter_key, spider_parameter_value)
                          ]

        env = os.environ.copy()
        env['SCRAPY_PROJECT'] = str(self.project_name)
        #env['SCRAPY_JOB'] = str(self.task.id)
        env['SCRAPY_FEED_URI'] = str(path_to_file_uri(items_file))

        p = Popen(pargs, env = env, stdout = f_output, cwd = self.project_workspace_dir, stderr = f_output)
        self.processes.append(p)

        def done(process):
            self.processes.remove(process)
            if process.returncode:
                return ret_future.set_exception(ProcessFailed())

            return ret_future.set_result(items_file)

        wait_process(p, done)
        return ret_future

    def find_project_settings(self, project):
        cwd = self.project_workspace_dir
        future = Future()
        try:
            env = os.environ.copy()
            env['SCRAPY_PROJECT'] = project
            process = Popen([self.python, '-m', 'scrapydd.utils.extract_settings', project], env = env, cwd=cwd, stdout = PIPE, stderr= PIPE)
            self.processes.append(process)
        except Exception as e:
            logger.error(e)
            future.set_exception(e)
            return future

        def done(process):
            retcode = process.poll()
            self.processes.remove(process)
            if retcode == 0:
                proc_output = process.stdout.read()
                logger.debug('find_project_settings output:')
                logger.debug(proc_output)
                return future.set_result(json.loads(proc_output))
            else:
                # future.set_exception(ProcessFailed(std_output=process.stdout.read(), err_output=process.stderr.read()))
                return future.set_exception(InvalidProjectEgg(detail=process.stderr.read()))

        wait_process(process, done)
        return future

    def put_egg(self, eggfile, version):
        eggfile.seek(0)
        self.egg_storage.put(eggfile=eggfile, project=self.project_name, version=version)

    def get_egg(self, version=None):
        return self.egg_storage.get(self.project_name, version=version)

    def delete_egg(self, project, version=None):
        logger.info('deleting project eggs')
        return self.egg_storage.delete(project, version)

    def list_versions(self, project):
        return self.egg_storage.list(project)

    @gen.coroutine
    def kill_process(self):
        logger.info('killing process')
        for process in self.processes:
            if process.poll() is not None:
                continue
            try:
                process.terminate()
            except OSError as e:
                logger.error('Caught OSError when try to terminate subprocess: %s' % e)

            gen.sleep(10)

            if process.poll() is not None:
                continue
            try:
                process.kill()
            except OSError as e:
                logger.error('Caught OSError when try to kill subprocess: %s' % e)

    def __del__(self):
        if self.project_workspace_dir and os.path.exists(self.project_workspace_dir):
            shutil.rmtree(self.project_workspace_dir)

def wait_process(process, callback):
    retcode = process.poll()
    if retcode is not None:
        return callback(process)

    IOLoop.current().call_later(1, wait_process, process, callback)<|endoftext|>


<commit_before>
__version__ = "0.4.2"
<commit_msg>Upgrade version<commit_after>
__version__ = "0.4.3"
<|endoftext|>


<commit_before>__author__ = 'Asish Mahapatra: asishkm@gmail.com'

import requests
import re
import os
import img2pdf
from multiprocessing import Process, Manager
import sys
from math import ceil
import time

json_pattern = re.compile(r'https.*scribdassets.*jsonp')
img_pattern = re.compile(r'orig.*(http.*scribd.{,45}jpg)')

patterns = {'jpg': re.compile(r'([0-9]*)-.*jpg'),
            'jsonp': re.compile(r'([0-9]*)-.*jsonp')}

DEBUG = False

def sort_json(link):
    '''
        Gets the page number of the image or the jsonp file.
        Urls have the forum 'pagenum-randomhash.jpg / jsonp'
    '''
    for pattern in patterns:
        if pattern in link:
            s = link.index(pattern)-16
            val = re.search(patterns[pattern], link[s:])
            return int(val.groups()[0])

def img_jsonp_urls(scribd_link):
    '''
        Gets a sorted list of URLs based on page number with both
        jsonp or jpg extensions
    '''
    print 'at get_images'

    scribd_conn = requests.get(scribd_link)

    json_list = re.findall(json_pattern, scribd_conn.content)
    img_links = re.findall(img_pattern, scribd_conn.content)
    json_list.extend(img_links)

    json_list = sorted(json_list, key = sort_json)

    return json_list

def write_image(img_file, img_link):
    '''
        Downloads the image from img_link onto the img_file
    '''
    f = open(img_file, 'wb')

    while 1:
        try:
            f.write(requests.get(img_link).content)
        except requests.ConnectionError:
            print 'break at getting img ', img_link, img_file
            continue
        except Exception as e:
            print e, img_file
            continue
        break
    f.close()

def get_img_url(jsonp_link):
    '''
        Gets the image url from a .jsonp link
    '''
    while 1:
        try:
            json_conn = requests.get(jsonp_link)
        except requests.ConnectionError:
            print 'breaking at getting json',
            continue
        break
    resp = json_conn.content

    try:
        img_link = re.findall(img_pattern, resp)[0]
    except IndexError:
        print 'no img link'
        img_link = ''

    return img_link

def convert_to_pdf(img_list, output_folder, pdf_file_name):
    '''
        Converts a list of image files into a pdf at the ouput directory
    '''

    pdf_bytes = img2pdf.convert(img_list, dpi = 100)

    with open(os.path.join(output_folder, pdf_file_name), 'wb') as f:
        f.write(pdf_bytes)

def download_img(urls, start, end, img_dir, pdf_name, shared):
    '''
        Downloads the images from the start index to end index
        of the list of urls (urls) to the img_dir

        Also appends the paths of the images to the shared list
        created my Manager.manager().list()
    '''

    print 'at downloading_img'
    image_list = set()

    print len(urls), 'files'
    print 'images downloaded to {}'.format(img_dir)

    i = start
    end = min(end, len(urls))
    n = str(len(str(len(urls))))

    while i < end:

        img_name = '{}{:0{k}d}.jpg'.format(pdf_name, i+1, k = n)
        img_path = os.path.join(img_dir, img_name)
        image_list.add(img_path)
        if os.path.exists(img_path):
            print img_path, 'exists'
            i += 1
            continue

        if 'jsonp' in urls[i]:
            img_url = get_img_url(urls[i])
        else:
            img_url = urls[i]

        write_image(img_path, img_url)
        i += 1
        if DEBUG:
            print 'Completed img {}'.format(i+1)
    shared.extend(image_list)

if __name__ == '__main__':

    processes = []

    a = raw_input('''scribd link, pdf file name, \
img_folder, out_folder separated by spaces \n''')

    print a.split()
    scribd, pdf, img_folder, out_folder = a.split()

    N = int(raw_input('Enter number of separate processes: '))

    s = time.clock()
    print 'started at', time.ctime()

    combined_list = img_jsonp_urls(scribd)
    n = len(combined_list)

    manager = Manager()
    img_files = manager.list()

    k = n/N

    for i in range(N):
        start = k*i
        end = k*(i+1)
        if i == N-1:
            end = max(end, n)
        process = Process(target = download_img, args =
                          [combined_list, start, end, img_folder, pdf,
                           img_files])

        print 'process {} started'.format(i)
        process.start()
        processes.append(process)

    for process in processes:
        process.join()

    print 'image file names secured'

    img_files = sorted(list(img_files))

    print 'converting'

    convert_to_pdf(img_files, out_folder, pdf + '.pdf')
    print 'finished at', time.ctime()
    print 'time taken: {} seconds'.format(time.clock() - s)
<commit_msg>removed some unused import statements<commit_after>__author__ = 'Asish Mahapatra: asishkm@gmail.com'

import requests
import re
import os
import img2pdf
from multiprocessing import Process, Manager
import time

json_pattern = re.compile(r'https.*scribdassets.*jsonp')
img_pattern = re.compile(r'orig.*(http.*scribd.{,45}jpg)')

patterns = {'jpg': re.compile(r'([0-9]*)-.*jpg'),
            'jsonp': re.compile(r'([0-9]*)-.*jsonp')}

DEBUG = False

def sort_json(link):
    '''
        Gets the page number of the image or the jsonp file.
        Urls have the forum 'pagenum-randomhash.jpg / jsonp'
    '''
    for pattern in patterns:
        if pattern in link:
            s = link.index(pattern)-16
            val = re.search(patterns[pattern], link[s:])
            return int(val.groups()[0])

def img_jsonp_urls(scribd_link):
    '''
        Gets a sorted list of URLs based on page number with both
        jsonp or jpg extensions
    '''
    print 'at get_images'

    scribd_conn = requests.get(scribd_link)

    json_list = re.findall(json_pattern, scribd_conn.content)
    img_links = re.findall(img_pattern, scribd_conn.content)
    json_list.extend(img_links)

    json_list = sorted(json_list, key = sort_json)

    return json_list

def write_image(img_file, img_link):
    '''
        Downloads the image from img_link onto the img_file
    '''
    f = open(img_file, 'wb')

    while 1:
        try:
            f.write(requests.get(img_link).content)
        except requests.ConnectionError:
            print 'break at getting img ', img_link, img_file
            continue
        except Exception as e:
            print e, img_file
            continue
        break
    f.close()

def get_img_url(jsonp_link):
    '''
        Gets the image url from a .jsonp link
    '''
    while 1:
        try:
            json_conn = requests.get(jsonp_link)
        except requests.ConnectionError:
            print 'breaking at getting json',
            continue
        break
    resp = json_conn.content

    try:
        img_link = re.findall(img_pattern, resp)[0]
    except IndexError:
        print 'no img link'
        img_link = ''

    return img_link

def convert_to_pdf(img_list, output_folder, pdf_file_name):
    '''
        Converts a list of image files into a pdf at the ouput directory
    '''

    pdf_bytes = img2pdf.convert(img_list, dpi = 100)

    with open(os.path.join(output_folder, pdf_file_name), 'wb') as f:
        f.write(pdf_bytes)

def download_img(urls, start, end, img_dir, pdf_name, shared):
    '''
        Downloads the images from the start index to end index
        of the list of urls (urls) to the img_dir

        Also appends the paths of the images to the shared list
        created my Manager.manager().list()
    '''

    print 'at downloading_img'
    image_list = set()

    print len(urls), 'files'
    print 'images downloaded to {}'.format(img_dir)

    i = start
    end = min(end, len(urls))
    n = str(len(str(len(urls))))

    while i < end:

        img_name = '{}{:0{k}d}.jpg'.format(pdf_name, i+1, k = n)
        img_path = os.path.join(img_dir, img_name)
        image_list.add(img_path)
        if os.path.exists(img_path):
            print img_path, 'exists'
            i += 1
            continue

        if 'jsonp' in urls[i]:
            img_url = get_img_url(urls[i])
        else:
            img_url = urls[i]

        write_image(img_path, img_url)
        i += 1
        if DEBUG:
            print 'Completed img {}'.format(i+1)
    shared.extend(image_list)

if __name__ == '__main__':

    processes = []

    a = raw_input('''scribd link, pdf file name, \
img_folder, out_folder separated by spaces \n''')

    print a.split()
    scribd, pdf, img_folder, out_folder = a.split()

    N = int(raw_input('Enter number of separate processes: '))

    s = time.clock()
    print 'started at', time.ctime()

    combined_list = img_jsonp_urls(scribd)
    n = len(combined_list)

    manager = Manager()
    img_files = manager.list()

    k = n/N

    for i in range(N):
        start = k*i
        end = k*(i+1)
        if i == N-1:
            end = max(end, n)
        process = Process(target = download_img, args =
                          [combined_list, start, end, img_folder, pdf,
                           img_files])

        print 'process {} started'.format(i)
        process.start()
        processes.append(process)

    for process in processes:
        process.join()

    print 'image file names secured'

    img_files = sorted(list(img_files))

    print 'converting'

    convert_to_pdf(img_files, out_folder, pdf + '.pdf')
    print 'finished at', time.ctime()
    print 'time taken: {} seconds'.format(time.clock() - s)
<|endoftext|>


<commit_before># -*- coding: utf8 -*-

from __future__ import absolute_import
from __future__ import division, print_function, unicode_literals

from .._compat import to_unicode
from ..utils import cached_property
from ..models.dom import Sentence, Paragraph, ObjectDocumentModel
from .parser import DocumentParser

class PlaintextParser(DocumentParser):
    @classmethod
    def from_string(cls, string, tokenizer):
        return cls(string, tokenizer)

    @classmethod
    def from_file(cls, file_path, tokenizer):
        with open(file_path) as file:
            return cls(file.read(), tokenizer)

    def __init__(self, text, tokenizer):
        super(PlaintextParser, self).__init__(tokenizer)
        self._text = to_unicode(text).strip()

    @cached_property
    def significant_words(self):
        words = []
        for paragraph in self.document.paragraphs:
            for sentence in paragraph:
                if sentence.is_heading:
                    words.extend(sentence.words)

        if words:
            return tuple(words)
        else:
            return self.SIGNIFICANT_WORDS

    @cached_property
    def stigma_words(self):
        return self.STIGMA_WORDS

    @cached_property
    def document(self):
        current_paragraph = []
        paragraphs = []
        for line in self._text.splitlines():
            line = line.strip()
            if line.isupper():
                words = self.tokenize_words(line)
                current_paragraph.append(Sentence(words, is_heading=True))
            elif not line and current_paragraph:
                sentences = self._to_sentences(current_paragraph)
                paragraphs.append(Paragraph(sentences))
                current_paragraph = []
            elif line:
                current_paragraph.append(line)

        sentences = self._to_sentences(current_paragraph)
        paragraphs.append(Paragraph(sentences))

        return ObjectDocumentModel(paragraphs)

    def _to_sentences(self, lines):
        text = ""
        sentence_objects = []

        for line in lines:
            if isinstance(line, Sentence):
                if text:
                    sentences = self.tokenize_sentences(text)
                    sentence_objects += map(self._to_sentence, sentences)

                sentence_objects.append(line)
                text = ""
            else:
                text += " " + line

        sentences = self.tokenize_sentences(text.strip())
        sentence_objects += map(self._to_sentence, sentences)

        return sentence_objects

    def _to_sentence(self, text):
        assert text.strip()
        return Sentence(self.tokenize_words(text))
<commit_msg>Don't try to split empty string into sentences<commit_after># -*- coding: utf8 -*-

from __future__ import absolute_import
from __future__ import division, print_function, unicode_literals

from .._compat import to_unicode
from ..utils import cached_property
from ..models.dom import Sentence, Paragraph, ObjectDocumentModel
from .parser import DocumentParser

class PlaintextParser(DocumentParser):
    @classmethod
    def from_string(cls, string, tokenizer):
        return cls(string, tokenizer)

    @classmethod
    def from_file(cls, file_path, tokenizer):
        with open(file_path) as file:
            return cls(file.read(), tokenizer)

    def __init__(self, text, tokenizer):
        super(PlaintextParser, self).__init__(tokenizer)
        self._text = to_unicode(text).strip()

    @cached_property
    def significant_words(self):
        words = []
        for paragraph in self.document.paragraphs:
            for sentence in paragraph:
                if sentence.is_heading:
                    words.extend(sentence.words)

        if words:
            return tuple(words)
        else:
            return self.SIGNIFICANT_WORDS

    @cached_property
    def stigma_words(self):
        return self.STIGMA_WORDS

    @cached_property
    def document(self):
        current_paragraph = []
        paragraphs = []
        for line in self._text.splitlines():
            line = line.strip()
            if line.isupper():
                words = self.tokenize_words(line)
                current_paragraph.append(Sentence(words, is_heading=True))
            elif not line and current_paragraph:
                sentences = self._to_sentences(current_paragraph)
                paragraphs.append(Paragraph(sentences))
                current_paragraph = []
            elif line:
                current_paragraph.append(line)

        sentences = self._to_sentences(current_paragraph)
        paragraphs.append(Paragraph(sentences))

        return ObjectDocumentModel(paragraphs)

    def _to_sentences(self, lines):
        text = ""
        sentence_objects = []

        for line in lines:
            if isinstance(line, Sentence):
                if text:
                    sentences = self.tokenize_sentences(text)
                    sentence_objects += map(self._to_sentence, sentences)

                sentence_objects.append(line)
                text = ""
            else:
                text += " " + line

        text = text.strip()
        if text:
            sentences = self.tokenize_sentences(text)
            sentence_objects += map(self._to_sentence, sentences)

        return sentence_objects

    def _to_sentence(self, text):
        assert text.strip()
        return Sentence(self.tokenize_words(text))
<|endoftext|>


<commit_before>"""Python implementation of Friedman's Supersmoother"""
from __future__ import absolute_import

__version__ = '0.3.2'

from .smoother import MovingAverageSmoother, LinearSmoother
from .supersmoother import SuperSmoother
<commit_msg>MAINT: bump version to 0.4.dev<commit_after>"""Python implementation of Friedman's Supersmoother"""
from __future__ import absolute_import

__version__ = '0.4.dev0'

from .smoother import MovingAverageSmoother, LinearSmoother
from .supersmoother import SuperSmoother
<|endoftext|>


<commit_before>"""This submodule contains tools for creating svg files from paths and path
segments."""

# External dependencies:
from __future__ import division, absolute_import, print_function
from math import ceil
from os import getcwd, path as os_path, makedirs
from xml.dom.minidom import parse as md_xml_parse
from svgwrite import Drawing, text as txt
from time import time
from warnings import warn

# Internal dependencies
from .path import Path, Line, is_path_segment
from .misctools import open_in_browser

# Used to convert a string colors (identified by single chars) to a list.
color_dict = {'a': 'aqua',
              'b': 'blue',
              'c': 'cyan',
              'd': 'darkblue',
              'e': '',
              'f': '',
              'g': 'green',
              'h': '',
              'i': '',
              'j': '',
              'k': 'black',
              'l': 'lime',
              'm': 'magenta',
              'n': 'brown',
              'o': 'orange',
              'p': 'pink',
              'q': 'turquoise',
              'r': 'red',
              's': 'salmon',
              't': 'tan',
              'u': 'purple',
              'v': 'violet',
              'w': 'white',
              'x': '',
              'y': 'yellow',
              'z': 'azure'}

def str2colorlist(s, default_color=None):
    color_list = [color_dict[ch] for ch in s]
    if default_color:
        for idx, c in enumerate(color_list):
            if not c:
                color_list[idx] = default_color
    return color_list

def is3tuple(c):
    return isinstance(c, tuple) and len(c) == 3

def big_bounding_box(paths_n_stuff):
    """Finds a BB containing a collection of paths, Bezier path segments, and
    points (given as complex numbers)."""
    bbs = []
    for thing in paths_n_stuff:
        if is_path_segment(thing) or isinstance(thing, Path):
            bbs.append(thing.bbox())
        elif isinstance(thing, complex):
            bbs.append((thing.real, thing.real, thing.imag, thing.imag))
        else:
            try:
                complexthing = complex(thing)
                bbs.append((complexthing.real, complexthing.real,
                            complexthing.imag, complexthing.imag))
            except ValueError:
                raise TypeError(
                    "paths_n_stuff can only contains Path, CubicBezier, "
                    "QuadraticBezier, Line, and complex objects.")
    xmins, xmaxs, ymins, ymaxs = list(zip(*bbs))
    xmin = min(xmins)
    xmax = max(xmaxs)
    ymin = min(ymins)
    ymax = max(ymaxs)
    return xmin, xmax, ymin, ymax

def disvg(paths=None, colors=None,
          filename=os_path.join(getcwd(), 'disvg_output.svg'),
          stroke_widths=None, nodes=None, node_colors=None, node_radii=None,
          openinbrowser=True, timestamp=False,
          margin_size=0.1, mindim=600, dimensions=None,
          viewbox=None, text=None, text_path=None, font_size=None,
          attributes=None, svg_attributes=None, svgwrite_debug=False):
    """Takes in a list of paths and creates an SVG file containing said paths.
    REQUIRED INPUTS:
        :param paths - a list of paths

    OPTIONAL INPUT:
        :param colors - specifies the path stroke color.  By default all paths
        will be black (#000000).  This paramater can be input in a few ways
        1) a list of strings that will be input into the path elements stroke
            attribute (so anything that is understood by the svg viewer).
        2) a string of single character colors -- e.g. setting colors='rrr' is
            equivalent to setting colors=['red', 'red', 'red'] (see the
            'color_dict' dictionary above for a list of possibilities).
        3) a list of rgb 3-tuples -- e.g. colors = [(255, 0, 0), ...].

        :param filename - the desired location/filename of the SVG file
        created (by default the SVG will be stored in the current working
        directory and named 'disvg_output.svg').

        :param stroke_widths - a list of stroke_widths to use for paths
        (default is 0.5% of the SVG's width or length)

        :param nodes - a list of points to draw as filled-in circles

        :param node_colors - a list of colors to use for the nodes (by default
        nodes will be red)

        :param node_radii - a list of radii to use for the nodes (by default
        nodes will be radius will be 1 percent of the svg's width/length)

        :param text - string or list of strings to be displayed

        :param text_path - if text is a list, then this should be a list of
        path (or path segments of the same length.  Note: the path must be
        long enough to display the text or the text will be cropped by the svg
        viewer.

        :param font_size - a single float of list of floats.

        :param openinbrowser -  Set to True to automatically open the created
        SVG in the user's default web browser.

        :param timestamp - if True, then the a timestamp will be appended to
        the output SVG's filename.  This will fix issues with rapidly opening
        multiple SVGs in your browser.

        :param margin_size - The min margin (empty area framing the collection
        of paths) size used for creating the canvas and background of the SVG.

        :param mindim - The minimum dimension (height or width) of the output
        SVG (default is 600).

        :param dimensions - The display dimensions of the output SVG.  Using
        this will override the mindim parameter.

        :param viewbox - This specifies what rectangular patch of R^2 will be
        viewable through the outputSVG.  It should be input in the form
        (min_x, min_y, width, height).  This is different from the display
        dimension of the svg, which can be set through mindim or dimensions.

        :param attributes - a list of dictionaries of attributes for the input
        paths.  Note: This will override any other conflicting settings.

        :param svg_attributes - a dictionary of attributes for output svg.

        :param svgwrite_debug - This parameter turns on/off `svgwrite`'s
        debugging mode.  By default svgwrite_debug=False.  This increases
        speed and also prevents `svgwrite` from raising of an error when not
        all `svg_attributes` key-value pairs are understood.

    NOTES:
        * The `svg_attributes` parameter will override any other conflicting
        settings.

        * Any `extra` parameters that `svgwrite.Drawing()` accepts can be
        controlled by passing them in through `svg_attributes`.

        * The unit of length here is assumed to be pixels in all variables.

        * If this function is used multiple times in quick succession to
        display multiple SVGs (all using the default filename), the
        svgviewer/browser will likely fail to load some of the SVGs in time.
        To fix this, use the timestamp attribute, or give the files unique
        names, or use a pause command (e.g. time.sleep(1)) between uses.
    """

    _default_relative_node_radius = 5e-3
    _default_relative_stroke_width = 1e-3
    _default_path_color = '#000000'  # black
    _default_node_color = '#ff0000'  # red
    _default_font_size = 12

    # append directory to filename (if not included)
    if os_path.dirname(filename) == '':
        filename = os_path.join(getcwd(), filename)

    # append time stamp to filename
    if timestamp:
        fbname, fext = os_path.splitext(filename)
        dirname = os_path.dirname(filename)
        tstamp = str(time()).replace('.', '')
        stfilename = os_path.split(fbname)[1] + '_' + tstamp + fext
        filename = os_path.join(dirname, stfilename)

    # check paths and colors are set
    if isinstance(paths, Path) or is_path_segment(paths):
        paths = [paths]
    if paths:
        if not colors:
            colors = [_default_path_color] * len(paths)
        else:
            assert len(colors) == len(paths)
            if isinstance(colors, str):
                colors = str2colorlist(colors,
                                       default_color=_default_path_color)
            elif isinstance(colors, list):
                for idx, c in enumerate(colors):
                    if is3tuple(c):
                        colors[idx] = "rgb" + str(c)

    # check nodes and nodes_colors are set (node_radii are set later)
    if nodes:
        if not node_colors:
            node_colors = [_default_node_color] * len(nodes)
        else:
            assert len(node_colors) == len(nodes)
            if isinstance(node_colors, str):
                node_colors = str2colorlist(node_colors,
                                            default_color=_default_node_color)
            elif isinstance(node_colors, list):
                for idx, c in enumerate(node_colors):
                    if is3tuple(c):
                        node_colors[idx] = "rgb" + str(c)

    # set up the viewBox and display dimensions of the output SVG
    # along the way, set stroke_widths and node_radii if not provided
    assert paths or nodes
    stuff2bound = []
    if viewbox:
        szx, szy = viewbox[2:4]
    else:
        if paths:
            stuff2bound += paths
        if nodes:
            stuff2bound += nodes
        if text_path:
            stuff2bound += text_path
        xmin, xmax, ymin, ymax = big_bounding_box(stuff2bound)
        dx = xmax - xmin
        dy = ymax - ymin

        if dx == 0:
            dx = 1
        if dy == 0:
            dy = 1

        # determine stroke_widths to use (if not provided) and max_stroke_width
        if paths:
            if not stroke_widths:
                sw = max(dx, dy) * _default_relative_stroke_width
                stroke_widths = [sw]*len(paths)
                max_stroke_width = sw
            else:
                assert len(paths) == len(stroke_widths)
                max_stroke_width = max(stroke_widths)
        else:
            max_stroke_width = 0

        # determine node_radii to use (if not provided) and max_node_diameter
        if nodes:
            if not node_radii:
                r = max(dx, dy) * _default_relative_node_radius
                node_radii = [r]*len(nodes)
                max_node_diameter = 2*r
            else:
                assert len(nodes) == len(node_radii)
                max_node_diameter = 2*max(node_radii)
        else:
            max_node_diameter = 0

        extra_space_for_style = max(max_stroke_width, max_node_diameter)
        xmin -= margin_size*dx + extra_space_for_style/2
        ymin -= margin_size*dy + extra_space_for_style/2
        dx += 2*margin_size*dx + extra_space_for_style
        dy += 2*margin_size*dy + extra_space_for_style
        viewbox = "%s %s %s %s" % (xmin, ymin, dx, dy)
        if dimensions:
            szx, szy = dimensions
        else:
            if dx > dy:
                szx = str(mindim) + 'px'
                szy = str(int(ceil(mindim * dy / dx))) + 'px'
            else:
                szx = str(int(ceil(mindim * dx / dy))) + 'px'
                szy = str(mindim) + 'px'

    # Create an SVG file
    if svg_attributes is not None:
        szx = svg_attributes.get("width", szx)
        szy = svg_attributes.get("height", szy)
        debug = svg_attributes.get("debug", svgwrite_debug)
        dwg = Drawing(filename=filename, size=(szx, szy), debug=debug,
                      **svg_attributes)
    else:
        dwg = Drawing(filename=filename, size=(szx, szy), debug=svgwrite_debug,
                      viewBox=viewbox)

    # add paths
    if paths:
        for i, p in enumerate(paths):
            if isinstance(p, Path):
                ps = p.d()
            elif is_path_segment(p):
                ps = Path(p).d()
            else:  # assume this path, p, was input as a Path d-string
                ps = p

            if attributes:
                good_attribs = {'d': ps}
                for key in attributes[i]:
                    val = attributes[i][key]
                    if key != 'd':
                        try:
                            dwg.path(ps, **{key: val})
                            good_attribs.update({key: val})
                        except Exception as e:
                            warn(str(e))

                dwg.add(dwg.path(**good_attribs))
            else:
                dwg.add(dwg.path(ps, stroke=colors[i],
                                 stroke_width=str(stroke_widths[i]),
                                 fill='none'))

    # add nodes (filled in circles)
    if nodes:
        for i_pt, pt in enumerate([(z.real, z.imag) for z in nodes]):
            dwg.add(dwg.circle(pt, node_radii[i_pt], fill=node_colors[i_pt]))

    # add texts
    if text:
        assert isinstance(text, str) or (isinstance(text, list) and
                                         isinstance(text_path, list) and
                                         len(text_path) == len(text))
        if isinstance(text, str):
            text = [text]
            if not font_size:
                font_size = [_default_font_size]
            if not text_path:
                pos = complex(xmin + margin_size*dx, ymin + margin_size*dy)
                text_path = [Line(pos, pos + 1).d()]
        else:
            if font_size:
                if isinstance(font_size, list):
                    assert len(font_size) == len(text)
                else:
                    font_size = [font_size] * len(text)
            else:
                font_size = [_default_font_size] * len(text)
        for idx, s in enumerate(text):
            p = text_path[idx]
            if isinstance(p, Path):
                ps = p.d()
            elif is_path_segment(p):
                ps = Path(p).d()
            else:  # assume this path, p, was input as a Path d-string
                ps = p

            # paragraph = dwg.add(dwg.g(font_size=font_size[idx]))
            # paragraph.add(dwg.textPath(ps, s))
            pathid = 'tp' + str(idx)
            dwg.defs.add(dwg.path(d=ps, id=pathid))
            txter = dwg.add(dwg.text('', font_size=font_size[idx]))
            txter.add(txt.TextPath('#'+pathid, s))

    # save svg
    if not os_path.exists(os_path.dirname(filename)):
        makedirs(os_path.dirname(filename))
    dwg.save()

    # re-open the svg, make the xml pretty, and save it again
    xmlstring = md_xml_parse(filename).toprettyxml()
    with open(filename, 'w') as f:
        f.write(xmlstring)

    # try to open in web browser
    if openinbrowser:
        try:
            open_in_browser(filename)
        except:
            print("Failed to open output SVG in browser.  SVG saved to:")
            print(filename)

def wsvg(paths=None, colors=None,
          filename=os_path.join(getcwd(), 'disvg_output.svg'),
          stroke_widths=None, nodes=None, node_colors=None, node_radii=None,
          openinbrowser=False, timestamp=False,
          margin_size=0.1, mindim=600, dimensions=None,
          viewbox=None, text=None, text_path=None, font_size=None,
          attributes=None, svg_attributes=None, svgwrite_debug=False):
    """Convenience function; identical to disvg() except that
    openinbrowser=False by default.  See disvg() docstring for more info."""
    disvg(paths, colors=colors, filename=filename,
          stroke_widths=stroke_widths, nodes=nodes,
          node_colors=node_colors, node_radii=node_radii,
          openinbrowser=openinbrowser, timestamp=timestamp,
          margin_size=margin_size, mindim=mindim, dimensions=dimensions,
          viewbox=viewbox, text=text, text_path=text_path, font_size=font_size,
          attributes=attributes, svg_attributes=svg_attributes,
          svgwrite_debug=svgwrite_debug)
<commit_msg>Fixed issue with viewbox being written as tuple<commit_after>"""This submodule contains tools for creating svg files from paths and path
segments."""

# External dependencies:
from __future__ import division, absolute_import, print_function
from math import ceil
from os import getcwd, path as os_path, makedirs
from xml.dom.minidom import parse as md_xml_parse
from svgwrite import Drawing, text as txt
from time import time
from warnings import warn

# Internal dependencies
from .path import Path, Line, is_path_segment
from .misctools import open_in_browser

# Used to convert a string colors (identified by single chars) to a list.
color_dict = {'a': 'aqua',
              'b': 'blue',
              'c': 'cyan',
              'd': 'darkblue',
              'e': '',
              'f': '',
              'g': 'green',
              'h': '',
              'i': '',
              'j': '',
              'k': 'black',
              'l': 'lime',
              'm': 'magenta',
              'n': 'brown',
              'o': 'orange',
              'p': 'pink',
              'q': 'turquoise',
              'r': 'red',
              's': 'salmon',
              't': 'tan',
              'u': 'purple',
              'v': 'violet',
              'w': 'white',
              'x': '',
              'y': 'yellow',
              'z': 'azure'}

def str2colorlist(s, default_color=None):
    color_list = [color_dict[ch] for ch in s]
    if default_color:
        for idx, c in enumerate(color_list):
            if not c:
                color_list[idx] = default_color
    return color_list

def is3tuple(c):
    return isinstance(c, tuple) and len(c) == 3

def big_bounding_box(paths_n_stuff):
    """Finds a BB containing a collection of paths, Bezier path segments, and
    points (given as complex numbers)."""
    bbs = []
    for thing in paths_n_stuff:
        if is_path_segment(thing) or isinstance(thing, Path):
            bbs.append(thing.bbox())
        elif isinstance(thing, complex):
            bbs.append((thing.real, thing.real, thing.imag, thing.imag))
        else:
            try:
                complexthing = complex(thing)
                bbs.append((complexthing.real, complexthing.real,
                            complexthing.imag, complexthing.imag))
            except ValueError:
                raise TypeError(
                    "paths_n_stuff can only contains Path, CubicBezier, "
                    "QuadraticBezier, Line, and complex objects.")
    xmins, xmaxs, ymins, ymaxs = list(zip(*bbs))
    xmin = min(xmins)
    xmax = max(xmaxs)
    ymin = min(ymins)
    ymax = max(ymaxs)
    return xmin, xmax, ymin, ymax

def disvg(paths=None, colors=None,
          filename=os_path.join(getcwd(), 'disvg_output.svg'),
          stroke_widths=None, nodes=None, node_colors=None, node_radii=None,
          openinbrowser=True, timestamp=False,
          margin_size=0.1, mindim=600, dimensions=None,
          viewbox=None, text=None, text_path=None, font_size=None,
          attributes=None, svg_attributes=None, svgwrite_debug=False):
    """Takes in a list of paths and creates an SVG file containing said paths.
    REQUIRED INPUTS:
        :param paths - a list of paths

    OPTIONAL INPUT:
        :param colors - specifies the path stroke color.  By default all paths
        will be black (#000000).  This paramater can be input in a few ways
        1) a list of strings that will be input into the path elements stroke
            attribute (so anything that is understood by the svg viewer).
        2) a string of single character colors -- e.g. setting colors='rrr' is
            equivalent to setting colors=['red', 'red', 'red'] (see the
            'color_dict' dictionary above for a list of possibilities).
        3) a list of rgb 3-tuples -- e.g. colors = [(255, 0, 0), ...].

        :param filename - the desired location/filename of the SVG file
        created (by default the SVG will be stored in the current working
        directory and named 'disvg_output.svg').

        :param stroke_widths - a list of stroke_widths to use for paths
        (default is 0.5% of the SVG's width or length)

        :param nodes - a list of points to draw as filled-in circles

        :param node_colors - a list of colors to use for the nodes (by default
        nodes will be red)

        :param node_radii - a list of radii to use for the nodes (by default
        nodes will be radius will be 1 percent of the svg's width/length)

        :param text - string or list of strings to be displayed

        :param text_path - if text is a list, then this should be a list of
        path (or path segments of the same length.  Note: the path must be
        long enough to display the text or the text will be cropped by the svg
        viewer.

        :param font_size - a single float of list of floats.

        :param openinbrowser -  Set to True to automatically open the created
        SVG in the user's default web browser.

        :param timestamp - if True, then the a timestamp will be appended to
        the output SVG's filename.  This will fix issues with rapidly opening
        multiple SVGs in your browser.

        :param margin_size - The min margin (empty area framing the collection
        of paths) size used for creating the canvas and background of the SVG.

        :param mindim - The minimum dimension (height or width) of the output
        SVG (default is 600).

        :param dimensions - The display dimensions of the output SVG.  Using
        this will override the mindim parameter.

        :param viewbox - This specifies what rectangular patch of R^2 will be
        viewable through the outputSVG.  It should be input as a 4-tuple,
        (min_x, min_y, width, height), or a string "min_x min_y width height".
        This is different from the display dimension of the svg, which can be
        set through mindim or dimensions.

        :param attributes - a list of dictionaries of attributes for the input
        paths.  Note: This will override any other conflicting settings.

        :param svg_attributes - a dictionary of attributes for output svg.

        :param svgwrite_debug - This parameter turns on/off `svgwrite`'s
        debugging mode.  By default svgwrite_debug=False.  This increases
        speed and also prevents `svgwrite` from raising of an error when not
        all `svg_attributes` key-value pairs are understood.

    NOTES:
        * The `svg_attributes` parameter will override any other conflicting
        settings.

        * Any `extra` parameters that `svgwrite.Drawing()` accepts can be
        controlled by passing them in through `svg_attributes`.

        * The unit of length here is assumed to be pixels in all variables.

        * If this function is used multiple times in quick succession to
        display multiple SVGs (all using the default filename), the
        svgviewer/browser will likely fail to load some of the SVGs in time.
        To fix this, use the timestamp attribute, or give the files unique
        names, or use a pause command (e.g. time.sleep(1)) between uses.
    """

    _default_relative_node_radius = 5e-3
    _default_relative_stroke_width = 1e-3
    _default_path_color = '#000000'  # black
    _default_node_color = '#ff0000'  # red
    _default_font_size = 12

    # append directory to filename (if not included)
    if os_path.dirname(filename) == '':
        filename = os_path.join(getcwd(), filename)

    # append time stamp to filename
    if timestamp:
        fbname, fext = os_path.splitext(filename)
        dirname = os_path.dirname(filename)
        tstamp = str(time()).replace('.', '')
        stfilename = os_path.split(fbname)[1] + '_' + tstamp + fext
        filename = os_path.join(dirname, stfilename)

    # check paths and colors are set
    if isinstance(paths, Path) or is_path_segment(paths):
        paths = [paths]
    if paths:
        if not colors:
            colors = [_default_path_color] * len(paths)
        else:
            assert len(colors) == len(paths)
            if isinstance(colors, str):
                colors = str2colorlist(colors,
                                       default_color=_default_path_color)
            elif isinstance(colors, list):
                for idx, c in enumerate(colors):
                    if is3tuple(c):
                        colors[idx] = "rgb" + str(c)

    # check nodes and nodes_colors are set (node_radii are set later)
    if nodes:
        if not node_colors:
            node_colors = [_default_node_color] * len(nodes)
        else:
            assert len(node_colors) == len(nodes)
            if isinstance(node_colors, str):
                node_colors = str2colorlist(node_colors,
                                            default_color=_default_node_color)
            elif isinstance(node_colors, list):
                for idx, c in enumerate(node_colors):
                    if is3tuple(c):
                        node_colors[idx] = "rgb" + str(c)

    # set up the viewBox and display dimensions of the output SVG
    # along the way, set stroke_widths and node_radii if not provided
    assert paths or nodes
    stuff2bound = []
    if viewbox:
        if isinstance(viewbox, str):
            szx, szy = viewbox.split(' ')[2:4]
        else:
            szx, szy = viewbox[2:4]
            viewbox = '%s %s %s %s' % viewbox
    else:
        if paths:
            stuff2bound += paths
        if nodes:
            stuff2bound += nodes
        if text_path:
            stuff2bound += text_path
        xmin, xmax, ymin, ymax = big_bounding_box(stuff2bound)
        dx = xmax - xmin
        dy = ymax - ymin

        if dx == 0:
            dx = 1
        if dy == 0:
            dy = 1

        # determine stroke_widths to use (if not provided) and max_stroke_width
        if paths:
            if not stroke_widths:
                sw = max(dx, dy) * _default_relative_stroke_width
                stroke_widths = [sw]*len(paths)
                max_stroke_width = sw
            else:
                assert len(paths) == len(stroke_widths)
                max_stroke_width = max(stroke_widths)
        else:
            max_stroke_width = 0

        # determine node_radii to use (if not provided) and max_node_diameter
        if nodes:
            if not node_radii:
                r = max(dx, dy) * _default_relative_node_radius
                node_radii = [r]*len(nodes)
                max_node_diameter = 2*r
            else:
                assert len(nodes) == len(node_radii)
                max_node_diameter = 2*max(node_radii)
        else:
            max_node_diameter = 0

        extra_space_for_style = max(max_stroke_width, max_node_diameter)
        xmin -= margin_size*dx + extra_space_for_style/2
        ymin -= margin_size*dy + extra_space_for_style/2
        dx += 2*margin_size*dx + extra_space_for_style
        dy += 2*margin_size*dy + extra_space_for_style
        viewbox = "%s %s %s %s" % (xmin, ymin, dx, dy)
        if dimensions:
            szx, szy = dimensions
        else:
            if dx > dy:
                szx = str(mindim) + 'px'
                szy = str(int(ceil(mindim * dy / dx))) + 'px'
            else:
                szx = str(int(ceil(mindim * dx / dy))) + 'px'
                szy = str(mindim) + 'px'

    # Create an SVG file
    if svg_attributes is not None:
        szx = svg_attributes.get("width", szx)
        szy = svg_attributes.get("height", szy)
        debug = svg_attributes.get("debug", svgwrite_debug)
        dwg = Drawing(filename=filename, size=(szx, szy), debug=debug,
                      **svg_attributes)
    else:
        dwg = Drawing(filename=filename, size=(szx, szy), debug=svgwrite_debug,
                      viewBox=viewbox)

    # add paths
    if paths:
        for i, p in enumerate(paths):
            if isinstance(p, Path):
                ps = p.d()
            elif is_path_segment(p):
                ps = Path(p).d()
            else:  # assume this path, p, was input as a Path d-string
                ps = p

            if attributes:
                good_attribs = {'d': ps}
                for key in attributes[i]:
                    val = attributes[i][key]
                    if key != 'd':
                        try:
                            dwg.path(ps, **{key: val})
                            good_attribs.update({key: val})
                        except Exception as e:
                            warn(str(e))

                dwg.add(dwg.path(**good_attribs))
            else:
                dwg.add(dwg.path(ps, stroke=colors[i],
                                 stroke_width=str(stroke_widths[i]),
                                 fill='none'))

    # add nodes (filled in circles)
    if nodes:
        for i_pt, pt in enumerate([(z.real, z.imag) for z in nodes]):
            dwg.add(dwg.circle(pt, node_radii[i_pt], fill=node_colors[i_pt]))

    # add texts
    if text:
        assert isinstance(text, str) or (isinstance(text, list) and
                                         isinstance(text_path, list) and
                                         len(text_path) == len(text))
        if isinstance(text, str):
            text = [text]
            if not font_size:
                font_size = [_default_font_size]
            if not text_path:
                pos = complex(xmin + margin_size*dx, ymin + margin_size*dy)
                text_path = [Line(pos, pos + 1).d()]
        else:
            if font_size:
                if isinstance(font_size, list):
                    assert len(font_size) == len(text)
                else:
                    font_size = [font_size] * len(text)
            else:
                font_size = [_default_font_size] * len(text)
        for idx, s in enumerate(text):
            p = text_path[idx]
            if isinstance(p, Path):
                ps = p.d()
            elif is_path_segment(p):
                ps = Path(p).d()
            else:  # assume this path, p, was input as a Path d-string
                ps = p

            # paragraph = dwg.add(dwg.g(font_size=font_size[idx]))
            # paragraph.add(dwg.textPath(ps, s))
            pathid = 'tp' + str(idx)
            dwg.defs.add(dwg.path(d=ps, id=pathid))
            txter = dwg.add(dwg.text('', font_size=font_size[idx]))
            txter.add(txt.TextPath('#'+pathid, s))

    # save svg
    if not os_path.exists(os_path.dirname(filename)):
        makedirs(os_path.dirname(filename))
    dwg.save()

    # re-open the svg, make the xml pretty, and save it again
    xmlstring = md_xml_parse(filename).toprettyxml()
    with open(filename, 'w') as f:
        f.write(xmlstring)

    # try to open in web browser
    if openinbrowser:
        try:
            open_in_browser(filename)
        except:
            print("Failed to open output SVG in browser.  SVG saved to:")
            print(filename)

def wsvg(paths=None, colors=None,
          filename=os_path.join(getcwd(), 'disvg_output.svg'),
          stroke_widths=None, nodes=None, node_colors=None, node_radii=None,
          openinbrowser=False, timestamp=False,
          margin_size=0.1, mindim=600, dimensions=None,
          viewbox=None, text=None, text_path=None, font_size=None,
          attributes=None, svg_attributes=None, svgwrite_debug=False):
    """Convenience function; identical to disvg() except that
    openinbrowser=False by default.  See disvg() docstring for more info."""
    disvg(paths, colors=colors, filename=filename,
          stroke_widths=stroke_widths, nodes=nodes,
          node_colors=node_colors, node_radii=node_radii,
          openinbrowser=openinbrowser, timestamp=timestamp,
          margin_size=margin_size, mindim=mindim, dimensions=dimensions,
          viewbox=viewbox, text=text, text_path=text_path, font_size=font_size,
          attributes=attributes, svg_attributes=svg_attributes,
          svgwrite_debug=svgwrite_debug)
<|endoftext|>


<commit_before>#Wall_Follow.py

class WallFollow(self):
	## Initialize Wall Following
	#
    # @param left   A Motor representing the left motor.
    # @param right  A Motor representing the right motor.
    # @param timer  A Timer for moderating data taking.
	def __init__(self,left,right,timer):
		self.leftmotor = left
		self.rightmotor = right
		self.timer = timer
		self.timer.reset()

	#IR input
	## Given value from LRIR sensor, makes bot move forward
    #
    # @param LRIR  A number proportional to the distance between a wall a LRIR sensor
    #
    # @param speed  A value from -255 to 255 that corresponds to the general
    #               speed of the robot.
	def Follow_Wall(self, LRIR, speed = 0):
		if self.timer.millis() > 100:
			self.timer.reset(0)

<commit_msg>copied pid into Follow_Wall<commit_after>#Wall_Follow.py

class WallFollow(self):
	## Initialize Wall Following
	#
    # @param left   A Motor representing the left motor.
    # @param right  A Motor representing the right motor.
    # @param timer  A Timer for moderating data taking.
	def __init__(self,left,right,timer):
		self.leftmotor = left
		self.rightmotor = right
		self.timer = timer
		self.timer.reset()

	#IR input
	## Given value from LRIR sensor, makes bot move forward
    #
    # @param LRIR  A number proportional to the distance between a wall a LRIR sensor
    #
    # @param speed  A value from -255 to 255 that corresponds to the general
    #               speed of the robot.
	def Follow_Wall(self, LRIR, speed = 0):
		if self.timer.millis() > 100:
			self.timer.reset(0)

			# error value
            err = 0 - theta

            # Integrate over the last several timesteps.
            self.record.insert(0, err)
            if len(self.record) > self.recordLen:
                self.record.pop()

            # Take the derivative over recorded history.
            deriv = self.record[0] - self.record[-1] if len(self.record) > 1 else 0

            power = self.kp * err + self.ki * sum(self.record) + kd * deriv

            self.leftMotor.write ((speed - power) > 0, min(abs(speed - power), 255))
            self.rightMotor.write((speed + power) > 0, min(abs(speed + power), 255))

            print(power)

    ## Reinitialize this class to start taking data over.
    def reset(self):
        self.timer.reset()
        self.record = []

<|endoftext|>


<commit_before>from __future__ import print_function

__author__  = "Mark Shroyer"
__email__   = "code@markshroyer.com"
__version__ = 0.1

__all__     = ['partial', 'pointfree', 'ignore', 'printfn']

import sys, inspect, types

# Gloss over differences in Python 2/3 dictionary methods...
if sys.version_info >= (3,0):
    def dict_has_key(dictionary, key):
        return (key in dictionary)
    def dict_items(dictionary):
        return dictionary.items()
else:
    def dict_has_key(dictionary, key):
        return dictionary.has_key(key)
    def dict_items(dictionary):
        return dictionary.iteritems()

class partial(object):
    """@partial function decorator

    Converts a regular Python function into one supporting a form of
    partial application.  Supports positional arguments only.

    """

    def __init__(self, f, argv={}, copy_sig=None):
        self.f = f
        self.argv = argv.copy()

        if copy_sig is not None:
            self.pargl     = list(copy_sig.pargl)
            self.kargl     = copy_sig.kargl.copy()
            self.def_argv  = copy_sig.def_argv.copy()
            self.var_pargs = copy_sig.var_pargs
            self.var_kargs = copy_sig.var_kargs
        else:
            if isinstance(f, types.MethodType):
                argspec = inspect.getargspec(f.__func__)
                self.pargl = (argspec[0])[1:]
            elif isinstance(f, classmethod):
                if hasattr(f, '__func__'):
                    argspec = inspect.getargspec(f.__func__)
                else:
                    # No classmethod.__func__ in Python 2.6
                    argspec = inspect.getargspec(f.__get__(1).__func__)
                self.pargl = (argspec[0])[1:]
            elif isinstance(f, staticmethod):
                if hasattr(f, '__func__'):
                    argspec = inspect.getargspec(f.__func__)
                else:
                    # No staticmethod.__func__ in Python 2.6
                    argspec = inspect.getargspec(f.__get__(1))
                self.pargl = (argspec[0])[:]
            else:
                argspec = inspect.getargspec(f)
                self.pargl = (argspec[0])[:]

            if argspec[3] is not None:
                def_offset = len(self.pargl) - len(argspec[3])
                self.def_argv = dict((self.pargl[def_offset+i],argspec[3][i]) \
                                         for i in range(len(argspec[3])))
            else:
                self.def_argv = {}

            # For future support of Python 3 keyword-only arguments
            self.kargl = {}

            self.var_pargs = argspec[1] is not None
            self.var_kargs = argspec[2] is not None

        if hasattr(f, '__doc__'):
            self.__doc__ = f.__doc__
        if hasattr(f, '__name__'):
            self.__name__ = f.__name__

    def __get__(self, inst, owner=None):
        return self.__class__(self.f.__get__(inst, owner))

    def __call__(self, *apply_pv, **apply_kv):
        new_argv = self.argv.copy()
        extra_argv = []

        for v in apply_pv:
            arg_i = None
            for name in self.pargl:
                if not dict_has_key(new_argv, name):
                    arg_i = name
                    break

            if arg_i:
                new_argv[arg_i] = v
            else:
                extra_argv.append(v)

        for k,v in dict_items(apply_kv):
            if not (self.var_kargs or (k in self.pargl) or (k in self.kargl.keys())):
                raise TypeError("%s() got an unexpected keyword argument '%s'" % (self.__name__, k))
            new_argv[k] = v

        app_argv = self.def_argv.copy()
        app_argv.update(new_argv)

        app_ready = True
        for name in self.pargl:
            if not dict_has_key(app_argv, name):
                app_ready = False
                break

        if app_ready:
            for name in self.kargl.keys():
                if not dict_has_key(app_argv, name):
                    app_ready = False
                    break

        if app_ready:
            fpargs = [new_argv[n] for n in self.pargl if dict_has_key(new_argv, n)] + extra_argv
            fkargs = dict((key,val) for key,val in dict_items(new_argv) if not key in self.pargl)
            return self.f(*fpargs, **fkargs)
        else:
            return self.__class__(self.f, argv=new_argv, copy_sig=self)

class pointfree(partial):
    """@pointfree function decorator

    Converts a regular Python function into one which can be composed with
    other Python functions using the * and >> operators.  Functions with
    this decorator also automatically support partial application.

    """

    def __mul__(self, g):
        return self.__class__(lambda *p,**k: self(g.f(*p,**k)), argv=g.argv, copy_sig=g)

    def __rshift__(self, g):
        return self.__class__(lambda *p,**k: g(self.f(*p,**k)), argv=self.argv, copy_sig=self)

@pointfree
def ignore(iterator):
    for x in iterator: pass

@pointfree
def printfn(output):
    print(output)
<commit_msg>Kill unnecessary compatibility wrappers<commit_after>from __future__ import print_function

__author__  = "Mark Shroyer"
__email__   = "code@markshroyer.com"
__version__ = 0.1

__all__     = ['partial', 'pointfree', 'ignore', 'printfn']

import sys, inspect, types

class partial(object):
    """@partial function decorator

    Converts a regular Python function into one supporting a form of
    partial application.  Supports positional arguments only.

    """

    def __init__(self, f, argv={}, copy_sig=None):
        self.f = f
        self.argv = argv.copy()

        if copy_sig is not None:
            self.pargl     = list(copy_sig.pargl)
            self.kargl     = copy_sig.kargl.copy()
            self.def_argv  = copy_sig.def_argv.copy()
            self.var_pargs = copy_sig.var_pargs
            self.var_kargs = copy_sig.var_kargs
        else:
            if isinstance(f, types.MethodType):
                argspec = inspect.getargspec(f.__func__)
                self.pargl = (argspec[0])[1:]
            elif isinstance(f, classmethod):
                if hasattr(f, '__func__'):
                    argspec = inspect.getargspec(f.__func__)
                else:
                    # No classmethod.__func__ in Python 2.6
                    argspec = inspect.getargspec(f.__get__(1).__func__)
                self.pargl = (argspec[0])[1:]
            elif isinstance(f, staticmethod):
                if hasattr(f, '__func__'):
                    argspec = inspect.getargspec(f.__func__)
                else:
                    # No staticmethod.__func__ in Python 2.6
                    argspec = inspect.getargspec(f.__get__(1))
                self.pargl = (argspec[0])[:]
            else:
                argspec = inspect.getargspec(f)
                self.pargl = (argspec[0])[:]

            if argspec[3] is not None:
                def_offset = len(self.pargl) - len(argspec[3])
                self.def_argv = dict((self.pargl[def_offset+i],argspec[3][i]) \
                                         for i in range(len(argspec[3])))
            else:
                self.def_argv = {}

            # For future support of Python 3 keyword-only arguments
            self.kargl = {}

            self.var_pargs = argspec[1] is not None
            self.var_kargs = argspec[2] is not None

        if hasattr(f, '__doc__'):
            self.__doc__ = f.__doc__
        if hasattr(f, '__name__'):
            self.__name__ = f.__name__

    def __get__(self, inst, owner=None):
        return self.__class__(self.f.__get__(inst, owner))

    def __call__(self, *apply_pv, **apply_kv):
        new_argv = self.argv.copy()
        extra_argv = []

        for v in apply_pv:
            arg_i = None
            for name in self.pargl:
                if not name in new_argv:
                    arg_i = name
                    break

            if arg_i:
                new_argv[arg_i] = v
            else:
                extra_argv.append(v)

        for k,v in apply_kv.items():
            if not (self.var_kargs or (k in self.pargl) or (k in self.kargl.keys())):
                raise TypeError("%s() got an unexpected keyword argument '%s'" % (self.__name__, k))
            new_argv[k] = v

        app_argv = self.def_argv.copy()
        app_argv.update(new_argv)

        app_ready = True
        for name in self.pargl:
            if not name in app_argv:
                app_ready = False
                break

        if app_ready:
            for name in self.kargl.keys():
                if not name in app_argv:
                    app_ready = False
                    break

        if app_ready:
            fpargs = [new_argv[n] for n in self.pargl if n in new_argv] + extra_argv
            fkargs = dict((key,val) for key,val in new_argv.items() if not key in self.pargl)
            return self.f(*fpargs, **fkargs)
        else:
            return self.__class__(self.f, argv=new_argv, copy_sig=self)

class pointfree(partial):
    """@pointfree function decorator

    Converts a regular Python function into one which can be composed with
    other Python functions using the * and >> operators.  Functions with
    this decorator also automatically support partial application.

    """

    def __mul__(self, g):
        return self.__class__(lambda *p,**k: self(g.f(*p,**k)), argv=g.argv, copy_sig=g)

    def __rshift__(self, g):
        return self.__class__(lambda *p,**k: g(self.f(*p,**k)), argv=self.argv, copy_sig=self)

@pointfree
def ignore(iterator):
    for x in iterator: pass

@pointfree
def printfn(output):
    print(output)
<|endoftext|>