<commit_before>from .base import LegistarScraper, LegistarAPIScraper
from pupa.scrape import Scraper
from lxml.etree import tostring
from collections import deque
from functools import partialmethod
import datetime
import pytz
import requests

class LegistarBillScraper(LegistarScraper):
    def legislation(self, search_text='', created_after=None,
                    created_before=None) :

        # If legislation is added to the the legistar system while we
        # are scraping, it will shift the list of legislation down and
        # we might revisit the same legislation. So, we keep track of
        # the last few pieces of legislation we've visited in order to
        # make sure we are not revisiting
        scraped_leg = deque([], maxlen=10)

        for page in self.searchLegislation(search_text, created_after,
                                           created_before) :
            for legislation_summary in self.parseSearchResults(page) :
                if not legislation_summary['url'] in scraped_leg :
                    yield legislation_summary
                    scraped_leg.append(legislation_summary['url'])

    def searchLegislation(self, search_text='', created_after=None,
                          created_before=None):
        """
        Submit a search query on the legislation search page, and return a list
        of summary results.
        """

        page = self.lxmlize(self.LEGISLATION_URL)

        page = self._advancedSearch(page)

        payload = {}

        # Enter the search parameters TODO: Each of the possible form
        # fields should be represented as keyword arguments to this
        # function. The default query string should be for the the
        # default 'Legislative text' field.
        payload['ctl00$ContentPlaceHolder1$txtText'] = search_text

        if created_after and created_before :
            payload.update(dateWithin(created_after, created_before))

        elif created_before :
            payload.update(dateBound(created_before))
            payload['ctl00$ContentPlaceHolder1$radFileCreated'] = '<'

        elif created_after :
            payload.update(dateBound(created_after))
            payload['ctl00$ContentPlaceHolder1$radFileCreated'] = '>'

        # Return up to one million search results
        payload['ctl00_ContentPlaceHolder1_lstMax_ClientState'] = '{"value":"1000000"}'
        payload['ctl00_ContentPlaceHolder1_lstYearsAdvanced_ClientState'] = '{"value":"All"}'
        payload['ctl00$ContentPlaceHolder1$btnSearch'] = 'Search Legislation'

        payload.update(self.sessionSecrets(page))

        return self.pages(self.LEGISLATION_URL, payload)

    def parseSearchResults(self, page) :
        """Take a page of search results and return a sequence of data
        of tuples about the legislation, of the form

        ('Document ID', 'Document URL', 'Type', 'Status', 'Introduction Date'
        'Passed Date', 'Main Sponsor', 'Title')
        """
        table = page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridMain_ctl00']")[0]
        for legislation, headers, row in self.parseDataTable(table):
            # Do legislation search-specific stuff
            # ------------------------------------
            # First column should be the ID of the record.
            id_key = headers[0]
            try:
                legislation_id = legislation[id_key]['label']
            except TypeError:
                continue
            legislation_url = legislation[id_key]['url'].split(self.BASE_URL)[-1]
            legislation[id_key] = legislation_id
            legislation['url'] = self.BASE_URL + legislation_url.split('&Options')[0] + '&FullText=1'

            yield legislation

    def _advancedSearch(self, page) :
        search_switcher = page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0]

        if 'simple search' in search_switcher.value.lower() :
            return page
        else :
            payload = self.sessionSecrets(page)
            payload[search_switcher.name] = search_switcher.value

            page = self.lxmlize(self.LEGISLATION_URL, payload)

            if 'simple search' not in page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0].value.lower() :
                raise ValueError('Not on the advanced search page')

            return page

    def details(self, detail_url, div_id) :
        detail_page = self.lxmlize(detail_url)

        detail_div = detail_page.xpath(".//div[@id='%s']" % div_id)[0]

        return self.parseDetails(detail_div)

    def legDetails(self, detail_url) :
        div_id = 'ctl00_ContentPlaceHolder1_pageDetails'
        return self.details(detail_url, div_id)

    def actionDetails(self, detail_url) :
        div_id = 'ctl00_ContentPlaceHolder1_pageTop1'
        return self.details(detail_url, div_id)

    def history(self, detail_url) :
        detail_page = self.lxmlize(detail_url)

        try :
            history_table = detail_page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridLegislation_ctl00']")[0]
        except IndexError :
            print(detail_url)
            raise

        history = [row[0] for row in self.parseDataTable(history_table)]

        try :
            history = sorted(history, key = self._actionSortKey)
        except (TypeError, ValueError) :
            pass

        for action in history :
            yield action

    def _actionSortKey(self, action) :
        action_date = self.toDate(action['Date'])
        action_url = action['Action\xa0Details']['url']

        return (action_date, action_url)

    def text(self, detail_url) :
        detail_page = self.lxmlize(detail_url)

        text_div = detail_page.xpath("//div[@id='ctl00_ContentPlaceHolder1_divText']")

        if len(text_div) :
            return tostring(text_div[0], pretty_print=True).decode()
        else :
            return None

    def extractVotes(self, action_detail_url) :
        action_detail_page = self.lxmlize(action_detail_url)
        try:
            vote_table = action_detail_page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridVote_ctl00']")[0]
        except IndexError:
            self.warning("No votes found in table")
            return None, []
        votes = list(self.parseDataTable(vote_table))
        vote_list = []
        for vote, _, _ in votes :
            raw_option = vote['Vote'].lower()
            vote_list.append((self.VOTE_OPTIONS.get(raw_option, raw_option),
                              vote['Person Name']['label']))

        action_detail_div = action_detail_page.xpath(".//div[@id='ctl00_ContentPlaceHolder1_pageTop1']")[0]
        action_details = self.parseDetails(action_detail_div)
        result = action_details['Result'].lower()

        return result, vote_list

def dateWithin(created_after, created_before) :
    payload = dateBound(created_after)

    payload['ctl00$ContentPlaceHolder1$txtFileCreated2'] =\
        '{d.year}-{d.month:02}-{d.day:02}'.format(d=created_before)
    payload['ctl00$ContentPlaceHolder1$txtFileCreated2$dateInput'] =\
        '{d.month}/{d.day}/{d.year}'.format(d=created_before)

    payload['ctl00_ContentPlaceHolder1_txtFileCreated2_dateInput_ClientState'] =\
        '{{"enabled":true, "emptyMessage":"","validationText":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","valueAsString":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","minDateStr":"1980-01-01-00-00-00","maxDateStr":"2099-12-31-00-00-00", "lastSetTextBoxValue":"{d.month}/{d.day}/{d.year}"}}'.format(d=created_before)

    payload['ctl00$ContentPlaceHolder1$radFileCreated'] = 'between'

    return payload

def dateBound(creation_date) :
    payload = {}

    payload['ctl00$ContentPlaceHolder1$txtFileCreated1'] =\
        '{d.year}-{d.month:02}-{d.day:02}'.format(d=creation_date)
    payload['ctl00$ContentPlaceHolder1$txtFileCreated1$dateInput'] =\
        '{d.month}/{d.day}/{d.year}'.format(d=creation_date)

    payload['ctl00_ContentPlaceHolder1_txtFileCreated1_dateInput_ClientState'] =\
        '{{"enabled":true, "emptyMessage":"","validationText":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","valueAsString":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","minDateStr":"1980-01-01-00-00-00","maxDateStr":"2099-12-31-00-00-00", "lastSetTextBoxValue":"{d.month}/{d.day}/{d.year}"}}'.format(d=creation_date)

    return payload

class LegistarAPIBillScraper(LegistarAPIScraper) :
    # Make parameter optional, as it is in events.py
    def matters(self, since_datetime=None) :
        if since_datetime:
            params = {'$filter' : "MatterLastModifiedUtc gt datetime'{since_datetime}'".format(since_datetime = since_datetime.isoformat())}
        else:
            params = {}

        matters_url = self.BASE_URL + '/matters'

        for matter in self.pages(matters_url,
                                 params=params,
                                 item_key="MatterId"):
            try:
                legistar_url = self.legislation_detail_url(matter['MatterId'])
            except KeyError:
                continue
            else:
                matter['legistar_url'] = legistar_url

            yield matter

    def endpoint(self, route, *args) :
        url = self.BASE_URL + route
        response = self.get(url.format(*args))
        return response.json()

    topics = partialmethod(endpoint, '/matters/{0}/indexes')
    attachments = partialmethod(endpoint, '/matters/{0}/attachments')
    code_sections = partialmethod(endpoint, 'matters/{0}/codesections')

    def votes(self, history_id) :
        url = self.BASE_URL + '/eventitems/{0}/votes'.format(history_id)
        response = requests.get(url)
        if response.status_code == 200 :
            return response.json()
        elif response.status_code == 500 and response.json().get('InnerException', {}).get('ExceptionMessage', '') == "The cast to value type 'System.Int32' failed because the materialized value is null. Either the result type's generic parameter or the query must use a nullable type." :
            return []
        else :
            response = self.get(url)
            return response.json()

    def history(self, matter_id) :
        actions = self.endpoint('/matters/{0}/histories', matter_id)
        for action in actions:
            action['MatterHistoryActionName'] = action['MatterHistoryActionName'].strip()

        actions = sorted((action for action in actions
                          if action['MatterHistoryActionDate'] and
                             action['MatterHistoryActionName'] and
                             action['MatterHistoryActionBodyName']),
                          key = lambda action : action['MatterHistoryActionDate'])

        return actions

    def sponsors(self, matter_id) :
        spons = self.endpoint('/matters/{0}/sponsors', matter_id)
        if spons:
            try:
                max_version = str(max(int(sponsor['MatterSponsorMatterVersion'])
                                  for sponsor in spons))

            except ValueError:
                if sponsor['MatterSponsorMatterVersion'] == '*':
                    spons = [sponsor for sponsor in spons]
                else:
                    raise

            else:
                spons = [sponsor for sponsor in spons
                         if sponsor['MatterSponsorMatterVersion'] == max_version]

            return sorted(spons,
                          key = lambda sponsor : sponsor["MatterSponsorSequence"])

        else:
            return []

    def relations(self, matter_id):
        relations = self.endpoint('/matters/{0}/relations', matter_id)
        if relations:
            highest_flag = max(int(relation['MatterRelationFlag'])
                               for relation in relations)

            relations = [relation for relation in relations
                         if relation['MatterRelationFlag'] == highest_flag]

            return relations
        else:
            return []

    def text(self, matter_id) :
        version_route = '/matters/{0}/versions'
        text_route = '/matters/{0}/texts/{1}'

        versions = self.endpoint(version_route, matter_id)

        latest_version = max(versions, key=lambda x : x['Value'])['Key']

        text_url = self.BASE_URL + text_route.format(matter_id, latest_version)
        response = self.get(text_url, stream=True)
        if int(response.headers['Content-Length']) < 21052630 :
            return response.json()

    def legislation_detail_url(self, matter_id) :
        gateway_url = self.BASE_WEB_URL + '/gateway.aspx?m=l&id={0}'

        legislation_detail_route = self.head(gateway_url.format(matter_id)).headers['Location']

        return self.BASE_WEB_URL + legislation_detail_route

<commit_msg>Add _version_rank<commit_after>from .base import LegistarScraper, LegistarAPIScraper
from pupa.scrape import Scraper
from lxml.etree import tostring
from collections import deque
from functools import partialmethod
import datetime
import pytz
import requests

class LegistarBillScraper(LegistarScraper):
    def legislation(self, search_text='', created_after=None,
                    created_before=None) :

        # If legislation is added to the the legistar system while we
        # are scraping, it will shift the list of legislation down and
        # we might revisit the same legislation. So, we keep track of
        # the last few pieces of legislation we've visited in order to
        # make sure we are not revisiting
        scraped_leg = deque([], maxlen=10)

        for page in self.searchLegislation(search_text, created_after,
                                           created_before) :
            for legislation_summary in self.parseSearchResults(page) :
                if not legislation_summary['url'] in scraped_leg :
                    yield legislation_summary
                    scraped_leg.append(legislation_summary['url'])

    def searchLegislation(self, search_text='', created_after=None,
                          created_before=None):
        """
        Submit a search query on the legislation search page, and return a list
        of summary results.
        """

        page = self.lxmlize(self.LEGISLATION_URL)

        page = self._advancedSearch(page)

        payload = {}

        # Enter the search parameters TODO: Each of the possible form
        # fields should be represented as keyword arguments to this
        # function. The default query string should be for the the
        # default 'Legislative text' field.
        payload['ctl00$ContentPlaceHolder1$txtText'] = search_text

        if created_after and created_before :
            payload.update(dateWithin(created_after, created_before))

        elif created_before :
            payload.update(dateBound(created_before))
            payload['ctl00$ContentPlaceHolder1$radFileCreated'] = '<'

        elif created_after :
            payload.update(dateBound(created_after))
            payload['ctl00$ContentPlaceHolder1$radFileCreated'] = '>'

        # Return up to one million search results
        payload['ctl00_ContentPlaceHolder1_lstMax_ClientState'] = '{"value":"1000000"}'
        payload['ctl00_ContentPlaceHolder1_lstYearsAdvanced_ClientState'] = '{"value":"All"}'
        payload['ctl00$ContentPlaceHolder1$btnSearch'] = 'Search Legislation'

        payload.update(self.sessionSecrets(page))

        return self.pages(self.LEGISLATION_URL, payload)

    def parseSearchResults(self, page) :
        """Take a page of search results and return a sequence of data
        of tuples about the legislation, of the form

        ('Document ID', 'Document URL', 'Type', 'Status', 'Introduction Date'
        'Passed Date', 'Main Sponsor', 'Title')
        """
        table = page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridMain_ctl00']")[0]
        for legislation, headers, row in self.parseDataTable(table):
            # Do legislation search-specific stuff
            # ------------------------------------
            # First column should be the ID of the record.
            id_key = headers[0]
            try:
                legislation_id = legislation[id_key]['label']
            except TypeError:
                continue
            legislation_url = legislation[id_key]['url'].split(self.BASE_URL)[-1]
            legislation[id_key] = legislation_id
            legislation['url'] = self.BASE_URL + legislation_url.split('&Options')[0] + '&FullText=1'

            yield legislation

    def _advancedSearch(self, page) :
        search_switcher = page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0]

        if 'simple search' in search_switcher.value.lower() :
            return page
        else :
            payload = self.sessionSecrets(page)
            payload[search_switcher.name] = search_switcher.value

            page = self.lxmlize(self.LEGISLATION_URL, payload)

            if 'simple search' not in page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0].value.lower() :
                raise ValueError('Not on the advanced search page')

            return page

    def details(self, detail_url, div_id) :
        detail_page = self.lxmlize(detail_url)

        detail_div = detail_page.xpath(".//div[@id='%s']" % div_id)[0]

        return self.parseDetails(detail_div)

    def legDetails(self, detail_url) :
        div_id = 'ctl00_ContentPlaceHolder1_pageDetails'
        return self.details(detail_url, div_id)

    def actionDetails(self, detail_url) :
        div_id = 'ctl00_ContentPlaceHolder1_pageTop1'
        return self.details(detail_url, div_id)

    def history(self, detail_url) :
        detail_page = self.lxmlize(detail_url)

        try :
            history_table = detail_page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridLegislation_ctl00']")[0]
        except IndexError :
            print(detail_url)
            raise

        history = [row[0] for row in self.parseDataTable(history_table)]

        try :
            history = sorted(history, key = self._actionSortKey)
        except (TypeError, ValueError) :
            pass

        for action in history :
            yield action

    def _actionSortKey(self, action) :
        action_date = self.toDate(action['Date'])
        action_url = action['Action\xa0Details']['url']

        return (action_date, action_url)

    def text(self, detail_url) :
        detail_page = self.lxmlize(detail_url)

        text_div = detail_page.xpath("//div[@id='ctl00_ContentPlaceHolder1_divText']")

        if len(text_div) :
            return tostring(text_div[0], pretty_print=True).decode()
        else :
            return None

    def extractVotes(self, action_detail_url) :
        action_detail_page = self.lxmlize(action_detail_url)
        try:
            vote_table = action_detail_page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridVote_ctl00']")[0]
        except IndexError:
            self.warning("No votes found in table")
            return None, []
        votes = list(self.parseDataTable(vote_table))
        vote_list = []
        for vote, _, _ in votes :
            raw_option = vote['Vote'].lower()
            vote_list.append((self.VOTE_OPTIONS.get(raw_option, raw_option),
                              vote['Person Name']['label']))

        action_detail_div = action_detail_page.xpath(".//div[@id='ctl00_ContentPlaceHolder1_pageTop1']")[0]
        action_details = self.parseDetails(action_detail_div)
        result = action_details['Result'].lower()

        return result, vote_list

def dateWithin(created_after, created_before) :
    payload = dateBound(created_after)

    payload['ctl00$ContentPlaceHolder1$txtFileCreated2'] =\
        '{d.year}-{d.month:02}-{d.day:02}'.format(d=created_before)
    payload['ctl00$ContentPlaceHolder1$txtFileCreated2$dateInput'] =\
        '{d.month}/{d.day}/{d.year}'.format(d=created_before)

    payload['ctl00_ContentPlaceHolder1_txtFileCreated2_dateInput_ClientState'] =\
        '{{"enabled":true, "emptyMessage":"","validationText":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","valueAsString":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","minDateStr":"1980-01-01-00-00-00","maxDateStr":"2099-12-31-00-00-00", "lastSetTextBoxValue":"{d.month}/{d.day}/{d.year}"}}'.format(d=created_before)

    payload['ctl00$ContentPlaceHolder1$radFileCreated'] = 'between'

    return payload

def dateBound(creation_date) :
    payload = {}

    payload['ctl00$ContentPlaceHolder1$txtFileCreated1'] =\
        '{d.year}-{d.month:02}-{d.day:02}'.format(d=creation_date)
    payload['ctl00$ContentPlaceHolder1$txtFileCreated1$dateInput'] =\
        '{d.month}/{d.day}/{d.year}'.format(d=creation_date)

    payload['ctl00_ContentPlaceHolder1_txtFileCreated1_dateInput_ClientState'] =\
        '{{"enabled":true, "emptyMessage":"","validationText":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","valueAsString":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","minDateStr":"1980-01-01-00-00-00","maxDateStr":"2099-12-31-00-00-00", "lastSetTextBoxValue":"{d.month}/{d.day}/{d.year}"}}'.format(d=creation_date)

    return payload

class LegistarAPIBillScraper(LegistarAPIScraper) :
    # Make parameter optional, as it is in events.py
    def matters(self, since_datetime=None) :
        if since_datetime:
            params = {'$filter' : "MatterLastModifiedUtc gt datetime'{since_datetime}'".format(since_datetime = since_datetime.isoformat())}
        else:
            params = {}

        matters_url = self.BASE_URL + '/matters'

        for matter in self.pages(matters_url,
                                 params=params,
                                 item_key="MatterId"):
            try:
                legistar_url = self.legislation_detail_url(matter['MatterId'])
            except KeyError:
                continue
            else:
                matter['legistar_url'] = legistar_url

            yield matter

    def endpoint(self, route, *args) :
        url = self.BASE_URL + route
        response = self.get(url.format(*args))
        return response.json()

    topics = partialmethod(endpoint, '/matters/{0}/indexes')
    attachments = partialmethod(endpoint, '/matters/{0}/attachments')
    code_sections = partialmethod(endpoint, 'matters/{0}/codesections')

    def votes(self, history_id) :
        url = self.BASE_URL + '/eventitems/{0}/votes'.format(history_id)
        response = requests.get(url)
        if response.status_code == 200 :
            return response.json()
        elif response.status_code == 500 and response.json().get('InnerException', {}).get('ExceptionMessage', '') == "The cast to value type 'System.Int32' failed because the materialized value is null. Either the result type's generic parameter or the query must use a nullable type." :
            return []
        else :
            response = self.get(url)
            return response.json()

    def history(self, matter_id) :
        actions = self.endpoint('/matters/{0}/histories', matter_id)
        for action in actions:
            action['MatterHistoryActionName'] = action['MatterHistoryActionName'].strip()

        actions = sorted((action for action in actions
                          if action['MatterHistoryActionDate'] and
                             action['MatterHistoryActionName'] and
                             action['MatterHistoryActionBodyName']),
                          key = lambda action : action['MatterHistoryActionDate'])

        return actions

    def sponsors(self, matter_id) :
        spons = self.endpoint('/matters/{0}/sponsors', matter_id)
        if spons:
            max_version = max(self._version_rank(sponsor['MatterSponsorMatterVersion'])
                              for sponsor in spons)

            spons = [sponsor for sponsor in spons
                     if sponsor['MatterSponsorMatterVersion'] == str(max_version)]

            return sorted(spons,
                          key = lambda sponsor : sponsor["MatterSponsorSequence"])

        else:
            return []

    def _version_rank(self, version) :
        '''
        In general, matter versions are numbers. This method provides an
        override opportunity for handling versions that are not numbers.
        '''
        return int(version)

    def relations(self, matter_id):
        relations = self.endpoint('/matters/{0}/relations', matter_id)
        if relations:
            highest_flag = max(int(relation['MatterRelationFlag'])
                               for relation in relations)

            relations = [relation for relation in relations
                         if relation['MatterRelationFlag'] == highest_flag]

            return relations
        else:
            return []

    def text(self, matter_id) :
        version_route = '/matters/{0}/versions'
        text_route = '/matters/{0}/texts/{1}'

        versions = self.endpoint(version_route, matter_id)

        latest_version = max(versions, key=lambda x : x['Value'])['Key']

        text_url = self.BASE_URL + text_route.format(matter_id, latest_version)
        response = self.get(text_url, stream=True)
        if int(response.headers['Content-Length']) < 21052630 :
            return response.json()

    def legislation_detail_url(self, matter_id) :
        gateway_url = self.BASE_WEB_URL + '/gateway.aspx?m=l&id={0}'

        legislation_detail_route = self.head(gateway_url.format(matter_id)).headers['Location']

        return self.BASE_WEB_URL + legislation_detail_route

<|endoftext|>


<commit_before># coding=utf8
from mytypes import types
from stitchpoles import stitch
from hashtable import hashtable
from coordinatesystems import systems
from bounds import bound

e = 1e-6
def isInfinit(n):
	return abs(n)==float('inf')

def topology (objects, options=False):
	Q = 1e4; # precision of quantization
	id = lambda d:d['id']
	def propertyTransform (outprop, key, inprop):
		outprop[key]=inprop
		return True
	stitchPoles = True;
	verbose = False;
	emax = 0
	system = False
	arcs = []
	if type(options)==type({}):
		if options.has_key('verbose'):
			verbose = not not options['verbose']
		if options.has_key('stitch-poles'):
			stitchPoles = not not options["stitch-poles"]
		if options.has_key('coordinate-system'):
			system = systems[options["coordinate-system"]]
		if options.has_key('quantization'):
			Q = float(options["quantization"])
		if options.has_key('id'):
			id = options['id']
		if options.has_key('property-transform'):
			propertyTransform = options["property-transform"]

	coincidences = hashtable(Q * 10);
	arcsByPoint = hashtable(Q * 10);
	pointsByPoint = hashtable(Q * 10);

	[x0,x1,y0,y1]=bound(objects)

	oversize = x0 < -180 - e or x1 > 180 + e or y0 < -90 - e or y1 > 90 + e
	if not system:
		if oversize:
			system = systems["cartesian"]
		else:
			system = systems["spherical"]
		if type(options)==type({}):
			options["coordinate-system"] = system['name']
	if system == systems['spherical']:
		if oversize:
			raise Exception(u"spherical coordinates outside of [±180°, ±90°]")
		if stitchPoles:
			stitch(objects)
			[x0,x1,y0,y1]=bound(objects)
		if x0 < -180 + e:
			x0 = -180
		if x1 > 180 - e:
			x1 = 180
		if y0 < -90 + e:
			y0 = -90
		if y1 > 90 - e:
			y1 = 90;
	if isInfinit(x0):
		x0 = 0
	if isInfinit(x1):
		x1 = 0;

	if isInfinit(y0):
		y0 = 0;
	if isInfinit(y1):
		y1 = 0;
	[kx,ky]=makeKs(Q,x0,x1,y0,y1)
	if not Q:
		Q = x1 + 1
		x0 = y0 = 0
	class findEmax(types):
		def __init__(self,obj):
			self.emax=0
			self.obj(obj)
		def point(self,point):
			x1 = point[0]
			y1 = point[1]
			x = round((x1 - x0) * kx)
			y = round((y1 - y0) * ky)
			ee = system['distance'](x1, y1, x / kx + x0, y / ky + y0)
			if ee > self.emax:
				self.emax = ee
			point[0] = x
			point[1] = y
	finde=findEmax(objects)
	emax = finde.emax
	class findCoincidences(types):
		def line(self,line):
			for point in line:
				lines = coincidences.get(point)
				if not line in lines:
					lines.append(line)
	fcInst = findCoincidences(objects)
	def line(points, open):
		lineArcs = [];
		n = len(points)
		a = []
		k = -1
		p=False
		if not open:
			points.pop()
			n-=1
		while k<n:
			k+=1
			t = coincidences.peak(points[k])
			if open:
				break
			if p and not linesEqual(p, t):
				tInP = all(map(lambda line: line in p,t))
				pInT = all(map(lambda line: line in t,p))
				if tInP and not pInT:
					k-=1
				break;
			p = t;
		# If no shared starting point is found for closed lines, rotate to minimum.
		if k == n and len(p) > 1:
			point0 = points[0]
			i = -1
			k=0
			while i<n:
				i+=1
				point = points[i];
				if pointCompare(point0, point) > 0:
					point0 = point
					k = i
		i = -1
		m = n if open else n + 1
		def matchForward(b,index):
			i = 0;
			if len(b) != n:
				return False
			while i < n:
				if pointCompare(a[i], b[i]):
					return False;
				i+=1
			lineArcs.append(b[index])
			return True;

		def matchBackward(b,index):
			i = 0
			if len(b) != n:
				return False
			while i<n:
				if pointCompare(a[i], b[n - i - 1]):
					return False
				i+=1
			lineArcs.append(~b[index])
			return True
		def arc(a, last=False):
			n = len(a)
			point=False
			index=-1
			if last and not len(lineArcs) and n == 1:
				point = a[0]
				index = pointsByPoint.get(point)
				if len(index):
					lineArcs.append(index[0])
				else:
					index[0] = len(arcs)
					lineArcs.append(index[0])
					arcs.append(a)
			elif n > 1:
				a0 = a[0]
				a1 = a[-1]
				point = a0 if pointCompare(a0, a1) < 0 else a1
				pointArcs = arcsByPoint.get(point)
				if any(map(lambda x:matchForward(x,index),pointArcs)) or any(map(lambda x:matchBackward(x,index),pointArcs)):
					return
				pointArcs.append(a)
				a[index]=len(arcs)
				lineArcs.append(a[index])
				arcs.append(a)
		while i < m:
			i+=1
			point = points[(i + k) % n]
			p = coincidences.peak(point)
			if not linesEqual(p, t):
				tInP = all(map(lambda line: line in p,t))
				pInT = all(map(lambda line: line in t,p))
				if tInP:
					a.append(point);
				arc(a)
				if not tInP and not pInT:
					arc([a[-1], point])
				if pInT:
					a = [a[-1]]
				else:
					a = [];
			if not len(a) or pointCompare(a[-1], point):
				a.append(point) # skip duplicate points
			t = p
		arc(a, True)

		return lineArcs
	polygon = lambda poly:map(lineClosed,poly)
	lineClosed = lambda points:line(points,False)
	lineOpen = lambda points:line(points,True)
	#Convert features to geometries, and stitch together arcs.
	class makeTopo(types):
		def Feature (self,feature):
			geometry = feature["geometry"]
			if feature['geometry'] == None:
				geometry = {};
			if feature.has_key('id'):
				geometry['id'] = feature['id']
			if feature.has_key('properties'):
				geometry['properties'] = feature['properties']
			return self.geometry(geometry);
		def FeatureCollection(self,collection):
			collection['type'] = "GeometryCollection";
			collection['geometries'] = map(self.Feature,collection['features'])
			del collection['features']
			return collection
		def GeometryCollection(self,collection):
			collection['geometries'] = map(self.geometry,collection['geometries'])
		def MultiPolygon(self,multiPolygon):
			multiPolygon['arcs'] = map(polygon,multiPolygon['coordinates'])
		def Polygon(self,polygon):
			 polygon['arcs'] = map(lineClosed,polygon['coordinates'])
		def MultiLineString(self,multiLineString):
			multiLineString['arcs'] = map(lineOpen,multiLineString['coordinates'])
		def LineString(self,lineString):
			lineString['arcs'] = lineOpen(lineString['coordinates'])
		def geometry(self,geometry):
			if geometry == None:
				geometry = {};
			else:
				types.geometry(self,geometry)
			geometry['id'] = id(geometry)
			if geometry['id'] == None:
				del geometry['id']
			properties0 = geometry['properties']
			if properties0:
				properties1 = {}
				del geometry['properties']
				for key0 in properties0:
					if propertyTransform(properties1, key0, properties0[key0]):
						geometry['properties'] = properties1
			if geometry.has_key('arcs'):
				del geometry['coordinates']
			return geometry;
	makeTopoInst = makeTopo(objects)
	objects = makeTopoInst.outObj

	coincidences = arcsByPoint = pointsByPoint = None

	return {
		'type': "Topology",
		'bbox': [x0, y0, x1, y1],
		'transform': {
			'scale': [1.0 / kx, 1.0 / ky],
			'translate': [x0, y0]
		},
		'objects': {'name':objects},
		'arcs': makeArcs(arcs)
	}

makeArcs = lambda arcs:map(mapFunc,arcs)

def mapFunc (arc):
	if len(arc)==2 and type(arc[0])==type(1):
		return [arc]
	i = 1;
	n = len(arc)
	point = arc[0]
	x1 = point[0]
	x2= dx =y2 = dy=False
	y1 = point[1]
	points = [[int(x1), int(y1)]]
	while i < n:
		point = arc[i]
		if not isPoint(point):
			i+=1
			continue
		x2 = point[0]
		y2 = point[1]
		dx = int(x2 - x1)
		dy = int(y2 - y1)
		if dx or dy:
			points.append([dx, dy])
			x1 = x2
			y1 = y2
		i+=1
	return points

def makeKs(Q,x0,x1,y0,y1):
	if Q:
		return [(Q - 1) / (x1 - x0) if x1 - x0 else 1,(Q - 1) / (y1 - y0) if y1 - y0 else 1]
	else:
		return [1,1]
def linesEqual(a, b):
	if not(type(a)==type(b)==type([])):
		return True
	n = len(a);
	i = 0
	if len(b) != n:
		return False
	while i < n:
		if a[i] != b[i]:
			return False;
		i+=1
	return True;

def pointCompare(a, b):
	if isPoint(a) and isPoint(b):
		return a[0] - b[0] or a[1] - b[1]

isPoint = lambda x : type(x)==type([]) and len(x)==2<commit_msg>error<commit_after># coding=utf8
from mytypes import types
from stitchpoles import stitch
from hashtable import hashtable
from coordinatesystems import systems
from bounds import bound

e = 1e-6
def isInfinit(n):
	return abs(n)==float('inf')

def topology (objects, options=False):
	Q = 1e4; # precision of quantization
	id = lambda d:d['id']
	def propertyTransform (outprop, key, inprop):
		outprop[key]=inprop
		return True
	stitchPoles = True;
	verbose = False;
	emax = 0
	system = False
	arcs = []
	if type(options)==type({}):
		if options.has_key('verbose'):
			verbose = not not options['verbose']
		if options.has_key('stitch-poles'):
			stitchPoles = not not options["stitch-poles"]
		if options.has_key('coordinate-system'):
			system = systems[options["coordinate-system"]]
		if options.has_key('quantization'):
			Q = float(options["quantization"])
		if options.has_key('id'):
			id = options['id']
		if options.has_key('property-transform'):
			propertyTransform = options["property-transform"]

	coincidences = hashtable(Q * 10);
	arcsByPoint = hashtable(Q * 10);
	pointsByPoint = hashtable(Q * 10);

	[x0,x1,y0,y1]=bound(objects)

	oversize = x0 < -180 - e or x1 > 180 + e or y0 < -90 - e or y1 > 90 + e
	if not system:
		if oversize:
			system = systems["cartesian"]
		else:
			system = systems["spherical"]
		if type(options)==type({}):
			options["coordinate-system"] = system['name']
	if system == systems['spherical']:
		if oversize:
			raise Exception(u"spherical coordinates outside of [±180°, ±90°]")
		if stitchPoles:
			stitch(objects)
			[x0,x1,y0,y1]=bound(objects)
		if x0 < -180 + e:
			x0 = -180
		if x1 > 180 - e:
			x1 = 180
		if y0 < -90 + e:
			y0 = -90
		if y1 > 90 - e:
			y1 = 90;
	if isInfinit(x0):
		x0 = 0
	if isInfinit(x1):
		x1 = 0;

	if isInfinit(y0):
		y0 = 0;
	if isInfinit(y1):
		y1 = 0;
	[kx,ky]=makeKs(Q,x0,x1,y0,y1)
	if not Q:
		Q = x1 + 1
		x0 = y0 = 0
	class findEmax(types):
		def __init__(self,obj):
			self.emax=0
			self.obj(obj)
		def point(self,point):
			x1 = point[0]
			y1 = point[1]
			x = round((x1 - x0) * kx)
			y = round((y1 - y0) * ky)
			ee = system['distance'](x1, y1, x / kx + x0, y / ky + y0)
			if ee > self.emax:
				self.emax = ee
			point[0] = x
			point[1] = y
	finde=findEmax(objects)
	emax = finde.emax
	class findCoincidences(types):
		def line(self,line):
			for point in line:
				lines = coincidences.get(point)
				if not line in lines:
					lines.append(line)
	fcInst = findCoincidences(objects)
	def line(points, open):
		lineArcs = [];
		n = len(points)
		a = []
		k = 0
		p=False
		if not open:
			points.pop()
			n-=1
		while k<n:
			t = coincidences.peak(points[k])
			if open:
				break
			if p and not linesEqual(p, t):
				tInP = all(map(lambda line: line in p,t))
				pInT = all(map(lambda line: line in t,p))
				if tInP and not pInT:
					k-=1
				break;
			p = t;
			k+=1
		# If no shared starting point is found for closed lines, rotate to minimum.
		if k == n and len(p) > 1:
			point0 = points[0]
			i = -1
			k=0
			while i<n:
				i+=1
				point = points[i];
				if pointCompare(point0, point) > 0:
					point0 = point
					k = i
		i = -1
		m = n if open else n + 1
		def matchForward(b,index):
			i = 0;
			if len(b) != n:
				return False
			while i < n:
				if pointCompare(a[i], b[i]):
					return False;
				i+=1
			lineArcs.append(b[index])
			return True;

		def matchBackward(b,index):
			i = 0
			if len(b) != n:
				return False
			while i<n:
				if pointCompare(a[i], b[n - i - 1]):
					return False
				i+=1
			lineArcs.append(~b[index])
			return True
		def arc(a, last=False):
			n = len(a)
			point=False
			index=-1
			if last and not len(lineArcs) and n == 1:
				point = a[0]
				index = pointsByPoint.get(point)
				if len(index):
					lineArcs.append(index[0])
				else:
					index[0] = len(arcs)
					lineArcs.append(index[0])
					arcs.append(a)
			elif n > 1:
				a0 = a[0]
				a1 = a[-1]
				point = a0 if pointCompare(a0, a1) < 0 else a1
				pointArcs = arcsByPoint.get(point)
				if any(map(lambda x:matchForward(x,index),pointArcs)) or any(map(lambda x:matchBackward(x,index),pointArcs)):
					return
				pointArcs.append(a)
				a[index]=len(arcs)
				lineArcs.append(a[index])
				arcs.append(a)
		while i < m:
			i+=1
			point = points[(i + k) % n]
			p = coincidences.peak(point)
			if not linesEqual(p, t):
				tInP = all(map(lambda line: line in p,t))
				pInT = all(map(lambda line: line in t,p))
				if tInP:
					a.append(point);
				arc(a)
				if not tInP and not pInT:
					arc([a[-1], point])
				if pInT:
					a = [a[-1]]
				else:
					a = [];
			if not len(a) or pointCompare(a[-1], point):
				a.append(point) # skip duplicate points
			t = p
		arc(a, True)
		return lineArcs
	polygon = lambda poly:map(lineClosed,poly)
	lineClosed = lambda points:line(points,False)
	lineOpen = lambda points:line(points,True)
	#Convert features to geometries, and stitch together arcs.
	class makeTopo(types):
		def Feature (self,feature):
			geometry = feature["geometry"]
			if feature['geometry'] == None:
				geometry = {};
			if feature.has_key('id'):
				geometry['id'] = feature['id']
			if feature.has_key('properties'):
				geometry['properties'] = feature['properties']
			return self.geometry(geometry);
		def FeatureCollection(self,collection):
			collection['type'] = "GeometryCollection";
			collection['geometries'] = map(self.Feature,collection['features'])
			del collection['features']
			return collection
		def GeometryCollection(self,collection):
			collection['geometries'] = map(self.geometry,collection['geometries'])
		def MultiPolygon(self,multiPolygon):
			multiPolygon['arcs'] = map(polygon,multiPolygon['coordinates'])
		def Polygon(self,polygon):
			 polygon['arcs'] = map(lineClosed,polygon['coordinates'])
		def MultiLineString(self,multiLineString):
			multiLineString['arcs'] = map(lineOpen,multiLineString['coordinates'])
		def LineString(self,lineString):
			lineString['arcs'] = lineOpen(lineString['coordinates'])
		def geometry(self,geometry):
			if geometry == None:
				geometry = {};
			else:
				types.geometry(self,geometry)
			geometry['id'] = id(geometry)
			if geometry['id'] == None:
				del geometry['id']
			properties0 = geometry['properties']
			if properties0:
				properties1 = {}
				del geometry['properties']
				for key0 in properties0:
					if propertyTransform(properties1, key0, properties0[key0]):
						geometry['properties'] = properties1
			if geometry.has_key('arcs'):
				del geometry['coordinates']
			return geometry;
	makeTopoInst = makeTopo(objects)
	objects = makeTopoInst.outObj

	coincidences = arcsByPoint = pointsByPoint = None

	return {
		'type': "Topology",
		'bbox': [x0, y0, x1, y1],
		'transform': {
			'scale': [1.0 / kx, 1.0 / ky],
			'translate': [x0, y0]
		},
		'objects': {'name':objects},
		'arcs': makeArcs(arcs)
	}

makeArcs = lambda arcs:map(mapFunc,arcs)

def mapFunc (arc):
	if len(arc)==2 and type(arc[0])==type(1):
		return [arc]
	i = 1;
	n = len(arc)
	point = arc[0]
	x1 = point[0]
	x2= dx =y2 = dy=False
	y1 = point[1]
	points = [[int(x1), int(y1)]]
	while i < n:
		point = arc[i]
		if not isPoint(point):
			i+=1
			continue
		x2 = point[0]
		y2 = point[1]
		dx = int(x2 - x1)
		dy = int(y2 - y1)
		if dx or dy:
			points.append([dx, dy])
			x1 = x2
			y1 = y2
		i+=1
	return points

def makeKs(Q,x0,x1,y0,y1):
	if Q:
		return [(Q - 1) / (x1 - x0) if x1 - x0 else 1,(Q - 1) / (y1 - y0) if y1 - y0 else 1]
	else:
		return [1,1]
def linesEqual(a, b):
	if not(type(a)==type(b)==type([])):
		return True
	n = len(a);
	i = 0
	if len(b) != n:
		return False
	while i < n:
		if a[i] != b[i]:
			return False;
		i+=1
	return True;

def pointCompare(a, b):
	if isPoint(a) and isPoint(b):
		return a[0] - b[0] or a[1] - b[1]

isPoint = lambda x : type(x)==type([]) and len(x)==2<|endoftext|>


<commit_before>

class parsingUtility:

    def __init__(self):
        pass

    def parseCommand(self,cmdString):
        #Directs the command processing to the appropriate function,
        #based on the type of command received.

        #Get the command type from the cmdString
        thisCommand = self.getCommand(cmdString)
        #Call the appropriate routine based on the command type
        if thisCommand == 'FROM':
            self.parseFROM(cmdString)
        elif thisCommand == 'MAINTAINER':
            self.parseMAINTAINER(cmdString)
        elif thisCommand == 'RUN':
            self.parseRUN(cmdString)
        elif thisCommand == 'CMD':
            self.parseCMD(cmdString)
        elif thisCommand == 'LABEL':
            self.parseLABEL(cmdString)
        elif thisCommand == 'EXPOSE':
            self.parseEXPOSE(cmdString)
        elif thisCommand == 'ENV':
            self.parseENV(cmdString)
        elif thisCommand == 'ADD':
            self.parseADD(cmdString)
        elif thisCommand == 'COPY':
            self.parseCOPY(cmdString)
        elif thisCommand == 'ENTRYPOINT':
            self.parseENTRYPOINT(cmdString)
        elif thisCommand == 'VOLUME':
            self.parseVOLUME(cmdString)
        elif thisCommand == 'USER':
            self.parseUSER(cmdString)
        elif thisCommand == 'WORKDIR':
            self.parseWORKDIR(cmdString)
        elif thisCommand == 'ARG':
            self.parseARG(cmdString)
        elif thisCommand == 'ONBUILD':
            self.parseONBUILD(cmdString)
        elif thisCommand == 'STOPSIGNAL':
            self.parseSTOPSIGNAL(cmdString)
        else:
            print "Error parsing command"

    def getCommand(self, cmdString):
        #returns command from cmdString, with the assumption
        #that the command is always the first word in the string
        return cmdString.split()[0]

    def get_command_data(self, command):
        """Obtains the second part of the command string.

        Args:
            command (str): The command string.

        Returns: The second part of the command string.

        """
        return command.split(" ", 1)[1]

    def parseFROM(self, cmdFROM):
        pass

    def parseMAINTAINER(self, cmdMAINTAINER):
        pass

    def parseRUN(self, cmdRUN):
        pass

    def parseCMD(self,cmdCMD):
        pass

    def parseLABEL(self, cmdLABEL):
        pass

    def parseEXPOSE(self, cmdEXPOSE):
        pass

    def parseENV(self, cmdENV):
        pass

    def parseADD(self, cmdADD):
        pass

    def parseCOPY(self,cmdCOPY):
        pass

    def parseENTRYPOINT(self, cmdENTRYPOINT):
        pass

    def parseVOLUME(self, cmdVOLUME):
        pass

    def parseUSER(self, cmdUSER):
        pass

    def parseWORKDIR(self, cmdWORKDIR):
        pass

    def parseARG(self, cmdARG):
        pass

    def parseONBUILD(self, cmdONBUILD):
        pass

    def parseSTOPSIGNAL(self, cmdSTOPSIGNAL):
        pass

<commit_msg>Add data dictionary instance variable to parsing utility.<commit_after>

class parsingUtility:

    def __init__(self):
        self.data = {}

    def parseCommand(self,cmdString):
        #Directs the command processing to the appropriate function,
        #based on the type of command received.

        #Get the command type from the cmdString
        thisCommand = self.getCommand(cmdString)
        #Call the appropriate routine based on the command type
        if thisCommand == 'FROM':
            self.parseFROM(cmdString)
        elif thisCommand == 'MAINTAINER':
            self.parseMAINTAINER(cmdString)
        elif thisCommand == 'RUN':
            self.parseRUN(cmdString)
        elif thisCommand == 'CMD':
            self.parseCMD(cmdString)
        elif thisCommand == 'LABEL':
            self.parseLABEL(cmdString)
        elif thisCommand == 'EXPOSE':
            self.parseEXPOSE(cmdString)
        elif thisCommand == 'ENV':
            self.parseENV(cmdString)
        elif thisCommand == 'ADD':
            self.parseADD(cmdString)
        elif thisCommand == 'COPY':
            self.parseCOPY(cmdString)
        elif thisCommand == 'ENTRYPOINT':
            self.parseENTRYPOINT(cmdString)
        elif thisCommand == 'VOLUME':
            self.parseVOLUME(cmdString)
        elif thisCommand == 'USER':
            self.parseUSER(cmdString)
        elif thisCommand == 'WORKDIR':
            self.parseWORKDIR(cmdString)
        elif thisCommand == 'ARG':
            self.parseARG(cmdString)
        elif thisCommand == 'ONBUILD':
            self.parseONBUILD(cmdString)
        elif thisCommand == 'STOPSIGNAL':
            self.parseSTOPSIGNAL(cmdString)
        else:
            print "Error parsing command"

    def getCommand(self, cmdString):
        #returns command from cmdString, with the assumption
        #that the command is always the first word in the string
        return cmdString.split()[0]

    def get_command_data(self, command):
        """Obtains the second part of the command string.

        Args:
            command (str): The command string.

        Returns: The second part of the command string.

        """
        return command.split(" ", 1)[1]

    def parseFROM(self, cmdFROM):
        pass

    def parseMAINTAINER(self, cmdMAINTAINER):
        pass

    def parseRUN(self, cmdRUN):
        pass

    def parseCMD(self,cmdCMD):
        pass

    def parseLABEL(self, cmdLABEL):
        pass

    def parseEXPOSE(self, cmdEXPOSE):
        pass

    def parseENV(self, cmdENV):
        pass

    def parseADD(self, cmdADD):
        pass

    def parseCOPY(self,cmdCOPY):
        pass

    def parseENTRYPOINT(self, cmdENTRYPOINT):
        pass

    def parseVOLUME(self, cmdVOLUME):
        pass

    def parseUSER(self, cmdUSER):
        pass

    def parseWORKDIR(self, cmdWORKDIR):
        pass

    def parseARG(self, cmdARG):
        pass

    def parseONBUILD(self, cmdONBUILD):
        pass

    def parseSTOPSIGNAL(self, cmdSTOPSIGNAL):
        pass

<|endoftext|>


<commit_before>from __future__ import with_statement
import os
import sys
import re
import codecs
import py
import tox
from .config import DepConfig

class CreationConfig:
    def __init__(self, md5, python, version, sitepackages,
                 usedevelop, deps):
        self.md5 = md5
        self.python = python
        self.version = version
        self.sitepackages = sitepackages
        self.usedevelop = usedevelop
        self.deps = deps

    def writeconfig(self, path):
        lines = ["%s %s" % (self.md5, self.python)]
        lines.append("%s %d %d" % (self.version, self.sitepackages, self.usedevelop))
        for dep in self.deps:
            lines.append("%s %s" % dep)
        path.ensure()
        path.write("\n".join(lines))

    @classmethod
    def readconfig(cls, path):
        try:
            lines = path.readlines(cr=0)
            value = lines.pop(0).split(None, 1)
            md5, python = value
            version, sitepackages, usedevelop = lines.pop(0).split(None, 3)
            sitepackages = bool(int(sitepackages))
            usedevelop = bool(int(usedevelop))
            deps = []
            for line in lines:
                md5, depstring = line.split(None, 1)
                deps.append((md5, depstring))
            return CreationConfig(md5, python, version, sitepackages, usedevelop, deps)
        except Exception:
            return None

    def matches(self, other):
        return (other and self.md5 == other.md5
                and self.python == other.python
                and self.version == other.version
                and self.sitepackages == other.sitepackages
                and self.usedevelop == other.usedevelop
                and self.deps == other.deps)

class VirtualEnv(object):
    def __init__(self, envconfig=None, session=None):
        self.envconfig = envconfig
        self.session = session
        self.path = envconfig.envdir
        self.path_config = self.path.join(".tox-config1")

    @property
    def name(self):
        return self.envconfig.envname

    def __repr__(self):
        return "<VirtualEnv at %r>" % (self.path)

    def getcommandpath(self, name=None, venv=True, cwd=None):
        if name is None:
            return self.envconfig.envpython
        name = str(name)
        if os.path.isabs(name):
            return name
        if os.path.split(name)[0] == ".":
            p = cwd.join(name)
            if p.check():
                return str(p)
        p = None
        if venv:
            p = py.path.local.sysfind(name, paths=[self.envconfig.envbindir])
        if p is not None:
            return p
        p = py.path.local.sysfind(name)
        if p is None:
            raise tox.exception.InvocationError(
                "could not find executable %r" % (name,))
        # p is not found in virtualenv script/bin dir
        if venv:
            if not self.is_allowed_external(p):
                self.session.report.warning(
                    "test command found but not installed in testenv\n"
                    "  cmd: %s\n"
                    "  env: %s\n"
                    "Maybe you forgot to specify a dependency? "
                    "See also the whitelist_externals envconfig setting." % (
                        p, self.envconfig.envdir))
        return str(p)  # will not be rewritten for reporting

    def is_allowed_external(self, p):
        tryadd = [""]
        if sys.platform == "win32":
            tryadd += [
                os.path.normcase(x)
                for x in os.environ['PATHEXT'].split(os.pathsep)
            ]
            p = py.path.local(os.path.normcase(str(p)))
        for x in self.envconfig.whitelist_externals:
            for add in tryadd:
                if p.fnmatch(x + add):
                    return True
        return False

    def _ispython3(self):
        return "python3" in str(self.envconfig.basepython)

    def update(self, action=None):
        """ return status string for updating actual venv to match configuration.
            if status string is empty, all is ok.
        """
        if action is None:
            action = self.session.newaction(self, "update")
        rconfig = CreationConfig.readconfig(self.path_config)
        if not self.envconfig.recreate and rconfig and \
           rconfig.matches(self._getliveconfig()):
            action.info("reusing", self.envconfig.envdir)
            return
        if rconfig is None:
            action.setactivity("create", self.envconfig.envdir)
        else:
            action.setactivity("recreate", self.envconfig.envdir)
        try:
            self.create(action)
        except tox.exception.UnsupportedInterpreter:
            return sys.exc_info()[1]
        except tox.exception.InterpreterNotFound:
            return sys.exc_info()[1]
        try:
            self.install_deps(action)
        except tox.exception.InvocationError:
            v = sys.exc_info()[1]
            return "could not install deps %s; v = %r" % (
                self.envconfig.deps, v)

    def _getliveconfig(self):
        python = self.envconfig.python_info.executable
        md5 = getdigest(python)
        version = tox.__version__
        sitepackages = self.envconfig.sitepackages
        develop = self.envconfig.usedevelop
        deps = []
        for dep in self._getresolvedeps():
            raw_dep = dep.name
            md5 = getdigest(raw_dep)
            deps.append((md5, raw_dep))
        return CreationConfig(md5, python, version,
                              sitepackages, develop, deps)

    def _getresolvedeps(self):
        l = []
        for dep in self.envconfig.deps:
            if dep.indexserver is None:
                res = self.session._resolve_pkg(dep.name)
                if res != dep.name:
                    dep = dep.__class__(res)
            l.append(dep)
        return l

    def getsupportedinterpreter(self):
        return self.envconfig.getsupportedinterpreter()

    def matching_platform(self):
        return re.match(self.envconfig.platform, sys.platform)

    def create(self, action=None):
        # if self.getcommandpath("activate").dirpath().check():
        #    return
        if action is None:
            action = self.session.newaction(self, "create")

        config_interpreter = self.getsupportedinterpreter()
        args = [sys.executable, '-m', 'virtualenv']
        if self.envconfig.sitepackages:
            args.append('--system-site-packages')
        # add interpreter explicitly, to prevent using
        # default (virtualenv.ini)
        args.extend(['--python', str(config_interpreter)])
        # if sys.platform == "win32":
        #    f, path, _ = py.std.imp.find_module("virtualenv")
        #    f.close()
        #    args[:1] = [str(config_interpreter), str(path)]
        # else:
        self.session.make_emptydir(self.path)
        basepath = self.path.dirpath()
        basepath.ensure(dir=1)
        args.append(self.path.basename)
        self._pcall(args, venv=False, action=action, cwd=basepath)
        self.just_created = True

    def finish(self):
        self._getliveconfig().writeconfig(self.path_config)

    def _needs_reinstall(self, setupdir, action):
        setup_py = setupdir.join('setup.py')
        setup_cfg = setupdir.join('setup.cfg')
        args = [self.envconfig.envpython, str(setup_py), '--name']
        output = action.popen(args, cwd=setupdir, redirect=False,
                              returnout=True)
        name = output.strip()
        egg_info = setupdir.join('.'.join((name, 'egg-info')))
        for conf_file in (setup_py, setup_cfg):
            if (not egg_info.check()
                    or (conf_file.check() and conf_file.mtime() > egg_info.mtime())):
                return True
        return False

    def developpkg(self, setupdir, action):
        assert action is not None
        if getattr(self, 'just_created', False):
            action.setactivity("develop-inst", setupdir)
            self.finish()
            extraopts = []
        else:
            if not self._needs_reinstall(setupdir, action):
                action.setactivity("develop-inst-noop", setupdir)
                return
            action.setactivity("develop-inst-nodeps", setupdir)
            extraopts = ['--no-deps']
        self._install(['-e', setupdir], extraopts=extraopts, action=action)

    def installpkg(self, sdistpath, action):
        assert action is not None
        if getattr(self, 'just_created', False):
            action.setactivity("inst", sdistpath)
            self.finish()
            extraopts = []
        else:
            action.setactivity("inst-nodeps", sdistpath)
            extraopts = ['-U', '--no-deps']
        self._install([sdistpath], extraopts=extraopts, action=action)

    def install_deps(self, action=None):
        if action is None:
            action = self.session.newaction(self, "install_deps")
        deps = self._getresolvedeps()
        if deps:
            depinfo = ", ".join(map(str, deps))
            action.setactivity("installdeps", "%s" % depinfo)
            self._install(deps, action=action)

    def _installopts(self, indexserver):
        l = []
        if indexserver:
            l += ["-i", indexserver]
        if self.envconfig.downloadcache:
            self.envconfig.downloadcache.ensure(dir=1)
            l.append("--download-cache=%s" % self.envconfig.downloadcache)
        if self.envconfig.pip_pre:
            l.append("--pre")
        return l

    def run_install_command(self, packages, options=(), action=None):
        argv = self.envconfig.install_command[:]
        # use pip-script on win32 to avoid the executable locking
        i = argv.index('{packages}')
        argv[i:i + 1] = packages
        if '{opts}' in argv:
            i = argv.index('{opts}')
            argv[i:i + 1] = list(options)

        for x in ('PIP_RESPECT_VIRTUALENV', 'PIP_REQUIRE_VIRTUALENV',
                  '__PYVENV_LAUNCHER__'):
            os.environ.pop(x, None)

        old_stdout = sys.stdout
        sys.stdout = codecs.getwriter('utf8')(sys.stdout)
        self._pcall(argv, cwd=self.envconfig.config.toxinidir, action=action)
        sys.stdout = old_stdout

    def _install(self, deps, extraopts=None, action=None):
        if not deps:
            return
        d = {}
        l = []
        for dep in deps:
            if isinstance(dep, (str, py.path.local)):
                dep = DepConfig(str(dep), None)
            assert isinstance(dep, DepConfig), dep
            if dep.indexserver is None:
                ixserver = self.envconfig.config.indexserver['default']
            else:
                ixserver = dep.indexserver
            d.setdefault(ixserver, []).append(dep.name)
            if ixserver not in l:
                l.append(ixserver)
            assert ixserver.url is None or isinstance(ixserver.url, str)

        for ixserver in l:
            packages = d[ixserver]
            options = self._installopts(ixserver.url)
            if extraopts:
                options.extend(extraopts)
            self.run_install_command(packages=packages, options=options,
                                     action=action)

    def _getenv(self, testcommand=False):
        if testcommand:
            # for executing tests we construct a clean environment
            env = {}
            for envname in self.envconfig.passenv:
                if envname in os.environ:
                    env[envname] = os.environ[envname]
        else:
            # for executing non-test commands we use the full
            # invocation environment
            env = os.environ.copy()

        # in any case we honor per-testenv setenv configuration
        env.update(self.envconfig.setenv)

        env['VIRTUAL_ENV'] = str(self.path)
        return env

    def test(self, redirect=False):
        action = self.session.newaction(self, "runtests")
        with action:
            self.status = 0
            self.session.make_emptydir(self.envconfig.envtmpdir)
            cwd = self.envconfig.changedir
            env = self._getenv(testcommand=True)
            # Display PYTHONHASHSEED to assist with reproducibility.
            action.setactivity("runtests", "PYTHONHASHSEED=%r" % env.get('PYTHONHASHSEED'))
            for i, argv in enumerate(self.envconfig.commands):
                # have to make strings as _pcall changes argv[0] to a local()
                # happens if the same environment is invoked twice
                message = "commands[%s] | %s" % (i, ' '.join(
                    [str(x) for x in argv]))
                action.setactivity("runtests", message)
                # check to see if we need to ignore the return code
                # if so, we need to alter the command line arguments
                if argv[0].startswith("-"):
                    ignore_ret = True
                    if argv[0] == "-":
                        del argv[0]
                    else:
                        argv[0] = argv[0].lstrip("-")
                else:
                    ignore_ret = False

                try:
                    self._pcall(argv, cwd=cwd, action=action, redirect=redirect,
                                ignore_ret=ignore_ret, testcommand=True)
                except tox.exception.InvocationError as err:
                    if self.envconfig.ignore_outcome:
                        self.session.report.warning(
                            "command failed but result from testenv is ignored\n"
                            "  cmd: %s" % (str(err),))
                        self.status = "ignored failed command"
                        continue  # keep processing commands

                    self.session.report.error(str(err))
                    self.status = "commands failed"
                    if not self.envconfig.ignore_errors:
                        break  # Don't process remaining commands
                except KeyboardInterrupt:
                    self.status = "keyboardinterrupt"
                    self.session.report.error(self.status)
                    raise

    def _pcall(self, args, cwd, venv=True, testcommand=False,
               action=None, redirect=True, ignore_ret=False):
        for name in ("VIRTUALENV_PYTHON", "PYTHONDONTWRITEBYTECODE"):
            os.environ.pop(name, None)

        cwd.ensure(dir=1)
        args[0] = self.getcommandpath(args[0], venv, cwd)
        env = self._getenv(testcommand=testcommand)
        bindir = str(self.envconfig.envbindir)
        env['PATH'] = p = os.pathsep.join([bindir, os.environ["PATH"]])
        self.session.report.verbosity2("setting PATH=%s" % p)
        return action.popen(args, cwd=cwd, env=env,
                            redirect=redirect, ignore_ret=ignore_ret)

def getdigest(path):
    path = py.path.local(path)
    if not path.check(file=1):
        return "0" * 32
    return path.computehash()
<commit_msg>use more explanatory variable names in VirtualEnv._install()<commit_after>from __future__ import with_statement
import os
import sys
import re
import codecs
import py
import tox
from .config import DepConfig

class CreationConfig:
    def __init__(self, md5, python, version, sitepackages,
                 usedevelop, deps):
        self.md5 = md5
        self.python = python
        self.version = version
        self.sitepackages = sitepackages
        self.usedevelop = usedevelop
        self.deps = deps

    def writeconfig(self, path):
        lines = ["%s %s" % (self.md5, self.python)]
        lines.append("%s %d %d" % (self.version, self.sitepackages, self.usedevelop))
        for dep in self.deps:
            lines.append("%s %s" % dep)
        path.ensure()
        path.write("\n".join(lines))

    @classmethod
    def readconfig(cls, path):
        try:
            lines = path.readlines(cr=0)
            value = lines.pop(0).split(None, 1)
            md5, python = value
            version, sitepackages, usedevelop = lines.pop(0).split(None, 3)
            sitepackages = bool(int(sitepackages))
            usedevelop = bool(int(usedevelop))
            deps = []
            for line in lines:
                md5, depstring = line.split(None, 1)
                deps.append((md5, depstring))
            return CreationConfig(md5, python, version, sitepackages, usedevelop, deps)
        except Exception:
            return None

    def matches(self, other):
        return (other and self.md5 == other.md5
                and self.python == other.python
                and self.version == other.version
                and self.sitepackages == other.sitepackages
                and self.usedevelop == other.usedevelop
                and self.deps == other.deps)

class VirtualEnv(object):
    def __init__(self, envconfig=None, session=None):
        self.envconfig = envconfig
        self.session = session
        self.path = envconfig.envdir
        self.path_config = self.path.join(".tox-config1")

    @property
    def name(self):
        return self.envconfig.envname

    def __repr__(self):
        return "<VirtualEnv at %r>" % (self.path)

    def getcommandpath(self, name=None, venv=True, cwd=None):
        if name is None:
            return self.envconfig.envpython
        name = str(name)
        if os.path.isabs(name):
            return name
        if os.path.split(name)[0] == ".":
            p = cwd.join(name)
            if p.check():
                return str(p)
        p = None
        if venv:
            p = py.path.local.sysfind(name, paths=[self.envconfig.envbindir])
        if p is not None:
            return p
        p = py.path.local.sysfind(name)
        if p is None:
            raise tox.exception.InvocationError(
                "could not find executable %r" % (name,))
        # p is not found in virtualenv script/bin dir
        if venv:
            if not self.is_allowed_external(p):
                self.session.report.warning(
                    "test command found but not installed in testenv\n"
                    "  cmd: %s\n"
                    "  env: %s\n"
                    "Maybe you forgot to specify a dependency? "
                    "See also the whitelist_externals envconfig setting." % (
                        p, self.envconfig.envdir))
        return str(p)  # will not be rewritten for reporting

    def is_allowed_external(self, p):
        tryadd = [""]
        if sys.platform == "win32":
            tryadd += [
                os.path.normcase(x)
                for x in os.environ['PATHEXT'].split(os.pathsep)
            ]
            p = py.path.local(os.path.normcase(str(p)))
        for x in self.envconfig.whitelist_externals:
            for add in tryadd:
                if p.fnmatch(x + add):
                    return True
        return False

    def _ispython3(self):
        return "python3" in str(self.envconfig.basepython)

    def update(self, action=None):
        """ return status string for updating actual venv to match configuration.
            if status string is empty, all is ok.
        """
        if action is None:
            action = self.session.newaction(self, "update")
        rconfig = CreationConfig.readconfig(self.path_config)
        if not self.envconfig.recreate and rconfig and \
           rconfig.matches(self._getliveconfig()):
            action.info("reusing", self.envconfig.envdir)
            return
        if rconfig is None:
            action.setactivity("create", self.envconfig.envdir)
        else:
            action.setactivity("recreate", self.envconfig.envdir)
        try:
            self.create(action)
        except tox.exception.UnsupportedInterpreter:
            return sys.exc_info()[1]
        except tox.exception.InterpreterNotFound:
            return sys.exc_info()[1]
        try:
            self.install_deps(action)
        except tox.exception.InvocationError:
            v = sys.exc_info()[1]
            return "could not install deps %s; v = %r" % (
                self.envconfig.deps, v)

    def _getliveconfig(self):
        python = self.envconfig.python_info.executable
        md5 = getdigest(python)
        version = tox.__version__
        sitepackages = self.envconfig.sitepackages
        develop = self.envconfig.usedevelop
        deps = []
        for dep in self._getresolvedeps():
            raw_dep = dep.name
            md5 = getdigest(raw_dep)
            deps.append((md5, raw_dep))
        return CreationConfig(md5, python, version,
                              sitepackages, develop, deps)

    def _getresolvedeps(self):
        l = []
        for dep in self.envconfig.deps:
            if dep.indexserver is None:
                res = self.session._resolve_pkg(dep.name)
                if res != dep.name:
                    dep = dep.__class__(res)
            l.append(dep)
        return l

    def getsupportedinterpreter(self):
        return self.envconfig.getsupportedinterpreter()

    def matching_platform(self):
        return re.match(self.envconfig.platform, sys.platform)

    def create(self, action=None):
        # if self.getcommandpath("activate").dirpath().check():
        #    return
        if action is None:
            action = self.session.newaction(self, "create")

        config_interpreter = self.getsupportedinterpreter()
        args = [sys.executable, '-m', 'virtualenv']
        if self.envconfig.sitepackages:
            args.append('--system-site-packages')
        # add interpreter explicitly, to prevent using
        # default (virtualenv.ini)
        args.extend(['--python', str(config_interpreter)])
        # if sys.platform == "win32":
        #    f, path, _ = py.std.imp.find_module("virtualenv")
        #    f.close()
        #    args[:1] = [str(config_interpreter), str(path)]
        # else:
        self.session.make_emptydir(self.path)
        basepath = self.path.dirpath()
        basepath.ensure(dir=1)
        args.append(self.path.basename)
        self._pcall(args, venv=False, action=action, cwd=basepath)
        self.just_created = True

    def finish(self):
        self._getliveconfig().writeconfig(self.path_config)

    def _needs_reinstall(self, setupdir, action):
        setup_py = setupdir.join('setup.py')
        setup_cfg = setupdir.join('setup.cfg')
        args = [self.envconfig.envpython, str(setup_py), '--name']
        output = action.popen(args, cwd=setupdir, redirect=False,
                              returnout=True)
        name = output.strip()
        egg_info = setupdir.join('.'.join((name, 'egg-info')))
        for conf_file in (setup_py, setup_cfg):
            if (not egg_info.check()
                    or (conf_file.check() and conf_file.mtime() > egg_info.mtime())):
                return True
        return False

    def developpkg(self, setupdir, action):
        assert action is not None
        if getattr(self, 'just_created', False):
            action.setactivity("develop-inst", setupdir)
            self.finish()
            extraopts = []
        else:
            if not self._needs_reinstall(setupdir, action):
                action.setactivity("develop-inst-noop", setupdir)
                return
            action.setactivity("develop-inst-nodeps", setupdir)
            extraopts = ['--no-deps']
        self._install(['-e', setupdir], extraopts=extraopts, action=action)

    def installpkg(self, sdistpath, action):
        assert action is not None
        if getattr(self, 'just_created', False):
            action.setactivity("inst", sdistpath)
            self.finish()
            extraopts = []
        else:
            action.setactivity("inst-nodeps", sdistpath)
            extraopts = ['-U', '--no-deps']
        self._install([sdistpath], extraopts=extraopts, action=action)

    def install_deps(self, action=None):
        if action is None:
            action = self.session.newaction(self, "install_deps")
        deps = self._getresolvedeps()
        if deps:
            depinfo = ", ".join(map(str, deps))
            action.setactivity("installdeps", "%s" % depinfo)
            self._install(deps, action=action)

    def _installopts(self, indexserver):
        l = []
        if indexserver:
            l += ["-i", indexserver]
        if self.envconfig.downloadcache:
            self.envconfig.downloadcache.ensure(dir=1)
            l.append("--download-cache=%s" % self.envconfig.downloadcache)
        if self.envconfig.pip_pre:
            l.append("--pre")
        return l

    def run_install_command(self, packages, options=(), action=None):
        argv = self.envconfig.install_command[:]
        # use pip-script on win32 to avoid the executable locking
        i = argv.index('{packages}')
        argv[i:i + 1] = packages
        if '{opts}' in argv:
            i = argv.index('{opts}')
            argv[i:i + 1] = list(options)

        for x in ('PIP_RESPECT_VIRTUALENV', 'PIP_REQUIRE_VIRTUALENV',
                  '__PYVENV_LAUNCHER__'):
            os.environ.pop(x, None)

        old_stdout = sys.stdout
        sys.stdout = codecs.getwriter('utf8')(sys.stdout)
        self._pcall(argv, cwd=self.envconfig.config.toxinidir, action=action)
        sys.stdout = old_stdout

    def _install(self, deps, extraopts=None, action=None):
        if not deps:
            return
        deps_by_ixserver = {}
        ixservers = []
        for dep in deps:
            if isinstance(dep, (str, py.path.local)):
                dep = DepConfig(str(dep), None)
            assert isinstance(dep, DepConfig), dep
            if dep.indexserver is None:
                ixserver = self.envconfig.config.indexserver['default']
            else:
                ixserver = dep.indexserver
            deps_by_ixserver.setdefault(ixserver, []).append(dep.name)
            if ixserver not in ixservers:
                ixservers.append(ixserver)
            assert ixserver.url is None or isinstance(ixserver.url, str)

        for ixserver in ixservers:
            packages = deps_by_ixserver[ixserver]
            options = self._installopts(ixserver.url)
            if extraopts:
                options.extend(extraopts)
            self.run_install_command(packages=packages, options=options,
                                     action=action)

    def _getenv(self, testcommand=False):
        if testcommand:
            # for executing tests we construct a clean environment
            env = {}
            for envname in self.envconfig.passenv:
                if envname in os.environ:
                    env[envname] = os.environ[envname]
        else:
            # for executing non-test commands we use the full
            # invocation environment
            env = os.environ.copy()

        # in any case we honor per-testenv setenv configuration
        env.update(self.envconfig.setenv)

        env['VIRTUAL_ENV'] = str(self.path)
        return env

    def test(self, redirect=False):
        action = self.session.newaction(self, "runtests")
        with action:
            self.status = 0
            self.session.make_emptydir(self.envconfig.envtmpdir)
            cwd = self.envconfig.changedir
            env = self._getenv(testcommand=True)
            # Display PYTHONHASHSEED to assist with reproducibility.
            action.setactivity("runtests", "PYTHONHASHSEED=%r" % env.get('PYTHONHASHSEED'))
            for i, argv in enumerate(self.envconfig.commands):
                # have to make strings as _pcall changes argv[0] to a local()
                # happens if the same environment is invoked twice
                message = "commands[%s] | %s" % (i, ' '.join(
                    [str(x) for x in argv]))
                action.setactivity("runtests", message)
                # check to see if we need to ignore the return code
                # if so, we need to alter the command line arguments
                if argv[0].startswith("-"):
                    ignore_ret = True
                    if argv[0] == "-":
                        del argv[0]
                    else:
                        argv[0] = argv[0].lstrip("-")
                else:
                    ignore_ret = False

                try:
                    self._pcall(argv, cwd=cwd, action=action, redirect=redirect,
                                ignore_ret=ignore_ret, testcommand=True)
                except tox.exception.InvocationError as err:
                    if self.envconfig.ignore_outcome:
                        self.session.report.warning(
                            "command failed but result from testenv is ignored\n"
                            "  cmd: %s" % (str(err),))
                        self.status = "ignored failed command"
                        continue  # keep processing commands

                    self.session.report.error(str(err))
                    self.status = "commands failed"
                    if not self.envconfig.ignore_errors:
                        break  # Don't process remaining commands
                except KeyboardInterrupt:
                    self.status = "keyboardinterrupt"
                    self.session.report.error(self.status)
                    raise

    def _pcall(self, args, cwd, venv=True, testcommand=False,
               action=None, redirect=True, ignore_ret=False):
        for name in ("VIRTUALENV_PYTHON", "PYTHONDONTWRITEBYTECODE"):
            os.environ.pop(name, None)

        cwd.ensure(dir=1)
        args[0] = self.getcommandpath(args[0], venv, cwd)
        env = self._getenv(testcommand=testcommand)
        bindir = str(self.envconfig.envbindir)
        env['PATH'] = p = os.pathsep.join([bindir, os.environ["PATH"]])
        self.session.report.verbosity2("setting PATH=%s" % p)
        return action.popen(args, cwd=cwd, env=env,
                            redirect=redirect, ignore_ret=ignore_ret)

def getdigest(path):
    path = py.path.local(path)
    if not path.check(file=1):
        return "0" * 32
    return path.computehash()
<|endoftext|>


<commit_before>{
    "targets":
    [
        {
            "target_name": "csound",
            "sources":
            [
                "jscsound.cpp",
            ],
            'conditions':
            [
                 ['OS=="linux"',
                    {
                        'libraries':
                        [
                            '-L$(CSOUND_HOME)/../cs6make -lcsound64',
                        ],
                        'include_dirs':
                        [
                            '$(CSOUND_HOME)/include',
                        ],
 			'cflags_cc!':
			[
              		'-fno-exceptions',
			'-std=c++11',
            		],
		    }
               ],
               ['OS=="win"',
                    {
                        'msbuild_toolset': 'v140_x0p',
                        'libraries':
                        [
                            '-l$(CSOUND_HOME)/mingw64/csound64.lib',
                        ],
                        'include_dirs':
                        [
                            '$(CSOUND_HOME)/include',
                        ],
                    }
                ]
            ]
        }
    ]
}
<commit_msg>More typos (#793).<commit_after>{
    "targets":
    [
        {
            "target_name": "csound",
            "sources":
            [
                "jscsound.cpp",
            ],
            'conditions':
            [
                 ['OS=="linux"',
                    {
                        'libraries':
                        [
                            '-L$(CSOUND_HOME)/../cs6make -lcsound64',
                        ],
                        'include_dirs':
                        [
                            '$(CSOUND_HOME)/include',
                        ],
 			'cflags_cc!':
			[
              		'-fno-exceptions',
			'-std=c++11',
            		],
		    }
               ],
               ['OS=="win"',
                    {
                        'msbuild_toolset': 'v140_xp',
                        'libraries':
                        [
                            '-l$(CSOUND_HOME)/mingw64/csound64.lib',
                        ],
                        'include_dirs':
                        [
                            '$(CSOUND_HOME)/include',
                        ],
                    }
                ]
            ]
        }
    ]
}
<|endoftext|>