<commit_before>"""Leetcode 516. Longest Palindromic Subsequence
Medium

URL: https://leetcode.com/problems/longest-palindromic-subsequence/

Given a string s, find the longest palindromic subsequence's length in s.
You may assume that the maximum length of s is 1000.

Example 1:
Input:
"bbbab"
Output:
4
One possible longest palindromic subsequence is "bbbb".

Example 2:
Input:
"cbbd"
Output:
2
One possible longest palindromic subsequence is "bb".
"""

class SolutionRecur(object):
    def _LPS(self, s, left, right):
        if left > right:
            return 0

        if left == right:
            return 1

        if s[left] == s[right]:
            # Check LPS in subsequence between s[left] and s[right].
            return self._LPS(s, left + 1, right - 1) + 2
        else:
            # Check max of LPS's in s[left+1:right-1] and s[left:right].
            return max(self._LPS(s, left + 1, right),
                       self._LPS(s, left, right - 1))

    def longestPalindromeSubseq(self, s):
        """
        :type s: str
        :rtype: int

        Note: Time Limit Exceeded.

        Time complexity: O(2^n).
        Space complexity: O(n).
        """
        # Apply top-down recursion.
        left, right = 0, len(s) - 1
        return self._LPS(s, left, right)

class SolutionMemo(object):
    def _LPS(self, s, left, right, T):
        if T[left][right]:
            return T[left][right]

        if left > right:
            return 0

        if s[left] == s[right]:
            # Check LPS in subsequence between s[left] and s[right].
            T[left][right] = self._LPS(s, left + 1, right - 1, T) + 2
        else:
            # Check max of LPS's in s[left+1:right-1] and s[left:right].
            T[left][right] = max(self._LPS(s, left + 1, right, T),
                                 self._LPS(s, left, right - 1, T))
        return T[left][right]

    def longestPalindromeSubseq(self, s):
        """
        :type s: str
        :rtype: int

        Time complexity: O(n^2).
        Space complexity: O(n^2).
        """
        # Apply top-down recursion with memoization.
        n = len(s)
        T = [[None] * n for _ in range(n)]

        for i in range(n):
            T[i][i] = 1

        left, right = 0, len(s) - 1
        return self._LPS(s, left, right, T)

class SolutionDp(object):
    def longestPalindromeSubseq(self, s):
        """
        :type s: str
        :rtype: int

        Time complexity: O(n^2).
        Space complexity: O(n^2).
        """
        # Apply bottom-up dynamic programming.
        n = len(s)
        T = [[0] * n for _ in range(n)]

        # Start left from tail for bottom-up.
        for left in range(n - 1, -1, -1):
            T[left][left] = 1

            for right in range(left + 1, n):
                if s[left] == s[right]:
                    T[left][right] = T[left + 1][right - 1] + 2
                else:
                    T[left][right] = max(T[left][right - 1], T[left + 1][right])

        return T[0][-1]

def main():
    # Output: 4.
    s = "bbbab"
    print SolutionRecur().longestPalindromeSubseq(s)
    print SolutionMemo().longestPalindromeSubseq(s)
    print SolutionDp().longestPalindromeSubseq(s)

    # Output: 2.
    s = "cbbd"
    print SolutionRecur().longestPalindromeSubseq(s)
    print SolutionMemo().longestPalindromeSubseq(s)
    print SolutionDp().longestPalindromeSubseq(s)

if __name__ == '__main__':
    main()
<commit_msg>Revise left/right to l/r<commit_after>"""Leetcode 516. Longest Palindromic Subsequence
Medium

URL: https://leetcode.com/problems/longest-palindromic-subsequence/

Given a string s, find the longest palindromic subsequence's length in s.
You may assume that the maximum length of s is 1000.

Example 1:
Input:
"bbbab"
Output:
4
One possible longest palindromic subsequence is "bbbb".

Example 2:
Input:
"cbbd"
Output:
2
One possible longest palindromic subsequence is "bb".
"""

class SolutionRecur(object):
    def _LPS(self, s, l, r):
        if l > r:
            return 0

        if l == r:
            return 1

        if s[l] == s[r]:
            # Check LPS in subsequence between s[l] and s[r].
            return self._LPS(s, l + 1, r - 1) + 2
        else:
            # Check max of LPS's in s[l+1:r-1] and s[l:r].
            return max(self._LPS(s, l + 1, r), self._LPS(s, l, r - 1))

    def longestPalindromeSubseq(self, s):
        """
        :type s: str
        :rtype: int

        Note: Time Limit Exceeded.

        Time complexity: O(2^n).
        Space complexity: O(n).
        """
        # Apply top-down recursion.
        l, r = 0, len(s) - 1
        return self._LPS(s, l, r)

class SolutionMemo(object):
    def _LPS(self, s, l, r, T):
        if T[l][r]:
            return T[l][r]

        if l > r:
            return 0

        if s[l] == s[r]:
            # Check LPS in subsequence between s[l] and s[r].
            T[l][r] = self._LPS(s, l + 1, r - 1, T) + 2
        else:
            # Check max of LPS's in s[l+1:r-1] and s[l:r].
            T[l][r] = max(self._LPS(s, l + 1, r, T), self._LPS(s, l, r - 1, T))
        return T[l][r]

    def longestPalindromeSubseq(self, s):
        """
        :type s: str
        :rtype: int

        Time complexity: O(n^2).
        Space complexity: O(n^2).
        """
        # Apply top-down recursion with memoization.
        n = len(s)
        T = [[None] * n for _ in range(n)]

        for i in range(n):
            T[i][i] = 1

        l, r = 0, len(s) - 1
        return self._LPS(s, l, r, T)

class SolutionDp(object):
    def longestPalindromeSubseq(self, s):
        """
        :type s: str
        :rtype: int

        Time complexity: O(n^2).
        Space complexity: O(n^2).
        """
        # Apply bottom-up dynamic programming.
        n = len(s)
        T = [[0] * n for _ in range(n)]

        # Start l from tail for bottom-up.
        for l in range(n - 1, -1, -1):
            T[l][l] = 1

            for r in range(l + 1, n):
                if s[l] == s[r]:
                    T[l][r] = T[l + 1][r - 1] + 2
                else:
                    T[l][r] = max(T[l][r - 1], T[l + 1][r])

        return T[0][-1]

def main():
    # Output: 4.
    s = "bbbab"
    print SolutionRecur().longestPalindromeSubseq(s)
    print SolutionMemo().longestPalindromeSubseq(s)
    print SolutionDp().longestPalindromeSubseq(s)

    # Output: 2.
    s = "cbbd"
    print SolutionRecur().longestPalindromeSubseq(s)
    print SolutionMemo().longestPalindromeSubseq(s)
    print SolutionDp().longestPalindromeSubseq(s)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># A simple upnp module to forward port for BitMessage
# Reference: http://mattscodecave.com/posts/using-python-and-upnp-to-forward-a-port
import socket
from struct import unpack, pack
import threading
import time
import shared

def createRequestXML(service, action, arguments=[]):
    from xml.dom.minidom import Document

    doc = Document()

    # create the envelope element and set its attributes
    envelope = doc.createElementNS('', 's:Envelope')
    envelope.setAttribute('xmlns:s', 'http://schemas.xmlsoap.org/soap/envelope/')
    envelope.setAttribute('s:encodingStyle', 'http://schemas.xmlsoap.org/soap/encoding/')

    # create the body element
    body = doc.createElementNS('', 's:Body')

    # create the function element and set its attribute
    fn = doc.createElementNS('', 'u:%s' % action)
    fn.setAttribute('xmlns:u', 'urn:schemas-upnp-org:service:%s' % service)

    # setup the argument element names and values
    # using a list of tuples to preserve order

    # container for created nodes
    argument_list = []

    # iterate over arguments, create nodes, create text nodes,
    # append text nodes to nodes, and finally add the ready product
    # to argument_list
    for k, v in arguments:
        tmp_node = doc.createElement(k)
        tmp_text_node = doc.createTextNode(v)
        tmp_node.appendChild(tmp_text_node)
        argument_list.append(tmp_node)

    # append the prepared argument nodes to the function element
    for arg in argument_list:
        fn.appendChild(arg)

    # append function element to the body element
    body.appendChild(fn)

    # append body element to envelope element
    envelope.appendChild(body)

    # append envelope element to document, making it the root element
    doc.appendChild(envelope)

    # our tree is ready, conver it to a string
    return doc.toxml()

class UPnPError(Exception):
    def __init__(self, message):
        self.message

class Router:
    name = ""
    path = ""
    address = None
    routerPath = None
    extPort = None

    def __init__(self, ssdpResponse, address):
        import urllib2
        from xml.dom.minidom import parseString
        from urlparse import urlparse

        self.address = address

        row = ssdpResponse.split('\r\n')
        header = {}
        for i in range(1, len(row)):
            part = row[i].split(': ')
            if len(part) == 2:
                header[part[0].lower()] = part[1]

        self.routerPath = urlparse(header['location'])

        # get the profile xml file and read it into a variable
        directory = urllib2.urlopen(header['location']).read()

        # create a DOM object that represents the `directory` document
        dom = parseString(directory)

        self.name = dom.getElementsByTagName('friendlyName')[0].childNodes[0].data
        # find all 'serviceType' elements
        service_types = dom.getElementsByTagName('serviceType')

        for service in service_types:
            if service.childNodes[0].data.find('WANIPConnection') > 0:
                self.path = service.parentNode.getElementsByTagName('controlURL')[0].childNodes[0].data

        # get local IP
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        try:
            logger.debug("Connecting to %s:%i", self.address, self.routerPath.port)
            s.connect ((self.address, self.routerPath.port))
        except:
            pass
        self.localAddress = s.getsockname()[0]
        logger.debug("Local IP: %s", self.localAddress)
        try:
            s.close()
        except:
            pass

    def AddPortMapping(self, externalPort, internalPort, internalClient, protocol, description, leaseDuration = 0, enabled = 1):
        from debug import logger
        resp = self.soapRequest('WANIPConnection:1', 'AddPortMapping', [
                ('NewExternalPort', str(externalPort)),
                ('NewProtocol', protocol),
                ('NewInternalPort', str(internalPort)),
                ('NewInternalClient', internalClient),
                ('NewEnabled', str(enabled)),
                ('NewPortMappingDescription', str(description)),
                ('NewLeaseDuration', str(leaseDuration))
            ])
        self.extPort = externalPort
        logger.info("Successfully established UPnP mapping for %s:%i on external port %i", internalClient, internalPort, externalPort)
        return resp

    def DeletePortMapping(self, externalPort, protocol):
        from debug import logger
        resp = self.soapRequest('WANIPConnection:1', 'DeletePortMapping', [
                ('NewExternalPort', str(externalPort)),
                ('NewProtocol', protocol),
            ])
        logger.info("Removed UPnP mapping on external port %i", externalPort)
        return resp

    def GetExternalIPAddress(self):
        from xml.dom.minidom import parseString
        resp = self.soapRequest('WANIPConnection:1', 'GetExternalIPAddress')
        dom = parseString(resp)
        return dom.getElementsByTagName('NewExternalIPAddress')[0].childNodes[0].data

    def soapRequest(self, service, action, arguments=[]):
        from xml.dom.minidom import parseString
        from debug import logger
        conn = httplib.HTTPConnection(self.routerPath.hostname, self.routerPath.port)
        conn.request(
            'POST',
            self.path,
            createRequestXML(service, action, arguments),
            {
                'SOAPAction': '"urn:schemas-upnp-org:service:%s#%s"' % (service, action),
                'Content-Type': 'text/xml'
                }
            )
        resp = conn.getresponse().read()
        dom = parseString(resp)
        errinfo = dom.getElementsByTagName('errorDescription')
        if len(errinfo) > 0:
            logger.error("UPnP error: %s", resp)
            raise UPnPError(errinfo[0].childNodes[0].data)
        return resp

class uPnPThread(threading.Thread):
    def __init__ (self):
        threading.Thread.__init__(self, name="uPnPThread")
        self.localPort = shared.config.getint('bitmessagesettings', 'port')
        self.extPort = None
        self.routers = []
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2)
        self.sock.settimeout(10)
        self.sendSleep = 60

    def run(self):
        from debug import logger

        logger.debug("Starting UPnP thread")
        lastSent = 0
        while shared.shutdown == 0:
            if time.time() - lastSent > self.sendSleep and len(self.routers) == 0:
                self.sendSearchRouter()
                lastSent = time.time()
            try:
                while shared.shutdown == 0:
                    resp,(ip,port) = self.sock.recvfrom(1000)
                    if resp is None:
                        continue
                    newRouter = Router(resp, ip)
                    for router in self.routers:
                        if router.location == newRouter.location:
                            break
                    else:
                        logger.debug("Found UPnP router at %s", ip)
                        self.routers.append(newRouter)
                        self.createPortMapping(newRouter)
                        break
            except socket.timeout as e:
                pass
            except:
                logger.error("Failure running UPnP router search.", exc_info=True)
            for router in self.routers:
                if router.extPort is None:
                    self.createPortMapping(router)
        for router in self.routers:
            if router.extPort is not None:
                self.deletePortMapping(router)
        logger.debug("UPnP thread done")

    def sendSearchRouter(self):
        from debug import logger
        SSDP_ADDR = "239.255.255.250"
        SSDP_PORT = 1900
        SSDP_MX = 2
        SSDP_ST = "urn:schemas-upnp-org:device:InternetGatewayDevice:1"
        ssdpRequest = "M-SEARCH * HTTP/1.1\r\n" + \
                    "HOST: %s:%d\r\n" % (SSDP_ADDR, SSDP_PORT) + \
                    "MAN: \"ssdp:discover\"\r\n" + \
                    "MX: %d\r\n" % (SSDP_MX, ) + \
                    "ST: %s\r\n" % (SSDP_ST, ) + "\r\n"

        try:
            logger.debug("Sending UPnP query")
            self.sock.sendto(ssdpRequest, (SSDP_ADDR, SSDP_PORT))
        except:
            logger.exception("UPnP send query failed")

    def createPortMapping(self, router):
        from debug import logger

        for i in range(50):
            try:
                routerIP, = unpack('>I', socket.inet_aton(router.address))
                localIP = router.localAddress
                if i == 0:
                    extPort = self.localPort # try same port first
                else:
                    extPort = randint(32767, 65535)
                logger.debug("Requesting UPnP mapping for %s:%i on external port %i", localIP, self.localPort,  extPort)
                router.AddPortMapping(extPort, self.localPort, localIP, 'TCP', 'BitMessage')
                shared.extPort = extPort
                break
            except UPnPError:
                logger.debug("UPnP error: ", exc_info=True)

    def deletePortMapping(self, router):
        router.DeletePortMapping(router.extPort, 'TCP')

<commit_msg>UPnP fixes<commit_after># A simple upnp module to forward port for BitMessage
# Reference: http://mattscodecave.com/posts/using-python-and-upnp-to-forward-a-port
import socket
from struct import unpack, pack
import threading
import time
import shared

def createRequestXML(service, action, arguments=[]):
    from xml.dom.minidom import Document

    doc = Document()

    # create the envelope element and set its attributes
    envelope = doc.createElementNS('', 's:Envelope')
    envelope.setAttribute('xmlns:s', 'http://schemas.xmlsoap.org/soap/envelope/')
    envelope.setAttribute('s:encodingStyle', 'http://schemas.xmlsoap.org/soap/encoding/')

    # create the body element
    body = doc.createElementNS('', 's:Body')

    # create the function element and set its attribute
    fn = doc.createElementNS('', 'u:%s' % action)
    fn.setAttribute('xmlns:u', 'urn:schemas-upnp-org:service:%s' % service)

    # setup the argument element names and values
    # using a list of tuples to preserve order

    # container for created nodes
    argument_list = []

    # iterate over arguments, create nodes, create text nodes,
    # append text nodes to nodes, and finally add the ready product
    # to argument_list
    for k, v in arguments:
        tmp_node = doc.createElement(k)
        tmp_text_node = doc.createTextNode(v)
        tmp_node.appendChild(tmp_text_node)
        argument_list.append(tmp_node)

    # append the prepared argument nodes to the function element
    for arg in argument_list:
        fn.appendChild(arg)

    # append function element to the body element
    body.appendChild(fn)

    # append body element to envelope element
    envelope.appendChild(body)

    # append envelope element to document, making it the root element
    doc.appendChild(envelope)

    # our tree is ready, conver it to a string
    return doc.toxml()

class UPnPError(Exception):
    def __init__(self, message):
        self.message

class Router:
    name = ""
    path = ""
    address = None
    routerPath = None
    extPort = None

    def __init__(self, ssdpResponse, address):
        import urllib2
        from xml.dom.minidom import parseString
        from urlparse import urlparse

        self.address = address

        row = ssdpResponse.split('\r\n')
        header = {}
        for i in range(1, len(row)):
            part = row[i].split(': ')
            if len(part) == 2:
                header[part[0].lower()] = part[1]

        self.routerPath = urlparse(header['location'])

        # get the profile xml file and read it into a variable
        directory = urllib2.urlopen(header['location']).read()

        # create a DOM object that represents the `directory` document
        dom = parseString(directory)

        self.name = dom.getElementsByTagName('friendlyName')[0].childNodes[0].data
        # find all 'serviceType' elements
        service_types = dom.getElementsByTagName('serviceType')

        for service in service_types:
            if service.childNodes[0].data.find('WANIPConnection') > 0:
                self.path = service.parentNode.getElementsByTagName('controlURL')[0].childNodes[0].data

        # get local IP
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        try:
            logger.debug("Connecting to %s:%i", self.address, self.routerPath.port)
            s.connect ((self.address, self.routerPath.port))
        except:
            pass
        self.localAddress = s.getsockname()[0]
        logger.debug("Local IP: %s", self.localAddress)
        try:
            s.close()
        except:
            pass

    def AddPortMapping(self, externalPort, internalPort, internalClient, protocol, description, leaseDuration = 0, enabled = 1):
        from debug import logger
        resp = self.soapRequest('WANIPConnection:1', 'AddPortMapping', [
                ('NewExternalPort', str(externalPort)),
                ('NewProtocol', protocol),
                ('NewInternalPort', str(internalPort)),
                ('NewInternalClient', internalClient),
                ('NewEnabled', str(enabled)),
                ('NewPortMappingDescription', str(description)),
                ('NewLeaseDuration', str(leaseDuration))
            ])
        self.extPort = externalPort
        logger.info("Successfully established UPnP mapping for %s:%i on external port %i", internalClient, internalPort, externalPort)
        return resp

    def DeletePortMapping(self, externalPort, protocol):
        from debug import logger
        resp = self.soapRequest('WANIPConnection:1', 'DeletePortMapping', [
                ('NewExternalPort', str(externalPort)),
                ('NewProtocol', protocol),
            ])
        logger.info("Removed UPnP mapping on external port %i", externalPort)
        return resp

    def GetExternalIPAddress(self):
        from xml.dom.minidom import parseString
        resp = self.soapRequest('WANIPConnection:1', 'GetExternalIPAddress')
        dom = parseString(resp)
        return dom.getElementsByTagName('NewExternalIPAddress')[0].childNodes[0].data

    def soapRequest(self, service, action, arguments=[]):
        from xml.dom.minidom import parseString
        from debug import logger
        conn = httplib.HTTPConnection(self.routerPath.hostname, self.routerPath.port)
        conn.request(
            'POST',
            self.path,
            createRequestXML(service, action, arguments),
            {
                'SOAPAction': '"urn:schemas-upnp-org:service:%s#%s"' % (service, action),
                'Content-Type': 'text/xml'
                }
            )
        resp = conn.getresponse().read()
        dom = parseString(resp)
        errinfo = dom.getElementsByTagName('errorDescription')
        if len(errinfo) > 0:
            logger.error("UPnP error: %s", resp)
            raise UPnPError(errinfo[0].childNodes[0].data)
        return resp

class uPnPThread(threading.Thread):
    def __init__ (self):
        threading.Thread.__init__(self, name="uPnPThread")
        self.localPort = shared.config.getint('bitmessagesettings', 'port')
        self.extPort = None
        self.routers = []
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2)
        self.sock.settimeout(10)
        self.sendSleep = 60

    def run(self):
        from debug import logger

        logger.debug("Starting UPnP thread")
        lastSent = 0
        while shared.shutdown == 0:
            if time.time() - lastSent > self.sendSleep and len(self.routers) == 0:
                self.sendSearchRouter()
                lastSent = time.time()
            try:
                while shared.shutdown == 0:
                    resp,(ip,port) = self.sock.recvfrom(1000)
                    if resp is None:
                        continue
                    newRouter = Router(resp, ip)
                    for router in self.routers:
                        if router.location == newRouter.location:
                            break
                    else:
                        logger.debug("Found UPnP router at %s", ip)
                        self.routers.append(newRouter)
                        self.createPortMapping(newRouter)
                        break
            except socket.timeout as e:
                pass
            except:
                logger.error("Failure running UPnP router search.", exc_info=True)
            for router in self.routers:
                if router.extPort is None:
                    self.createPortMapping(router)
        self.sock.close()
        for router in self.routers:
            if router.extPort is not None:
                self.deletePortMapping(router)
        logger.debug("UPnP thread done")

    def sendSearchRouter(self):
        from debug import logger
        SSDP_ADDR = "239.255.255.250"
        SSDP_PORT = 1900
        SSDP_MX = 2
        SSDP_ST = "urn:schemas-upnp-org:device:InternetGatewayDevice:1"
        ssdpRequest = "M-SEARCH * HTTP/1.1\r\n" + \
                    "HOST: %s:%d\r\n" % (SSDP_ADDR, SSDP_PORT) + \
                    "MAN: \"ssdp:discover\"\r\n" + \
                    "MX: %d\r\n" % (SSDP_MX, ) + \
                    "ST: %s\r\n" % (SSDP_ST, ) + "\r\n"

        try:
            logger.debug("Sending UPnP query")
            self.sock.sendto(ssdpRequest, (SSDP_ADDR, SSDP_PORT))
        except:
            logger.exception("UPnP send query failed")

    def createPortMapping(self, router):
        from debug import logger

        for i in range(50):
            try:
                routerIP, = unpack('>I', socket.inet_aton(router.address))
                localIP = router.localAddress
                if i == 0:
                    extPort = self.localPort # try same port first
                else:
                    extPort = randint(32767, 65535)
                logger.debug("Requesting UPnP mapping for %s:%i on external port %i", localIP, self.localPort,  extPort)
                router.AddPortMapping(extPort, self.localPort, localIP, 'TCP', 'BitMessage')
                shared.extPort = extPort
                break
            except UPnPError:
                logger.debug("UPnP error: ", exc_info=True)

    def deletePortMapping(self, router):
        router.DeletePortMapping(router.extPort, 'TCP')

<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
USPS Address Lookup client
'''

from urllib2 import urlopen, URLError
from urllib import urlencode
from lxml.html import fromstring

ADDRESS_LOOKUP_URL = 'https://tools.usps.com/go/ZipLookupResultsAction!input.action?'

def _parse(text):
    'Give html text from the web page, extract the list of addresses.'

    if '''
Unfortunately, this address wasn't found.
''' in text and \
        '''
Please double-check it and try again.
''' in text:
        return []

    elif 'The address you provided is not recognized by the US Postal Service as an address we serve. Mail sent to this address may be returned.' in text:
        return []

    resultstart = '
'
    text = resultstart + text.split(resultstart)[1].split('
')[0].replace('™', '')
    html = fromstring(text)
    out = []
    for std_address in html.cssselect('#result-list p.std-address'):
        spans = std_address.xpath('span[@class != "address1 range" and @class != "hyphen"]')
        outaddress = {unicode(span.attrib['class'].replace(' range', '')): unicode(span.text) for span in spans}

        addresses1 = std_address.xpath('span[@class="address1 range"]')
        outaddress[u'address1'] = addresses1[-1].text
        if len(addresses1) == 2:
            outaddress[u'name'] = addresses1[0].text

        if outaddress[u'zip4'] == 'None':
            outaddress[u'zip4'] = None

        # Strip padding
        for k, v in outaddress.items():
            outaddress[k] = v.strip()

        out.append(outaddress)

    return out

def lookup(address1, address2, city, state, zipcode):
    'Look up an address, and return the usps version.'
    address = {
        "address1": address1,
        "address2": address2,
        "city": city,
        "state": state,
        "zipcode": zipcode,
        "resultMode": 0,
        "companyName": "",
        "urbanCode": "",
        "postalCode": "",
    }
    url = ADDRESS_LOOKUP_URL + urlencode(address)

    try:
        handle = urlopen(url)
    except:
        print url
        raise

    return self._parse(handle.read())
<commit_msg>remove self<commit_after>#!/usr/bin/env python
'''
USPS Address Lookup client
'''

from urllib2 import urlopen, URLError
from urllib import urlencode
from lxml.html import fromstring

ADDRESS_LOOKUP_URL = 'https://tools.usps.com/go/ZipLookupResultsAction!input.action?'

def _parse(text):
    'Give html text from the web page, extract the list of addresses.'

    if '''
Unfortunately, this address wasn't found.
''' in text and \
        '''
Please double-check it and try again.
''' in text:
        return []

    elif 'The address you provided is not recognized by the US Postal Service as an address we serve. Mail sent to this address may be returned.' in text:
        return []

    resultstart = '
'
    text = resultstart + text.split(resultstart)[1].split('
')[0].replace('™', '')
    html = fromstring(text)
    out = []
    for std_address in html.cssselect('#result-list p.std-address'):
        spans = std_address.xpath('span[@class != "address1 range" and @class != "hyphen"]')
        outaddress = {unicode(span.attrib['class'].replace(' range', '')): unicode(span.text) for span in spans}

        addresses1 = std_address.xpath('span[@class="address1 range"]')
        outaddress[u'address1'] = addresses1[-1].text
        if len(addresses1) == 2:
            outaddress[u'name'] = addresses1[0].text

        if outaddress[u'zip4'] == 'None':
            outaddress[u'zip4'] = None

        # Strip padding
        for k, v in outaddress.items():
            outaddress[k] = v.strip()

        out.append(outaddress)

    return out

def lookup(address1, address2, city, state, zipcode):
    'Look up an address, and return the usps version.'
    address = {
        "address1": address1,
        "address2": address2,
        "city": city,
        "state": state,
        "zipcode": zipcode,
        "resultMode": 0,
        "companyName": "",
        "urbanCode": "",
        "postalCode": "",
    }
    url = ADDRESS_LOOKUP_URL + urlencode(address)

    try:
        handle = urlopen(url)
    except:
        print url
        raise

    return _parse(handle.read())
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2019 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
"""Build modified projects."""

from __future__ import print_function

import enum
import os
import re
import sys
import subprocess
import yaml

# pylint: disable=wrong-import-position,import-error
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import constants

CANARY_PROJECT = 'skcms'

DEFAULT_ARCHITECTURES = ['x86_64']
DEFAULT_ENGINES = ['afl', 'honggfuzz', 'libfuzzer']
DEFAULT_SANITIZERS = ['address', 'undefined']

def get_changed_files_output():
  """Returns the output of a git command that discovers changed files."""
  branch_commit_hash = subprocess.check_output(
      ['git', 'merge-base', 'HEAD', 'origin/HEAD']).strip().decode()

  return subprocess.check_output(
      ['git', 'diff', '--name-only', branch_commit_hash + '..']).decode()

def get_modified_buildable_projects():
  """Returns a list of all the projects modified in this commit that have a
  build.sh file."""
  git_output = get_changed_files_output()
  projects_regex = '.*projects/(?P<name>.*)/.*\n'
  modified_projects = set(re.findall(projects_regex, git_output))
  projects_dir = os.path.join(get_oss_fuzz_root(), 'projects')
  # Filter out projects without Dockerfile files since new projects and reverted
  # projects frequently don't have them. In these cases we don't want Travis's
  # builds to fail.
  modified_buildable_projects = []
  for project in modified_projects:
    if not os.path.exists(os.path.join(projects_dir, project, 'Dockerfile')):
      print('Project {0} does not have Dockerfile. skipping build.'.format(
          project))
      continue
    modified_buildable_projects.append(project)
  return modified_buildable_projects

def get_oss_fuzz_root():
  """Get the absolute path of the root of the oss-fuzz checkout."""
  script_path = os.path.realpath(__file__)
  return os.path.abspath(
      os.path.dirname(os.path.dirname(os.path.dirname(script_path))))

def execute_helper_command(helper_command):
  """Execute |helper_command| using helper.py."""
  root = get_oss_fuzz_root()
  script_path = os.path.join(root, 'infra', 'helper.py')
  command = ['python', script_path] + helper_command
  print('Running command: %s' % ' '.join(command))
  subprocess.check_call(command)

def build_fuzzers(project, engine, sanitizer, architecture):
  """Execute helper.py's build_fuzzers command on |project|. Build the fuzzers
  with |engine| and |sanitizer| for |architecture|."""
  execute_helper_command([
      'build_fuzzers', project, '--engine', engine, '--sanitizer', sanitizer,
      '--architecture', architecture
  ])

def check_build(project, engine, sanitizer, architecture):
  """Execute helper.py's check_build command on |project|, assuming it was most
  recently built with |engine| and |sanitizer| for |architecture|."""
  execute_helper_command([
      'check_build', project, '--engine', engine, '--sanitizer', sanitizer,
      '--architecture', architecture
  ])

def should_build_coverage(project_yaml):
  """Returns True if a coverage build should be done based on project.yaml
  contents."""
  # Enable coverage builds on projects that use engines. Those that don't use
  # engines shouldn't get coverage builds.
  engines = project_yaml.get('fuzzing_engines', DEFAULT_ENGINES)
  engineless = 'none' in engines
  if engineless:
    assert_message = ('Forbidden to specify multiple engines for '
                      '"fuzzing_engines" if "none" is specified.')
    assert len(engines) == 1, assert_message
    return False

  language = project_yaml.get('language')
  if language not in constants.LANGUAGES_WITH_COVERAGE_SUPPORT:
    print(('Project is written in "{language}", '
           'coverage is not supported yet.').format(language=language))
    return False

  return True

def flatten_options(option_list):
  """Generator that flattens |option_list| (a list of sanitizers, architectures
  or fuzzing engines) by returning each element in the list that isn't a
  dictionary. For elements that are dictionaries, the sole key is returned."""
  result = []
  for option in option_list:
    if isinstance(option, dict):
      keys = list(option.keys())
      assert len(keys) == 1
      result.append(keys[0])
      continue
    result.append(option)
  print(result)
  return result

def should_build(project_yaml):
  """Returns True on if the build specified is enabled in the project.yaml."""

  if os.getenv('SANITIZER') == 'coverage':
    # This assumes we only do coverage builds with libFuzzer on x86_64.
    return should_build_coverage(project_yaml)

  def is_enabled(env_var, yaml_name, defaults):
    """Is the value of |env_var| enabled in |project_yaml| (in the |yaml_name|
    section)? Uses |defaults| if |yaml_name| section is unspecified."""
    return os.getenv(env_var) in flatten_options(
        project_yaml.get(yaml_name, defaults))

  return (is_enabled('ENGINE', 'fuzzing_engines', DEFAULT_ENGINES) and
          is_enabled('SANITIZER', 'sanitizers', DEFAULT_SANITIZERS) and
          is_enabled('ARCHITECTURE', 'architectures', DEFAULT_ARCHITECTURES))

def build_project(project):
  """Do the build of |project| that is specified by the environment variables -
  SANITIZER, ENGINE, and ARCHITECTURE."""
  root = get_oss_fuzz_root()
  project_yaml_path = os.path.join(root, 'projects', project, 'project.yaml')
  with open(project_yaml_path) as file_handle:
    project_yaml = yaml.safe_load(file_handle)

  if project_yaml.get('disabled', False):
    print('Project {0} is disabled, skipping build.'.format(project))
    return

  engine = os.getenv('ENGINE')
  sanitizer = os.getenv('SANITIZER')
  architecture = os.getenv('ARCHITECTURE')

  if not should_build(project_yaml):
    print(('Specified build: engine: {0}, sanitizer: {1}, architecture: {2} '
           'not enabled for this project: {3}. Skipping build.').format(
               engine, sanitizer, architecture, project))

    return

  print('Building project', project)
  build_fuzzers(project, engine, sanitizer, architecture)

  if engine != 'none' and sanitizer != 'coverage':
    check_build(project, engine, sanitizer, architecture)

class BuildModifiedProjectsResult(enum.Enum):
  """Enum containing the return values of build_modified_projects()."""
  NONE_BUILT = 0
  BUILD_SUCCESS = 1
  BUILD_FAIL = 2

def build_modified_projects():
  """Build modified projects. Returns BuildModifiedProjectsResult.NONE_BUILT if
  no builds were attempted. Returns BuildModifiedProjectsResult.BUILD_SUCCESS if
  all attempts succeed, otherwise returns
  BuildModifiedProjectsResult.BUILD_FAIL."""
  projects = get_modified_buildable_projects()
  if not projects:
    return BuildModifiedProjectsResult.NONE_BUILT

  failed_projects = []
  for project in projects:
    try:
      build_project(project)
    except subprocess.CalledProcessError:
      failed_projects.append(project)

  if failed_projects:
    print('Failed projects:', ' '.join(failed_projects))
    return BuildModifiedProjectsResult.BUILD_FAIL

  return BuildModifiedProjectsResult.BUILD_SUCCESS

def is_infra_changed():
  """Returns True if the infra directory was changed."""
  git_output = get_changed_files_output()
  infra_code_regex = '.*infra/.*\n'
  return re.search(infra_code_regex, git_output) is not None

def build_base_images():
  """Builds base images."""
  # TODO(jonathanmetzman): Investigate why caching fails so often and
  # when we improve it, build base-clang as well. Also, move this function
  # to a helper command when we can support base-clang.
  execute_helper_command(['pull_images'])
  images = [
      'base-image',
      'base-builder',
      'base-builder-go',
      'base-builder-go-codeintelligencetesting',
      'base-builder-javascript',
      'base-builder-jvm',
      'base-builder-python',
      'base-builder-rust',
      'base-builder-swift',
      'base-runner',
  ]
  for image in images:
    try:
      execute_helper_command(['build_image', image, '--no-pull', '--cache'])
    except subprocess.CalledProcessError:
      return 1

  return 0

def build_canary_project():
  """Builds a specific project when infra/ is changed to verify that infra/
  changes don't break things. Returns False if build was attempted but
  failed."""

  try:
    build_project('skcms')
  except subprocess.CalledProcessError:
    return False

  return True

def main():
  """Build modified projects or canary project."""
  os.environ['OSS_FUZZ_CI'] = '1'
  infra_changed = is_infra_changed()
  if infra_changed:
    print('Pulling and building base images first.')
    if build_base_images():
      return 1

  result = build_modified_projects()
  if result == BuildModifiedProjectsResult.BUILD_FAIL:
    return 1

  # It's unnecessary to build the canary if we've built any projects already.
  no_projects_built = result == BuildModifiedProjectsResult.NONE_BUILT
  should_build_canary = no_projects_built and infra_changed
  if should_build_canary and not build_canary_project():
    return 1

  return 0

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>[CI] Allow projects to opt-out of running tests. (#8518)<commit_after>#!/usr/bin/env python
# Copyright 2019 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
"""Build modified projects."""

from __future__ import print_function

import enum
import os
import re
import sys
import subprocess
import yaml

# pylint: disable=wrong-import-position,import-error
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import constants

CANARY_PROJECT = 'skcms'

DEFAULT_ARCHITECTURES = ['x86_64']
DEFAULT_ENGINES = ['afl', 'honggfuzz', 'libfuzzer']
DEFAULT_SANITIZERS = ['address', 'undefined']

def get_changed_files_output():
  """Returns the output of a git command that discovers changed files."""
  branch_commit_hash = subprocess.check_output(
      ['git', 'merge-base', 'HEAD', 'origin/HEAD']).strip().decode()

  return subprocess.check_output(
      ['git', 'diff', '--name-only', branch_commit_hash + '..']).decode()

def get_modified_buildable_projects():
  """Returns a list of all the projects modified in this commit that have a
  build.sh file."""
  git_output = get_changed_files_output()
  projects_regex = '.*projects/(?P<name>.*)/.*\n'
  modified_projects = set(re.findall(projects_regex, git_output))
  projects_dir = os.path.join(get_oss_fuzz_root(), 'projects')
  # Filter out projects without Dockerfile files since new projects and reverted
  # projects frequently don't have them. In these cases we don't want Travis's
  # builds to fail.
  modified_buildable_projects = []
  for project in modified_projects:
    if not os.path.exists(os.path.join(projects_dir, project, 'Dockerfile')):
      print('Project {0} does not have Dockerfile. skipping build.'.format(
          project))
      continue
    modified_buildable_projects.append(project)
  return modified_buildable_projects

def get_oss_fuzz_root():
  """Get the absolute path of the root of the oss-fuzz checkout."""
  script_path = os.path.realpath(__file__)
  return os.path.abspath(
      os.path.dirname(os.path.dirname(os.path.dirname(script_path))))

def execute_helper_command(helper_command):
  """Execute |helper_command| using helper.py."""
  root = get_oss_fuzz_root()
  script_path = os.path.join(root, 'infra', 'helper.py')
  command = ['python', script_path] + helper_command
  print('Running command: %s' % ' '.join(command))
  subprocess.check_call(command)

def build_fuzzers(project, engine, sanitizer, architecture):
  """Execute helper.py's build_fuzzers command on |project|. Build the fuzzers
  with |engine| and |sanitizer| for |architecture|."""
  execute_helper_command([
      'build_fuzzers', project, '--engine', engine, '--sanitizer', sanitizer,
      '--architecture', architecture
  ])

def check_build(project, engine, sanitizer, architecture):
  """Execute helper.py's check_build command on |project|, assuming it was most
  recently built with |engine| and |sanitizer| for |architecture|."""
  execute_helper_command([
      'check_build', project, '--engine', engine, '--sanitizer', sanitizer,
      '--architecture', architecture
  ])

def should_build_coverage(project_yaml):
  """Returns True if a coverage build should be done based on project.yaml
  contents."""
  # Enable coverage builds on projects that use engines. Those that don't use
  # engines shouldn't get coverage builds.
  engines = project_yaml.get('fuzzing_engines', DEFAULT_ENGINES)
  engineless = 'none' in engines
  if engineless:
    assert_message = ('Forbidden to specify multiple engines for '
                      '"fuzzing_engines" if "none" is specified.')
    assert len(engines) == 1, assert_message
    return False

  language = project_yaml.get('language')
  if language not in constants.LANGUAGES_WITH_COVERAGE_SUPPORT:
    print(('Project is written in "{language}", '
           'coverage is not supported yet.').format(language=language))
    return False

  return True

def flatten_options(option_list):
  """Generator that flattens |option_list| (a list of sanitizers, architectures
  or fuzzing engines) by returning each element in the list that isn't a
  dictionary. For elements that are dictionaries, the sole key is returned."""
  result = []
  for option in option_list:
    if isinstance(option, dict):
      keys = list(option.keys())
      assert len(keys) == 1
      result.append(keys[0])
      continue
    result.append(option)
  print(result)
  return result

def should_build(project_yaml):
  """Returns True on if the build specified is enabled in the project.yaml."""

  if os.getenv('SANITIZER') == 'coverage':
    # This assumes we only do coverage builds with libFuzzer on x86_64.
    return should_build_coverage(project_yaml)

  def is_enabled(env_var, yaml_name, defaults):
    """Is the value of |env_var| enabled in |project_yaml| (in the |yaml_name|
    section)? Uses |defaults| if |yaml_name| section is unspecified."""
    return os.getenv(env_var) in flatten_options(
        project_yaml.get(yaml_name, defaults))

  return (is_enabled('ENGINE', 'fuzzing_engines', DEFAULT_ENGINES) and
          is_enabled('SANITIZER', 'sanitizers', DEFAULT_SANITIZERS) and
          is_enabled('ARCHITECTURE', 'architectures', DEFAULT_ARCHITECTURES))

def build_project(project):
  """Do the build of |project| that is specified by the environment variables -
  SANITIZER, ENGINE, and ARCHITECTURE."""
  root = get_oss_fuzz_root()
  project_yaml_path = os.path.join(root, 'projects', project, 'project.yaml')
  with open(project_yaml_path) as file_handle:
    project_yaml = yaml.safe_load(file_handle)

  if project_yaml.get('disabled', False):
    print('Project {0} is disabled, skipping build.'.format(project))
    return

  engine = os.getenv('ENGINE')
  sanitizer = os.getenv('SANITIZER')
  architecture = os.getenv('ARCHITECTURE')

  if not should_build(project_yaml):
    print(('Specified build: engine: {0}, sanitizer: {1}, architecture: {2} '
           'not enabled for this project: {3}. Skipping build.').format(
               engine, sanitizer, architecture, project))

    return

  print('Building project', project)
  build_fuzzers(project, engine, sanitizer, architecture)

  run_tests = project_yaml.get('run_tests', True)
  if engine != 'none' and sanitizer != 'coverage' and run_tests:
    check_build(project, engine, sanitizer, architecture)

class BuildModifiedProjectsResult(enum.Enum):
  """Enum containing the return values of build_modified_projects()."""
  NONE_BUILT = 0
  BUILD_SUCCESS = 1
  BUILD_FAIL = 2

def build_modified_projects():
  """Build modified projects. Returns BuildModifiedProjectsResult.NONE_BUILT if
  no builds were attempted. Returns BuildModifiedProjectsResult.BUILD_SUCCESS if
  all attempts succeed, otherwise returns
  BuildModifiedProjectsResult.BUILD_FAIL."""
  projects = get_modified_buildable_projects()
  if not projects:
    return BuildModifiedProjectsResult.NONE_BUILT

  failed_projects = []
  for project in projects:
    try:
      build_project(project)
    except subprocess.CalledProcessError:
      failed_projects.append(project)

  if failed_projects:
    print('Failed projects:', ' '.join(failed_projects))
    return BuildModifiedProjectsResult.BUILD_FAIL

  return BuildModifiedProjectsResult.BUILD_SUCCESS

def is_infra_changed():
  """Returns True if the infra directory was changed."""
  git_output = get_changed_files_output()
  infra_code_regex = '.*infra/.*\n'
  return re.search(infra_code_regex, git_output) is not None

def build_base_images():
  """Builds base images."""
  # TODO(jonathanmetzman): Investigate why caching fails so often and
  # when we improve it, build base-clang as well. Also, move this function
  # to a helper command when we can support base-clang.
  execute_helper_command(['pull_images'])
  images = [
      'base-image',
      'base-builder',
      'base-builder-go',
      'base-builder-go-codeintelligencetesting',
      'base-builder-javascript',
      'base-builder-jvm',
      'base-builder-python',
      'base-builder-rust',
      'base-builder-swift',
      'base-runner',
  ]
  for image in images:
    try:
      execute_helper_command(['build_image', image, '--no-pull', '--cache'])
    except subprocess.CalledProcessError:
      return 1

  return 0

def build_canary_project():
  """Builds a specific project when infra/ is changed to verify that infra/
  changes don't break things. Returns False if build was attempted but
  failed."""

  try:
    build_project('skcms')
  except subprocess.CalledProcessError:
    return False

  return True

def main():
  """Build modified projects or canary project."""
  os.environ['OSS_FUZZ_CI'] = '1'
  infra_changed = is_infra_changed()
  if infra_changed:
    print('Pulling and building base images first.')
    if build_base_images():
      return 1

  result = build_modified_projects()
  if result == BuildModifiedProjectsResult.BUILD_FAIL:
    return 1

  # It's unnecessary to build the canary if we've built any projects already.
  no_projects_built = result == BuildModifiedProjectsResult.NONE_BUILT
  should_build_canary = no_projects_built and infra_changed
  if should_build_canary and not build_canary_project():
    return 1

  return 0

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2021 Google LLC..
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# python3
"""Cloud Environment setup module for initial installation.

This module automates the following steps:
  1. Create GMC BigQuery Data Transfer
  2. Setup tables and views in BigQuery
"""
from dataclasses import dataclass
from datetime import date, datetime, time, timedelta
import logging
import argparse
from typing import NamedTuple, Dict, Union, List
from google.auth import credentials
from google.cloud.bigquery_datatransfer_v1.types import TransferConfig
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from google.cloud import storage, pubsub_v1, exceptions
from google.api_core import exceptions
from pprint import pprint
# if you're getting an error on the next line "ModuleNotFound",
# make sure you define env var PYTHONPATH="."
from common import auth, config_utils, cloud_utils, bigquery_utils, file_utils
from install import cloud_data_transfer

# Set logging level.
logging.getLogger().setLevel(logging.INFO)
logging.getLogger('googleapiclient.discovery').setLevel(logging.WARNING)
logging.getLogger('googleapiclient.discovery_cache').setLevel(logging.ERROR)
logging.getLogger('google.cloud.pubsub_v1.subscriber').setLevel(logging.WARNING)

def create_views(bigquery_util: bigquery_utils.CloudBigQueryUtils,
                 config: config_utils.Config,
                 target: config_utils.ConfigTarget):
  if not target.name:
    raise ValueError("Target has no name")
  sql_files = [
      'filter-products.sql',
  ]
  SEARCH_CONDITIONS = "SEARCH_CONDITIONS"
  # AND merchant_id IN ({merchant_id})
  params = {SEARCH_CONDITIONS: ''}
  if target.merchant_id:
    if isinstance(target.merchant_id, list):
      condition = f'AND merchant_id IN {tuple(target.merchant_id)}'
    elif isinstance(target.merchant_id, str) and ',' in target.merchant_id:
      condition = f'AND merchant_id IN {tuple(target.merchant_id.split(", "))}'
    else:
      condition = f'AND merchant_id = {target.merchant_id}'
    params[SEARCH_CONDITIONS] = condition
  params['merchant_id'] = config.merchant_id
  params['target'] = target.name
  bigquery_util.execute_queries(sql_files, config.dataset_id, params)

def set_permission_on_drive(fileId, email, credentials):
  driveAPI = build('drive', 'v3', credentials=credentials)
  logging.info(f'Adding write permissions to {email} on {fileId}')
  try:
    access = driveAPI.permissions().create(
        fileId=fileId,
        body={
            'type': 'user',
            'role': 'writer',
            'emailAddress': email
        },
        fields='id',
        #transferOwnership=True  - use it if you want to transfer ownership (change role  to 'owner' then, and comment out sendNotificationEmail=False)
        sendNotificationEmail=False).execute()
  except HttpError as e:
    # NOTE: if the user doesn't exist then we'll get 500 "Internal Error"
    if e.status_code == 500:
      raise Exception(
          f'Error occured on adding drive permissions, please make sure that account \'{email}\' really exists ({e.reason})'
      ) from e
    raise

def get_service_account_email(project_id: str):
  return f"{project_id}@appspot.gserviceaccount.com"

def create_spreadsheets(config: config_utils.Config, target: config_utils.ConfigTarget,
                        credentials: credentials.Credentials, share_with: str) -> bool:
  created = False
  merchant_id = target.merchant_id or config.merchant_id
  if not share_with:
    share_with = get_service_account_email(config.project_id)
  if target.page_feed_spreadsheetid:
    logging.info(
        'Skipped page feed spreadsheet creation as config contains a docid: ' +
        target.page_feed_spreadsheetid)
  else:
    logging.info('Creating a spreadsheet for page feed data')
    title = f"Product page feed (GMC {merchant_id} / GCP project {config.project_id})"
    target.page_feed_spreadsheetid = create_spreadsheet(title, credentials)
    created = True
  # set permission on the created spreadsheet for GAE default service account
  set_permission_on_drive(target.page_feed_spreadsheetid, share_with, credentials)

  if target.adcustomizer_spreadsheetid:
    logging.info(
        'Skipped adcustomizer spreadsheet creation as config contains a docid: '
        + target.adcustomizer_spreadsheetid)
  else:
    logging.info('Creating a spreadsheet for ad customizer')
    title = f"Ad customizers feed (GMC {merchant_id} / GCP project {config.project_id})"
    target.adcustomizer_spreadsheetid = create_spreadsheet(title, credentials)
    created = True
  # set permission on the created spreadsheet for GAE default service account
  set_permission_on_drive(target.adcustomizer_spreadsheetid, share_with,
                          credentials)
  return created

def create_spreadsheet(title: str, credentials: credentials.Credentials) -> str:
  """Create a Google Spreadsheet (either for page feed or adcustomizers data)"""
  sheetsAPI = build('sheets', 'v4', credentials=credentials)
  result = sheetsAPI.spreadsheets().create(body={
      "sheets": [{
          "properties": {
              "title": "Main"
          }
      }],
      "properties": {
          "title": title
      }
  }).execute()
  spreadsheet_id = result['spreadsheetId']
  logging.info('Created spreadsheet: ' + spreadsheet_id)

  return spreadsheet_id

def backup_config(config_file_name: str, config: config_utils.Config,
                  credentials):
  """Backs up config file onto GCS (bucket "{project_id}-pdsa", will be created if doesn't exist)"""
  if not config_file_name.startswith('gs://'):
    file_utils.upload_file_to_gcs(config.project_id, credentials,
                                  config_file_name)
    logging.info(
        f'config file ({config_file_name}) has been copied to gs://{config.project_id}-pdsa'
    )

def enable_apis(apis: List[str], config: config_utils.Config, credentials):
  """Enables multiple Cloud APIs for a GCP project.

  Args:
    apis: The list of APIs to be enabled.

  Raises:
      Error: If the request was not processed successfully.
  """
  parent = f'projects/{config.project_id}'
  request_body = {'serviceIds': apis}
  client = build('serviceusage',
                 'v1',
                 credentials=credentials,
                 cache_discovery=False)
  operation = client.services().batchEnable(parent=parent,
                                            body=request_body).execute()
  cloud_utils.wait_for_operation(client.operations(), operation)

def create_pubsub_topic(config: config_utils.Config, credentials):
  """Creates a pub/sub topic for data transfer signal completion to"""
  topic_name = f'projects/{config.project_id}/topics/{config.pubsub_topic_dt_finish}'
  publisher = pubsub_v1.PublisherClient(credentials=credentials)

  # creating a topic with exising name will cause 409 Resource already exists in the project error,
  # so check first
  try:
    topic = publisher.get_topic(topic=topic_name)
    not_found = False
  except exceptions.NotFound:
    not_found = True
  if not_found:
    publisher.create_topic(name=topic_name)
  return topic_name

def wait_for_transfer_completion(pubsub_topic: str, config: config_utils.Config,
                                 credentials, data_transfer: cloud_data_transfer.CloudDataTransferUtils,
                                 transfer_config: TransferConfig):
  """Checks data transfer completed via creating an async pull subscription pub/sub completion topic"""

  def callback(message):
    message.ack()
    #print(message.data.decode())
    future.cancel()

  subscription_name = f'projects/{config.project_id}/subscriptions/install_script'
  with pubsub_v1.SubscriberClient(credentials=credentials) as subscriber:
    try:
      subscriber.delete_subscription(subscription=subscription_name)
    except exceptions.NotFound:
      pass  # expected
    subscriber.create_subscription(name=subscription_name, topic=pubsub_topic)
    future = subscriber.subscribe(subscription_name, callback)
    # if DT fails then waiting for a pub/sub will last forever, so first we'll wait the DT actually started
    # and only after that wait for a pub/sub
    data_transfer.wait_for_transfer_started(transfer_config)
    try:
      # NOTE: the next line blocks current thread in waiting and
      # only in callback on a message it'll be unblocked (future.cancel)
      future.result()
    except KeyboardInterrupt:
      future.cancel()
    subscriber.delete_subscription(subscription=subscription_name)

def create_subscription_to_update_feeds(pubsub_topic: str,
                                        config: config_utils.Config,
                                        credentials):
  """Creates a pub/sub push subscription to data transfer completion topic
     with signaling to GAE application to update page feed and adcustomizer feed"""
  subscription_name = f'projects/{config.project_id}/subscriptions/update_feeds'
  with pubsub_v1.SubscriberClient(credentials=credentials) as subscriber:
    try:
      subscriber.get_subscription(subscription=subscription_name)
      not_found = False
    except exceptions.NotFound:
      not_found = True
    if not_found:
      # TODO: by default pubsub creates a subscription with expriration of 31 days of inactivity,
      # in UI it can be set as 'never exprire' but how to do it via API?
      # TODO: do not hard-code GAE url, use AppEngine Admin API to get app url
      # let gae = (await google.appengine("v1").apps.get({ appsId: `${projectId}` })).data;
      # there could be several application instances in a GCP project, each one in its own GAE service
      # we'll need to distintuish them somehow. Ideally we need to detect a current service when running in GAE
      subscriber.create_subscription(
          name=subscription_name,
          topic=pubsub_topic,
          push_config=pubsub_v1.types.PushConfig(
              push_endpoint=
              f'https://{config.project_id}.ew.r.appspot.com/api/update',
              oidc_token=pubsub_v1.types.PushConfig.OidcToken(
                  service_account_email=get_service_account_email(config.project_id))))

@dataclass
class DeployOptions:
  skip_dt_run: bool
  user_email: str
  skip_spreadsheets: bool

def deploy(config: config_utils.Config, credentials: credentials.Credentials,
           options: DeployOptions) -> bool:
  """Deploys cloud components.
    Return: bool - whether config was updated (true)
  """
  errors = config.validate()
  if len(errors):
    raise ValueError(f'Configuration is invalid: {errors}')

  logging.info('Enabling apis')
  enable_apis([
      'bigquery.googleapis.com', 'bigquerydatatransfer.googleapis.com',
      'sheets.googleapis.com', 'drive.googleapis.com', 'pubsub.googleapis.com',
      'iap.googleapis.com', 'cloudresourcemanager.googleapis.com'
  ], config, credentials)
  logging.info('apis have been enabled')

  logging.info('Creating %s dataset.', config.dataset_id)
  bigquery_client = bigquery_utils.CloudBigQueryUtils(config.project_id,
                                                      credentials)
  bigquery_client.create_dataset_if_not_exists(config.dataset_id,
                                               config.dataset_location)

  data_transfer = cloud_data_transfer.CloudDataTransferUtils(
      config.project_id, config.dataset_location, credentials)
  pubsub_topic = create_pubsub_topic(config, credentials)
  # create or reuse data transfer for GMC
  transfer_config = data_transfer.create_merchant_center_transfer(
      config.merchant_id, config.dataset_id, config.dt_schedule, pubsub_topic,
      options.skip_dt_run)

  if not options.skip_dt_run:
    logging.info('Waiting for Data Transfer to complete...')
    wait_for_transfer_completion(pubsub_topic, config, credentials, data_transfer,
                                 transfer_config)
    logging.info('Data Transfer has completed')

  # ads_config = data_transfer.create_google_ads_transfer(ads_customer_id, args.dataset_id)

  if not config.targets:
    logging.warning('No targets found in configuration, exiting')
    exit()

  created = False
  for target in config.targets:
    logging.info(
        f'Creating solution specific views for \'{target.name}\' target.')
    create_views(bigquery_client, config, target)

    # creating spreadsheets for page feed data and adcustomizers
    if not options.skip_spreadsheets:
      created = created or create_spreadsheets(config, target, credentials,
                                               options.user_email)

  create_subscription_to_update_feeds(pubsub_topic, config, credentials)
  logging.info(
      'Created pub/sub subscription for data transfer to call GAE app for updating feeds'
  )

  logging.info('Installation is complete!')
  return created

def add_args(parser: argparse.ArgumentParser):
  parser.add_argument('--skip-dt-run',
                      dest='skip_dt_run',
                      action="store_true",
                      help='Suppress running data transfer if it exists')
  parser.add_argument('--user-email',
                      dest='user_email',
                      help='User email to share created spreadsheets with')
  parser.add_argument('--skip-spreadsheets',
                      dest='skip_spreadsheets',
                      action='store_true')
  parser.add_argument('--create-spreadsheets',
                      dest='create_spreadsheets',
                      action='store_true')

def main():
  args = config_utils.parse_arguments(only_known=False, func=add_args)
  config = config_utils.get_config(args)
  pprint(vars(config))
  credentials = auth.get_credentials(args)

  if args.service_account_file and getattr(credentials, "project_id",
                                           None) != config.project_id:
    logging.error(
        f'Provided service account belongs to a different project ({getattr(credentials, "project_id", None)}) '
        f'than your current project in config ({config.project_id})'
    )
    exit()
  if args.service_account_file and not args.user_email:
    logging.error(f'You supplied a service account but not a user email')
    exit()

  if args.create_spreadsheets:
    for target in config.targets:
      create_spreadsheets(config, target, credentials, args.user_email)
    return

  created = deploy(
      config, credentials,
      DeployOptions(args.skip_dt_run, args.user_email, args.skip_spreadsheets))

  config_file_name = args.config or 'config.json'
  if created:
    # overwrite config file with new data
    config_utils.save_config(config, config_file_name)

  # As we could have modified the config (e.g. put a spreadsheet id),
  # we need to save it to a well-known location - GCS bucket {project_id}-pdsa:
  backup_config(config_file_name, config, credentials)

if __name__ == '__main__':
  main()
<commit_msg>[setup] Do not hard-code GAE region while creating a pub/sub subscription for DT to update feeds<commit_after># coding=utf-8
# Copyright 2021 Google LLC..
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# python3
"""Cloud Environment setup module for initial installation.

This module automates the following steps:
  1. Create GMC BigQuery Data Transfer
  2. Setup tables and views in BigQuery
"""
from dataclasses import dataclass
from datetime import date, datetime, time, timedelta
import logging
import argparse
from typing import NamedTuple, Dict, Union, List
from google.auth import credentials
from google.cloud.bigquery_datatransfer_v1.types import TransferConfig
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from google.cloud import storage, pubsub_v1, exceptions
from google.api_core import exceptions
from googleapiclient import discovery
from pprint import pprint
# if you're getting an error on the next line "ModuleNotFound",
# make sure you define env var PYTHONPATH="."
from common import auth, config_utils, cloud_utils, bigquery_utils, file_utils
from install import cloud_data_transfer

# Set logging level.
logging.getLogger().setLevel(logging.INFO)
logging.getLogger('googleapiclient.discovery').setLevel(logging.WARNING)
logging.getLogger('googleapiclient.discovery_cache').setLevel(logging.ERROR)
logging.getLogger('google.cloud.pubsub_v1.subscriber').setLevel(logging.WARNING)

def create_views(bigquery_util: bigquery_utils.CloudBigQueryUtils,
                 config: config_utils.Config,
                 target: config_utils.ConfigTarget):
  if not target.name:
    raise ValueError("Target has no name")
  sql_files = [
      'filter-products.sql',
  ]
  SEARCH_CONDITIONS = "SEARCH_CONDITIONS"
  # AND merchant_id IN ({merchant_id})
  params = {SEARCH_CONDITIONS: ''}
  if target.merchant_id:
    if isinstance(target.merchant_id, list):
      condition = f'AND merchant_id IN {tuple(target.merchant_id)}'
    elif isinstance(target.merchant_id, str) and ',' in target.merchant_id:
      condition = f'AND merchant_id IN {tuple(target.merchant_id.split(", "))}'
    else:
      condition = f'AND merchant_id = {target.merchant_id}'
    params[SEARCH_CONDITIONS] = condition
  params['merchant_id'] = config.merchant_id
  params['target'] = target.name
  bigquery_util.execute_queries(sql_files, config.dataset_id, params)

def set_permission_on_drive(fileId, email, credentials):
  driveAPI = build('drive', 'v3', credentials=credentials)
  logging.info(f'Adding write permissions to {email} on {fileId}')
  try:
    access = driveAPI.permissions().create(
        fileId=fileId,
        body={
            'type': 'user',
            'role': 'writer',
            'emailAddress': email
        },
        fields='id',
        #transferOwnership=True  - use it if you want to transfer ownership (change role  to 'owner' then, and comment out sendNotificationEmail=False)
        sendNotificationEmail=False).execute()
  except HttpError as e:
    # NOTE: if the user doesn't exist then we'll get 500 "Internal Error"
    if e.status_code == 500:
      raise Exception(
          f'Error occured on adding drive permissions, please make sure that account \'{email}\' really exists ({e.reason})'
      ) from e
    raise

def get_service_account_email(project_id: str):
  return f"{project_id}@appspot.gserviceaccount.com"

def create_spreadsheets(config: config_utils.Config, target: config_utils.ConfigTarget,
                        credentials: credentials.Credentials, share_with: str) -> bool:
  created = False
  merchant_id = target.merchant_id or config.merchant_id
  if not share_with:
    share_with = get_service_account_email(config.project_id)
  if target.page_feed_spreadsheetid:
    logging.info(
        'Skipped page feed spreadsheet creation as config contains a docid: ' +
        target.page_feed_spreadsheetid)
  else:
    logging.info('Creating a spreadsheet for page feed data')
    title = f"Product page feed (GMC {merchant_id} / GCP project {config.project_id})"
    target.page_feed_spreadsheetid = create_spreadsheet(title, credentials)
    created = True
  # set permission on the created spreadsheet for GAE default service account
  set_permission_on_drive(target.page_feed_spreadsheetid, share_with, credentials)

  if target.adcustomizer_spreadsheetid:
    logging.info(
        'Skipped adcustomizer spreadsheet creation as config contains a docid: '
        + target.adcustomizer_spreadsheetid)
  else:
    logging.info('Creating a spreadsheet for ad customizer')
    title = f"Ad customizers feed (GMC {merchant_id} / GCP project {config.project_id})"
    target.adcustomizer_spreadsheetid = create_spreadsheet(title, credentials)
    created = True
  # set permission on the created spreadsheet for GAE default service account
  set_permission_on_drive(target.adcustomizer_spreadsheetid, share_with,
                          credentials)
  return created

def create_spreadsheet(title: str, credentials: credentials.Credentials) -> str:
  """Create a Google Spreadsheet (either for page feed or adcustomizers data)"""
  sheetsAPI = build('sheets', 'v4', credentials=credentials)
  result = sheetsAPI.spreadsheets().create(body={
      "sheets": [{
          "properties": {
              "title": "Main"
          }
      }],
      "properties": {
          "title": title
      }
  }).execute()
  spreadsheet_id = result['spreadsheetId']
  logging.info('Created spreadsheet: ' + spreadsheet_id)

  return spreadsheet_id

def backup_config(config_file_name: str, config: config_utils.Config,
                  credentials):
  """Backs up config file onto GCS (bucket "{project_id}-pdsa", will be created if doesn't exist)"""
  if not config_file_name.startswith('gs://'):
    file_utils.upload_file_to_gcs(config.project_id, credentials,
                                  config_file_name)
    logging.info(
        f'config file ({config_file_name}) has been copied to gs://{config.project_id}-pdsa'
    )

def enable_apis(apis: List[str], config: config_utils.Config, credentials):
  """Enables multiple Cloud APIs for a GCP project.

  Args:
    apis: The list of APIs to be enabled.

  Raises:
      Error: If the request was not processed successfully.
  """
  parent = f'projects/{config.project_id}'
  request_body = {'serviceIds': apis}
  client = build('serviceusage',
                 'v1',
                 credentials=credentials,
                 cache_discovery=False)
  operation = client.services().batchEnable(parent=parent,
                                            body=request_body).execute()
  cloud_utils.wait_for_operation(client.operations(), operation)

def create_pubsub_topic(config: config_utils.Config, credentials):
  """Creates a pub/sub topic for data transfer signal completion to"""
  topic_name = f'projects/{config.project_id}/topics/{config.pubsub_topic_dt_finish}'
  publisher = pubsub_v1.PublisherClient(credentials=credentials)

  # creating a topic with exising name will cause 409 Resource already exists in the project error,
  # so check first
  try:
    topic = publisher.get_topic(topic=topic_name)
    not_found = False
  except exceptions.NotFound:
    not_found = True
  if not_found:
    publisher.create_topic(name=topic_name)
  return topic_name

def wait_for_transfer_completion(pubsub_topic: str, config: config_utils.Config,
                                 credentials, data_transfer: cloud_data_transfer.CloudDataTransferUtils,
                                 transfer_config: TransferConfig):
  """Checks data transfer completed via creating an async pull subscription pub/sub completion topic"""

  def callback(message):
    message.ack()
    #print(message.data.decode())
    future.cancel()

  subscription_name = f'projects/{config.project_id}/subscriptions/install_script'
  with pubsub_v1.SubscriberClient(credentials=credentials) as subscriber:
    try:
      subscriber.delete_subscription(subscription=subscription_name)
    except exceptions.NotFound:
      pass  # expected
    subscriber.create_subscription(name=subscription_name, topic=pubsub_topic)
    future = subscriber.subscribe(subscription_name, callback)
    # if DT fails then waiting for a pub/sub will last forever, so first we'll wait the DT actually started
    # and only after that wait for a pub/sub
    data_transfer.wait_for_transfer_started(transfer_config)
    try:
      # NOTE: the next line blocks current thread in waiting and
      # only in callback on a message it'll be unblocked (future.cancel)
      future.result()
    except KeyboardInterrupt:
      future.cancel()
    subscriber.delete_subscription(subscription=subscription_name)

def create_subscription_to_update_feeds(pubsub_topic: str,
                                        config: config_utils.Config,
                                        credentials):
  """Creates a pub/sub push subscription to data transfer completion topic
     with signaling to GAE application to update page feed and adcustomizer feed"""
  subscription_name = f'projects/{config.project_id}/subscriptions/update_feeds'
  with pubsub_v1.SubscriberClient(credentials=credentials) as subscriber:
    try:
      subscriber.get_subscription(subscription=subscription_name)
      not_found = False
    except exceptions.NotFound:
      not_found = True
    if not_found:
      # TODO: by default pubsub creates a subscription with expriration of 31 days of inactivity,
      # in UI it can be set as 'never exprire' but how to do it via API?

      # fetch GAE app hostname (it can be different depending on the chosen region)
      service = discovery.build("appengine", "v1", credentials=credentials)
      gae_info = service.apps().get(appsId=config.project_id).execute()
      hostname = gae_info['defaultHostname']
      # TODO: there could be several application instances in a GCP project, each one in its own GAE service
      # we'll need to distintuish them somehow. Ideally we need to detect a current service when running in GAE
      subscriber.create_subscription(
          name=subscription_name,
          topic=pubsub_topic,
          push_config=pubsub_v1.types.PushConfig(
              push_endpoint=f'https://{hostname}/api/update',
              oidc_token=pubsub_v1.types.PushConfig.OidcToken(
                  service_account_email=get_service_account_email(
                      config.project_id))))

@dataclass
class DeployOptions:
  skip_dt_run: bool
  user_email: str
  skip_spreadsheets: bool

def deploy(config: config_utils.Config, credentials: credentials.Credentials,
           options: DeployOptions) -> bool:
  """Deploys cloud components.
    Return: bool - whether config was updated (true)
  """
  errors = config.validate()
  if len(errors):
    raise ValueError(f'Configuration is invalid: {errors}')

  logging.info('Enabling apis')
  enable_apis([
      'bigquery.googleapis.com', 'bigquerydatatransfer.googleapis.com',
      'sheets.googleapis.com', 'drive.googleapis.com', 'pubsub.googleapis.com',
      'iap.googleapis.com', 'cloudresourcemanager.googleapis.com'
  ], config, credentials)
  logging.info('apis have been enabled')

  logging.info('Creating %s dataset.', config.dataset_id)
  bigquery_client = bigquery_utils.CloudBigQueryUtils(config.project_id,
                                                      credentials)
  bigquery_client.create_dataset_if_not_exists(config.dataset_id,
                                               config.dataset_location)

  data_transfer = cloud_data_transfer.CloudDataTransferUtils(
      config.project_id, config.dataset_location, credentials)
  pubsub_topic = create_pubsub_topic(config, credentials)
  # create or reuse data transfer for GMC
  transfer_config = data_transfer.create_merchant_center_transfer(
      config.merchant_id, config.dataset_id, config.dt_schedule, pubsub_topic,
      options.skip_dt_run)

  if not options.skip_dt_run:
    logging.info('Waiting for Data Transfer to complete...')
    wait_for_transfer_completion(pubsub_topic, config, credentials, data_transfer,
                                 transfer_config)
    logging.info('Data Transfer has completed')

  # ads_config = data_transfer.create_google_ads_transfer(ads_customer_id, args.dataset_id)

  if not config.targets:
    logging.warning('No targets found in configuration, exiting')
    exit()

  created = False
  for target in config.targets:
    logging.info(
        f'Creating solution specific views for \'{target.name}\' target.')
    create_views(bigquery_client, config, target)

    # creating spreadsheets for page feed data and adcustomizers
    if not options.skip_spreadsheets:
      created = created or create_spreadsheets(config, target, credentials,
                                               options.user_email)

  create_subscription_to_update_feeds(pubsub_topic, config, credentials)
  logging.info(
      'Created pub/sub subscription for data transfer to call GAE app for updating feeds'
  )

  logging.info('Installation is complete!')
  return created

def add_args(parser: argparse.ArgumentParser):
  parser.add_argument('--skip-dt-run',
                      dest='skip_dt_run',
                      action="store_true",
                      help='Suppress running data transfer if it exists')
  parser.add_argument('--user-email',
                      dest='user_email',
                      help='User email to share created spreadsheets with')
  parser.add_argument('--skip-spreadsheets',
                      dest='skip_spreadsheets',
                      action='store_true')
  parser.add_argument('--create-spreadsheets',
                      dest='create_spreadsheets',
                      action='store_true')

def main():
  args = config_utils.parse_arguments(only_known=False, func=add_args)
  config = config_utils.get_config(args)
  pprint(vars(config))
  credentials = auth.get_credentials(args)

  if args.service_account_file and getattr(credentials, "project_id",
                                           None) != config.project_id:
    logging.error(
        f'Provided service account belongs to a different project ({getattr(credentials, "project_id", None)}) '
        f'than your current project in config ({config.project_id})'
    )
    exit()
  if args.service_account_file and not args.user_email:
    logging.error(f'You supplied a service account but not a user email')
    exit()

  if args.create_spreadsheets:
    for target in config.targets:
      create_spreadsheets(config, target, credentials, args.user_email)
    return

  created = deploy(
      config, credentials,
      DeployOptions(args.skip_dt_run, args.user_email, args.skip_spreadsheets))

  config_file_name = args.config or 'config.json'
  if created:
    # overwrite config file with new data
    config_utils.save_config(config, config_file_name)

  # As we could have modified the config (e.g. put a spreadsheet id),
  # we need to save it to a well-known location - GCS bucket {project_id}-pdsa:
  backup_config(config_file_name, config, credentials)

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>import os
import shutil

from spec import Spec, skip
from invoke import run

class integration(Spec):
    def setup(self):
        self.cwd = os.getcwd()

    def teardown(self):
        os.chdir(self.cwd)

    def _build(self, folder, opts, target, asserts=None, conf='.', warn=False):
        # Dynamic sphinx opt overrides
        if opts:
            pairs = map(lambda x: '='.join(x), (opts or {}).items())
            flags = map(lambda x: '-D {}'.format(x), pairs)
            flagstr = ' '.join(flags)
        else:
            flagstr = ''
        # Setup
        os.chdir(os.path.join('integration', '_support'))
        build = os.path.join(folder, '_build')
        try:
            # Build
            cmd = 'sphinx-build {} -c {} -W {} {}'.format(
                flagstr, conf, folder, build)
            result = run(cmd, warn=warn, hide=True)
            if callable(asserts):
                asserts(result, build, target)
            return result
        finally:
            shutil.rmtree(build)

    def _assert_worked(self, folder, opts=None, target='changelog', conf='.'):
        self._build(
            folder, opts, target, asserts=self._basic_asserts, conf=conf,
        )

    def _basic_asserts(self, result, build, target):
        # Check for errors
        msg = "Build failed w/ stderr: {}"
        assert result.ok, msg.format(result.stderr)
        # Check for vaguely correct output
        changelog = os.path.join(build, '{}.html'.format(target))
        with open(changelog) as fd:
            text = fd.read()
            assert "1.0.1" in text
            assert "#1" in text

    def vanilla_invocation(self):
        # Doctree with just-a-changelog-named-changelog
        self._assert_worked('vanilla')

    def customized_filename_with_identical_title(self):
        # Changelog named not 'changelog', same title
        self._assert_worked(
            folder='custom_identical',
            opts={'releases_document_name': 'notachangelog'},
            target='notachangelog',
        )

    def customized_filename_with_different_title(self):
        # Changelog named not 'changelog', title distinct
        # NOTE: the difference here is in the fixture!
        self._assert_worked(
            folder='custom_different',
            opts={'releases_document_name': 'notachangelog'},
            target='notachangelog',
        )

    def useful_error_if_changelog_is_missed(self):
        # Don't barf with unknown-node errors if changelog not found.
        skip()

    def useful_error_if_buried_issue_nodes(self):
        # Don't unknown-node error if broken ReST causes 'hidden' issue nodes.
        # E.g. an accidental definition-list wrapping one.
        result = self._build(
            folder='hidden_issues',
            opts=None,
            target='changelog',
            warn=True,
        )
        assert result.failed
        assert "ValueError" in result.stderr
        assert "double-check" in result.stderr
        assert "innocuous" in result.stderr

    def multiple_changelogs(self):
        # support multiple changelogs
        self._assert_worked(
            folder='multiple_changelogs',
            opts=None,
            conf='multiple_changelogs',
            target='a_changelog')
<commit_msg>Make integration test re #59 actually test both changelogs :D<commit_after>import os
import shutil

from spec import Spec, skip
from invoke import run

from invoke.vendor.six import string_types

class integration(Spec):
    def setup(self):
        self.cwd = os.getcwd()

    def teardown(self):
        os.chdir(self.cwd)

    def _build(self, folder, opts, target, asserts=None, conf='.', warn=False):
        # Dynamic sphinx opt overrides
        if opts:
            pairs = map(lambda x: '='.join(x), (opts or {}).items())
            flags = map(lambda x: '-D {}'.format(x), pairs)
            flagstr = ' '.join(flags)
        else:
            flagstr = ''
        # Setup
        os.chdir(os.path.join('integration', '_support'))
        build = os.path.join(folder, '_build')
        try:
            # Build
            cmd = 'sphinx-build {} -c {} -W {} {}'.format(
                flagstr, conf, folder, build)
            result = run(cmd, warn=warn, hide=True)
            if callable(asserts):
                if isinstance(target, string_types):
                    targets = [target]
                else:
                    targets = target
                for target in targets:
                    asserts(result, build, target)
            return result
        finally:
            shutil.rmtree(build)

    def _assert_worked(self, folder, opts=None, target='changelog', conf='.'):
        self._build(
            folder, opts, target, asserts=self._basic_asserts, conf=conf,
        )

    def _basic_asserts(self, result, build, target):
        # Check for errors
        msg = "Build failed w/ stderr: {}"
        assert result.ok, msg.format(result.stderr)
        # Check for vaguely correct output
        changelog = os.path.join(build, '{}.html'.format(target))
        with open(changelog) as fd:
            text = fd.read()
            assert "1.0.1" in text
            assert "#1" in text

    def vanilla_invocation(self):
        # Doctree with just-a-changelog-named-changelog
        self._assert_worked('vanilla')

    def customized_filename_with_identical_title(self):
        # Changelog named not 'changelog', same title
        self._assert_worked(
            folder='custom_identical',
            opts={'releases_document_name': 'notachangelog'},
            target='notachangelog',
        )

    def customized_filename_with_different_title(self):
        # Changelog named not 'changelog', title distinct
        # NOTE: the difference here is in the fixture!
        self._assert_worked(
            folder='custom_different',
            opts={'releases_document_name': 'notachangelog'},
            target='notachangelog',
        )

    def useful_error_if_changelog_is_missed(self):
        # Don't barf with unknown-node errors if changelog not found.
        skip()

    def useful_error_if_buried_issue_nodes(self):
        # Don't unknown-node error if broken ReST causes 'hidden' issue nodes.
        # E.g. an accidental definition-list wrapping one.
        result = self._build(
            folder='hidden_issues',
            opts=None,
            target='changelog',
            warn=True,
        )
        assert result.failed
        assert "ValueError" in result.stderr
        assert "double-check" in result.stderr
        assert "innocuous" in result.stderr

    def multiple_changelogs(self):
        # support multiple changelogs
        self._assert_worked(
            folder='multiple_changelogs',
            opts=None,
            conf='multiple_changelogs',
            # Ensure the asserts check both changelogs
            target=['a_changelog', 'b_changelog'],
        )
<|endoftext|>


<commit_before>import requests

import json
import urllib.request as req
"""
Functions for better usage of queries
================================================
Prompts the user to enter the API token and mine corresponding to the account
example:

    >>>from intermine import query_manager as qm
    Enter the api token: <enter api token from account>
    Enter the mine name: <enter mine name>
"""

def get_all_query_names():
    """
    A function to list all the queries that are saved in a user account
    ================================================
    example:

        >>>from intermine import query_manager as qm
        >>>qm.get_all_query_names()
        <returns the names of all the saved queries in user account>

    """
    x = "http://registry.intermine.org/service/instances/" + mine
    r = requests.get(x)
    dict = json.loads(r.text)
    link = dict["instance"]["url"] + "/service/user/queries?token=" + token
    r = requests.get(link)
    dict = json.loads(r.text)
    count = 0
    for key in dict['queries'].keys():
        count = count + 1
        print(key)

    if count == 0:
        print("No saved queries")

def get_query(name):
    """
    A function that returns the columns that a given query constitutes of
    ================================================
    example:

        >>>from intermine import query_manager as qm
        >>>qm.get_query('queryName')
        <returns information about the query whose name is 'queryName'>

    """
    x = "http://registry.intermine.org/service/instances/" + mine

    r = requests.get(x)
    dict = json.loads(r.text)
    link = dict["instance"]["url"] + "/service/user/queries?token=" + token

    r = requests.get(link)
    dict = json.loads(r.text)
    count = 0
    for key in dict['queries'].keys():
        if name == key:
            count = count + 1
            print("Columns:")
            for i in range(len(dict['queries'][name]['select'])):
                print(dict['queries'][name]['select'][i])
    if count == 0:
        print("No such query available")

def delete_query(name):
    """
    A function that deletes a given query
    ================================================
    example:

        >>>from intermine import query_manager as qm
        >>>qm.delete_query('queryName')
        <deletes the query whose name is 'queryName' from user's account>

    """
    x = "http://registry.intermine.org/service/instances/" + mine
    r = requests.get(x)
    dict = json.loads(r.text)

    y = dict["instance"]["url"] + "/service/user/queries?token=" + token
    r = requests.get(y)
    z = json.loads(r.text)
    count = 0
    for key in z['queries'].keys():
        if key == name:
            count = count + 1
    if count == 0:
        print("No such query available")

    else:
        link = dict["instance"]["url"] + "/service/user/queries/" + name +\
         "?token=" + token
        requests.delete(link)

def post_query(xml):
    """
    A function to post a query(in the form of string containg xml) to a user account
    ================================================
    example:

        >>>qm.post_query('<query name="" model="genomic" view="Gene.length Gene.symbol" \
           longDescription="" sortOrder="Gene.length asc"></query>')

    Note that the name should be defined first
    """
    x = "http://registry.intermine.org/service/instances/" + mine
    r = requests.get(x)
    dict = json.loads(r.text)
    name = []
    for i in range(position,len(xml)):
        if xml[i]!='"':
            name.append(xml[i])
        else:
            break
    name = "".join(name)
    link = dict["instance"]["url"] + "/service/user/queries?token=" + token
    r = requests.get(link)
    raw = json.loads(r.text)
    count = 0
    for key in raw['queries'].keys():
        if key == name:
            count = count + 1
            print("Use another query name")
            resp = input("Or do you want to replace the old query? [y/n]")
            if resp == 'y':
                count = 0

    if count == 0:

        link = dict["instance"]["url"] + "/service/user/queries?xml=" + \
        req.pathname2url(xml) + "&token=" + token
        requests.put(link)
        flag = 0
        r = requests.get(link)
        raw = json.loads(r.text)
        for key in raw['queries'].keys():
            if key == name:
                flag = 1
        if flag == 0:
            print("Incorrect xml (Note: name should contain no special symbol and should be defined first)")

position = 13
mine = input("Enter the mine name: ")
l = "http://registry.intermine.org/service/instances/" + mine
try:
    m = requests.get(l)
    d = json.loads(m.text)
    n = d["instance"]["url"]
    token = input("Enter the api token: ")
    n = d["instance"]["url"] + "/service/user/queries?token=" + token
    try:
        o = requests.get(n)
        d = json.loads(o.text)
        p = d['queries'].keys()
    except:
        print ("Invalid token")
except:
    print ("Invalid mine name")
<commit_msg>made formatting changes according to pep8<commit_after>import requests

import json
import urllib.request as req
"""
Functions for better usage of queries
================================================
Prompts the user to enter the API token and mine corresponding to the account
example:

    >>>from intermine import query_manager as qm
    Enter the api token: <enter api token from account>
    Enter the mine name: <enter mine name>
"""

def get_all_query_names():
    """
    A function to list all the queries that are saved in a user account
    ================================================
    example:

        >>>from intermine import query_manager as qm
        >>>qm.get_all_query_names()
        <returns the names of all the saved queries in user account>

    """
    x = "http://registry.intermine.org/service/instances/" + mine
    r = requests.get(x)
    dict = json.loads(r.text)
    link = dict["instance"]["url"] + "/service/user/queries?token=" + token
    r = requests.get(link)
    dict = json.loads(r.text)
    count = 0
    for key in dict['queries'].keys():
        count = count + 1
        print(key)

    if count == 0:
        print("No saved queries")

def get_query(name):
    """
    A function that returns the columns that a given query constitutes of
    ================================================
    example:

        >>>from intermine import query_manager as qm
        >>>qm.get_query('queryName')
        <returns information about the query whose name is 'queryName'>

    """
    x = "http://registry.intermine.org/service/instances/" + mine

    r = requests.get(x)
    dict = json.loads(r.text)
    link = dict["instance"]["url"] + "/service/user/queries?token=" + token

    r = requests.get(link)
    dict = json.loads(r.text)
    count = 0
    for key in dict['queries'].keys():
        if name == key:
            count = count + 1
            print("Columns:")
            for i in range(len(dict['queries'][name]['select'])):
                print(dict['queries'][name]['select'][i])
    if count == 0:
        print("No such query available")

def delete_query(name):
    """
    A function that deletes a given query
    ================================================
    example:

        >>>from intermine import query_manager as qm
        >>>qm.delete_query('queryName')
        <deletes the query whose name is 'queryName' from user's account>

    """
    x = "http://registry.intermine.org/service/instances/" + mine
    r = requests.get(x)
    dict = json.loads(r.text)

    y = dict["instance"]["url"] + "/service/user/queries?token=" + token
    r = requests.get(y)
    z = json.loads(r.text)
    count = 0
    for key in z['queries'].keys():
        if key == name:
            count = count + 1
    if count == 0:
        print("No such query available")

    else:
        link = dict["instance"]["url"] + "/service/user/queries/" + name +\
         "?token=" + token
        requests.delete(link)

def post_query(xml):
    """
    A function to post a query(in the form of string containing xml)
    to a user account
    ================================================
    example:

        >>>qm.post_query('<query name="" model="genomic" view="Gene.length\
            Gene.symbol" longDescription="" sortOrder="Gene.length asc">\
            </query>')
    Note that the name should be defined first
    """
    x = "http://registry.intermine.org/service/instances/" + mine
    r = requests.get(x)
    dict = json.loads(r.text)
    name = []
    for i in range(position, len(xml)):
        if xml[i] != '"':
            name.append(xml[i])
        else:
            break
    name = "".join(name)
    link = dict["instance"]["url"] + "/service/user/queries?token=" + token
    r = requests.get(link)
    raw = json.loads(r.text)
    count = 0
    for key in raw['queries'].keys():
        if key == name:
            count = count + 1
            print("Use another query name")
            resp = input("Or do you want to replace the old query? [y/n]")
            if resp == 'y':
                count = 0

    if count == 0:

        link = dict["instance"]["url"] + "/service/user/queries?xml=" + \
        req.pathname2url(xml) + "&token=" + token
        requests.put(link)
        flag = 0
        r = requests.get(link)
        raw = json.loads(r.text)
        for key in raw['queries'].keys():
            if key == name:
                flag = 1
        if flag == 0:
            print("Incorrect xml (Note: name should contain no special symbol\
             and should be defined first)")

position = 13
mine = input("Enter the mine name: ")
l = "http://registry.intermine.org/service/instances/" + mine
try:
    m = requests.get(l)
    d = json.loads(m.text)
    n = d["instance"]["url"]
    token = input("Enter the api token: ")
    n = d["instance"]["url"] + "/service/user/queries?token=" + token
    try:
        o = requests.get(n)
        d = json.loads(o.text)
        p = d['queries'].keys()
    except:
        print("Invalid token")
except:
    print("Invalid mine name")
<|endoftext|>