<commit_before>import time
import os
import re
import urllib.request
import shutil
import json
import optparse
from collections import defaultdict
import boto3
import socket
import traceback

from selenium import webdriver

optparser = optparse.OptionParser()
optparser.add_option("-l", "--language", dest="language", default="French", help="Language to scrape")
optparser.add_option("-n", "--num_images", dest="num_images", default=100, type=int, help="Number of images to harvest per word")
optparser.add_option("-d", "--dictionary", dest="dictionary", default="dict.fr", help="Google languages json file")
optparser.add_option("-L", "--language-map", dest="language_map", default="google-languages.json", help="Google languages json file")
optparser.add_option("-s", "--start-index", dest="start_index", default=None, type=int, help="Word index to start iterating at")
optparser.add_option("-p", "--base-image-path", dest="base_image_path", default='/mnt/storage/images/', help="Base path where to store image output")
optparser.add_option("-v", action="store_true", dest="verbose_mode", help="Verbose mode")
(opts, _) = optparser.parse_args()

# tbm=isch sets this to be image search, start=0 will give us 100 results on page 1
BASE_GOOGLE_IMAGE_SEARCH_LINK = 'https://www.google.com/search?#tbm=isch&start=0'

# regex for extracting source link component: http://www.inddist.com/sites/inddist.com/files/Dollar-Sign.jpg
# from compound links like:
# https://www.google.com/imgres?imgurl=http://www.inddist.com/sites/inddist.com/files/Dollar-Sign.jpg&imgrefurl=http://www.inddist.com/article/2015/05/put-dollar-sign-next-your-service-value&h=1600&w=1200&tbnid=9XAphqXNraTVnM:&docid=Hm9c-cTh-Hl_DM&ei=43-xVsTKMsixeNrCoNAL&tbm=isch&ved=0ahUKEwiEyMjt3trKAhXIGB4KHVohCLoQMwgdKAAwAA
# Note: question marks are sometimes encoded as `%253F` at the end of the link to add question marks before an ampersand
# while ampersands are sometimes encoded as `%253D`
# Note: photobucket appends ~c200 before the ampersand.. strange
IMAGE_URL_REGEX = r'imgres\?imgurl=(?P<url>.*?)(&|%253F|%253D|~c200)'

# XPATH statement that finds all of the links on the page that correspond to the original image links
# Note: this is probably subject to change on google's part
GOOGLE_IMAGE_LINK_XPATH = "//a[@class='rg_l']"

# XPATH statement that pulls the div adjacent to the image link path that contains google-created metadata
# Note: this is probably subject to change on google's part
GOOGLE_METADATA_XPATH = "//div[@class='rg_meta']"

# user agent string from a recent version of firefox, override the default urllib User-Agent value
USER_AGENT_STRING = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:44.0) Gecko/20100101 Firefox/44.0"

# List of Valid file extensions to check against, if any of these don't match, this means our regex didn't quite
# parse the link correctly to pull out the real file link.
# Note: check for presence in list is also lowercased
VALID_FILE_EXTENSIONS = ['jpg', 'jpeg', 'gif', 'png', 'ico', 'bmp', 'svg']

DEBUG_MODE = False

# class that handles image scraping in google
class GoogleImageScraper(object):
    def __init__(self, opts):
        # creates the driver for running selenium
        self.driver = None # dummy call to ensure instance var is created in init
        self.create_selenium_browser()

        # track a dictionary with the errors for all words
        self.all_word_download_errors = defaultdict(int)

        # read in the json file with the arguments (hl and lr) for each language in google
        with open(opts.language_map, encoding='utf-8') as data_file:
            full_language_arg_map = json.loads(data_file.read())

        # pull the foreign words out of the bilingual dictionary, this assumes the format foreign\tenglish\n
        self.foreign_word_list = [line.strip().split('\t')[0] for line in open(opts.dictionary)]

        # build up a language-specific base link to start out with, before modifying it per each individual search term
        # add on the hl field for all languages because if it is in our JSON file, it has a hl field
        current_language_entry = full_language_arg_map[opts.language]
        self.base_language_search_url = BASE_GOOGLE_IMAGE_SEARCH_LINK + '&hl=' + current_language_entry['hl']

        # the lr field is present in some but not all of the language possibilities in the JSON config file
        if len(current_language_entry['lr']) > 0:
            self.base_language_search_url += '&lr=' + current_language_entry['lr']

    # creates a selenium browser instance with Firefox
    def create_selenium_browser(self):
        self.driver = webdriver.Firefox()
        self.driver.implicitly_wait(10)#10 is arbitrary

    # takes the link element from xpath, dissects out the href attribute, and runs a regex to extract the source URL
    def get_image_link(self, href_attribute):
        regex_result = re.search(IMAGE_URL_REGEX, href_attribute)
        return regex_result.group('url')

    # get the href attributes that contain the underlying image links
    def get_href_attributes_for_word(self, word):
        # initialize an array of href attributes that are extracted via selenium
        href_attributes = []

        # create the search url for this word by appending the search term onto our base search URL for the language
        url = self.base_language_search_url + '&q=' + urllib.parse.quote(word)

        # retry search query up to 10 times if we hit failures
        # deliberately encapsulates all of the interaction with the selenium library in this retry loop and
        # try/except blocks so that we can handle selenium library errors in a common way
        for attempt in range(10):
            try:
                self.driver.get(url)
                time.sleep(2) #2 is arbitrary

                # pull all elements out of the page using the google-specific xpath for the elements that contain
                # the underlying image links
                link_elements = self.driver.find_elements_by_xpath(GOOGLE_IMAGE_LINK_XPATH)

                # also pull the google-created metadatas out of the page, stored as JSON in a hidden div
                google_metadatas = [s.get_attribute('innerHTML') for s in self.driver.find_elements_by_xpath(GOOGLE_METADATA_XPATH)]

                # pull the href attribute out of each element that contains the underlying image link
                # note: need to keep this inside the retries and try/except catch blocks.
                # there is an occasional selenium issue where even get_attribute can inexplicably fail
                # TODO for this occasional failure, we may have to add a timeout to all of the selenium code as a whole,
                # as it caused the scraper to hang endlessly
                href_attributes = [ link_element.get_attribute('href') for link_element in link_elements ]

            # catch what seem to be BadStatusLine and ConnectionRefusedError's, quit the webdriver, and then create a new
            # instance. there is something really funky going on here that seems to work when resetting selenium
            except Exception as e:
                self.driver.quit()
                self.driver.stop_client()
                self.create_selenium_browser()
                print('Search query failed for:' + url + ' due to: ' + type(e).__name__ + ' retry:'+str(attempt))
            # break for loop on success
            else: break

        return [href_attributes, google_metadatas]

    # function that drives processing every word in the parsed list of words
    def process_all_words(self):
        for word_index, foreign_word in enumerate(self.foreign_word_list):
            # if a start index was passed in, this is a case where we are resuming a previously failed run, skip every word
            # that successfully completed and just pick up where we left off
            if opts.start_index and word_index < opts.start_index: continue

            if opts.verbose_mode:
                print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
                print('Current word: ' + foreign_word + ' at index: ' + str(word_index))
                print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')

            # track a dictionary with the errors for the current words
            current_word_download_errors = defaultdict(int)

            base_path_for_index = opts.base_image_path+str(word_index)+'/'
            # ensure directory is created for this word index
            if not os.path.exists(base_path_for_index): os.makedirs(base_path_for_index)

            # array to store the metadata dictionaries for each image we download
            list_of_image_metadata = []

            href_attributes, metadatas = self.get_href_attributes_for_word(foreign_word)

            for link_index, href_attribute in enumerate(href_attributes):
                # take the list href attribute, and extract the actual image link from that compound string
                actual_image_link = self.get_image_link(href_attribute)

                # force the link index string to be two digits (like 01, 02, etc), and then store the
                link_index_str = "{0:0=2d}".format(link_index+1)

                # take the metadata stored in an adjacent div as JSON, load it as a hash, and add it to our metadata
                google_metadata_for_image = json.loads(metadatas[link_index])

                metadata_for_image = {'image_link': actual_image_link, 'google': google_metadata_for_image}

                # extract the actual file name and the file extension
                actual_file_name = actual_image_link.split('/')[-1]
                metadata_for_image['original_filename'] = actual_file_name
                file_extension = actual_file_name.split('.')[-1]
                no_extension = False

                # ggpht images seem to be internal to the search engine results, skipping them
                if actual_image_link.find('ggpht.com/') != -1:
                    print('Skipped: '+actual_image_link)
                    continue

                # if the file path does not have a valid extension, mark this as such so that we can set it later
                # based on the actual content type (which typically falls in our allowed list)
                if not file_extension.lower() in VALID_FILE_EXTENSIONS:
                    no_extension = True
                    if DEBUG_MODE:
                        print(link_index)
                        print(actual_file_name)
                        print(file_extension)
                        print(actual_image_link)
                        print(href_attribute)
                else:
                    if DEBUG_MODE: continue

                # when in debug mode, just print the link out, otherwise download the file
                if DEBUG_MODE:
                    print(actual_image_link)
                else:
                    if opts.verbose_mode: print('Downloading... ' + actual_image_link)

                    metadata_for_image['filename'] = link_index_str+'.'+file_extension

                    # get the full path where we will store the image, e.g. base_path/01.jpg
                    full_path = base_path_for_index+metadata_for_image['filename']

                    try:
                        # Crucial, fudge the user-agent string here so that network admins don't think we are
                        # urllib, since anything "programmatic" is automatically conflated with an "attack"
                        request = urllib.request.Request(actual_image_link, None, {
                            'User-agent' : USER_AGENT_STRING
                        })
                        with urllib.request.urlopen(request, timeout=30) as response, open(full_path, 'wb') as out_file:
                            content_type = response.info().get_content_type()
                            shutil.copyfileobj(response, out_file)

                        # if there was no file extension and this is a content-type of image,
                        # lets use the content-type from the request to set it, [6:] takes everything after image/
                        if no_extension and content_type.startswith('image/'):
                            original_file_path = full_path
                            new_file_path = link_index_str + '.' + content_type[6:]
                            shutil.move(original_file_path, new_file_path)

                    # catch some 503/504 type errors and also if the link points to a directory rather than a file
                    # typically errors like:
                    # (urllib.error.HTTPError, IsADirectoryError, socket.timeout, urllib.error.URLError, ConnectionResetError)
                    # extended to be indiscriminate since from the perspective of the full scrape, we probably don't care whether
                    # we know about the error yet
                    # TODO log error counts?
                    except Exception as e:
                        error_class = type(e).__name__
                        self.all_word_download_errors[error_class] += 1
                        current_word_download_errors[error_class] += 1
                        print('Failed to fetch:' + actual_image_link + ' due to: ' + error_class)
                        if DEBUG_MODE: print(traceback.format_exc())

                list_of_image_metadata.append(metadata_for_image)

            json.dump(list_of_image_metadata, open(base_path_for_index+'metadata.json', 'w'))
            json.dump(current_word_download_errors, open(base_path_for_index+'errors.json', 'w'))

            # exit after first word in debug mode
            if DEBUG_MODE:
                # sleep 10 seconds between google searches in debug mode to prevent hammering
                time.sleep(10)

                # arbitrary testing point of 100 iterations before exiting in debug mode
                if word_index > 100: exit()

        json.dump(self.all_word_download_errors, open(opts.base_image_path+'all_errors.json', 'w'))

# initialize the image scraper class with the comand line options, then process all the words
image_scraper = GoogleImageScraper(opts)
image_scraper.process_all_words()

#  when S3 mode is turned on, upload the file over to S3 and delete the local copy
# TODO can we simplify this so it doesn't need to write out to a tmp location?
# TODO add metadata like original link to storage?
# S3_BUCKET_NAME = 'brendan.callahan'
# S3_BASE_PATH = 'thesis/images/'+opts.language+'/'
# if STORAGE_MODE == 'S3':
#     word_index_str = "{0:0=2d}".format(word_index)
#     destination_path = S3_BASE_PATH+word_index_str+'/'+actual_file_name
#     s3_client = boto3.resource('s3')
#     s3_client.meta.client.upload_file(full_path, S3_BUCKET_NAME, destination_path)
#     os.remove(full_path)
<commit_msg>simplify the regex for extracting the link before it gets out of hand, adds triple unquote step to make sense of some of the junk links that are being extracted, minor change to capitalize A in User-Agent<commit_after>import time
import os
import re
import urllib.request
import shutil
import json
import optparse
from collections import defaultdict
import boto3
import socket
import traceback

from selenium import webdriver

optparser = optparse.OptionParser()
optparser.add_option("-l", "--language", dest="language", default="French", help="Language to scrape")
optparser.add_option("-n", "--num_images", dest="num_images", default=100, type=int, help="Number of images to harvest per word")
optparser.add_option("-d", "--dictionary", dest="dictionary", default="dict.fr", help="Google languages json file")
optparser.add_option("-L", "--language-map", dest="language_map", default="google-languages.json", help="Google languages json file")
optparser.add_option("-s", "--start-index", dest="start_index", default=None, type=int, help="Word index to start iterating at")
optparser.add_option("-p", "--base-image-path", dest="base_image_path", default='/mnt/storage/images/', help="Base path where to store image output")
optparser.add_option("-v", action="store_true", dest="verbose_mode", help="Verbose mode")
(opts, _) = optparser.parse_args()

# tbm=isch sets this to be image search, start=0 will give us 100 results on page 1
BASE_GOOGLE_IMAGE_SEARCH_LINK = 'https://www.google.com/search?#tbm=isch&start=0'

# regex for extracting source link component: http://www.inddist.com/sites/inddist.com/files/Dollar-Sign.jpg
# from compound links like:
# https://www.google.com/imgres?imgurl=http://www.inddist.com/sites/inddist.com/files/Dollar-Sign.jpg&imgrefurl=http://www.inddist.com/article/2015/05/put-dollar-sign-next-your-service-value&h=1600&w=1200&tbnid=9XAphqXNraTVnM:&docid=Hm9c-cTh-Hl_DM&ei=43-xVsTKMsixeNrCoNAL&tbm=isch&ved=0ahUKEwiEyMjt3trKAhXIGB4KHVohCLoQMwgdKAAwAA
IMAGE_URL_REGEX = r'imgres\?imgurl=(?P<url>.*?)(&imgrefurl)'

# XPATH statement that finds all of the links on the page that correspond to the original image links
# Note: this is probably subject to change on google's part
GOOGLE_IMAGE_LINK_XPATH = "//a[@class='rg_l']"

# XPATH statement that pulls the div adjacent to the image link path that contains google-created metadata
# Note: this is probably subject to change on google's part
GOOGLE_METADATA_XPATH = "//div[@class='rg_meta']"

# user agent string from a recent version of firefox, override the default urllib User-Agent value
USER_AGENT_STRING = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:44.0) Gecko/20100101 Firefox/44.0"

# List of Valid file extensions to check against, if any of these don't match, this means our regex didn't quite
# parse the link correctly to pull out the real file link.
# Note: check for presence in list is also lowercased
VALID_FILE_EXTENSIONS = ['jpg', 'jpeg', 'gif', 'png', 'ico', 'bmp', 'svg']

DEBUG_MODE = True

# class that handles image scraping in google
class GoogleImageScraper(object):
    def __init__(self, opts):
        # creates the driver for running selenium
        self.driver = None # dummy call to ensure instance var is created in init
        self.create_selenium_browser()

        # track a dictionary with the errors for all words
        self.all_word_download_errors = defaultdict(int)

        # read in the json file with the arguments (hl and lr) for each language in google
        with open(opts.language_map, encoding='utf-8') as data_file:
            full_language_arg_map = json.loads(data_file.read())

        # pull the foreign words out of the bilingual dictionary, this assumes the format foreign\tenglish\n
        self.foreign_word_list = [line.strip().split('\t')[0] for line in open(opts.dictionary)]

        # build up a language-specific base link to start out with, before modifying it per each individual search term
        # add on the hl field for all languages because if it is in our JSON file, it has a hl field
        current_language_entry = full_language_arg_map[opts.language]
        self.base_language_search_url = BASE_GOOGLE_IMAGE_SEARCH_LINK + '&hl=' + current_language_entry['hl']

        # the lr field is present in some but not all of the language possibilities in the JSON config file
        if len(current_language_entry['lr']) > 0:
            self.base_language_search_url += '&lr=' + current_language_entry['lr']

    # creates a selenium browser instance with Firefox
    def create_selenium_browser(self):
        self.driver = webdriver.Firefox()
        self.driver.implicitly_wait(10)#10 is arbitrary

    # takes the link element from xpath, dissects out the href attribute, and runs a regex to extract the source URL
    def get_image_link(self, href_attribute):
        # some links are double or triple quoted, by chaining this call 3 times, we seem to unquote most of it
        # there is no functional downside that i know of for unquoting more than need be
        triple_decoded_attr = urllib.parse.unquote(urllib.parse.unquote(urllib.parse.unquote(href_attribute)))
        regex_result = re.search(IMAGE_URL_REGEX, triple_decoded_attr)
        return regex_result.group('url')

    # get the href attributes that contain the underlying image links
    def get_href_attributes_for_word(self, word):
        # initialize an array of href attributes that are extracted via selenium
        href_attributes = []

        # create the search url for this word by appending the search term onto our base search URL for the language
        url = self.base_language_search_url + '&q=' + urllib.parse.quote(word)

        # retry search query up to 10 times if we hit failures
        # deliberately encapsulates all of the interaction with the selenium library in this retry loop and
        # try/except blocks so that we can handle selenium library errors in a common way
        for attempt in range(10):
            try:
                self.driver.get(url)
                time.sleep(2) #2 is arbitrary

                # pull all elements out of the page using the google-specific xpath for the elements that contain
                # the underlying image links
                link_elements = self.driver.find_elements_by_xpath(GOOGLE_IMAGE_LINK_XPATH)

                # also pull the google-created metadatas out of the page, stored as JSON in a hidden div
                google_metadatas = [s.get_attribute('innerHTML') for s in self.driver.find_elements_by_xpath(GOOGLE_METADATA_XPATH)]

                # pull the href attribute out of each element that contains the underlying image link
                # note: need to keep this inside the retries and try/except catch blocks.
                # there is an occasional selenium issue where even get_attribute can inexplicably fail
                # TODO for this occasional failure, we may have to add a timeout to all of the selenium code as a whole,
                # as it caused the scraper to hang endlessly
                href_attributes = [ link_element.get_attribute('href') for link_element in link_elements ]

            # catch what seem to be BadStatusLine and ConnectionRefusedError's, quit the webdriver, and then create a new
            # instance. there is something really funky going on here that seems to work when resetting selenium
            except Exception as e:
                self.driver.quit()
                self.driver.stop_client()
                self.create_selenium_browser()
                print('Search query failed for:' + url + ' due to: ' + type(e).__name__ + ' retry:'+str(attempt))
            # break for loop on success
            else: break

        return [href_attributes, google_metadatas]

    # function that drives processing every word in the parsed list of words
    def process_all_words(self):
        for word_index, foreign_word in enumerate(self.foreign_word_list):
            # if a start index was passed in, this is a case where we are resuming a previously failed run, skip every word
            # that successfully completed and just pick up where we left off
            if opts.start_index and word_index < opts.start_index: continue

            if opts.verbose_mode:
                print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
                print('Current word: ' + foreign_word + ' at index: ' + str(word_index))
                print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')

            # track a dictionary with the errors for the current words
            current_word_download_errors = defaultdict(int)

            base_path_for_index = opts.base_image_path+str(word_index)+'/'
            # ensure directory is created for this word index
            if not os.path.exists(base_path_for_index): os.makedirs(base_path_for_index)

            # array to store the metadata dictionaries for each image we download
            list_of_image_metadata = []

            href_attributes, metadatas = self.get_href_attributes_for_word(foreign_word)

            for link_index, href_attribute in enumerate(href_attributes):
                # take the list href attribute, and extract the actual image link from that compound string
                actual_image_link = self.get_image_link(href_attribute)

                # force the link index string to be two digits (like 01, 02, etc), and then store the
                link_index_str = "{0:0=2d}".format(link_index+1)

                # take the metadata stored in an adjacent div as JSON, load it as a hash, and add it to our metadata
                google_metadata_for_image = json.loads(metadatas[link_index])

                metadata_for_image = {'image_link': actual_image_link, 'google': google_metadata_for_image}

                # extract the actual file name and the file extension
                actual_file_name = actual_image_link.split('/')[-1]
                metadata_for_image['original_filename'] = actual_file_name
                file_extension = actual_file_name.split('.')[-1]
                no_extension = False

                # ggpht images seem to be internal to the search engine results, skipping them
                if actual_image_link.find('ggpht.com/') != -1:
                    print('Skipped: '+actual_image_link)
                    continue

                # if the file path does not have a valid extension, mark this as such so that we can set it later
                # based on the actual content type (which typically falls in our allowed list)
                if not file_extension.lower() in VALID_FILE_EXTENSIONS:
                    no_extension = True
                    if DEBUG_MODE:
                        #print(link_index)
                        #print(actual_file_name)
                        print(actual_image_link)
                        print(urllib.parse.unquote(urllib.parse.unquote(urllib.parse.unquote(href_attribute))))
                else:
                    if DEBUG_MODE: continue

                # when in debug mode, just print the link out, otherwise download the file
                if DEBUG_MODE:
                    print(actual_image_link)
                else:
                    if opts.verbose_mode: print('Downloading... ' + actual_image_link)

                    metadata_for_image['filename'] = link_index_str+'.'+file_extension

                    # get the full path where we will store the image, e.g. base_path/01.jpg
                    full_path = base_path_for_index+metadata_for_image['filename']

                    try:
                        # Crucial, fudge the user-agent string here so that network admins don't think we are
                        # urllib, since anything "programmatic" is automatically conflated with an "attack"
                        request = urllib.request.Request(actual_image_link, None, {
                            'User-Agent': USER_AGENT_STRING
                        })
                        with urllib.request.urlopen(request, timeout=30) as response, open(full_path, 'wb') as out_file:
                            content_type = response.info().get_content_type()
                            shutil.copyfileobj(response, out_file)

                        # if there was no file extension and this is a content-type of image,
                        # lets use the content-type from the request to set it, [6:] takes everything after image/
                        # note: with the 1-liner above for the file path, it seems we need this workaround
                        # TODO: should probably deconstruct it
                        if no_extension and content_type.startswith('image/'):
                            original_file_path = full_path
                            new_file_path = link_index_str + '.' + content_type[6:]
                            shutil.move(original_file_path, new_file_path)

                    # catch some 503/504 type errors and also if the link points to a directory rather than a file
                    # typically errors like:
                    # (urllib.error.HTTPError, IsADirectoryError, socket.timeout, urllib.error.URLError, ConnectionResetError)
                    # extended to be indiscriminate since from the perspective of the full scrape, we probably don't care whether
                    # we know about the error yet
                    # TODO log error counts?
                    except Exception as e:
                        error_class = type(e).__name__
                        self.all_word_download_errors[error_class] += 1
                        current_word_download_errors[error_class] += 1
                        print('Failed to fetch:' + actual_image_link + ' due to: ' + error_class)
                        if DEBUG_MODE: print(traceback.format_exc())

                list_of_image_metadata.append(metadata_for_image)

            json.dump(list_of_image_metadata, open(base_path_for_index+'metadata.json', 'w'))
            json.dump(current_word_download_errors, open(base_path_for_index+'errors.json', 'w'))

            # exit after first word in debug mode
            if DEBUG_MODE:
                # sleep 10 seconds between google searches in debug mode to prevent hammering
                time.sleep(10)

                # arbitrary testing point of 100 iterations before exiting in debug mode
                if word_index > 100: exit()

        json.dump(self.all_word_download_errors, open(opts.base_image_path+'all_errors.json', 'w'))

# initialize the image scraper class with the comand line options, then process all the words
image_scraper = GoogleImageScraper(opts)
image_scraper.process_all_words()

#  when S3 mode is turned on, upload the file over to S3 and delete the local copy
# TODO can we simplify this so it doesn't need to write out to a tmp location?
# TODO add metadata like original link to storage?
# S3_BUCKET_NAME = 'brendan.callahan'
# S3_BASE_PATH = 'thesis/images/'+opts.language+'/'
# if STORAGE_MODE == 'S3':
#     word_index_str = "{0:0=2d}".format(word_index)
#     destination_path = S3_BASE_PATH+word_index_str+'/'+actual_file_name
#     s3_client = boto3.resource('s3')
#     s3_client.meta.client.upload_file(full_path, S3_BUCKET_NAME, destination_path)
#     os.remove(full_path)
<|endoftext|>


<commit_before>import time
import os
import re
import urllib.request
import shutil
import json
import optparse
import boto3
import socket

from selenium import webdriver

optparser = optparse.OptionParser()
optparser.add_option("-l", "--language", dest="language", default="French", help="Language to scrape")
optparser.add_option("-n", "--num_images", dest="num_images", default=100, type=int, help="Number of images to harvest per word")
optparser.add_option("-d", "--dictionary", dest="dictionary", default="dict.fr", help="Google languages json file")
optparser.add_option("-L", "--language-map", dest="language_map", default="google-languages.json", help="Google languages json file")
optparser.add_option("-s", "--start_index", dest="start_index", default=None, type=int, help="Word index to start iterating at")
(opts, _) = optparser.parse_args()

# tbm=isch sets this to be image search, start=0 will give us 100 results on page 1
BASE_GOOGLE_IMAGE_SEARCH_LINK = 'https://www.google.com/search?#tbm=isch&start=0'

# regex for extracting source link component: http://www.inddist.com/sites/inddist.com/files/Dollar-Sign.jpg
# from compound links like:
# https://www.google.com/imgres?imgurl=http://www.inddist.com/sites/inddist.com/files/Dollar-Sign.jpg&imgrefurl=http://www.inddist.com/article/2015/05/put-dollar-sign-next-your-service-value&h=1600&w=1200&tbnid=9XAphqXNraTVnM:&docid=Hm9c-cTh-Hl_DM&ei=43-xVsTKMsixeNrCoNAL&tbm=isch&ved=0ahUKEwiEyMjt3trKAhXIGB4KHVohCLoQMwgdKAAwAA
IMAGE_URL_REGEX = r'imgres\?imgurl=(?P<url>.*?)&'

# XPATH statement that finds all of the links on the page that correspond to the original image links
# this is probably subject to change on google's part
GOOGLE_IMAGE_LINK_XPATH = "//a[@class='rg_l']"

BASE_IMAGE_PATH = 'images/'
S3_BUCKET_NAME = 'brendan.callahan'
S3_BASE_PATH = 'thesis/images/'+opts.language+'/'
STORAGE_MODE = 'S3' # or 'FILESYSTEM'
DEBUG_MODE = False
VERBOSE_MOSE = True

# read in the json file with the arguments (hl and lr) for each language in google
with open(opts.language_map, encoding='utf-8') as data_file:
    full_language_arg_map = json.loads(data_file.read())

# pull the foreign words out of the bilingual dictionary, this assumes the format foreign\tenglish\n
foreign_word_list = [line.strip().split('\t')[0] for line in open(opts.dictionary)]

# creates a selenium browser instance with Firefox
def create_selenium_browser():
    driver = webdriver.Firefox()
    driver.implicitly_wait(10)#10 is arbitrary
    return driver

# takes the link element from xpath, dissects out the href attribute, and runs a regex to extract the source URL
def get_image_link(link_element):
    href = link_element.get_attribute('href')
    regex_result = re.search(IMAGE_URL_REGEX, href)
    return regex_result.group('url')

# takes the image link, pulls out the filename, downloads it. when in S3 mode, it will push the result onto S3 as well
def download_image(actual_image_link, word_index):
    actual_file_name = actual_image_link.split('/')[-1]
    if VERBOSE_MOSE: print('Downloading... ' + actual_image_link)

    # ggpht images seem to be internal to the search engine results, skipping them
    if actual_image_link.find('ggpht.com/') == -1:
        full_path = BASE_IMAGE_PATH+actual_file_name
        try:
            with urllib.request.urlopen(actual_image_link, timeout=30) as response, open(full_path, 'wb') as out_file:
                shutil.copyfileobj(response, out_file)

            # when S3 mode is turned on, upload the file over to S3 and delete the local copy
            # TODO can we simplify this so it doesn't need to write out to a tmp location?
            # TODO add metadata like original link to storage?
            if STORAGE_MODE == 'S3':
                word_index_str = "{0:0=2d}".format(word_index)
                destination_path = S3_BASE_PATH+word_index_str+'/'+actual_file_name
                s3_client = boto3.resource('s3')
                s3_client.meta.client.upload_file(full_path, S3_BUCKET_NAME, destination_path)
                os.remove(full_path)

        # catch some 503/504 type errors and also if the link points to a directory rather than a file
        # typically errors like:
        # (urllib.error.HTTPError, IsADirectoryError, socket.timeout, urllib.error.URLError, ConnectionResetError)
        # extended to be indiscriminate since from the perspective of the full scrape, we probably don't care whether
        # we know about the error yet
        # TODO log error counts?
        except Exception as e:
            print('Failed to fetch:' + actual_image_link + ' due to: ' + str(type(e)))

    else:
        print('Skipped: '+actual_image_link)

driver = create_selenium_browser()

# build up a language-specific base link to start out with, before modifying it per each individual search term
# add on the hl field for all languages because if it is in our JSON file, it has a hl field
current_language_entry = full_language_arg_map[opts.language]
base_language_search_url = BASE_GOOGLE_IMAGE_SEARCH_LINK + '&hl=' + current_language_entry['hl']

# the lr field is present in some but not all of the language possibilities in the JSON config file
if len(current_language_entry['lr']) > 0:
    base_language_search_url += '&lr=' + current_language_entry['lr']

for word_index, foreign_word in enumerate(foreign_word_list):
    if VERBOSE_MOSE:
        print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
        print('Current word: ' + foreign_word + ' at index: ' + str(word_index))
        print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')

    # if a start index was passed in, this is a case where we are resuming a previously failed run, skip every word
    # that successfully completed and just pick up where we left off
    if opts.start_index and word_index < opts.start_index: continue

    # retry search query up to 10 times if we hit failures
    for attempt in range(10):
        try:
            url = base_language_search_url + '&q=' + urllib.parse.quote(foreign_word)
            driver.get(url)
            time.sleep(2) #2 is arbitrary

            link_elements = driver.find_elements_by_xpath(GOOGLE_IMAGE_LINK_XPATH)

        # catch what seem to be BadStatusLine and ConnectionRefusedError's, quit the webdriver, and then create a new
        # instance. there is something really funky going on here that seems to work when resetting selenium
        except Exception as e:
            driver.quit()
            driver.stop_client()
            driver = create_selenium_browser()
            print('Search query failed for:' + url + ' due to: ' + str(type(e)) + ' retry:'+str(attempt))
        # break for loop on success
        else: break

    for link_element in link_elements:
        actual_image_link = get_image_link(link_element)

        # when in debug mode, just print the link out, otherwise download the file
        if DEBUG_MODE:
            print(actual_image_link)
        else:
            download_image(actual_image_link, word_index)

    # exit after first word in debug mode
    if DEBUG_MODE: exit()

<commit_msg>make verbose mode togglable at command line, turn s3 transfers off for now<commit_after>import time
import os
import re
import urllib.request
import shutil
import json
import optparse
import boto3
import socket

from selenium import webdriver

optparser = optparse.OptionParser()
optparser.add_option("-l", "--language", dest="language", default="French", help="Language to scrape")
optparser.add_option("-n", "--num_images", dest="num_images", default=100, type=int, help="Number of images to harvest per word")
optparser.add_option("-d", "--dictionary", dest="dictionary", default="dict.fr", help="Google languages json file")
optparser.add_option("-L", "--language-map", dest="language_map", default="google-languages.json", help="Google languages json file")
optparser.add_option("-s", "--start_index", dest="start_index", default=None, type=int, help="Word index to start iterating at")
optparser.add_option("-v", action="store_true", dest="verbose_mode", help="Verbose mode")
(opts, _) = optparser.parse_args()

# tbm=isch sets this to be image search, start=0 will give us 100 results on page 1
BASE_GOOGLE_IMAGE_SEARCH_LINK = 'https://www.google.com/search?#tbm=isch&start=0'

# regex for extracting source link component: http://www.inddist.com/sites/inddist.com/files/Dollar-Sign.jpg
# from compound links like:
# https://www.google.com/imgres?imgurl=http://www.inddist.com/sites/inddist.com/files/Dollar-Sign.jpg&imgrefurl=http://www.inddist.com/article/2015/05/put-dollar-sign-next-your-service-value&h=1600&w=1200&tbnid=9XAphqXNraTVnM:&docid=Hm9c-cTh-Hl_DM&ei=43-xVsTKMsixeNrCoNAL&tbm=isch&ved=0ahUKEwiEyMjt3trKAhXIGB4KHVohCLoQMwgdKAAwAA
IMAGE_URL_REGEX = r'imgres\?imgurl=(?P<url>.*?)&'

# XPATH statement that finds all of the links on the page that correspond to the original image links
# this is probably subject to change on google's part
GOOGLE_IMAGE_LINK_XPATH = "//a[@class='rg_l']"

BASE_IMAGE_PATH = 'images/'
S3_BUCKET_NAME = 'brendan.callahan'
S3_BASE_PATH = 'thesis/images/'+opts.language+'/'
DEBUG_MODE = False

# read in the json file with the arguments (hl and lr) for each language in google
with open(opts.language_map, encoding='utf-8') as data_file:
    full_language_arg_map = json.loads(data_file.read())

# pull the foreign words out of the bilingual dictionary, this assumes the format foreign\tenglish\n
foreign_word_list = [line.strip().split('\t')[0] for line in open(opts.dictionary)]

# creates a selenium browser instance with Firefox
def create_selenium_browser():
    driver = webdriver.Firefox()
    driver.implicitly_wait(10)#10 is arbitrary
    return driver

# takes the link element from xpath, dissects out the href attribute, and runs a regex to extract the source URL
def get_image_link(link_element):
    href = link_element.get_attribute('href')
    regex_result = re.search(IMAGE_URL_REGEX, href)
    return regex_result.group('url')

# takes the image link, pulls out the filename, downloads it. when in S3 mode, it will push the result onto S3 as well
def download_image(actual_image_link, word_index):
    actual_file_name = actual_image_link.split('/')[-1]
    if opts.verbose_mode: print('Downloading... ' + actual_image_link)

    # ggpht images seem to be internal to the search engine results, skipping them
    if actual_image_link.find('ggpht.com/') == -1:
        full_path = BASE_IMAGE_PATH+actual_file_name
        try:
            with urllib.request.urlopen(actual_image_link, timeout=30) as response, open(full_path, 'wb') as out_file:
                shutil.copyfileobj(response, out_file)

            # when S3 mode is turned on, upload the file over to S3 and delete the local copy
            # TODO can we simplify this so it doesn't need to write out to a tmp location?
            # TODO add metadata like original link to storage?
            # if STORAGE_MODE == 'S3':
            #     word_index_str = "{0:0=2d}".format(word_index)
            #     destination_path = S3_BASE_PATH+word_index_str+'/'+actual_file_name
            #     s3_client = boto3.resource('s3')
            #     s3_client.meta.client.upload_file(full_path, S3_BUCKET_NAME, destination_path)
            #     os.remove(full_path)

        # catch some 503/504 type errors and also if the link points to a directory rather than a file
        # typically errors like:
        # (urllib.error.HTTPError, IsADirectoryError, socket.timeout, urllib.error.URLError, ConnectionResetError)
        # extended to be indiscriminate since from the perspective of the full scrape, we probably don't care whether
        # we know about the error yet
        # TODO log error counts?
        except Exception as e:
            print('Failed to fetch:' + actual_image_link + ' due to: ' + str(type(e)))

    else:
        print('Skipped: '+actual_image_link)

driver = create_selenium_browser()

# build up a language-specific base link to start out with, before modifying it per each individual search term
# add on the hl field for all languages because if it is in our JSON file, it has a hl field
current_language_entry = full_language_arg_map[opts.language]
base_language_search_url = BASE_GOOGLE_IMAGE_SEARCH_LINK + '&hl=' + current_language_entry['hl']

# the lr field is present in some but not all of the language possibilities in the JSON config file
if len(current_language_entry['lr']) > 0:
    base_language_search_url += '&lr=' + current_language_entry['lr']

for word_index, foreign_word in enumerate(foreign_word_list):
    if opts.verbose_mode:
        print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
        print('Current word: ' + foreign_word + ' at index: ' + str(word_index))
        print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')

    # if a start index was passed in, this is a case where we are resuming a previously failed run, skip every word
    # that successfully completed and just pick up where we left off
    if opts.start_index and word_index < opts.start_index: continue

    # retry search query up to 10 times if we hit failures
    for attempt in range(10):
        try:
            url = base_language_search_url + '&q=' + urllib.parse.quote(foreign_word)
            driver.get(url)
            time.sleep(2) #2 is arbitrary

            link_elements = driver.find_elements_by_xpath(GOOGLE_IMAGE_LINK_XPATH)

        # catch what seem to be BadStatusLine and ConnectionRefusedError's, quit the webdriver, and then create a new
        # instance. there is something really funky going on here that seems to work when resetting selenium
        except Exception as e:
            driver.quit()
            driver.stop_client()
            driver = create_selenium_browser()
            print('Search query failed for:' + url + ' due to: ' + str(type(e)) + ' retry:'+str(attempt))
        # break for loop on success
        else: break

    for link_element in link_elements:
        actual_image_link = get_image_link(link_element)

        # when in debug mode, just print the link out, otherwise download the file
        if DEBUG_MODE:
            print(actual_image_link)
        else:
            download_image(actual_image_link, word_index)

    # exit after first word in debug mode
    if DEBUG_MODE: exit()

<|endoftext|>


<commit_before>from __future__ import unicode_literals

import logging
import inspect
from functools import wraps

from fluent import event

from raven import Client

from scrapi import settings

logger = logging.getLogger(__name__)
sentry = Client(dsn=settings.SENTRY_DSN)

if not settings.USE_FLUENTD:
    logger.warning('USE_FLUENTD is set to False; logs will not be stored')

# Events
PROCESSING = 'processing'
HARVESTER_RUN = 'runHarvester'
CHECK_ARCHIVE = 'checkArchive'
NORMALIZATION = 'normalization'

# statuses
FAILED = 'failed'
SKIPPED = 'skipped'
CREATED = 'created'
STARTED = 'started'
COMPLETED = 'completed'

def log_to_sentry(message, **kwargs):
    if not settings.SENTRY_DSN:
        return logger.warn('send_to_raven called with no SENTRY_DSN')
    return sentry.captureMessage(message, extra=kwargs)

class Skip(Exception):
    pass

def serialize_fluent_data(data):
    if isinstance(data, dict):
        return {
            key: serialize_fluent_data(val)
            for key, val in data.items()
        }
    elif isinstance(data, list):
        return [
            serialize_fluent_data(item)
            for item in data
        ]
    elif isinstance(data, (str, unicode)):
        return data
    else:
        return repr(data)

# Ues _index here as to not clutter the namespace for kwargs
def dispatch(_event, status, _index=None, **kwargs):
    if not settings.USE_FLUENTD:
        return

    evnt = {
        'event': _event,
        'status': status
    }

    evnt.update(serialize_fluent_data(kwargs))

    if _index:
        _event = '{}.{}'.format(_event, _index)

    logger.debug('[{}][{}]{!r}'.format(_event, status, kwargs))
    event.Event(_event, evnt)

def logged(event, index=None):
    def _logged(func):
        @wraps(func)
        def wrapped(*args, **kwargs):
            context = extract_context(func, *args, **kwargs)
            dispatch(event, STARTED, _index=index, **context)
            try:
                res = func(*args, **kwargs)
            except Skip as e:
                dispatch(event, SKIPPED, _index=index, reason=e.message, **context)
                return None
            except Exception as e:
                if settings.SENTRY_DSN:
                    sentry.captureException()
                dispatch(event, FAILED, _index=index, exception=e, **context)
                raise
            else:
                dispatch(event, COMPLETED, _index=index, **context)
            return res
        return wrapped
    return _logged

def extract_context(func, *args, **kwargs):
    args = list(reversed(args))
    arginfo = inspect.getargspec(func)

    if arginfo.defaults:
        arg_names = arginfo.args[:len(arginfo.defaults) - 1]
        kwarg_names = arginfo.args[len(arginfo.defaults) - 1:]
    else:
        kwarg_names = []
        arg_names = arginfo.args

    real_args = {
        key: kwargs.pop(key, val)
        for key, val
        in zip(kwarg_names, arginfo.defaults or [])
    }

    for name in arg_names:
        real_args[name] = args.pop()

    if arginfo.varargs:
        real_args[arginfo.varargs] = list(reversed(args))

    if arginfo.keywords:
        real_args[arginfo.keywords] = kwargs

    return real_args

def creates_task(event):
    def _creates_task(func):
        @wraps(func)
        def wrapped(*args, **kwargs):
            res = func(*args, **kwargs)
            dispatch(event, CREATED, **extract_context(func, *args, **kwargs))
            return res
        return wrapped
    return _creates_task
<commit_msg>Update extract context to work with multiple kwargs, but breaks tests<commit_after>from __future__ import unicode_literals

import logging
import inspect
from functools import wraps

from fluent import event

from raven import Client

from scrapi import settings

logger = logging.getLogger(__name__)
sentry = Client(dsn=settings.SENTRY_DSN)

if not settings.USE_FLUENTD:
    logger.warning('USE_FLUENTD is set to False; logs will not be stored')

# Events
PROCESSING = 'processing'
HARVESTER_RUN = 'runHarvester'
CHECK_ARCHIVE = 'checkArchive'
NORMALIZATION = 'normalization'

# statuses
FAILED = 'failed'
SKIPPED = 'skipped'
CREATED = 'created'
STARTED = 'started'
COMPLETED = 'completed'

def log_to_sentry(message, **kwargs):
    if not settings.SENTRY_DSN:
        return logger.warn('send_to_raven called with no SENTRY_DSN')
    return sentry.captureMessage(message, extra=kwargs)

class Skip(Exception):
    pass

def serialize_fluent_data(data):
    if isinstance(data, dict):
        return {
            key: serialize_fluent_data(val)
            for key, val in data.items()
        }
    elif isinstance(data, list):
        return [
            serialize_fluent_data(item)
            for item in data
        ]
    elif isinstance(data, (str, unicode)):
        return data
    else:
        return repr(data)

# Ues _index here as to not clutter the namespace for kwargs
def dispatch(_event, status, _index=None, **kwargs):
    if not settings.USE_FLUENTD:
        return

    evnt = {
        'event': _event,
        'status': status
    }

    evnt.update(serialize_fluent_data(kwargs))

    if _index:
        _event = '{}.{}'.format(_event, _index)

    logger.debug('[{}][{}]{!r}'.format(_event, status, kwargs))
    event.Event(_event, evnt)

def logged(event, index=None):
    def _logged(func):
        @wraps(func)
        def wrapped(*args, **kwargs):
            context = extract_context(func, *args, **kwargs)
            dispatch(event, STARTED, _index=index, **context)
            try:
                res = func(*args, **kwargs)
            except Skip as e:
                dispatch(event, SKIPPED, _index=index, reason=e.message, **context)
                return None
            except Exception as e:
                if settings.SENTRY_DSN:
                    sentry.captureException()
                dispatch(event, FAILED, _index=index, exception=e, **context)
                raise
            else:
                dispatch(event, COMPLETED, _index=index, **context)
            return res
        return wrapped
    return _logged

def extract_context(func, *args, **kwargs):
    arginfo = inspect.getargspec(func)
    arg_names = arginfo.args
    defaults = {
        arg_names.pop(-1): kwarg
        for kwarg in (arginfo.defaults or [])
    }

    return dict(zip(arg_names, args), **dict(defaults, **kwargs))

def creates_task(event):
    def _creates_task(func):
        @wraps(func)
        def wrapped(*args, **kwargs):
            res = func(*args, **kwargs)
            dispatch(event, CREATED, **extract_context(func, *args, **kwargs))
            return res
        return wrapped
    return _creates_task
<|endoftext|>


<commit_before>import helpers

from settings import *
from neopixel import *

S = Settings()

instance = None

class Screen:
    def __init__(self,
                 width=int(S.get('screen', 'matrix_width')),
                 height=int(S.get('screen', 'matrix_height')),
                 led_pin=int(S.get('screen', 'led_pin')),
                 led_freq_hz=int(S.get('screen', 'led_freq')),
                 led_dma=int(S.get('screen', 'led_dma')),
                 led_invert=(True if S.get('screen', 'led_invert').lower() == 'true' else False),
                 led_brightness=int(S.get('screen', 'brightness'))):
        self.width = width
        self.height = height

        self.strip = Adafruit_NeoPixel(width * height, led_pin, led_freq_hz, led_dma, led_invert, led_brightness)

        # pygame.display.init()  # needed for events

        try:
            self.strip.begin()
        except RuntimeError:
            print('\033[38;5;196merror: did you run it with sudo?\033[0m')

            self.update_brightness()
            self.pixel = [[helpers.Color(0, 0, 0) for y in range(height)] for x in range(width)]

            global instance
            instance = self

    def clear(self, color=helpers.Color(0, 0, 0)):
        for x in range(self.width):
            for y in range(self.height):
                self.pixel[x][y] = color

    def update(self):
        for y in range(self.height):
            for x in range(self.width):
                if y % 2 == 0:
                    self.strip.setPixelColor(y * self.width + x, self.pixel[x][y])
                else:
                    self.strip.setPixelColor(y * self.width + self.width - 1 - x, self.pixel[x][y])
        self.strip.show()

    def update_brightness(self):
        self.strip.setBrightness(int(4 + 3.1 * (int(S.get('screen', 'brightness')) + 1) ** 2))

    def set_brightness(self, value):
        value = min(max(value, 0), 8)
        S.set('screen', 'brightness', value)
        self.update_brightness()

    def get_brightness(self):
        return int(S.get('screen', 'brightness'))
<commit_msg>fixed ident<commit_after>import helpers

from settings import *
from neopixel import *

S = Settings()

instance = None

class Screen:
    def __init__(self,
                 width=int(S.get('screen', 'matrix_width')),
                 height=int(S.get('screen', 'matrix_height')),
                 led_pin=int(S.get('screen', 'led_pin')),
                 led_freq_hz=int(S.get('screen', 'led_freq')),
                 led_dma=int(S.get('screen', 'led_dma')),
                 led_invert=(True if S.get('screen', 'led_invert').lower() == 'true' else False),
                 led_brightness=int(S.get('screen', 'brightness'))):
        self.width = width
        self.height = height

        self.strip = Adafruit_NeoPixel(width * height, led_pin, led_freq_hz, led_dma, led_invert, led_brightness)

        # pygame.display.init()  # needed for events

        try:
            self.strip.begin()
        except RuntimeError:
            print('\033[38;5;196merror: did you run it with sudo?\033[0m')

        self.update_brightness()
        self.pixel = [[helpers.Color(0, 0, 0) for y in range(height)] for x in range(width)]

        global instance
        instance = self

    def clear(self, color=helpers.Color(0, 0, 0)):
        for x in range(self.width):
            for y in range(self.height):
                self.pixel[x][y] = color

    def update(self):
        for y in range(self.height):
            for x in range(self.width):
                if y % 2 == 0:
                    self.strip.setPixelColor(y * self.width + x, self.pixel[x][y])
                else:
                    self.strip.setPixelColor(y * self.width + self.width - 1 - x, self.pixel[x][y])
        self.strip.show()

    def update_brightness(self):
        self.strip.setBrightness(int(4 + 3.1 * (int(S.get('screen', 'brightness')) + 1) ** 2))

    def set_brightness(self, value):
        value = min(max(value, 0), 8)
        S.set('screen', 'brightness', value)
        self.update_brightness()

    def get_brightness(self):
        return int(S.get('screen', 'brightness'))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import functions.py
print "Welcome to Chapter 3: Hung Jury"
print " "
print "You realize that you’re not cut out for a life on the run, so you decide to head back to the palace and stand trial. As the judge concludes your charges, you standand defiantly plead 'Not Guilty'\n"
print " "
player_input(prompt="What is your argument? Do you claim you were taking notes in such a way to establish a new metadata schemeand usher in the educational practice of information science OR do you claim your experiences in the forest will make you a valuable warrior for the kingdom?)
king_rebuttal - raw_input("What is your argument? Do you claim you were taking notes in such a way to establish a new metadata schemeand usher in the educational practice of " + "\"information science\"" + ' OR do you claim your experiences in the forest will make you a " + "\"valuable warrior\"" + ' for the kingdom?)
if 'Information science' in king_rebuttal:
  print " "
  print "The king scoffs at your notion. '98% of our kingdom cannot even read. INFORMATION SCIENCE PAH?! I condemn you to the guillotine! Ouch, sounds like the king isn't the only one losing his head over this."
  print " "
  guillotine = raw_input("As you approach the guillotine, the neighboring kingdom suddenly attacks the courtyard. You still have your quill in the seat of your pants. Do you " + "\"stay and defend the kingdom\"" + ' or '+ "\"take the opportunity to run\""+'?'))
  if 'stay and defend the kingdom' in guillotine:
      print " "
      print "You withdraw your quill and line up next to the king's guard. 'FOR HONOR' you yell. Congratulations! Your heroic deeds have successfully cleared your name throughout all of Karflooglesville. But will you be able to fend off the guards? Stay tuned to the project to find out!"
      print " "
  if 'take the opportunity to run' in guillotine:
      print " "
      print "The king catches you running away in the melee and orders a knight after you. You're no match for the horse's speed, and the king orders an execution on the spot. Game over. Just kidding, care to try again?"
      print "import scribe_level3.py"

if 'Valuable warrior' in action:
  print "The king considers your offer thoroughly. 'Hm' he ponders, 'We have been short a few heroes recently.' Suddenly a loud crash is heard outside. 'MILORD' the king's page screams, 'We are besieged by the neighboring kingdom!' The king turns to you, 'Alrighty lad, you say you'll be a valuable warrior, now's the time to prove it.'\n"
  courtyard = raw_input("Do you " + "\"Meet the enemy head on\"" + ' or do you stay back, nervously observing your surroundings and making a break for the " + "\"back gate\"" + ' while the king is distracted?")
  if 'Meet the enemy head on' in courtyard:
      print " "
      print "You cautiously but confidently approach the enemy in the company of the king's guard. Quickly, you drawyour quill (no pun intended), causing the enemy to burst into laughter. Congratulations! Your heroic deeds have successfully cleared your name throughout all of Karflooglesville. But will you be able to fend off the guards? Stay tuned to the project to find out!"\n"
  if 'back gate' in courtyard:
      print " "
      print "The back gate is right behind the dungeon. Unfortunately, the dungeon master is privy to your plan and halts you. Your quill barely scratches his imposing armor and thick mask 'Back in the cell ya go, laddie,' he says, tossing you in the familiarity of your cell. Dammit, not again\n"
      print "import scribe_level1.py"
<commit_msg>Updated scribe_level3.py<commit_after># -*- coding: utf-8 -*-

import functions.py #this imports several crucial functions
print "Welcome to Chapter 3: Hung Jury" #this is the title of the chapter
print " "
print "You realize that you’re not cut out for a life on the run, so you decide to head back to the palace and stand trial. As the judge concludes your charges, you standand defiantly plead 'Not Guilty'\n"
print " "
player_input(prompt="What is your argument? Do you claim you were taking notes in such a way to establish a new metadata schemeand usher in the educational practice of information science OR do you claim your experiences in the forest will make you a valuable warrior for the kingdom?)
king_rebuttal - raw_input("What is your argument? Do you claim you were taking notes in such a way to establish a new metadata schemeand usher in the educational practice of " + "\"information science\"" + ' OR do you claim your experiences in the forest will make you a " + "\"valuable warrior\"" + ' for the kingdom?) #this gives the players their two options to pick
if 'Information science' in king_rebuttal: #this is one of the possible answers, it needs to be case sensitive so keep it in mind
  print " "
  print "The king scoffs at your notion. '98% of our kingdom cannot even read. INFORMATION SCIENCE PAH?! I condemn you to the guillotine! Ouch, sounds like the king isn't the only one losing his head over this." #this input continues the story depending on what the player chose
  print " "
  guillotine = raw_input("As you approach the guillotine, the neighboring kingdom suddenly attacks the courtyard. You still have your quill in the seat of your pants. Do you " + "\"stay and defend the kingdom\"" + ' or '+ "\"take the opportunity to run\""+'?'))
  if 'stay and defend the kingdom' in guillotine: #this is a specific choice a player can make once they've chosen this branch
      print " "
      print "You withdraw your quill and line up next to the king's guard. 'FOR HONOR' you yell. Congratulations! Your heroic deeds have successfully cleared your name throughout all of Karflooglesville. But will you be able to fend off the guards? Stay tuned to the project to find out!"
      print " "
  if 'take the opportunity to run' in guillotine: #this is the alternate choice
      print " "
      print "The king catches you running away in the melee and orders a knight after you. You're no match for the horse's speed, and the king orders an execution on the spot. Game over. Just kidding, care to try again?"
      print "import scribe_level3.py" #it was the 'wrong' choice so it forces the player to restart the level

if 'Valuable warrior' in action: #this lays out the story for players that chose the second option
  print "The king considers your offer thoroughly. 'Hm' he ponders, 'We have been short a few heroes recently.' Suddenly a loud crash is heard outside. 'MILORD' the king's page screams, 'We are besieged by the neighboring kingdom!' The king turns to you, 'Alrighty lad, you say you'll be a valuable warrior, now's the time to prove it.'\n"
  courtyard = raw_input("Do you " + "\"Meet the enemy head on\"" + ' or do you stay back, nervously observing your surroundings and making a break for the " + "\"back gate\"" + ' while the king is distracted?")
  if 'Meet the enemy head on' in courtyard: #here's the first choice
      print " "
      print "You cautiously but confidently approach the enemy in the company of the king's guard. Quickly, you drawyour quill (no pun intended), causing the enemy to burst into laughter. Congratulations! Your heroic deeds have successfully cleared your name throughout all of Karflooglesville. But will you be able to fend off the guards? Stay tuned to the project to find out!"\n"
  if 'back gate' in courtyard:
      print " "
      print "The back gate is right behind the dungeon. Unfortunately, the dungeon master is privy to your plan and halts you. Your quill barely scratches his imposing armor and thick mask 'Back in the cell ya go, laddie,' he says, tossing you in the familiarity of your cell. Dammit, not again\n" #this is the end of our first go at the project, but if you're creating a new level obviously end it however you want
      print "import scribe_level1.py" #...it sends you back to the dungeon, so you restart the whole game.
<|endoftext|>


<commit_before>import os
import sys

os.environ['DJANGO_SETTINGS_MODULE'] = 'democracy.settings'
sys.path.append('/var/git/emo')
sys.path.append('/var/git/emo/democracy')
sys.path.append('/var/git/emo/external_apps')

import django.core.handlers.wsgi
application = django.core.handlers.wsgi.WSGIHandler()
<commit_msg>made portability changes<commit_after>import os
import sys

from os.path import abspath , dirname , join

# redirect prints to error log
sys.stdout = sys.stderr

os.environ['DJANGO_SETTINGS_MODULE'] = 'democracy.settings'

sys.path.append(abspath(join(dirname(__file__), "../../")))
sys.path.append(abspath(join(dirname(__file__), "../../external_apps")))
sys.path.append(abspath(join(dirname(__file__), "../")))

import django.core.handlers.wsgi
application = django.core.handlers.wsgi.WSGIHandler()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import errno
import os
import subprocess
import sys
import tempfile

from lib.config import PLATFORM, get_target_arch, get_chromedriver_version, \
                       get_platform_key, get_env_var
from lib.util import electron_gyp, execute, get_electron_version, parse_version, \
                     scoped_cwd
from lib.github import GitHub

ELECTRON_REPO = 'electron/electron'
ELECTRON_VERSION = get_electron_version()

PROJECT_NAME = electron_gyp()['project_name%']
PRODUCT_NAME = electron_gyp()['product_name%']

SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'R')
DIST_DIR = os.path.join(SOURCE_ROOT, 'dist')
DIST_NAME = '{0}-{1}-{2}-{3}.zip'.format(PROJECT_NAME,
                                         ELECTRON_VERSION,
                                         get_platform_key(),
                                         get_target_arch())
SYMBOLS_NAME = '{0}-{1}-{2}-{3}-symbols.zip'.format(PROJECT_NAME,
                                                    ELECTRON_VERSION,
                                                    get_platform_key(),
                                                    get_target_arch())
DSYM_NAME = '{0}-{1}-{2}-{3}-dsym.zip'.format(PROJECT_NAME,
                                              ELECTRON_VERSION,
                                              get_platform_key(),
                                              get_target_arch())

def main():
  args = parse_args()

  if not args.publish_release:
    if not dist_newer_than_head():
      create_dist = os.path.join(SOURCE_ROOT, 'script', 'create-dist.py')
      execute([sys.executable, create_dist])

    build_version = get_electron_build_version()
    if not ELECTRON_VERSION.startswith(build_version):
      error = 'Tag name ({0}) should match build version ({1})\n'.format(
          ELECTRON_VERSION, build_version)
      sys.stderr.write(error)
      sys.stderr.flush()
      return 1

  github = GitHub(auth_token())
  releases = github.repos(ELECTRON_REPO).releases.get()
  tag_exists = False
  for release in releases:
    if not release['draft'] and release['tag_name'] == args.version:
      tag_exists = True
      break

  release = create_or_get_release_draft(github, releases, args.version,
                                        tag_exists)

  if args.publish_release:
    # Upload the SHASUMS.txt.
    execute([sys.executable,
             os.path.join(SOURCE_ROOT, 'script', 'upload-checksums.py'),
             '-v', ELECTRON_VERSION])

    # Upload the index.json.
    execute([sys.executable,
             os.path.join(SOURCE_ROOT, 'script', 'upload-index-json.py')])

    # Press the publish button.
    publish_release(github, release['id'])

    # Do not upload other files when passed "-p".
    return

  # Upload Electron with GitHub Releases API.
  upload_electron(github, release, os.path.join(DIST_DIR, DIST_NAME))
  upload_electron(github, release, os.path.join(DIST_DIR, SYMBOLS_NAME))
  if PLATFORM == 'darwin':
    upload_electron(github, release, os.path.join(DIST_DIR, DSYM_NAME))

  # Upload free version of ffmpeg.
  ffmpeg = 'ffmpeg-{0}-{1}-{2}.zip'.format(
      ELECTRON_VERSION, get_platform_key(), get_target_arch())
  upload_electron(github, release, os.path.join(DIST_DIR, ffmpeg))

  # Upload chromedriver and mksnapshot for minor version update.
  if parse_version(args.version)[2] == '0':
    chromedriver = 'chromedriver-{0}-{1}-{2}.zip'.format(
        get_chromedriver_version(), get_platform_key(), get_target_arch())
    upload_electron(github, release, os.path.join(DIST_DIR, chromedriver))
    mksnapshot = 'mksnapshot-{0}-{1}-{2}.zip'.format(
        ELECTRON_VERSION, get_platform_key(), get_target_arch())
    upload_electron(github, release, os.path.join(DIST_DIR, mksnapshot))

  if PLATFORM == 'win32' and not tag_exists:
    # Upload PDBs to Windows symbol server.
    execute([sys.executable,
             os.path.join(SOURCE_ROOT, 'script', 'upload-windows-pdb.py')])

    # Upload node headers.
    execute([sys.executable,
             os.path.join(SOURCE_ROOT, 'script', 'upload-node-headers.py'),
             '-v', args.version])

def parse_args():
  parser = argparse.ArgumentParser(description='upload distribution file')
  parser.add_argument('-v', '--version', help='Specify the version',
                      default=ELECTRON_VERSION)
  parser.add_argument('-p', '--publish-release',
                      help='Publish the release',
                      action='store_true')
  return parser.parse_args()

def get_electron_build_version():
  if get_target_arch() == 'arm' or os.environ.has_key('CI'):
    # In CI we just build as told.
    return ELECTRON_VERSION
  if PLATFORM == 'darwin':
    electron = os.path.join(SOURCE_ROOT, 'out', 'R',
                              '{0}.app'.format(PRODUCT_NAME), 'Contents',
                              'MacOS', PRODUCT_NAME)
  elif PLATFORM == 'win32':
    electron = os.path.join(SOURCE_ROOT, 'out', 'R',
                              '{0}.exe'.format(PROJECT_NAME))
  else:
    electron = os.path.join(SOURCE_ROOT, 'out', 'R', PROJECT_NAME)

  return subprocess.check_output([electron, '--version']).strip()

def dist_newer_than_head():
  with scoped_cwd(SOURCE_ROOT):
    try:
      head_time = subprocess.check_output(['git', 'log', '--pretty=format:%at',
                                           '-n', '1']).strip()
      dist_time = os.path.getmtime(os.path.join(DIST_DIR, DIST_NAME))
    except OSError as e:
      if e.errno != errno.ENOENT:
        raise
      return False

  return dist_time > int(head_time)

def get_text_with_editor(name):
  editor = os.environ.get('EDITOR', 'nano')
  initial_message = '\n# Please enter the body of your release note for %s.' \
                    % name

  t = tempfile.NamedTemporaryFile(suffix='.tmp', delete=False)
  t.write(initial_message)
  t.close()
  subprocess.call([editor, t.name])

  text = ''
  for line in open(t.name, 'r'):
    if len(line) == 0 or line[0] != '#':
      text += line

  os.unlink(t.name)
  return text

def create_or_get_release_draft(github, releases, tag, tag_exists):
  # Search for existing draft.
  for release in releases:
    if release['draft']:
      return release

  if tag_exists:
    tag = 'do-not-publish-me'
  return create_release_draft(github, tag)

def create_release_draft(github, tag):
  name = '{0} {1}'.format(PROJECT_NAME, tag)
  if os.environ.has_key('CI'):
    body = '(placeholder)'
  else:
    body = get_text_with_editor(name)
  if body == '':
    sys.stderr.write('Quit due to empty release note.\n')
    sys.exit(0)

  data = dict(tag_name=tag, name=name, body=body, draft=True)
  r = github.repos(ELECTRON_REPO).releases.post(data=data)
  return r

def upload_electron(github, release, file_path):
  # Delete the original file before uploading in CI.
  if os.environ.has_key('CI'):
    try:
      for asset in release['assets']:
        if asset['name'] == os.path.basename(file_path):
          github.repos(ELECTRON_REPO).releases.assets(asset['id']).delete()
          break
    except Exception:
      pass

  # Upload the file.
  params = {'name': os.path.basename(file_path)}
  headers = {'Content-Type': 'application/zip'}
  with open(file_path, 'rb') as f:
    github.repos(ELECTRON_REPO).releases(release['id']).assets.post(
        params=params, headers=headers, data=f, verify=False)

def publish_release(github, release_id):
  data = dict(draft=False)
  github.repos(ELECTRON_REPO).releases(release_id).patch(data=data)

def auth_token():
  token = get_env_var('GITHUB_TOKEN')
  message = ('Error: Please set the $ELECTRON_GITHUB_TOKEN '
             'environment variable, which is your personal token')
  assert token, message
  return token

if __name__ == '__main__':
  import sys
  sys.exit(main())
<commit_msg>Fix pylint warnings<commit_after>#!/usr/bin/env python

import argparse
import errno
import os
import subprocess
import sys
import tempfile

from lib.config import PLATFORM, get_target_arch, get_chromedriver_version, \
                       get_platform_key, get_env_var
from lib.util import electron_gyp, execute, get_electron_version, \
                     parse_version, scoped_cwd
from lib.github import GitHub

ELECTRON_REPO = 'electron/electron'
ELECTRON_VERSION = get_electron_version()

PROJECT_NAME = electron_gyp()['project_name%']
PRODUCT_NAME = electron_gyp()['product_name%']

SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'R')
DIST_DIR = os.path.join(SOURCE_ROOT, 'dist')
DIST_NAME = '{0}-{1}-{2}-{3}.zip'.format(PROJECT_NAME,
                                         ELECTRON_VERSION,
                                         get_platform_key(),
                                         get_target_arch())
SYMBOLS_NAME = '{0}-{1}-{2}-{3}-symbols.zip'.format(PROJECT_NAME,
                                                    ELECTRON_VERSION,
                                                    get_platform_key(),
                                                    get_target_arch())
DSYM_NAME = '{0}-{1}-{2}-{3}-dsym.zip'.format(PROJECT_NAME,
                                              ELECTRON_VERSION,
                                              get_platform_key(),
                                              get_target_arch())

def main():
  args = parse_args()

  if not args.publish_release:
    if not dist_newer_than_head():
      create_dist = os.path.join(SOURCE_ROOT, 'script', 'create-dist.py')
      execute([sys.executable, create_dist])

    build_version = get_electron_build_version()
    if not ELECTRON_VERSION.startswith(build_version):
      error = 'Tag name ({0}) should match build version ({1})\n'.format(
          ELECTRON_VERSION, build_version)
      sys.stderr.write(error)
      sys.stderr.flush()
      return 1

  github = GitHub(auth_token())
  releases = github.repos(ELECTRON_REPO).releases.get()
  tag_exists = False
  for release in releases:
    if not release['draft'] and release['tag_name'] == args.version:
      tag_exists = True
      break

  release = create_or_get_release_draft(github, releases, args.version,
                                        tag_exists)

  if args.publish_release:
    # Upload the SHASUMS.txt.
    execute([sys.executable,
             os.path.join(SOURCE_ROOT, 'script', 'upload-checksums.py'),
             '-v', ELECTRON_VERSION])

    # Upload the index.json.
    execute([sys.executable,
             os.path.join(SOURCE_ROOT, 'script', 'upload-index-json.py')])

    # Press the publish button.
    publish_release(github, release['id'])

    # Do not upload other files when passed "-p".
    return

  # Upload Electron with GitHub Releases API.
  upload_electron(github, release, os.path.join(DIST_DIR, DIST_NAME))
  upload_electron(github, release, os.path.join(DIST_DIR, SYMBOLS_NAME))
  if PLATFORM == 'darwin':
    upload_electron(github, release, os.path.join(DIST_DIR, DSYM_NAME))

  # Upload free version of ffmpeg.
  ffmpeg = 'ffmpeg-{0}-{1}-{2}.zip'.format(
      ELECTRON_VERSION, get_platform_key(), get_target_arch())
  upload_electron(github, release, os.path.join(DIST_DIR, ffmpeg))

  # Upload chromedriver and mksnapshot for minor version update.
  if parse_version(args.version)[2] == '0':
    chromedriver = 'chromedriver-{0}-{1}-{2}.zip'.format(
        get_chromedriver_version(), get_platform_key(), get_target_arch())
    upload_electron(github, release, os.path.join(DIST_DIR, chromedriver))
    mksnapshot = 'mksnapshot-{0}-{1}-{2}.zip'.format(
        ELECTRON_VERSION, get_platform_key(), get_target_arch())
    upload_electron(github, release, os.path.join(DIST_DIR, mksnapshot))

  if PLATFORM == 'win32' and not tag_exists:
    # Upload PDBs to Windows symbol server.
    execute([sys.executable,
             os.path.join(SOURCE_ROOT, 'script', 'upload-windows-pdb.py')])

    # Upload node headers.
    execute([sys.executable,
             os.path.join(SOURCE_ROOT, 'script', 'upload-node-headers.py'),
             '-v', args.version])

def parse_args():
  parser = argparse.ArgumentParser(description='upload distribution file')
  parser.add_argument('-v', '--version', help='Specify the version',
                      default=ELECTRON_VERSION)
  parser.add_argument('-p', '--publish-release',
                      help='Publish the release',
                      action='store_true')
  return parser.parse_args()

def get_electron_build_version():
  if get_target_arch() == 'arm' or os.environ.has_key('CI'):
    # In CI we just build as told.
    return ELECTRON_VERSION
  if PLATFORM == 'darwin':
    electron = os.path.join(SOURCE_ROOT, 'out', 'R',
                              '{0}.app'.format(PRODUCT_NAME), 'Contents',
                              'MacOS', PRODUCT_NAME)
  elif PLATFORM == 'win32':
    electron = os.path.join(SOURCE_ROOT, 'out', 'R',
                              '{0}.exe'.format(PROJECT_NAME))
  else:
    electron = os.path.join(SOURCE_ROOT, 'out', 'R', PROJECT_NAME)

  return subprocess.check_output([electron, '--version']).strip()

def dist_newer_than_head():
  with scoped_cwd(SOURCE_ROOT):
    try:
      head_time = subprocess.check_output(['git', 'log', '--pretty=format:%at',
                                           '-n', '1']).strip()
      dist_time = os.path.getmtime(os.path.join(DIST_DIR, DIST_NAME))
    except OSError as e:
      if e.errno != errno.ENOENT:
        raise
      return False

  return dist_time > int(head_time)

def get_text_with_editor(name):
  editor = os.environ.get('EDITOR', 'nano')
  initial_message = '\n# Please enter the body of your release note for %s.' \
                    % name

  t = tempfile.NamedTemporaryFile(suffix='.tmp', delete=False)
  t.write(initial_message)
  t.close()
  subprocess.call([editor, t.name])

  text = ''
  for line in open(t.name, 'r'):
    if len(line) == 0 or line[0] != '#':
      text += line

  os.unlink(t.name)
  return text

def create_or_get_release_draft(github, releases, tag, tag_exists):
  # Search for existing draft.
  for release in releases:
    if release['draft']:
      return release

  if tag_exists:
    tag = 'do-not-publish-me'
  return create_release_draft(github, tag)

def create_release_draft(github, tag):
  name = '{0} {1}'.format(PROJECT_NAME, tag)
  if os.environ.has_key('CI'):
    body = '(placeholder)'
  else:
    body = get_text_with_editor(name)
  if body == '':
    sys.stderr.write('Quit due to empty release note.\n')
    sys.exit(0)

  data = dict(tag_name=tag, name=name, body=body, draft=True)
  r = github.repos(ELECTRON_REPO).releases.post(data=data)
  return r

def upload_electron(github, release, file_path):
  # Delete the original file before uploading in CI.
  if os.environ.has_key('CI'):
    try:
      for asset in release['assets']:
        if asset['name'] == os.path.basename(file_path):
          github.repos(ELECTRON_REPO).releases.assets(asset['id']).delete()
          break
    except Exception:
      pass

  # Upload the file.
  params = {'name': os.path.basename(file_path)}
  headers = {'Content-Type': 'application/zip'}
  with open(file_path, 'rb') as f:
    github.repos(ELECTRON_REPO).releases(release['id']).assets.post(
        params=params, headers=headers, data=f, verify=False)

def publish_release(github, release_id):
  data = dict(draft=False)
  github.repos(ELECTRON_REPO).releases(release_id).patch(data=data)

def auth_token():
  token = get_env_var('GITHUB_TOKEN')
  message = ('Error: Please set the $ELECTRON_GITHUB_TOKEN '
             'environment variable, which is your personal token')
  assert token, message
  return token

if __name__ == '__main__':
  import sys
  sys.exit(main())
<|endoftext|>


<commit_before>
class MinNodesNotRunning(Exception):
    """
    Exception corresponding to nodes not in running state.
    """
    def __init__(self, name, version, min_units, units):
        """
        Constructor.

        :param name: Name of the application
        :type name: str
        :param version: Application version
        :type version: str
        :param min_units: Minimum units required for application (in running
            state)
        :type min_units: int
        :param units: Actual deployed units.
        :type units: dict
        """
        self.name = name
        self.version = version
        self.min_units = min_units
        self.units = units
        self.message = 'Minimum of %d units for application:%s version:%s ' \
                       'were not found in running state.' % \
                       (self.min_units, self.name, self.version)
        super(MinNodesNotRunning, self).__init__(
            name, version, min_units, units)

    def to_dict(self):
        return {
            'message': 'Minimum of %d units for application:%s version:%s '
                       'were not found in running state.' %
                       (self.min_units, self.name, self.version),
            'code': 'MIN_NODES_NOT_RUNNING',
            'details': {
                'name': self.name,
                'version': self.version,
                'min-units': self.min_units,
                'units': self.units
                }
        }

class MinNodesNotDiscovered(Exception):
    """
    Exception corresponding to minimum nodes not being discovered.
    """
    def __init__(self, name, version, min_nodes, discovered_nodes):
        """
        Constructor.

        :param name: Name of the application
        :type name: str
        :param version: Application version
        :type version: str
        :param min_nodes: Minimum nodes required for application
        :type min_nodes: int
        :param discovered_nodes: Discovered nodes so far.
        :type discovered_nodes: dict
        """
        self.name = name
        self.version = version
        self.min_nodes = min_nodes
        self.discovered_nodes = discovered_nodes
        self.message = 'Minimum of %d nodes for application:%s version:%s ' \
                       'were not registered to yoda proxy.' % \
                       (self.min_nodes, self.name, self.version)
        super(MinNodesNotDiscovered, self).__init__(
            name, version, min_nodes, discovered_nodes)

    def to_dict(self):
        return {
            'message': self.message,
            'code': 'MIN_NODES_NOT_DISCOVERED',
            'details': {
                'name': self.name,
                'version': self.version,
                'min-nodes': self.min_nodes,
                'discovered-nodes': self.discovered_nodes
            }
        }

    def __str__(self):
        return self.message

class NodeNotUndeployed(Exception):
    """
    Exception corresponding to node still in deployed state.
    """

    def __init__(self, name, version, deployed_units):
        self.name = name
        self.version = version
        self.deployed_units = deployed_units
        self.message = 'Nodes for application:%s version:%s did not get ' \
                       'un-deployed' % (self.name, self.version)
        super(NodeNotUndeployed, self).__init__(
            name, version, deployed_units)

    def to_dict(self):
        return {
            'message': self.message,
            'code': 'NODE_NOT_UNDEPLOYED',
            'details': {
                'name': self.name,
                'version': self.version
            }
        }

    def __str__(self):
        return self.message

class NodeNotStopped(Exception):
    """
    Exception corresponding to node has not yet been stopped.
    """

    def __init__(self, name, version, active_units):
        self.name = name
        self.version = version
        self.deployed_units = active_units
        self.message = 'Nodes for application:%s version:%s did not get ' \
                       'stopped' % (self.name, self.version)
        super(NodeNotUndeployed, self).__init__(
            name, version, active_units)

    def to_dict(self):
        return {
            'message': self.message,
            'code': 'NODE_NOT_STOPPED',
            'details': {
                'name': self.name,
                'version': self.version
            }
        }

    def __str__(self):
        return self.message

class TaskExecutionException(Exception):
    """
    Exception wrapping the final exception returned.
    """

    def __init__(self, cause, traceback=None):
        try:
            dict_repr = cause.to_dict()
        except AttributeError:
            dict_repr = {}

        self.message = dict_repr.get('message', str(cause))
        self.code = dict_repr.get('code', 'INTERNAL')
        self.traceback = traceback
        self.details = dict_repr.get('details', None)
        super(TaskExecutionException, self).__init__(cause, traceback)

    def to_dict(self):
        return {
            'message': self.message,
            'code': self.code,
            'details': self.details,
            'traceback': self.traceback
        }

    def __str__(self):
        return self.message

class NodeCheckFailed(Exception):
    """
    Exception corresponding to failed deployment check for a given node.
    """

    def __init__(self, url, reason, status=None, response=None):
        self.url = url
        self.message = 'Deployment check failed for url: {0} due to: {1}'\
            .format(url, reason)
        self.status = status
        self.response = response
        super(NodeCheckFailed, self).__init__(url, reason, status, response)

    def to_dict(self):
        return {
            'message': self.message,
            'code': 'NODE_CHECK_FAILED',
            'details': {
                'url': self.url,
                'status': self.status,
                'response': self.response
            }
        }

    def __str__(self):
        return self.message

    def __eq__(self, other):
        return self.status == other.status and \
            self.message == other.message and \
            self.response == other.response and \
            self.url == other.url
<commit_msg>Fix exception: NodeNotStopped<commit_after>
class MinNodesNotRunning(Exception):
    """
    Exception corresponding to nodes not in running state.
    """
    def __init__(self, name, version, min_units, units):
        """
        Constructor.

        :param name: Name of the application
        :type name: str
        :param version: Application version
        :type version: str
        :param min_units: Minimum units required for application (in running
            state)
        :type min_units: int
        :param units: Actual deployed units.
        :type units: dict
        """
        self.name = name
        self.version = version
        self.min_units = min_units
        self.units = units
        self.message = 'Minimum of %d units for application:%s version:%s ' \
                       'were not found in running state.' % \
                       (self.min_units, self.name, self.version)
        super(MinNodesNotRunning, self).__init__(
            name, version, min_units, units)

    def to_dict(self):
        return {
            'message': 'Minimum of %d units for application:%s version:%s '
                       'were not found in running state.' %
                       (self.min_units, self.name, self.version),
            'code': 'MIN_NODES_NOT_RUNNING',
            'details': {
                'name': self.name,
                'version': self.version,
                'min-units': self.min_units,
                'units': self.units
                }
        }

class MinNodesNotDiscovered(Exception):
    """
    Exception corresponding to minimum nodes not being discovered.
    """
    def __init__(self, name, version, min_nodes, discovered_nodes):
        """
        Constructor.

        :param name: Name of the application
        :type name: str
        :param version: Application version
        :type version: str
        :param min_nodes: Minimum nodes required for application
        :type min_nodes: int
        :param discovered_nodes: Discovered nodes so far.
        :type discovered_nodes: dict
        """
        self.name = name
        self.version = version
        self.min_nodes = min_nodes
        self.discovered_nodes = discovered_nodes
        self.message = 'Minimum of %d nodes for application:%s version:%s ' \
                       'were not registered to yoda proxy.' % \
                       (self.min_nodes, self.name, self.version)
        super(MinNodesNotDiscovered, self).__init__(
            name, version, min_nodes, discovered_nodes)

    def to_dict(self):
        return {
            'message': self.message,
            'code': 'MIN_NODES_NOT_DISCOVERED',
            'details': {
                'name': self.name,
                'version': self.version,
                'min-nodes': self.min_nodes,
                'discovered-nodes': self.discovered_nodes
            }
        }

    def __str__(self):
        return self.message

class NodeNotUndeployed(Exception):
    """
    Exception corresponding to node still in deployed state.
    """

    def __init__(self, name, version, deployed_units):
        self.name = name
        self.version = version
        self.deployed_units = deployed_units
        self.message = 'Nodes for application:%s version:%s did not get ' \
                       'un-deployed' % (self.name, self.version)
        super(NodeNotUndeployed, self).__init__(
            name, version, deployed_units)

    def to_dict(self):
        return {
            'message': self.message,
            'code': 'NODE_NOT_UNDEPLOYED',
            'details': {
                'name': self.name,
                'version': self.version
            }
        }

    def __str__(self):
        return self.message

class NodeNotStopped(Exception):
    """
    Exception corresponding to node has not yet been stopped.
    """

    def __init__(self, name, version, active_units):
        self.name = name
        self.version = version
        self.deployed_units = active_units
        self.message = 'Nodes for application:%s version:%s did not get ' \
                       'stopped' % (self.name, self.version)
        super(NodeNotStopped, self).__init__(
            name, version, active_units)

    def to_dict(self):
        return {
            'message': self.message,
            'code': 'NODE_NOT_STOPPED',
            'details': {
                'name': self.name,
                'version': self.version
            }
        }

    def __str__(self):
        return self.message

class TaskExecutionException(Exception):
    """
    Exception wrapping the final exception returned.
    """

    def __init__(self, cause, traceback=None):
        try:
            dict_repr = cause.to_dict()
        except AttributeError:
            dict_repr = {}

        self.message = dict_repr.get('message', str(cause))
        self.code = dict_repr.get('code', 'INTERNAL')
        self.traceback = traceback
        self.details = dict_repr.get('details', None)
        super(TaskExecutionException, self).__init__(cause, traceback)

    def to_dict(self):
        return {
            'message': self.message,
            'code': self.code,
            'details': self.details,
            'traceback': self.traceback
        }

    def __str__(self):
        return self.message

class NodeCheckFailed(Exception):
    """
    Exception corresponding to failed deployment check for a given node.
    """

    def __init__(self, url, reason, status=None, response=None):
        self.url = url
        self.message = 'Deployment check failed for url: {0} due to: {1}'\
            .format(url, reason)
        self.status = status
        self.response = response
        super(NodeCheckFailed, self).__init__(url, reason, status, response)

    def to_dict(self):
        return {
            'message': self.message,
            'code': 'NODE_CHECK_FAILED',
            'details': {
                'url': self.url,
                'status': self.status,
                'response': self.response
            }
        }

    def __str__(self):
        return self.message

    def __eq__(self, other):
        return self.status == other.status and \
            self.message == other.message and \
            self.response == other.response and \
            self.url == other.url
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2018 The Google AI Language Team Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import unittest

import torch

from pytorch_pretrained_bert import BertAdam

class OptimizationTest(unittest.TestCase):

    def assertListAlmostEqual(self, list1, list2, tol):
        self.assertEqual(len(list1), len(list2))
        for a, b in zip(list1, list2):
            self.assertAlmostEqual(a, b, delta=tol)

    def test_adam(self):
        w = torch.tensor([0.1, -0.2, -0.1], requires_grad=True)
        target = torch.tensor([0.4, 0.2, -0.5])
        criterion = torch.nn.MSELoss(reduction='elementwise_mean')
        # No warmup, constant schedule, no gradient clipping
        optimizer = BertAdam(params=[w], lr=2e-1,
                                          weight_decay=0.0,
                                          max_grad_norm=-1)
        for _ in range(100):
            loss = criterion(w, target)
            loss.backward()
            optimizer.step()
            w.grad.detach_() # No zero_grad() function on simple tensors. we do it ourselves.
            w.grad.zero_()
        self.assertListAlmostEqual(w.tolist(), [0.4, 0.2, -0.5], tol=1e-2)

if __name__ == "__main__":
    unittest.main()
<commit_msg>compatibility PT 1.0 and 0.4.1<commit_after># coding=utf-8
# Copyright 2018 The Google AI Language Team Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import unittest

import torch

from pytorch_pretrained_bert import BertAdam

class OptimizationTest(unittest.TestCase):

    def assertListAlmostEqual(self, list1, list2, tol):
        self.assertEqual(len(list1), len(list2))
        for a, b in zip(list1, list2):
            self.assertAlmostEqual(a, b, delta=tol)

    def test_adam(self):
        w = torch.tensor([0.1, -0.2, -0.1], requires_grad=True)
        target = torch.tensor([0.4, 0.2, -0.5])
        criterion = torch.nn.MSELoss()
        # No warmup, constant schedule, no gradient clipping
        optimizer = BertAdam(params=[w], lr=2e-1,
                                          weight_decay=0.0,
                                          max_grad_norm=-1)
        for _ in range(100):
            loss = criterion(w, target)
            loss.backward()
            optimizer.step()
            w.grad.detach_() # No zero_grad() function on simple tensors. we do it ourselves.
            w.grad.zero_()
        self.assertListAlmostEqual(w.tolist(), [0.4, 0.2, -0.5], tol=1e-2)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>