<commit_before># -*- coding: utf-8 -*-

from bs4 import BeautifulSoup
from datetime import datetime
import re

"""A module encapsulating the functions used by the media parsers to scrape data from websites."""

def decompose( html_element ):
    """Decomposes a html element if the element's type is not None.

    Parameters:
        html_element - A BeautifulSoup tag.
    """
    if html_element != None:
        html_element.decompose()
    else:
        return

def decompose_all( html_elements ):
    """Iterates over a list of html elements and decomposes the elements if their type is not None.

    Parameters:
        html_elements - A list of BeautifulSoup tags.
    """
    for element in html_elements:
        if element != None:
            element.decompose()
        else:
            continue

def collect_categories( html_elements, decompose ):
    """Iterates over a list of html elements and collects their contained text using the collect_text() function.

    Returns the collected texts as a list of strings.

    Parameters:
        html_elements - A list of BeautifulSoup tags.
        decompose - A boolean determining whether the html elements should be decomposed after handling.
    """
    categories = [None]

    for element in html_elements:
        if element != None:
            categories.append( collect_text( element, decompose ) )

    categories.pop(0)
    return categories

def collect_datetime( html_element, datetime_format ):
    """Collects all datetime strings contained within a html element using the re module,
    and returns them as a list of datetime objects.

    Uses the list_datetime_objects() function to create the datetime object list from datetime strings.

    Parameters:
        html_element - A BeautifulSoup tag containing datetime strings.
        datetime_format - A string determining the format of the datetime strings on the website.
    """
    if datetime_format == 'timedate':
        datetime_expression = r'([0-2]?[0-9][.:][0-6][0-9])[^0-9]*([0-3]?[0-9]\.[0-1]?[0-9]\.[1-2]?[09]?[0-9]?[0-9]?)'
    elif datetime_format == 'datetime date':
        datetime_expression = r'([0-3]?[0-9]\.[0-1]?[0-9]\.[1-2]?[09]?[0-9]?[0-9]?)?[^0-9]*([0-2]?[0-9][.:][0-6][0-9])?'
    else:
        datetime_expression = r'([0-3]?[0-9]\.[0-1]?[0-9]\.[1-2]?[09]?[0-9]?[0-9]?)[^0-9]*([0-2]?[0-9][.:][0-6][0-9])?'

    datetime_list = []
    if html_element != None:
        datetime_string = html_element.get_text( ' ' , strip = True )
        match_list = re.findall( datetime_expression, datetime_string )
        datetime_list = list_datetime_objects( match_list, datetime_format )

    return datetime_list

def collect_datetime_objects( html_elements, attribute ):
    """Iterates over a list of html_elements and returns their timestamp attributes as a list of datetime objects.

    Uses the function prepare_datetime_list().

    Parameters:
        html_elements - A list of BeautifulSoup tags.
        attribute - A string name of the html element attribute containing the timestamp.
    """
    datetime_list = [None]

    for element in html_elements:
        if element != None and element.has_attr( attribute ):
            datetime_object = element[attribute].replace('T', ' ').replace('Z', '')
            datetime_object = datetime_object.split( '+' )[0].split('.')[0]
            datetime_list.append( datetime_object )
        else:
            continue

    return prepare_datetime_list( datetime_list )

def collect_datetime_json( json, published, updated ):
    """Collects the publish and update datetime objects from json and returns them as a list.

    Parameters:
        json - JSON containing the publish and update datetime objects.
        published - A string name of the attribute in the JSON with the publish datetime as its value.
        updated - A string name of the attribute in the JSON with the update datetime as its value.
    """
    datetime_list = [None]
    if published in json:
        datetime_list.append( strip_datetime_object( json[published] ) )
    if updated in json:
        datetime_list.append( strip_datetime_object( json[updated] ) )
    return prepare_datetime_list( datetime_list )

def collect_text( html_element, decompose ):
    """Returns the text contained in a html element if the type of the element is not None.

    Uses the function process().

    Parameters:
        html_element - A BeautifulSoup tag containing the text.
        decompose - A boolean determining whether the html elements should be decomposed after handling.
    """
    if html_element != None:
        text = html_element.get_text( ' ', strip=True )
        text = process(text)

        if decompose == True:
            html_element.decompose()
    else:
        text = ''

    return text

def collect_images( images, attribute, url_base ):
    """Iterates over a list of html image elements and returns their corresponding urls as a list of strings.

    Parameters:
        images - A list of BeautifulSoup tags.
        attribute - A string that gives the image element attribute which contains the image's url.
        url_base - A string that gives the base part of the image urls.
    """
    image_src = [None]
    for img in images:
        if img != None and img.has_attr( attribute ):
            image_src.append( '' + str( url_base + img[attribute].encode('utf8') ) )
    image_src.pop(0)
    return image_src

def collect_images_by_parent( html_elements, url_base ):
    """Iterates over a list of html elements containing image hyperlinks and collects their urls.

    Parameters:
        html_elements - A list of BeautifulSoup tags.
        url_base - A string that gives the base part of the image urls.
    """
    image_src = [None]
    for parent in html_elements:
        image_link = parent.find( 'a' )
        if image_link != None:
            image_src.append( '' + str( url_base + image_link['href'].encode('utf8') ) )
        else:
            continue
    image_src.pop(0)
    return image_src

def collect_image_captions( captions, decompose = False ):
    """Iterates over a list of html elements containing image captions and returns them as a list of strings.

    Uses the function collect_text().

    Parameters:
        captions - A list of BeautifulSoup tags containing the caption texts.
        decompose - A boolean determining whether the html elements should be decomposed after handling.
    """
    captions_text = [None]
    for caption in captions:
        if caption != None:
            captions_text.append( '' + collect_text( caption, decompose ) )

    captions_text.pop(0)
    return captions_text

def list_datetime_objects( match_list, datetime_format ):
    """Iterates over a list of datetime strings and returns a list of datetime objects created from them.

    Uses the functions create_datetime_object() and prepare_datetime_list().

    Parameters:
        match_list - A list containing datetime strings.
        datetime_format - A string determining the format of the datetime strings on the website.
    """
    datetime_list = [None]
    published_date = ''
    for match in match_list:
        if datetime_format == 'timedate':
            date = match[1]
            time = match[0]
        else:
            date = match[0]
            time = match[1]

        if date == '' and time == '':
            continue
        elif date == '':
            datetime_list.append( create_datetime_object( published_date + ' ' + time ) )
        elif time == '':
            datetime_list.append( create_datetime_object( date ) )
        else:
            datetime_list.append( create_datetime_object( date + ' ' + time ) )
            published_date = date

    return prepare_datetime_list( datetime_list )

def create_datetime_object( datetime_string ):
    """Returns a datetime object created from a datetime string.

    Parameters:
        datetime_string - A datetime in string format.
    """
    datetime_parts = datetime_string.split( ' ' )
    if len( datetime_parts[0] ) < 7 :
        datetime_parts[0] = datetime_parts[0] + '2016'

    if len( datetime_parts ) > 1:
        datetime_parts[1] = datetime_parts[1].replace( ':', '.' )
        datetime_object = datetime.strptime( datetime_parts[0] + ' ' + datetime_parts[1], '%d.%m.%Y %H.%M' )
    else:
        datetime_object = datetime.date( datetime.strptime( datetime_parts[0], '%d.%m.%Y' ) )
    return datetime_object

def strip_datetime_object( datetime_object ):
    """Formats a datetime object and returns the formatted object.

    Parameters:
        datetime_object - A datetime object to format.
    """
    datetime_object = datetime_object.replace('T', ' ').replace('Z', '')
    datetime_object = datetime_object.split( '+' )[0].split('.')[0]
    return datetime_object

def prepare_datetime_list( datetime_list ):
    """Removes the first member of a datetime_list and returns the list reversed.

    Parameters:
        datetime_list - A list of datetime objects.
    """
    datetime_list.pop(0)
    datetime_list.reverse()
    return datetime_list

def create_dictionary(domain, url, http_status, categories, datetime_list, author, title, ingress, text, images, captions):
    """Returns a dictionary containing the scraped data from the website, given as parameters.

    Parameters:
        domain - The domain of the media source
        url - The website url as a string object.
        http_status - The parser http request status.
        categories - A list of collected news item categories as strings.
        datetime_list - A list of datetime objects.
        author - The author of the news item as a string.
        title - The title of the news item as a string.
        ingress - The ingress of the news item as a string.
        text - The main text content of the news item as a string.
        images - A list of the source urls of the images in the news item as strings.
        captions - A list of the image captions in the news item as strings.
    """
    media_content = { 'domain' : domain,
                      'url' : url,
					  'http' : str( http_status ),
					  'categories' : categories,
					  'datetime_list' : datetime_list,
					  'author' : author,
					  'title' : title,
					  'ingress' : ingress,
					  'text' : text,
					  'images' : images,
					  'captions' : captions }
    return media_content

def process(content):
    """Performs character replace functions on scraped text and returns the text as a string.

    Parameters:
        content - Content text scraped from a website.
    """
    content = content.replace(' .', '.')
    content = content.replace(' ,', ',')
    content = content.replace('( ', '(')
    content = content.replace(' )', ')')
    content = content.replace(' !', '!')
    content = content.replace(' ?', '?')
    content = content.replace(' ;', ';')
    content = content.replace(' ]', ']')
    content = content.replace('[ ', '[')
    content = content.replace(' :', ':')
    content = content.replace('   ', ' ')
    content = content.replace('    ', '')
    content = content.replace('      ', ' ' )
    content = content.replace('\n', ' ')
    content = content.replace(' '.decode('utf8'), ' ')
    content = content.replace('  '.decode('utf8'), ' ')
    return str( content.encode('utf8') )

def convert_month(datetime_string):
    """Converts month substrings in a datetime string into their numeral equivalents.

    Parameters:
        datetime_string - A string expressing a datetime.
    """
    conversions = { 'syyskuu' : '09' }
    for conversion in conversions:
        datetime_string = datetime_string.replace( conversion, conversions[conversion] )
    return datetime_string
<commit_msg>Move utf-8 encoding to file writing method in common.py<commit_after># -*- coding: utf-8 -*-

from bs4 import BeautifulSoup
from datetime import datetime
import re

"""A module encapsulating the functions used by the media parsers to scrape data from websites."""

def decompose( html_element ):
    """Decomposes a html element if the element's type is not None.

    Parameters:
        html_element - A BeautifulSoup tag.
    """
    if html_element != None:
        html_element.decompose()
    else:
        return

def decompose_all( html_elements ):
    """Iterates over a list of html elements and decomposes the elements if their type is not None.

    Parameters:
        html_elements - A list of BeautifulSoup tags.
    """
    for element in html_elements:
        if element != None:
            element.decompose()
        else:
            continue

def collect_categories( html_elements, decompose ):
    """Iterates over a list of html elements and collects their contained text using the collect_text() function.

    Returns the collected texts as a list of strings.

    Parameters:
        html_elements - A list of BeautifulSoup tags.
        decompose - A boolean determining whether the html elements should be decomposed after handling.
    """
    categories = [None]

    for element in html_elements:
        if element != None:
            categories.append( collect_text( element, decompose ) )

    categories.pop(0)
    return categories

def collect_datetime( html_element, datetime_format ):
    """Collects all datetime strings contained within a html element using the re module,
    and returns them as a list of datetime objects.

    Uses the list_datetime_objects() function to create the datetime object list from datetime strings.

    Parameters:
        html_element - A BeautifulSoup tag containing datetime strings.
        datetime_format - A string determining the format of the datetime strings on the website.
    """
    if datetime_format == 'timedate':
        datetime_expression = r'([0-2]?[0-9][.:][0-6][0-9])[^0-9]*([0-3]?[0-9]\.[0-1]?[0-9]\.[1-2]?[09]?[0-9]?[0-9]?)'
    elif datetime_format == 'datetime date':
        datetime_expression = r'([0-3]?[0-9]\.[0-1]?[0-9]\.[1-2]?[09]?[0-9]?[0-9]?)?[^0-9]*([0-2]?[0-9][.:][0-6][0-9])?'
    else:
        datetime_expression = r'([0-3]?[0-9]\.[0-1]?[0-9]\.[1-2]?[09]?[0-9]?[0-9]?)[^0-9]*([0-2]?[0-9][.:][0-6][0-9])?'

    datetime_list = []
    if html_element != None:
        datetime_string = html_element.get_text( ' ' , strip = True )
        match_list = re.findall( datetime_expression, datetime_string )
        datetime_list = list_datetime_objects( match_list, datetime_format )

    return datetime_list

def collect_datetime_objects( html_elements, attribute ):
    """Iterates over a list of html_elements and returns their timestamp attributes as a list of datetime objects.

    Uses the function prepare_datetime_list().

    Parameters:
        html_elements - A list of BeautifulSoup tags.
        attribute - A string name of the html element attribute containing the timestamp.
    """
    datetime_list = [None]

    for element in html_elements:
        if element != None and element.has_attr( attribute ):
            datetime_object = element[attribute].replace('T', ' ').replace('Z', '')
            datetime_object = datetime_object.split( '+' )[0].split('.')[0]
            datetime_list.append( datetime_object )
        else:
            continue

    return prepare_datetime_list( datetime_list )

def collect_datetime_json( json, published, updated ):
    """Collects the publish and update datetime objects from json and returns them as a list.

    Parameters:
        json - JSON containing the publish and update datetime objects.
        published - A string name of the attribute in the JSON with the publish datetime as its value.
        updated - A string name of the attribute in the JSON with the update datetime as its value.
    """
    datetime_list = [None]
    if published in json:
        datetime_list.append( strip_datetime_object( json[published] ) )
    if updated in json:
        datetime_list.append( strip_datetime_object( json[updated] ) )
    return prepare_datetime_list( datetime_list )

def collect_text( html_element, decompose ):
    """Returns the text contained in a html element if the type of the element is not None.

    Uses the function process().

    Parameters:
        html_element - A BeautifulSoup tag containing the text.
        decompose - A boolean determining whether the html elements should be decomposed after handling.
    """
    if html_element != None:
        text = html_element.get_text( ' ', strip=True )
        text = process(text)

        if decompose == True:
            html_element.decompose()
    else:
        text = ''

    return u'' + text

def collect_images( images, attribute, url_base ):
    """Iterates over a list of html image elements and returns their corresponding urls as a list of strings.

    Parameters:
        images - A list of BeautifulSoup tags.
        attribute - A string that gives the image element attribute which contains the image's url.
        url_base - A string that gives the base part of the image urls.
    """
    image_src = [None]
    for img in images:
        if img != None and img.has_attr( attribute ):
            image_src.append( url_base + img[attribute] )
    image_src.pop(0)
    return image_src

def collect_images_by_parent( html_elements, url_base ):
    """Iterates over a list of html elements containing image hyperlinks and collects their urls.

    Parameters:
        html_elements - A list of BeautifulSoup tags.
        url_base - A string that gives the base part of the image urls.
    """
    image_src = [None]
    for parent in html_elements:
        image_link = parent.find( 'a' )
        if image_link != None:
            image_src.append( url_base + image_link['href'] )
        else:
            continue
    image_src.pop(0)
    return image_src

def collect_image_captions( captions, decompose = False ):
    """Iterates over a list of html elements containing image captions and returns them as a list of strings.

    Uses the function collect_text().

    Parameters:
        captions - A list of BeautifulSoup tags containing the caption texts.
        decompose - A boolean determining whether the html elements should be decomposed after handling.
    """
    captions_text = [None]
    for caption in captions:
        if caption != None:
            captions_text.append( '' + collect_text( caption, decompose ) )

    captions_text.pop(0)
    return captions_text

def list_datetime_objects( match_list, datetime_format ):
    """Iterates over a list of datetime strings and returns a list of datetime objects created from them.

    Uses the functions create_datetime_object() and prepare_datetime_list().

    Parameters:
        match_list - A list containing datetime strings.
        datetime_format - A string determining the format of the datetime strings on the website.
    """
    datetime_list = [None]
    published_date = ''
    for match in match_list:
        if datetime_format == 'timedate':
            date = match[1]
            time = match[0]
        else:
            date = match[0]
            time = match[1]

        if date == '' and time == '':
            continue
        elif date == '':
            datetime_list.append( create_datetime_object( published_date + ' ' + time ) )
        elif time == '':
            datetime_list.append( create_datetime_object( date ) )
        else:
            datetime_list.append( create_datetime_object( date + ' ' + time ) )
            published_date = date

    return prepare_datetime_list( datetime_list )

def create_datetime_object( datetime_string ):
    """Returns a datetime object created from a datetime string.

    Parameters:
        datetime_string - A datetime in string format.
    """
    datetime_parts = datetime_string.split( ' ' )
    if len( datetime_parts[0] ) < 7 :
        datetime_parts[0] = datetime_parts[0] + '2016'

    if len( datetime_parts ) > 1:
        datetime_parts[1] = datetime_parts[1].replace( ':', '.' )
        datetime_object = datetime.strptime( datetime_parts[0] + ' ' + datetime_parts[1], '%d.%m.%Y %H.%M' )
    else:
        datetime_object = datetime.date( datetime.strptime( datetime_parts[0], '%d.%m.%Y' ) )
    return datetime_object

def strip_datetime_object( datetime_object ):
    """Formats a datetime object and returns the formatted object.

    Parameters:
        datetime_object - A datetime object to format.
    """
    datetime_object = datetime_object.replace('T', ' ').replace('Z', '')
    datetime_object = datetime_object.split( '+' )[0].split('.')[0]
    return datetime_object

def prepare_datetime_list( datetime_list ):
    """Removes the first member of a datetime_list and returns the list reversed.

    Parameters:
        datetime_list - A list of datetime objects.
    """
    datetime_list.pop(0)
    datetime_list.reverse()
    return datetime_list

def create_dictionary(domain, url, http_status, categories, datetime_list, author, title, ingress, text, images, captions):
    """Returns a dictionary containing the scraped data from the website, given as parameters.

    Parameters:
        domain - The domain of the media source
        url - The website url as a string object.
        http_status - The parser http request status.
        categories - A list of collected news item categories as strings.
        datetime_list - A list of datetime objects.
        author - The author of the news item as a string.
        title - The title of the news item as a string.
        ingress - The ingress of the news item as a string.
        text - The main text content of the news item as a string.
        images - A list of the source urls of the images in the news item as strings.
        captions - A list of the image captions in the news item as strings.
    """
    media_content = { 'domain' : domain,
                      'url' : url,
					  'http' : http_status,
					  'categories' : categories,
					  'datetime_list' : datetime_list,
					  'author' : author,
					  'title' : title,
					  'ingress' : ingress,
					  'text' : text,
					  'images' : images,
					  'captions' : captions }
    return media_content

def process(content):
    """Performs character replace functions on scraped text and returns the text as a string.

    Parameters:
        content - Content text scraped from a website.
    """
    content = content.replace(' .', '.')
    content = content.replace(' ,', ',')
    content = content.replace('( ', '(')
    content = content.replace(' )', ')')
    content = content.replace(' !', '!')
    content = content.replace(' ?', '?')
    content = content.replace(' ;', ';')
    content = content.replace(' ]', ']')
    content = content.replace('[ ', '[')
    content = content.replace(' :', ':')
    content = content.replace('   ', ' ')
    content = content.replace('    ', '')
    content = content.replace('      ', ' ' )
    content = content.replace('\n', ' ')
    content = content.replace(' '.decode('utf-8'), ' ')
    content = content.replace('  '.decode('utf-8'), ' ')
    return content

def convert_month(datetime_string):
    """Converts month substrings in a datetime string into their numeral equivalents.

    Parameters:
        datetime_string - A string expressing a datetime.
    """
    conversions = { 'syyskuu' : '09' }
    for conversion in conversions:
        datetime_string = datetime_string.replace( conversion, conversions[conversion] )
    return datetime_string
<|endoftext|>


<commit_before>import unittest

from chainer import testing
import numpy

import einsum

def _from_str_subscript(subscript):
    # subscript should be lower case (a-z)
    return [
        (Ellipsis if char == '@' else ord(char) - ord('a'))
        for char in subscript.replace('...', '@')
    ]

@testing.parameterize(*testing.product_dict(
    [
        {'subscripts': 'ij,jk->ik', 'shapes': ((2, 3), (3, 4))},
        {'subscripts': ',ij->i', 'shapes': ((), (3, 4),)},
        {'subscripts': 'kj,ji->ik', 'shapes': ((2, 3), (3, 4))},
        {'subscripts': 'ij,jk,kl->il', 'shapes': ((5, 2), (2, 3), (3, 4))},
        {'subscripts': 'ij,ij->i', 'shapes': ((2, 3), (2, 3))},
        {'subscripts': 'ij,jk', 'shapes': ((2, 3), (3, 4))},
        {'subscripts': 'i->', 'shapes': ((3,),)},
        {'subscripts': 'ii', 'shapes': ((2, 2),)},
        {'subscripts': 'ii->i', 'shapes': ((2, 2),)},
        {'subscripts': 'j,j', 'shapes': ((3,), (3))},
        {'subscripts': 'j,ij', 'shapes': ((3,), (2, 3))},
        {'subscripts': 'j,iij', 'shapes': ((3,), (2, 2, 3))},
        {'subscripts': 'iij,kkj', 'shapes': ((2, 2, 3), (4, 4, 3))},
        {'subscripts': '...ij,...jk->...ik',
         'shapes': ((2, 1, 2, 3), (2, 1, 3, 4))},
        {'subscripts': 'i...j,jk...->k...i', 'shapes': ((4, 2, 3), (3, 5, 2))},
        {'subscripts': 'ii...,...jj', 'shapes': ((2, 2, 4), (4, 3, 3))},
        {'subscripts': '...i,i', 'shapes': ((2, 2, 3), (3,))},
        {'subscripts': 'i...,i->...i', 'shapes': ((3, 2, 2), (3,))},
    ],
    testing.product({
        'dtype': [numpy.float32, numpy.float64],
        'subscript_type': ['str', 'int'],
    }),
))
class TestEinSum(unittest.TestCase):

    def setUp(self):
        self.inputs = tuple([
            self._setup_tensor(-1, 1, shape, self.dtype)
            for shape in self.shapes
        ])

    def _get_args(self, xs):
        if self.subscript_type == 'str':
            return (self.subscripts,) + xs
        else:
            args = []
            subscripts = self.subscripts.split('->')
            for in_subscript, x in zip(subscripts[0].split(','), xs):
                args.extend([x, _from_str_subscript(in_subscript)])
            if len(subscripts) == 2:
                args.append(_from_str_subscript(subscripts[1]))
            return tuple(args)

    def _setup_tensor(self, _min, _max, shape, dtype):
        return numpy.random.uniform(_min, _max, shape).astype(dtype)

    def test_forward(self, atol=1e-4, rtol=1e-5):
        forward_answer = numpy.einsum(*self._get_args(self.inputs))
        out = einsum.einsum(*self._get_args(self.inputs))
        testing.assert_allclose(out, forward_answer, atol, rtol)

        if isinstance(forward_answer, numpy.ndarray):  # not 0-dim
            # test views
            forward_answer[...] = 0
            out[...] = 0
            testing.assert_allclose(out, forward_answer, atol, rtol)

            forward_answer[...] = 1
            out[...] = 1
            testing.assert_allclose(out, forward_answer, atol, rtol)

testing.run_module(__name__, __file__)
<commit_msg>Fix view test<commit_after>import unittest

from chainer import testing
import numpy

import einsum

def _from_str_subscript(subscript):
    # subscript should be lower case (a-z)
    return [
        (Ellipsis if char == '@' else ord(char) - ord('a'))
        for char in subscript.replace('...', '@')
    ]

@testing.parameterize(*testing.product_dict(
    [
        {'subscripts': 'ij,jk->ik', 'shapes': ((2, 3), (3, 4))},
        {'subscripts': ',ij->i', 'shapes': ((), (3, 4),)},
        {'subscripts': 'kj,ji->ik', 'shapes': ((2, 3), (3, 4))},
        {'subscripts': 'ij,jk,kl->il', 'shapes': ((5, 2), (2, 3), (3, 4))},
        {'subscripts': 'ij,ij->i', 'shapes': ((2, 3), (2, 3))},
        {'subscripts': 'ij,jk', 'shapes': ((2, 3), (3, 4))},
        {'subscripts': 'i->', 'shapes': ((3,),)},
        {'subscripts': 'ii', 'shapes': ((2, 2),)},
        {'subscripts': 'ii->i', 'shapes': ((2, 2),)},
        {'subscripts': 'ij', 'shapes': ((2, 3),)},
        {'subscripts': 'j,j', 'shapes': ((3,), (3))},
        {'subscripts': 'j,ij', 'shapes': ((3,), (2, 3))},
        {'subscripts': 'j,iij', 'shapes': ((3,), (2, 2, 3))},
        {'subscripts': 'iij,kkj', 'shapes': ((2, 2, 3), (4, 4, 3))},
        {'subscripts': '...ij,...jk->...ik',
         'shapes': ((2, 1, 2, 3), (2, 1, 3, 4))},
        {'subscripts': 'i...j,jk...->k...i', 'shapes': ((4, 2, 3), (3, 5, 2))},
        {'subscripts': 'ii...,...jj', 'shapes': ((2, 2, 4), (4, 3, 3))},
        {'subscripts': '...i,i', 'shapes': ((2, 2, 3), (3,))},
        {'subscripts': 'i...,i->...i', 'shapes': ((3, 2, 2), (3,))},
    ],
    testing.product({
        'dtype': [numpy.float32, numpy.float64],
        'subscript_type': ['str', 'int'],
    }),
))
class TestEinSum(unittest.TestCase):

    def setUp(self):
        self.inputs = tuple([
            self._setup_tensor(-1, 1, shape, self.dtype)
            for shape in self.shapes
        ])

    def _get_args(self, xs):
        if self.subscript_type == 'str':
            return (self.subscripts,) + xs
        else:
            args = []
            subscripts = self.subscripts.split('->')
            for in_subscript, x in zip(subscripts[0].split(','), xs):
                args.extend([x, _from_str_subscript(in_subscript)])
            if len(subscripts) == 2:
                args.append(_from_str_subscript(subscripts[1]))
            return tuple(args)

    def _setup_tensor(self, _min, _max, shape, dtype):
        return numpy.random.uniform(_min, _max, shape).astype(dtype)

    def test_forward(self, atol=1e-4, rtol=1e-5):
        inputs_ans = tuple(arr.copy() for arr in self.inputs)
        forward_answer = numpy.einsum(*self._get_args(inputs_ans))
        inputs = self.inputs
        out = einsum.einsum(*self._get_args(inputs))
        testing.assert_allclose(out, forward_answer, atol, rtol)

        if isinstance(forward_answer, numpy.ndarray):  # not 0-dim
            # test views
            forward_answer[...] = 0
            out[...] = 0
            testing.assert_allclose(inputs[0], inputs_ans[0], atol, rtol)

            forward_answer[...] = 1
            out[...] = 1
            testing.assert_allclose(inputs[0], inputs_ans[0], atol, rtol)

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>#!/usr/bin/python

import http.client
import logging
import datetime
import re
import urllib
from . import register, Site, Section, Chapter

logger = logging.getLogger(__name__)

@register
class RoyalRoad(Site):
    """Royal Road: a place where people write novels, mostly seeming to be light-novel in tone."""
    @staticmethod
    def matches(url):
        # e.g. https://royalroadl.com/fiction/6752/lament-of-the-fallen
        match = re.match(r'^(https?://(?:www\.)?royalroadl\.com/fiction/\d+)/?.*', url)
        if match:
            return match.group(1) + '/'

    def extract(self, url):
        workid = re.match(r'^https?://(?:www\.)?royalroadl\.com/fiction/(\d+)/?.*', url).group(1)
        soup = self._soup('https://www.royalroadl.com/fiction/{}'.format(workid))
        # should have gotten redirected, for a valid title

        original_maxheaders = http.client._MAXHEADERS
        http.client._MAXHEADERS = 1000

        story = Section(
            title=soup.find('h1', property='name').string.strip(),
            author=soup.find('meta', property='books:author').get('content').strip(),
            url=soup.find('meta', property='og:url').get('content').strip()
        )

        for chapter in soup.select('#chapters tbody tr[data-url]'):
            chapter_url = str(urllib.parse.urljoin(story.url, str(chapter.get('data-url'))))

            # Have to get exact publishing time from the chapter page
            chapter_soup = self._soup(chapter_url)
            contents, updated = self._chapter(chapter_url)

            story.add(Chapter(title=chapter.find('a', href=True).string.strip(), contents=contents, date=updated))

        http.client._MAXHEADERS = original_maxheaders

        return story

    def _chapter(self, url):
        logger.info("Extracting chapter @ %s", url)
        soup = self._soup(url)
        content = soup.find('div', class_='chapter-content')

        # TODO: this could be more robust, and I don't know if there's post-chapter notes anywhere as well.
        author_note = soup.find('div', class_='author-note-portlet')

        updated = int(soup.find(class_="profile-info").find('time').get('unixtime'))

        return (author_note and (author_note.prettify() + '
') or '') + content.prettify(), updated
<commit_msg>Fix spacing<commit_after>#!/usr/bin/python

import http.client
import logging
import datetime
import re
import urllib
from . import register, Site, Section, Chapter

logger = logging.getLogger(__name__)

@register
class RoyalRoad(Site):
    """Royal Road: a place where people write novels, mostly seeming to be light-novel in tone."""
    @staticmethod
    def matches(url):
        # e.g. https://royalroadl.com/fiction/6752/lament-of-the-fallen
        match = re.match(r'^(https?://(?:www\.)?royalroadl\.com/fiction/\d+)/?.*', url)
        if match:
            return match.group(1) + '/'

    def extract(self, url):
        workid = re.match(r'^https?://(?:www\.)?royalroadl\.com/fiction/(\d+)/?.*', url).group(1)
        soup = self._soup('https://www.royalroadl.com/fiction/{}'.format(workid))
        # should have gotten redirected, for a valid title

        original_maxheaders = http.client._MAXHEADERS
        http.client._MAXHEADERS = 1000

        story = Section(
            title=soup.find('h1', property='name').string.strip(),
            author=soup.find('meta', property='books:author').get('content').strip(),
            url=soup.find('meta', property='og:url').get('content').strip()
        )

        for chapter in soup.select('#chapters tbody tr[data-url]'):
            chapter_url = str(urllib.parse.urljoin(story.url, str(chapter.get('data-url'))))

            contents, updated = self._chapter(chapter_url)

            story.add(Chapter(title=chapter.find('a', href=True).string.strip(), contents=contents, date=updated))

        http.client._MAXHEADERS = original_maxheaders

        return story

    def _chapter(self, url):
        logger.info("Extracting chapter @ %s", url)
        soup = self._soup(url)
        content = soup.find('div', class_='chapter-content')

        # TODO: this could be more robust, and I don't know if there's post-chapter notes anywhere as well.
        author_note = soup.find('div', class_='author-note-portlet')

        updated = int(soup.find(class_="profile-info").find('time').get('unixtime'))

        return (author_note and (author_note.prettify() + '
') or '') + content.prettify(), updated
<|endoftext|>


<commit_before>"""Allow users to set and activate scenes."""
from __future__ import annotations

import logging
from typing import Any, NamedTuple

import voluptuous as vol

from homeassistant import config as conf_util
from homeassistant.components.light import ATTR_TRANSITION
from homeassistant.components.scene import DOMAIN as SCENE_DOMAIN, STATES, Scene
from homeassistant.const import (
    ATTR_ENTITY_ID,
    ATTR_STATE,
    CONF_ENTITIES,
    CONF_ICON,
    CONF_ID,
    CONF_NAME,
    CONF_PLATFORM,
    SERVICE_RELOAD,
    STATE_OFF,
    STATE_ON,
)
from homeassistant.core import (
    DOMAIN as HA_DOMAIN,
    HomeAssistant,
    ServiceCall,
    State,
    callback,
)
from homeassistant.exceptions import HomeAssistantError
from homeassistant.helpers import (
    config_per_platform,
    config_validation as cv,
    entity_platform,
)
from homeassistant.helpers.state import async_reproduce_state
from homeassistant.loader import async_get_integration

def _convert_states(states):
    """Convert state definitions to State objects."""
    result = {}

    for entity_id, info in states.items():
        entity_id = cv.entity_id(entity_id)

        if isinstance(info, dict):
            entity_attrs = info.copy()
            state = entity_attrs.pop(ATTR_STATE, None)
            attributes = entity_attrs
        else:
            state = info
            attributes = {}

        # YAML translates 'on' to a boolean
        # http://yaml.org/type/bool.html
        if isinstance(state, bool):
            state = STATE_ON if state else STATE_OFF
        elif not isinstance(state, str):
            raise vol.Invalid(f"State for {entity_id} should be a string")

        result[entity_id] = State(entity_id, state, attributes)

    return result

def _ensure_no_intersection(value):
    """Validate that entities and snapshot_entities do not overlap."""
    if (
        CONF_SNAPSHOT not in value
        or CONF_ENTITIES not in value
        or all(
            entity_id not in value[CONF_SNAPSHOT] for entity_id in value[CONF_ENTITIES]
        )
    ):
        return value

    raise vol.Invalid("entities and snapshot_entities must not overlap")

CONF_SCENE_ID = "scene_id"
CONF_SNAPSHOT = "snapshot_entities"
DATA_PLATFORM = "homeassistant_scene"
EVENT_SCENE_RELOADED = "scene_reloaded"
STATES_SCHEMA = vol.All(dict, _convert_states)

PLATFORM_SCHEMA = vol.Schema(
    {
        vol.Required(CONF_PLATFORM): HA_DOMAIN,
        vol.Required(STATES): vol.All(
            cv.ensure_list,
            [
                vol.Schema(
                    {
                        vol.Optional(CONF_ID): cv.string,
                        vol.Required(CONF_NAME): cv.string,
                        vol.Optional(CONF_ICON): cv.icon,
                        vol.Required(CONF_ENTITIES): STATES_SCHEMA,
                    }
                )
            ],
        ),
    },
    extra=vol.ALLOW_EXTRA,
)

CREATE_SCENE_SCHEMA = vol.All(
    cv.has_at_least_one_key(CONF_ENTITIES, CONF_SNAPSHOT),
    _ensure_no_intersection,
    vol.Schema(
        {
            vol.Required(CONF_SCENE_ID): cv.slug,
            vol.Optional(CONF_ENTITIES, default={}): STATES_SCHEMA,
            vol.Optional(CONF_SNAPSHOT, default=[]): cv.entity_ids,
        }
    ),
)

SERVICE_APPLY = "apply"
SERVICE_CREATE = "create"

_LOGGER = logging.getLogger(__name__)

class SceneConfig(NamedTuple):
    """Object for storing scene config."""

    id: str | None
    name: str
    icon: str | None
    states: dict

@callback
def scenes_with_entity(hass: HomeAssistant, entity_id: str) -> list[str]:
    """Return all scenes that reference the entity."""
    if DATA_PLATFORM not in hass.data:
        return []

    platform = hass.data[DATA_PLATFORM]

    return [
        scene_entity.entity_id
        for scene_entity in platform.entities.values()
        if entity_id in scene_entity.scene_config.states
    ]

@callback
def entities_in_scene(hass: HomeAssistant, entity_id: str) -> list[str]:
    """Return all entities in a scene."""
    if DATA_PLATFORM not in hass.data:
        return []

    platform = hass.data[DATA_PLATFORM]

    if (entity := platform.entities.get(entity_id)) is None:
        return []

    return list(entity.scene_config.states)

async def async_setup_platform(hass, config, async_add_entities, discovery_info=None):
    """Set up Home Assistant scene entries."""
    _process_scenes_config(hass, async_add_entities, config)

    # This platform can be loaded multiple times. Only first time register the service.
    if hass.services.has_service(SCENE_DOMAIN, SERVICE_RELOAD):
        return

    # Store platform for later.
    platform = hass.data[DATA_PLATFORM] = entity_platform.async_get_current_platform()

    async def reload_config(call):
        """Reload the scene config."""
        try:
            conf = await conf_util.async_hass_config_yaml(hass)
        except HomeAssistantError as err:
            _LOGGER.error(err)
            return

        integration = await async_get_integration(hass, SCENE_DOMAIN)

        conf = await conf_util.async_process_component_config(hass, conf, integration)

        if not (conf and platform):
            return

        await platform.async_reset()

        # Extract only the config for the Home Assistant platform, ignore the rest.
        for p_type, p_config in config_per_platform(conf, SCENE_DOMAIN):
            if p_type != HA_DOMAIN:
                continue

            _process_scenes_config(hass, async_add_entities, p_config)

        hass.bus.async_fire(EVENT_SCENE_RELOADED, context=call.context)

    hass.helpers.service.async_register_admin_service(
        SCENE_DOMAIN, SERVICE_RELOAD, reload_config
    )

    async def apply_service(call: ServiceCall) -> None:
        """Apply a scene."""
        reproduce_options = {}

        if ATTR_TRANSITION in call.data:
            reproduce_options[ATTR_TRANSITION] = call.data.get(ATTR_TRANSITION)

        await async_reproduce_state(
            hass,
            call.data[CONF_ENTITIES].values(),
            context=call.context,
            reproduce_options=reproduce_options,
        )

    hass.services.async_register(
        SCENE_DOMAIN,
        SERVICE_APPLY,
        apply_service,
        vol.Schema(
            {
                vol.Optional(ATTR_TRANSITION): vol.All(
                    vol.Coerce(float), vol.Clamp(min=0, max=6553)
                ),
                vol.Required(CONF_ENTITIES): STATES_SCHEMA,
            }
        ),
    )

    async def create_service(call: ServiceCall) -> None:
        """Create a scene."""
        snapshot = call.data[CONF_SNAPSHOT]
        entities = call.data[CONF_ENTITIES]

        for entity_id in snapshot:
            if (state := hass.states.get(entity_id)) is None:
                _LOGGER.warning(
                    "Entity %s does not exist and therefore cannot be snapshotted",
                    entity_id,
                )
                continue
            entities[entity_id] = State(entity_id, state.state, state.attributes)

        if not entities:
            _LOGGER.warning("Empty scenes are not allowed")
            return

        scene_config = SceneConfig(None, call.data[CONF_SCENE_ID], None, entities)
        entity_id = f"{SCENE_DOMAIN}.{scene_config.name}"
        if (old := platform.entities.get(entity_id)) is not None:
            if not old.from_service:
                _LOGGER.warning("The scene %s already exists", entity_id)
                return
            await platform.async_remove_entity(entity_id)
        async_add_entities([HomeAssistantScene(hass, scene_config, from_service=True)])

    hass.services.async_register(
        SCENE_DOMAIN, SERVICE_CREATE, create_service, CREATE_SCENE_SCHEMA
    )

def _process_scenes_config(hass, async_add_entities, config):
    """Process multiple scenes and add them."""
    # Check empty list
    if not (scene_config := config[STATES]):
        return

    async_add_entities(
        HomeAssistantScene(
            hass,
            SceneConfig(
                scene.get(CONF_ID),
                scene[CONF_NAME],
                scene.get(CONF_ICON),
                scene[CONF_ENTITIES],
            ),
        )
        for scene in scene_config
    )

class HomeAssistantScene(Scene):
    """A scene is a group of entities and the states we want them to be."""

    def __init__(self, hass, scene_config, from_service=False):
        """Initialize the scene."""
        self.hass = hass
        self.scene_config = scene_config
        self.from_service = from_service

    @property
    def name(self):
        """Return the name of the scene."""
        return self.scene_config.name

    @property
    def icon(self):
        """Return the icon of the scene."""
        return self.scene_config.icon

    @property
    def unique_id(self):
        """Return unique ID."""
        return self.scene_config.id

    @property
    def extra_state_attributes(self):
        """Return the scene state attributes."""
        attributes = {ATTR_ENTITY_ID: list(self.scene_config.states)}
        if (unique_id := self.unique_id) is not None:
            attributes[CONF_ID] = unique_id
        return attributes

    async def async_activate(self, **kwargs: Any) -> None:
        """Activate scene. Try to get entities into requested state."""
        await async_reproduce_state(
            self.hass,
            self.scene_config.states.values(),
            context=self._context,
            reproduce_options=kwargs,
        )
<commit_msg>Add type hint to reload_config in homeassistant scene (#63094)<commit_after>"""Allow users to set and activate scenes."""
from __future__ import annotations

import logging
from typing import Any, NamedTuple

import voluptuous as vol

from homeassistant import config as conf_util
from homeassistant.components.light import ATTR_TRANSITION
from homeassistant.components.scene import DOMAIN as SCENE_DOMAIN, STATES, Scene
from homeassistant.const import (
    ATTR_ENTITY_ID,
    ATTR_STATE,
    CONF_ENTITIES,
    CONF_ICON,
    CONF_ID,
    CONF_NAME,
    CONF_PLATFORM,
    SERVICE_RELOAD,
    STATE_OFF,
    STATE_ON,
)
from homeassistant.core import (
    DOMAIN as HA_DOMAIN,
    HomeAssistant,
    ServiceCall,
    State,
    callback,
)
from homeassistant.exceptions import HomeAssistantError
from homeassistant.helpers import (
    config_per_platform,
    config_validation as cv,
    entity_platform,
)
from homeassistant.helpers.state import async_reproduce_state
from homeassistant.loader import async_get_integration

def _convert_states(states):
    """Convert state definitions to State objects."""
    result = {}

    for entity_id, info in states.items():
        entity_id = cv.entity_id(entity_id)

        if isinstance(info, dict):
            entity_attrs = info.copy()
            state = entity_attrs.pop(ATTR_STATE, None)
            attributes = entity_attrs
        else:
            state = info
            attributes = {}

        # YAML translates 'on' to a boolean
        # http://yaml.org/type/bool.html
        if isinstance(state, bool):
            state = STATE_ON if state else STATE_OFF
        elif not isinstance(state, str):
            raise vol.Invalid(f"State for {entity_id} should be a string")

        result[entity_id] = State(entity_id, state, attributes)

    return result

def _ensure_no_intersection(value):
    """Validate that entities and snapshot_entities do not overlap."""
    if (
        CONF_SNAPSHOT not in value
        or CONF_ENTITIES not in value
        or all(
            entity_id not in value[CONF_SNAPSHOT] for entity_id in value[CONF_ENTITIES]
        )
    ):
        return value

    raise vol.Invalid("entities and snapshot_entities must not overlap")

CONF_SCENE_ID = "scene_id"
CONF_SNAPSHOT = "snapshot_entities"
DATA_PLATFORM = "homeassistant_scene"
EVENT_SCENE_RELOADED = "scene_reloaded"
STATES_SCHEMA = vol.All(dict, _convert_states)

PLATFORM_SCHEMA = vol.Schema(
    {
        vol.Required(CONF_PLATFORM): HA_DOMAIN,
        vol.Required(STATES): vol.All(
            cv.ensure_list,
            [
                vol.Schema(
                    {
                        vol.Optional(CONF_ID): cv.string,
                        vol.Required(CONF_NAME): cv.string,
                        vol.Optional(CONF_ICON): cv.icon,
                        vol.Required(CONF_ENTITIES): STATES_SCHEMA,
                    }
                )
            ],
        ),
    },
    extra=vol.ALLOW_EXTRA,
)

CREATE_SCENE_SCHEMA = vol.All(
    cv.has_at_least_one_key(CONF_ENTITIES, CONF_SNAPSHOT),
    _ensure_no_intersection,
    vol.Schema(
        {
            vol.Required(CONF_SCENE_ID): cv.slug,
            vol.Optional(CONF_ENTITIES, default={}): STATES_SCHEMA,
            vol.Optional(CONF_SNAPSHOT, default=[]): cv.entity_ids,
        }
    ),
)

SERVICE_APPLY = "apply"
SERVICE_CREATE = "create"

_LOGGER = logging.getLogger(__name__)

class SceneConfig(NamedTuple):
    """Object for storing scene config."""

    id: str | None
    name: str
    icon: str | None
    states: dict

@callback
def scenes_with_entity(hass: HomeAssistant, entity_id: str) -> list[str]:
    """Return all scenes that reference the entity."""
    if DATA_PLATFORM not in hass.data:
        return []

    platform = hass.data[DATA_PLATFORM]

    return [
        scene_entity.entity_id
        for scene_entity in platform.entities.values()
        if entity_id in scene_entity.scene_config.states
    ]

@callback
def entities_in_scene(hass: HomeAssistant, entity_id: str) -> list[str]:
    """Return all entities in a scene."""
    if DATA_PLATFORM not in hass.data:
        return []

    platform = hass.data[DATA_PLATFORM]

    if (entity := platform.entities.get(entity_id)) is None:
        return []

    return list(entity.scene_config.states)

async def async_setup_platform(hass, config, async_add_entities, discovery_info=None):
    """Set up Home Assistant scene entries."""
    _process_scenes_config(hass, async_add_entities, config)

    # This platform can be loaded multiple times. Only first time register the service.
    if hass.services.has_service(SCENE_DOMAIN, SERVICE_RELOAD):
        return

    # Store platform for later.
    platform = hass.data[DATA_PLATFORM] = entity_platform.async_get_current_platform()

    async def reload_config(call: ServiceCall) -> None:
        """Reload the scene config."""
        try:
            config = await conf_util.async_hass_config_yaml(hass)
        except HomeAssistantError as err:
            _LOGGER.error(err)
            return

        integration = await async_get_integration(hass, SCENE_DOMAIN)

        conf = await conf_util.async_process_component_config(hass, config, integration)

        if not (conf and platform):
            return

        await platform.async_reset()

        # Extract only the config for the Home Assistant platform, ignore the rest.
        for p_type, p_config in config_per_platform(conf, SCENE_DOMAIN):
            if p_type != HA_DOMAIN:
                continue

            _process_scenes_config(hass, async_add_entities, p_config)

        hass.bus.async_fire(EVENT_SCENE_RELOADED, context=call.context)

    hass.helpers.service.async_register_admin_service(
        SCENE_DOMAIN, SERVICE_RELOAD, reload_config
    )

    async def apply_service(call: ServiceCall) -> None:
        """Apply a scene."""
        reproduce_options = {}

        if ATTR_TRANSITION in call.data:
            reproduce_options[ATTR_TRANSITION] = call.data.get(ATTR_TRANSITION)

        await async_reproduce_state(
            hass,
            call.data[CONF_ENTITIES].values(),
            context=call.context,
            reproduce_options=reproduce_options,
        )

    hass.services.async_register(
        SCENE_DOMAIN,
        SERVICE_APPLY,
        apply_service,
        vol.Schema(
            {
                vol.Optional(ATTR_TRANSITION): vol.All(
                    vol.Coerce(float), vol.Clamp(min=0, max=6553)
                ),
                vol.Required(CONF_ENTITIES): STATES_SCHEMA,
            }
        ),
    )

    async def create_service(call: ServiceCall) -> None:
        """Create a scene."""
        snapshot = call.data[CONF_SNAPSHOT]
        entities = call.data[CONF_ENTITIES]

        for entity_id in snapshot:
            if (state := hass.states.get(entity_id)) is None:
                _LOGGER.warning(
                    "Entity %s does not exist and therefore cannot be snapshotted",
                    entity_id,
                )
                continue
            entities[entity_id] = State(entity_id, state.state, state.attributes)

        if not entities:
            _LOGGER.warning("Empty scenes are not allowed")
            return

        scene_config = SceneConfig(None, call.data[CONF_SCENE_ID], None, entities)
        entity_id = f"{SCENE_DOMAIN}.{scene_config.name}"
        if (old := platform.entities.get(entity_id)) is not None:
            if not old.from_service:
                _LOGGER.warning("The scene %s already exists", entity_id)
                return
            await platform.async_remove_entity(entity_id)
        async_add_entities([HomeAssistantScene(hass, scene_config, from_service=True)])

    hass.services.async_register(
        SCENE_DOMAIN, SERVICE_CREATE, create_service, CREATE_SCENE_SCHEMA
    )

def _process_scenes_config(hass, async_add_entities, config):
    """Process multiple scenes and add them."""
    # Check empty list
    if not (scene_config := config[STATES]):
        return

    async_add_entities(
        HomeAssistantScene(
            hass,
            SceneConfig(
                scene.get(CONF_ID),
                scene[CONF_NAME],
                scene.get(CONF_ICON),
                scene[CONF_ENTITIES],
            ),
        )
        for scene in scene_config
    )

class HomeAssistantScene(Scene):
    """A scene is a group of entities and the states we want them to be."""

    def __init__(self, hass, scene_config, from_service=False):
        """Initialize the scene."""
        self.hass = hass
        self.scene_config = scene_config
        self.from_service = from_service

    @property
    def name(self):
        """Return the name of the scene."""
        return self.scene_config.name

    @property
    def icon(self):
        """Return the icon of the scene."""
        return self.scene_config.icon

    @property
    def unique_id(self):
        """Return unique ID."""
        return self.scene_config.id

    @property
    def extra_state_attributes(self):
        """Return the scene state attributes."""
        attributes = {ATTR_ENTITY_ID: list(self.scene_config.states)}
        if (unique_id := self.unique_id) is not None:
            attributes[CONF_ID] = unique_id
        return attributes

    async def async_activate(self, **kwargs: Any) -> None:
        """Activate scene. Try to get entities into requested state."""
        await async_reproduce_state(
            self.hass,
            self.scene_config.states.values(),
            context=self._context,
            reproduce_options=kwargs,
        )
<|endoftext|>


<commit_before>#From https://gist.github.com/EndingCredits/b5f35e84df10d46cfa716178d9c862a3
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import state_ops
from tensorflow.python.framework import ops
from tensorflow.python.training import optimizer
import tensorflow as tf
import hyperchamber as hc
import inspect
import numpy as np
from hypergan.train_hooks.base_train_hook import BaseTrainHook

class WeightConstraintTrainHook(BaseTrainHook):
  def __init__(self, gan=None, config=None, trainer=None, name="WeightConstraintTrainHook"):
    super().__init__(config=config, gan=gan, trainer=trainer, name=name)
    allvars = gan.variables()
    self.update_weight_constraints = [self._update_weight_constraint(v,i) for i,v in enumerate(allvars)]
    self.update_weight_constraints = [v for v in self.update_weight_constraints if v is not None]

  def _update_ortho(self,v,i):
    if len(v.shape) == 4:
      w=v
      w = tf.reshape(v, [-1, self.ops.shape(w)[-1]])
      identity = tf.cast(tf.diag(np.ones(self.ops.shape(w)[0])), tf.float32)
      wt = tf.transpose(w)
      #s = self.ops.shape(v_transpose)
      #identity = tf.reshape(identity, [s[0],s[1],1,1])
      #identity = tf.tile(identity, [1,1,s[2],s[3]])
      decay = self.config.ortho_decay or 0.01
      newv = tf.matmul(w, tf.matmul(wt,w))
      newv = tf.reshape(newv,self.ops.shape(v))
      newv=(1+decay)*v - decay*(newv)

      #newv = tf.transpose(v, perm=[1,0,2,3])
      return tf.assign(v, newv)
    return None
  def _update_lipschitz(self,v,i):
    config = self.config
    if len(v.shape) > 1:
      k = self.config.weight_constraint_k or 100.0000
      wi_hat = v
      if len(v.shape) == 4:
        #fij = tf.reduce_sum(tf.abs(wi_hat),  axis=[0,1])
        fij = wi_hat
        fij = tf.reduce_sum(tf.abs(fij),  axis=[1])
        fij = tf.reduce_max(fij,  axis=[0])
      else:
        fij = wi_hat

      if self.config.ortho_pnorm == "inf":
        wp = tf.reduce_max(tf.reduce_sum(tf.abs(fij), axis=0), axis=0)
      else:
        # conv
        wp = tf.reduce_max(tf.reduce_sum(tf.abs(fij), axis=1), axis=0)
      ratio = (1.0/tf.maximum(1.0, wp/k))

      if self.config.weight_bounce:
        bounce = tf.minimum(1.0, tf.ceil(wp/k-0.999))
        ratio -= tf.maximum(0.0, bounce) * 0.2

      if self.config.weight_scaleup:
        up = tf.minimum(1.0, tf.ceil(0.02-wp/k))
        ratio += tf.maximum(0.0, up) * k/wp * 0.2

      wi = ratio*(wi_hat)
      #self.gan.metrics['wi'+str(i)]=wp
      #self.gan.metrics['wk'+str(i)]=ratio
      #self.gan.metrics['bouce'+str(i)]=bounce
      return tf.assign(v, wi)
    return None

  def _update_l2nn(self,v,i):
    config = self.config
    if len(v.shape) > 1:
      w=v
      w = tf.reshape(w, [-1, self.ops.shape(v)[-1]])
      wt = tf.transpose(w)
      #wt = tf.transpose(v, perm=[0,1,3,2])
      #wt = tf.transpose(w, perm=[1,0,2,3])
      def _r(m):
        s = self.ops.shape(m)
        m = tf.abs(m)
        m = tf.reduce_sum(m, axis=0,keep_dims=True)
        m = tf.reduce_max(m, axis=1,keep_dims=True)
        #m = tf.tile(m,[s[0],s[1],1,1])
        return m
      wtw = tf.matmul(wt,w)
      wwt = tf.matmul(w,wt)
      bw = tf.minimum(_r(wtw), _r(wwt))
      decay = self.config.l2nn_decay
      wi = (v/tf.sqrt(bw))
      if decay is not None:
        wi = (1-decay)*v+(decay*wi)
      return tf.assign(v, wi)
    return None
  def _update_weight_constraint(self,v,i):
    config = self.config
    #skipped = [gan.generator.ops.weights[0], gan.generator.ops.weights[-1], gan.discriminator.ops.weights[0], gan.discriminator.ops.weights[-1]]
    #skipped = [gan.discriminator.ops.weights[-1]]
    skipped=[]
    for skip in skipped:
      if self.ops.shape(v) == self.ops.shape(skip):
        print("Skipping constraints on", v)
        return None
    constraints = self.config.weight_constraint or []
    result = []
    if "ortho" in constraints:
      result.append(self._update_ortho(v,i))
    if "lipschitz" in constraints:
      result.append(self._update_lipschitz(v,i))
    if "l2nn" in constraints:
      result.append(self._update_l2nn(v,i))
    if "l2nn-d" in constraints:
      if v in d_vars:
        result.append(self._update_l2nn(v,i))
    result = [r for r in result if r is not None]
    if(len(result) == 0):
      return None
    return tf.group(result)

  def before_step(self, step, feed_dict):
    if self.config.order == "after":
        pass
    else:
        if ((step % (self.config.constraint_every or 100)) == 0):
            print("Applying weight constraint (pre)")
            self.gan.session.run(self.update_weight_constraints, feed_dict)

  def after_step(self, step, feed_dict):
    if self.config.order == "after":
        if ((step % (self.config.constraint_every or 100)) == 0):
            print("Applying weight constraint (post)")
            self.gan.session.run(self.update_weight_constraints, feed_dict)
<commit_msg>[fix] ortho from https://arxiv.org/pdf/1811.05381v1.pdf<commit_after>#From https://gist.github.com/EndingCredits/b5f35e84df10d46cfa716178d9c862a3
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import state_ops
from tensorflow.python.framework import ops
from tensorflow.python.training import optimizer
import tensorflow as tf
import hyperchamber as hc
import inspect
import numpy as np
from hypergan.train_hooks.base_train_hook import BaseTrainHook

class WeightConstraintTrainHook(BaseTrainHook):
  def __init__(self, gan=None, config=None, trainer=None, name="WeightConstraintTrainHook"):
    super().__init__(config=config, gan=gan, trainer=trainer, name=name)
    allvars = gan.variables()
    self.update_weight_constraints = [self._update_weight_constraint(v,i) for i,v in enumerate(allvars)]
    self.update_weight_constraints = [v for v in self.update_weight_constraints if v is not None]
  def _update_ortho(self,v,i):
    s = self.gan.ops.shape(v)
    if len(s) == 4 and s[0] == s[1]:
      w=v
      newv = []
      #s = self.ops.shape(v_transpose)
      #identity = tf.reshape(identity, [s[0],s[1],1,1])
      #identity = tf.tile(identity, [1,1,s[2],s[3]])
      decay = self.config.ortho_decay or 0.01
      w = tf.transpose(w, perm=[2,3,0,1])
      for i in range(self.config.iterations or 3):
          wt = tf.transpose(w, perm=[1,0,2,3])
          w2 = tf.reshape(w,[-1, s[0],s[1]])
          wt2 = tf.reshape(wt,[-1, s[0],s[1]])
          orth = tf.matmul(wt2,w2)
          eye = tf.eye(s[0],s[1])
          eye = tf.tile(eye, [1,s[2]*s[3]])
          eye = tf.reshape(eye, self.gan.ops.shape(w))
          orth = tf.reshape(orth, self.gan.ops.shape(w))
          qk = eye - orth
          w = w * (eye + 0.5*qk + 3.0/8*tf.square(qk))
      w = tf.transpose(w, perm=[2,3,0,1])
      newv = w

      newv = tf.reshape(newv,self.ops.shape(v))

  #def _update_ortho(self,v,i):
  #  if len(v.shape) == 4:
  #    w=v
  #    w = tf.reshape(v, [-1, self.ops.shape(w)[-1]])
  #    identity = tf.cast(tf.diag(np.ones(self.ops.shape(w)[0])), tf.float32)
  #    wt = tf.transpose(w)
  #    #s = self.ops.shape(v_transpose)
  #    #identity = tf.reshape(identity, [s[0],s[1],1,1])
  #    #identity = tf.tile(identity, [1,1,s[2],s[3]])
  #    decay = self.config.ortho_decay or 0.01
  #    newv = tf.matmul(w, tf.matmul(wt,w))
  #    newv = tf.reshape(newv,self.ops.shape(v))
  #    newv=(1+decay)*v - decay*(newv)

  #    #newv = tf.transpose(v, perm=[1,0,2,3])
  #    return tf.assign(v, newv)
  #  return None
  def _update_lipschitz(self,v,i):
    config = self.config
    if len(v.shape) > 1:
      k = self.config.weight_constraint_k or 100.0000
      wi_hat = v
      if len(v.shape) == 4:
        #fij = tf.reduce_sum(tf.abs(wi_hat),  axis=[0,1])
        fij = wi_hat
        fij = tf.reduce_sum(tf.abs(fij),  axis=[1])
        fij = tf.reduce_max(fij,  axis=[0])
      else:
        fij = wi_hat

      if self.config.ortho_pnorm == "inf":
        wp = tf.reduce_max(tf.reduce_sum(tf.abs(fij), axis=0), axis=0)
      else:
        # conv
        wp = tf.reduce_max(tf.reduce_sum(tf.abs(fij), axis=1), axis=0)
      ratio = (1.0/tf.maximum(1.0, wp/k))

      if self.config.weight_bounce:
        bounce = tf.minimum(1.0, tf.ceil(wp/k-0.999))
        ratio -= tf.maximum(0.0, bounce) * 0.2

      if self.config.weight_scaleup:
        up = tf.minimum(1.0, tf.ceil(0.02-wp/k))
        ratio += tf.maximum(0.0, up) * k/wp * 0.2

      wi = ratio*(wi_hat)
      #self.gan.metrics['wi'+str(i)]=wp
      #self.gan.metrics['wk'+str(i)]=ratio
      #self.gan.metrics['bouce'+str(i)]=bounce
      return tf.assign(v, wi)
    return None

  def _update_l2nn(self,v,i):
    config = self.config
    if len(v.shape) > 1:
      w=v
      w = tf.reshape(w, [-1, self.ops.shape(v)[-1]])
      wt = tf.transpose(w)
      #wt = tf.transpose(v, perm=[0,1,3,2])
      #wt = tf.transpose(w, perm=[1,0,2,3])
      def _r(m):
        s = self.ops.shape(m)
        m = tf.abs(m)
        m = tf.reduce_sum(m, axis=0,keep_dims=True)
        m = tf.reduce_max(m, axis=1,keep_dims=True)
        #m = tf.tile(m,[s[0],s[1],1,1])
        return m
      wtw = tf.matmul(wt,w)
      wwt = tf.matmul(w,wt)
      bw = tf.minimum(_r(wtw), _r(wwt))
      decay = self.config.l2nn_decay
      wi = (v/tf.sqrt(bw))
      if decay is not None:
        wi = (1-decay)*v+(decay*wi)
      return tf.assign(v, wi)
    return None
  def _update_weight_constraint(self,v,i):
    config = self.config
    #skipped = [gan.generator.ops.weights[0], gan.generator.ops.weights[-1], gan.discriminator.ops.weights[0], gan.discriminator.ops.weights[-1]]
    #skipped = [gan.discriminator.ops.weights[-1]]
    skipped=[]
    for skip in skipped:
      if self.ops.shape(v) == self.ops.shape(skip):
        print("Skipping constraints on", v)
        return None
    constraints = self.config.weight_constraint or []
    result = []
    if "ortho" in constraints:
      result.append(self._update_ortho(v,i))
    if "lipschitz" in constraints:
      result.append(self._update_lipschitz(v,i))
    if "l2nn" in constraints:
      result.append(self._update_l2nn(v,i))
    if "l2nn-d" in constraints:
      if v in d_vars:
        result.append(self._update_l2nn(v,i))
    result = [r for r in result if r is not None]
    if(len(result) == 0):
      return None
    return tf.group(result)

  def before_step(self, step, feed_dict):
    if self.config.order == "after":
        pass
    else:
        if ((step % (self.config.constraint_every or 100)) == 0):
            #print("Applying weight constraint (pre)")
            self.gan.session.run(self.update_weight_constraints, feed_dict)

  def after_step(self, step, feed_dict):
    if self.config.order == "after":
        if ((step % (self.config.constraint_every or 100)) == 0):
            #print("Applying weight constraint (post)")
            self.gan.session.run(self.update_weight_constraints, feed_dict)
<|endoftext|>


<commit_before>import db
from werkzeug.wrappers import Request, Response
from werkzeug.routing import Map, Rule
from werkzeug.exceptions import HTTPException, NotFound
from werkzeug.utils import redirect
import json
from models.experiment import Experiment
#from models.alternative import Alternative

class Sixpack(object):

    def __init__(self):
        self.redis = db.REDIS

        self.url_map = Map([
            Rule('/', endpoint='status'),
            Rule('/experiment', endpoint='start_experiment'),
            Rule('/convert', endpoint='convert')
        ])

    def dispatch_request(self, request):
        adapter = self.url_map.bind_to_environ(request.environ)
        try:
            endpoint, values = adapter.match()
            return getattr(self, 'on_' + endpoint)(request, **values)
        except HTTPException, e:
            return e

    def wsgi_app(self, environ, start_response):
        request = Request(environ)
        response = self.dispatch_request(request)
        return response(environ, start_response)

    def __call__(self, environ, start_response):
        return self.wsgi_app(environ, start_response)

    # Routes are below, investigate moving these out of this file
    def on_status(self, request):
        status = {
            'status': 'ok'
        }
        return json_resp(status)

    def on_convert(self, request):
        experiment_name = request.args.get('experiment')
        client_id = request.args.get('client_id')

        experiment = Experiment.find(experiment_name)

        if client_id is None or experiment_name is None:
            raise Exception('You forgot something, bro')

        return json_resp({'status': 'ok'})

    def on_start_experiment(self, request):
        alts = request.args.getlist('alternatives')
        experiment_name = request.args.get('experiment')
        force = request.args.get('force', None)
        client_id = request.args.get('client_id')

        if client_id is None or experiment_name is None or alts is None:
            raise Exception('You forgot something, bro')

        if force and force in alts:
            return json_resp(force)

        # This should be wrapped up and moved out of the 'controller'
        seq_id = db.sequential_id('sequential_ids', client_id)
        experiment = Experiment.find_or_create(experiment_name, alts)
        alternative = experiment.get_alternative(seq_id)

        resp = {
            'chosen_alt': alternative,
            'experiment': experiment_name,
            'client_id': client_id,
            'seq_id': seq_id
        }
        return json_resp(resp)

# troll helper
def json_resp(thign):
    resp = Response(json.dumps(thign))
    resp.headers['Context-Type'] = 'application/json'
    return resp

def create_app():
    app = Sixpack()

    return app

if __name__ == '__main__':
    from werkzeug.serving import run_simple
    app = create_app()
    run_simple('127.0.0.1', 5000, app, use_debugger=True, use_reloader=True)<commit_msg>scratch: call experiment.convert on 'on_convert'<commit_after>import db
from werkzeug.wrappers import Request, Response
from werkzeug.routing import Map, Rule
from werkzeug.exceptions import HTTPException, NotFound
from werkzeug.utils import redirect
import json
from models.experiment import Experiment
#from models.alternative import Alternative

class Sixpack(object):

    def __init__(self):
        self.redis = db.REDIS

        self.url_map = Map([
            Rule('/', endpoint='status'),
            Rule('/experiment', endpoint='start_experiment'),
            Rule('/convert', endpoint='convert')
        ])

    def dispatch_request(self, request):
        adapter = self.url_map.bind_to_environ(request.environ)
        try:
            endpoint, values = adapter.match()
            return getattr(self, 'on_' + endpoint)(request, **values)
        except HTTPException, e:
            return e

    def wsgi_app(self, environ, start_response):
        request = Request(environ)
        response = self.dispatch_request(request)
        return response(environ, start_response)

    def __call__(self, environ, start_response):
        return self.wsgi_app(environ, start_response)

    # Routes are below, investigate moving these out of this file
    def on_status(self, request):
        status = {
            'status': 'ok'
        }
        return json_resp(status)

    def on_convert(self, request):
        experiment_name = request.args.get('experiment')

        client_id = request.args.get('client_id')

        experiment = Experiment.find(experiment_name)
        experiment.convert(client_id)

        if client_id is None or experiment_name is None:
            raise Exception('You forgot something, bro')

        return json_resp({'status': 'ok'})

    def on_start_experiment(self, request):
        alts = request.args.getlist('alternatives')
        experiment_name = request.args.get('experiment')
        force = request.args.get('force', None)
        client_id = request.args.get('client_id')

        if client_id is None or experiment_name is None or alts is None:
            raise Exception('You forgot something, bro')

        if force and force in alts:
            return json_resp(force)

        # This should be wrapped up and moved out of the 'controller'
        seq_id = db.sequential_id('sequential_ids', client_id)
        experiment = Experiment.find_or_create(experiment_name, alts)
        alternative = experiment.get_alternative(seq_id)

        resp = {
            'chosen_alt': alternative,
            'experiment': experiment_name,
            'client_id': client_id,
            'seq_id': seq_id
        }
        return json_resp(resp)

# troll helper
def json_resp(thign):
    resp = Response(json.dumps(thign))
    resp.headers['Context-Type'] = 'application/json'
    return resp

def create_app():
    app = Sixpack()

    return app

if __name__ == '__main__':
    from werkzeug.serving import run_simple
    app = create_app()
    run_simple('127.0.0.1', 5000, app, use_debugger=True, use_reloader=True)<|endoftext|>


<commit_before>from unittest.case import SkipTest

from batchy.runloop import coro_return, runloop_coroutine
from batchy.batch_coroutine import batch_coroutine, class_batch_coroutine

try:
    import gevent
    from gevent.coros import BoundedSemaphore

    import batchy.gevent as batchy_gevent
except ImportError:
    batchy_gevent = None
    print 'Gevent not installed; skipping gevent tests.'

from . import BaseTestCase

CALL_COUNT = 0

@batch_coroutine()
def increment(arg_lists):
    def increment_single(n):
        return n + 1

    global CALL_COUNT
    CALL_COUNT += 1
    coro_return([increment_single(*ar, **kw) for ar, kw in arg_lists])
    yield

class BatchTests(BaseTestCase):
    def setup(self):
        if not batchy_gevent:
            raise SkipTest()

        # Quiet gevent's internal exception printing.
        self.old_print_exception = gevent.get_hub().print_exception
        gevent.get_hub().print_exception = lambda context, type, value, tb: None

        global CALL_COUNT
        CALL_COUNT = 0

    def tear_down(self):
        gevent.get_hub().print_exception = self.old_print_exception

    def test_simple_gevent(self):
        sema = BoundedSemaphore()

        def acq():
            sema.acquire()
            return 1

        def rel():
            sema.release()
            return 2

        @runloop_coroutine()
        def test():
            r1, r2 = yield batchy_gevent.spawn(acq), batchy_gevent.spawn(rel)
            coro_return(r1 + r2)

        self.assert_equals(3, test())

    def test_gevent_exceptions(self):
        def throw():
            raise ValueError()

        @runloop_coroutine()
        def test():
            yield batchy_gevent.spawn(throw)

        self.assert_raises(ValueError, test)
<commit_msg>Slightly better gevent test.<commit_after>from unittest.case import SkipTest

from batchy.runloop import coro_return, runloop_coroutine
from batchy.batch_coroutine import batch_coroutine, class_batch_coroutine

try:
    import gevent
    from gevent.coros import Semaphore

    import batchy.gevent as batchy_gevent
except ImportError:
    batchy_gevent = None
    print 'Gevent not installed; skipping gevent tests.'

from . import BaseTestCase

CALL_COUNT = 0

@batch_coroutine()
def increment(arg_lists):
    def increment_single(n):
        return n + 1

    global CALL_COUNT
    CALL_COUNT += 1
    coro_return([increment_single(*ar, **kw) for ar, kw in arg_lists])
    yield

class BatchTests(BaseTestCase):
    def setup(self):
        if not batchy_gevent:
            raise SkipTest()

        # Quiet gevent's internal exception printing.
        self.old_print_exception = gevent.get_hub().print_exception
        gevent.get_hub().print_exception = lambda context, type, value, tb: None

        global CALL_COUNT
        CALL_COUNT = 0

    def tear_down(self):
        gevent.get_hub().print_exception = self.old_print_exception

    def test_simple_gevent(self):
        sema = Semaphore(0)

        def acq():
            sema.acquire()
            return 1

        @runloop_coroutine()
        def rel():
            sema.release()
            coro_return(2)
            yield

        @runloop_coroutine()
        def test():
            r1, r2 = yield batchy_gevent.spawn(acq), rel()
            coro_return(r1 + r2)

        self.assert_equals(3, test())

    def test_gevent_exceptions(self):
        def throw():
            raise ValueError()

        @runloop_coroutine()
        def test():
            yield batchy_gevent.spawn(throw)

        self.assert_raises(ValueError, test)

    # TODO: gevent concurrently with batch
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
新版本打包自动化
'''
# pip3 install semver

import argparse
import codecs
import glob
import os
import re
import semver
import time

import plib.utils as utils
import plib.git as git
from plib.environment import get_current_packet_id, get_interface_path, get_packet_path, set_packet_as_cwd
from plib.language.converter import Converter
import plib.environment as env

TIME_TAG = time.strftime('%Y%m%d%H%M%S', time.localtime())

def __compress(addon):
	'''
	Compress and concat addon source into one file.

	Args:
		addon: Addon name
	'''
	print('--------------------------------')
	print('Compressing: %s' % addon)
	file_count = 0
	converter = Converter('zh-TW')
	srcname = 'src.' + TIME_TAG + '.lua'
	# Remove debug codes in source
	for line in open('%s/info.ini' % addon):
		parts = line.strip().split('=')
		if parts[0].find('lua_') == 0:
			source_file = os.path.join(addon, parts[1])
			source_code = codecs.open(source_file,'r',encoding='gbk').read()
			source_code = re.sub(r'(?is)[^\n]*--\[\[#DEBUG LINE\]\][^\n]*\n?', '', source_code)
			source_code = re.sub(r'(?is)\n?--\[\[#DEBUG BEGIN\]\].*?--\[\[#DEBUG END\]\]\n?', '', source_code)
			codecs.open(source_file,'w',encoding='gbk').write(source_code)
	# Delete old files
	acientFileList = glob.glob('./%s/src*.lua' % addon, recursive=False)
	for filePath in acientFileList:
		try:
			os.remove(filePath)
		except OSError:
			print('Error while deleting file: ' + filePath)
	# Generate squishy file and execute squish
	with open('squishy', 'w') as squishy:
		squishy.write('Output "./%s/%s"\n' % (addon, srcname))
		for line in open('%s/info.ini' % addon):
			parts = line.strip().split('=')
			if parts[0].find('lua_') == 0:
				if parts[1].startswith('src.') and parts[1].endswith('.lua'): # src.lua
					print('Already compressed...')
					return
				file_path = os.path.join('.', addon, parts[1]).replace('\\', '/')
				file_count = file_count + 1
				squishy.write('Module "%d" "%s"\n' % (file_count, file_path))
	os.popen('lua "./!src-dist/tools/react/squish" --minify-level=full').read()
	os.remove('squishy')
	# Modify dist file for loading modules
	with open('./%s/%s' % (addon, srcname), 'r+') as src:
		content = src.read()
		src.seek(0, 0)
		src.write('local __INIT_TIME__ = GetTime()\n')
		src.write('local package={preload={}}\n')
		src.write(content)
	with open('./%s/%s' % (addon, srcname), 'a') as src:
		src.write('\nfor _, k in ipairs({')
		for i in range(1, file_count + 1):
			src.write('\'%d\',' % i)
		src.write('}) do package.preload[k]() end\n')
		src.write('Log("[ADDON] Module %s v%s loaded during " .. (GetTime() - __INIT_TIME__) .. "ms.")' % (addon, TIME_TAG))
	print('Compress done...')
	# Modify info.ini file
	info_content = ''
	for _, line in enumerate(codecs.open('%s/info.ini' % addon,'r',encoding='gbk')):
		parts = line.split('=')
		if parts[0].find('lua_') == 0:
			if parts[0] == 'lua_0':
				info_content = info_content + 'lua_0=' + srcname + '\n'
		else:
			info_content = info_content + line
	with codecs.open('%s/info.ini' % addon,'w',encoding='gbk') as f:
		f.write(info_content)
	with codecs.open('%s/info.ini.zh_TW' % addon,'w',encoding='utf8') as f:
		f.write(converter.convert(info_content))
	print('Update info done...')

def __get_version_info(diff_ver):
	'''Get version information'''
	# Read version from Base.lua
	current_version = ''
	for line in open('%s_!Base/src/lib/Base.lua' % get_current_packet_id()):
		if line[0:16] == 'local _VERSION_ ':
			current_version = re.sub(r'(?is)^local _VERSION_\s+=', '', line).strip()[1:-1]
	# Read max and previous release commit
	current_hash = os.popen('git log -n 1 --pretty=format:"%h"').read().strip()
	commit_list = os.popen('git log --grep release: --pretty=format:"SUCCESS|%h|%s"').read().split('\n')
	if diff_ver:
		commit_list += os.popen('git log ' + diff_ver + ' -n 1 --pretty=format:"SUCCESS|%h|%s"').read().split('\n')
	commit_list = filter(lambda x: x and x.startswith('SUCCESS|'), commit_list)
	commit_list = map(lambda x: x[8:], commit_list)
	max_version, prev_version, prev_version_message, prev_version_hash = '', '', '', ''
	for commit in commit_list:
		try:
			version = re.sub(r'(?is)^\w+\|release:\s+', '', commit).strip()
			version_message = re.sub(r'(?is)^\w+\|', '', commit).strip()
			version_hash = re.sub(r'(?is)\|.+$', '', commit).strip()
			if diff_ver:
				if diff_ver == version and semver.compare(version, '0.0.0') == 1:
					max_version = version
					prev_version = version
					prev_version_message = version_message
					prev_version_hash = version_hash
					continue
				if diff_ver.startswith(version_hash):
					max_version = '0.0.0'
					prev_version = '0.0.0'
					prev_version_message = version_message
					prev_version_hash = version_hash
					continue
			else:
				if max_version == '' and semver.compare(version, '0.0.0') == 1:
					max_version = version
					prev_version = version
					prev_version_message = version_message
					prev_version_hash = version_hash
					continue
				if semver.compare(version, current_version) == -1 and semver.compare(version, prev_version) == 1:
					prev_version = version
					prev_version_message = version_message
					prev_version_hash = version_hash
				if semver(version, max_version) == 1:
					max_version = version
		except:
			pass
	return {
		'current': current_version,
		'current_hash': current_hash,
		'max': max_version,
		'previous': prev_version,
		'previous_message': prev_version_message,
		'previous_hash': prev_version_hash,
	}

def __7zip(file_name, base_message, base_hash, extra_ignore_file):
	cmd_suffix = ''
	if extra_ignore_file:
		cmd_suffix = cmd_suffix + ' -x@' + extra_ignore_file
	if base_hash != '':
		# Generate file change list since previous release commit
		def pathToModule(path):
			return re.sub('(?:^\\!src-dist/data/|["/].*$)', '', path)
		paths = {
			'package.ini': True,
			'package.ini.*': True,
		}
		print('File change list:')
		filelist = os.popen('git diff ' + base_hash + ' HEAD --name-status').read().strip().split('\n')
		for file in filelist:
			lst = file.split('\t')
			if lst[0] == 'A' or lst[0] == 'M' or lst[0] == 'D':
				paths[pathToModule(lst[1])] = True
			elif lst[0][0] == 'R':
				paths[pathToModule(lst[1])] = True
				paths[pathToModule(lst[2])] = True
			print(file)
		print('')
		# Print addon change list
		print('Subpath change list:')
		for path in paths:
			print('/' + path)
			cmd_suffix = cmd_suffix + ' "' + path + '"'
		print('')

	# Prepare for 7z compressing
	print('zippping...')
	os.system('start /wait /b ./!src-dist/bin/7z.exe a -t7z ' + file_name + ' -xr!manifest.dat -xr!manifest.key -xr!publisher.key -x@.7zipignore' + cmd_suffix)
	print('File(s) compressing acomplished!')
	print('Url: ' + file_name)
	print('Based on git commit "%s(%s)".' % (base_message, base_hash) if base_hash != '' else 'Full package.')

def run(diff_ver, is_source):
	print('> DIFF VERSION: %s' % (diff_ver or 'auto'))
	print('> RELEASE MODE: %s' % ('source' if is_source else 'dist'))
	version_info = __get_version_info(diff_ver)

	if diff_ver and version_info.get('previous_hash') == '':
		print('Error: Specified diff commit not found (release: %s).' % diff_ver)
		exit()

	if not is_source:
		# Merge master into prelease
		if git.get_current_branch() == 'master':
			utils.assert_exit(git.is_clean(), 'Error: master branch has uncommited file change(s)!')
			os.system('git checkout prelease || git checkout -b prelease')
			os.system('git rebase master')
			os.system('code "%s"' % os.path.join(get_packet_path(), './%s_!Base/src/lib/Base.lua' % get_current_packet_id()))
			os.system('code "%s"' % os.path.join(get_packet_path(), './CHANGELOG.md'))
			utils.exit_with_message('Switched to prelease branch. Please commit release info and then run this script again!')

		# Merge prelease into stable
		if git.get_current_branch() == 'prelease':
			os.system('git reset master')
			version_info = __get_version_info(diff_ver)
			utils.assert_exit(version_info.get('max') == '' or semver.compare(version_info.get('current'), version_info.get('max')) == 1,
				'Error: current version(%s) must be larger than max history version(%s)!' % (version_info.get('current'), version_info.get('max')))
			os.system('git add * && git commit -m "release: %s"' % version_info.get('current'))
			os.system('git checkout stable || git checkout -b stable')
			os.system('git reset origin/stable --hard')
			os.system('git rebase prelease')

		# Check if branch
		utils.assert_exit(git.is_clean(), 'Error: resolve conflict and remove uncommited changes first!')
		utils.assert_exit(git.get_current_branch() == 'stable', 'Error: current branch is not on stable!')

	# Compress and concat source file
	for addon in os.listdir('./'):
		if os.path.exists(os.path.join('./', addon, 'info.ini')):
			__compress(addon)

	dist_root = os.path.abspath(os.path.join(get_interface_path(), os.pardir))
	if os.path.isfile(os.path.abspath(os.path.join(dist_root, 'gameupdater.exe'))):
		dist_root = os.path.abspath(os.path.join(get_packet_path(), '!src-dist', 'dist'))
	else:
		dist_root = os.path.abspath(os.path.join(dist_root, os.pardir, 'dist'))

	# Package files
	if version_info.get('previous_hash'):
		file_name_fmt = os.path.abspath(os.path.join(dist_root, '%s_%s_v%s.%sdiff-%s-%s.7z' % (
			get_current_packet_id(),
			TIME_TAG,
			version_info.get('current'),
			'%s',
			version_info.get('previous_hash'),
			version_info.get('current_hash'),
		)))
		base_message = ''
		base_hash = ''
		if version_info.get('current') != '' and version_info.get('previous_hash') != '':
			base_message = version_info.get('previous_message')
			base_hash = version_info.get('previous_hash')
		__7zip(file_name_fmt % '', base_message, base_hash, '')
		__7zip(file_name_fmt % 'remake-', base_message, base_hash, '.7zipignore-remake')
		__7zip(file_name_fmt % 'classic-', base_message, base_hash, '.7zipignore-classic')

	file_name_fmt = os.path.abspath(os.path.join(dist_root, '%s_%s_v%s.%sfull.7z' % (
		get_current_packet_id(),
		TIME_TAG,
		version_info.get('current'),
		'%s',
	)))
	__7zip(file_name_fmt % '', '', '', '')
	__7zip(file_name_fmt % 'remake-', '', '', '.7zipignore-remake')
	__7zip(file_name_fmt % 'classic-', '', '', '.7zipignore-classic')

	# Revert source code modify by compressing
	if not is_source:
		os.system('git reset HEAD --hard')
		os.system('git checkout master')
	time.sleep(5)
	print('Exiting...')

if __name__ == '__main__':
	parser = argparse.ArgumentParser(description='One-key release packet product helper.')
	parser.add_argument('--diff', type=str, help='Package diff version.')
	parser.add_argument('--no-build', action='store_true', help='Package source code.')
	args = parser.parse_args()

	env.set_packet_as_cwd()

	run(args.diff, args.no_build)
<commit_msg>fix: 打包目录有空格会失败的问题<commit_after># -*- coding: utf-8 -*-
'''
新版本打包自动化
'''
# pip3 install semver

import argparse
import codecs
import glob
import os
import re
import semver
import time

import plib.utils as utils
import plib.git as git
from plib.environment import get_current_packet_id, get_interface_path, get_packet_path, set_packet_as_cwd
from plib.language.converter import Converter
import plib.environment as env

TIME_TAG = time.strftime('%Y%m%d%H%M%S', time.localtime())

def __compress(addon):
	'''
	Compress and concat addon source into one file.

	Args:
		addon: Addon name
	'''
	print('--------------------------------')
	print('Compressing: %s' % addon)
	file_count = 0
	converter = Converter('zh-TW')
	srcname = 'src.' + TIME_TAG + '.lua'
	# Remove debug codes in source
	for line in open('%s/info.ini' % addon):
		parts = line.strip().split('=')
		if parts[0].find('lua_') == 0:
			source_file = os.path.join(addon, parts[1])
			source_code = codecs.open(source_file,'r',encoding='gbk').read()
			source_code = re.sub(r'(?is)[^\n]*--\[\[#DEBUG LINE\]\][^\n]*\n?', '', source_code)
			source_code = re.sub(r'(?is)\n?--\[\[#DEBUG BEGIN\]\].*?--\[\[#DEBUG END\]\]\n?', '', source_code)
			codecs.open(source_file,'w',encoding='gbk').write(source_code)
	# Delete old files
	acientFileList = glob.glob('./%s/src*.lua' % addon, recursive=False)
	for filePath in acientFileList:
		try:
			os.remove(filePath)
		except OSError:
			print('Error while deleting file: ' + filePath)
	# Generate squishy file and execute squish
	with open('squishy', 'w') as squishy:
		squishy.write('Output "./%s/%s"\n' % (addon, srcname))
		for line in open('%s/info.ini' % addon):
			parts = line.strip().split('=')
			if parts[0].find('lua_') == 0:
				if parts[1].startswith('src.') and parts[1].endswith('.lua'): # src.lua
					print('Already compressed...')
					return
				file_path = os.path.join('.', addon, parts[1]).replace('\\', '/')
				file_count = file_count + 1
				squishy.write('Module "%d" "%s"\n' % (file_count, file_path))
	os.popen('lua "./!src-dist/tools/react/squish" --minify-level=full').read()
	os.remove('squishy')
	# Modify dist file for loading modules
	with open('./%s/%s' % (addon, srcname), 'r+') as src:
		content = src.read()
		src.seek(0, 0)
		src.write('local __INIT_TIME__ = GetTime()\n')
		src.write('local package={preload={}}\n')
		src.write(content)
	with open('./%s/%s' % (addon, srcname), 'a') as src:
		src.write('\nfor _, k in ipairs({')
		for i in range(1, file_count + 1):
			src.write('\'%d\',' % i)
		src.write('}) do package.preload[k]() end\n')
		src.write('Log("[ADDON] Module %s v%s loaded during " .. (GetTime() - __INIT_TIME__) .. "ms.")' % (addon, TIME_TAG))
	print('Compress done...')
	# Modify info.ini file
	info_content = ''
	for _, line in enumerate(codecs.open('%s/info.ini' % addon,'r',encoding='gbk')):
		parts = line.split('=')
		if parts[0].find('lua_') == 0:
			if parts[0] == 'lua_0':
				info_content = info_content + 'lua_0=' + srcname + '\n'
		else:
			info_content = info_content + line
	with codecs.open('%s/info.ini' % addon,'w',encoding='gbk') as f:
		f.write(info_content)
	with codecs.open('%s/info.ini.zh_TW' % addon,'w',encoding='utf8') as f:
		f.write(converter.convert(info_content))
	print('Update info done...')

def __get_version_info(diff_ver):
	'''Get version information'''
	# Read version from Base.lua
	current_version = ''
	for line in open('%s_!Base/src/lib/Base.lua' % get_current_packet_id()):
		if line[0:16] == 'local _VERSION_ ':
			current_version = re.sub(r'(?is)^local _VERSION_\s+=', '', line).strip()[1:-1]
	# Read max and previous release commit
	current_hash = os.popen('git log -n 1 --pretty=format:"%h"').read().strip()
	commit_list = os.popen('git log --grep release: --pretty=format:"SUCCESS|%h|%s"').read().split('\n')
	if diff_ver:
		commit_list += os.popen('git log ' + diff_ver + ' -n 1 --pretty=format:"SUCCESS|%h|%s"').read().split('\n')
	commit_list = filter(lambda x: x and x.startswith('SUCCESS|'), commit_list)
	commit_list = map(lambda x: x[8:], commit_list)
	max_version, prev_version, prev_version_message, prev_version_hash = '', '', '', ''
	for commit in commit_list:
		try:
			version = re.sub(r'(?is)^\w+\|release:\s+', '', commit).strip()
			version_message = re.sub(r'(?is)^\w+\|', '', commit).strip()
			version_hash = re.sub(r'(?is)\|.+$', '', commit).strip()
			if diff_ver:
				if diff_ver == version and semver.compare(version, '0.0.0') == 1:
					max_version = version
					prev_version = version
					prev_version_message = version_message
					prev_version_hash = version_hash
					continue
				if diff_ver.startswith(version_hash):
					max_version = '0.0.0'
					prev_version = '0.0.0'
					prev_version_message = version_message
					prev_version_hash = version_hash
					continue
			else:
				if max_version == '' and semver.compare(version, '0.0.0') == 1:
					max_version = version
					prev_version = version
					prev_version_message = version_message
					prev_version_hash = version_hash
					continue
				if semver.compare(version, current_version) == -1 and semver.compare(version, prev_version) == 1:
					prev_version = version
					prev_version_message = version_message
					prev_version_hash = version_hash
				if semver(version, max_version) == 1:
					max_version = version
		except:
			pass
	return {
		'current': current_version,
		'current_hash': current_hash,
		'max': max_version,
		'previous': prev_version,
		'previous_message': prev_version_message,
		'previous_hash': prev_version_hash,
	}

def __7zip(file_name, base_message, base_hash, extra_ignore_file):
	cmd_suffix = ''
	if extra_ignore_file:
		cmd_suffix = cmd_suffix + ' -x@' + extra_ignore_file
	if base_hash != '':
		# Generate file change list since previous release commit
		def pathToModule(path):
			return re.sub('(?:^\\!src-dist/data/|["/].*$)', '', path)
		paths = {
			'package.ini': True,
			'package.ini.*': True,
		}
		print('File change list:')
		filelist = os.popen('git diff ' + base_hash + ' HEAD --name-status').read().strip().split('\n')
		for file in filelist:
			lst = file.split('\t')
			if lst[0] == 'A' or lst[0] == 'M' or lst[0] == 'D':
				paths[pathToModule(lst[1])] = True
			elif lst[0][0] == 'R':
				paths[pathToModule(lst[1])] = True
				paths[pathToModule(lst[2])] = True
			print(file)
		print('')
		# Print addon change list
		print('Subpath change list:')
		for path in paths:
			print('/' + path)
			cmd_suffix = cmd_suffix + ' "' + path + '"'
		print('')

	# Prepare for 7z compressing
	print('zippping...')
	os.system('start /wait /b ./!src-dist/bin/7z.exe a -t7z "' + file_name + '" -xr!manifest.dat -xr!manifest.key -xr!publisher.key -x@.7zipignore' + cmd_suffix)
	print('File(s) compressing acomplished!')
	print('Url: ' + file_name)
	print('Based on git commit "%s(%s)".' % (base_message, base_hash) if base_hash != '' else 'Full package.')

def run(diff_ver, is_source):
	print('> DIFF VERSION: %s' % (diff_ver or 'auto'))
	print('> RELEASE MODE: %s' % ('source' if is_source else 'dist'))
	version_info = __get_version_info(diff_ver)

	if diff_ver and version_info.get('previous_hash') == '':
		print('Error: Specified diff commit not found (release: %s).' % diff_ver)
		exit()

	if not is_source:
		# Merge master into prelease
		if git.get_current_branch() == 'master':
			utils.assert_exit(git.is_clean(), 'Error: master branch has uncommited file change(s)!')
			os.system('git checkout prelease || git checkout -b prelease')
			os.system('git rebase master')
			os.system('code "%s"' % os.path.join(get_packet_path(), './%s_!Base/src/lib/Base.lua' % get_current_packet_id()))
			os.system('code "%s"' % os.path.join(get_packet_path(), './CHANGELOG.md'))
			utils.exit_with_message('Switched to prelease branch. Please commit release info and then run this script again!')

		# Merge prelease into stable
		if git.get_current_branch() == 'prelease':
			os.system('git reset master')
			version_info = __get_version_info(diff_ver)
			utils.assert_exit(version_info.get('max') == '' or semver.compare(version_info.get('current'), version_info.get('max')) == 1,
				'Error: current version(%s) must be larger than max history version(%s)!' % (version_info.get('current'), version_info.get('max')))
			os.system('git add * && git commit -m "release: %s"' % version_info.get('current'))
			os.system('git checkout stable || git checkout -b stable')
			os.system('git reset origin/stable --hard')
			os.system('git rebase prelease')

		# Check if branch
		utils.assert_exit(git.is_clean(), 'Error: resolve conflict and remove uncommited changes first!')
		utils.assert_exit(git.get_current_branch() == 'stable', 'Error: current branch is not on stable!')

	# Compress and concat source file
	for addon in os.listdir('./'):
		if os.path.exists(os.path.join('./', addon, 'info.ini')):
			__compress(addon)

	dist_root = os.path.abspath(os.path.join(get_interface_path(), os.pardir))
	if os.path.isfile(os.path.abspath(os.path.join(dist_root, 'gameupdater.exe'))):
		dist_root = os.path.abspath(os.path.join(get_packet_path(), '!src-dist', 'dist'))
	else:
		dist_root = os.path.abspath(os.path.join(dist_root, os.pardir, 'dist'))

	# Package files
	if version_info.get('previous_hash'):
		file_name_fmt = os.path.abspath(os.path.join(dist_root, '%s_%s_v%s.%sdiff-%s-%s.7z' % (
			get_current_packet_id(),
			TIME_TAG,
			version_info.get('current'),
			'%s',
			version_info.get('previous_hash'),
			version_info.get('current_hash'),
		)))
		base_message = ''
		base_hash = ''
		if version_info.get('current') != '' and version_info.get('previous_hash') != '':
			base_message = version_info.get('previous_message')
			base_hash = version_info.get('previous_hash')
		__7zip(file_name_fmt % '', base_message, base_hash, '')
		__7zip(file_name_fmt % 'remake-', base_message, base_hash, '.7zipignore-remake')
		__7zip(file_name_fmt % 'classic-', base_message, base_hash, '.7zipignore-classic')

	file_name_fmt = os.path.abspath(os.path.join(dist_root, '%s_%s_v%s.%sfull.7z' % (
		get_current_packet_id(),
		TIME_TAG,
		version_info.get('current'),
		'%s',
	)))
	__7zip(file_name_fmt % '', '', '', '')
	__7zip(file_name_fmt % 'remake-', '', '', '.7zipignore-remake')
	__7zip(file_name_fmt % 'classic-', '', '', '.7zipignore-classic')

	# Revert source code modify by compressing
	if not is_source:
		os.system('git reset HEAD --hard')
		os.system('git checkout master')
	time.sleep(5)
	print('Exiting...')

if __name__ == '__main__':
	parser = argparse.ArgumentParser(description='One-key release packet product helper.')
	parser.add_argument('--diff', type=str, help='Package diff version.')
	parser.add_argument('--no-build', action='store_true', help='Package source code.')
	args = parser.parse_args()

	env.set_packet_as_cwd()

	run(args.diff, args.no_build)
<|endoftext|>