<commit_before>#!/usr/bin/python
# Made with <3 by Fran González (@spaceisstrange)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a
#  copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>

class ResultItem(object):
    """
    Defines a general result item
    """
    def __init__(self, json):
        """
        Initializes the ResultItem class from the JSON provided
        :param json: String. Raw JSON data to fetch information from
        """
        self.artist_name = json['artistName']
        self.type = None

        if 'wrapperType' in json:
            self.type = json['wrapperType']

            if 'collectionType' in json:
                self.collection_type = json['collectionType']
            elif 'artistType' in json:
                self.artist_type = json['artistType']
            elif 'kind' in json:
                self.track_type = json['kind']
        elif 'kind' in json:
            self.type = json['kind']

        if 'primaryGenreName' in json:
            self.primary_genre_name = json['primaryGenreName']

        if 'artistId' in json:
            self.artist_id = json['artistId']

        if 'artistLinkUrl' in json:
            self.artist_link_url = json['artistLinkUrl']

        if 'radioStationUrl' in json:
            self.artist_radio_url = json['radioStationUrl']

        if 'primaryGenreId' in json:
            self.primary_genre_id = json['primaryGenreId']

        if 'amgArtistId' in json:
            self.artist_amg_id = json['amgArtistId']

        if 'artistViewUrl' in json:
            self.artist_view_url = json['artistViewUrl']

        if 'trackName' in json:
            self.track_name = json['trackName']

        if 'trackId' in json:
            self.track_id = json['trackId']

        if 'trackCensoredName' in json:
            self.track_censored_name = json['trackCensoredName']

        if 'trackViewUrl' in json:
            self.track_view_url = json['trackViewUrl']

        if 'trackCount' in json:
            self.track_count = json['trackCount']

        if 'trackNumber' in json:
            self.track_number = json['trackNumber']

        if 'previewUrl' in json:
            self.preview_url = json['previewUrl']

        if 'artworkUrl30' in json:
            self.artwork_url_30 = json['artworkUrl30']

        if 'artworkUrl60' in json:
            self.artwork_url_60 = json['artworkUrl60']

        if 'artworkUrl100' in json:
            self.artwork_url_100 = json['artworkUrl100']

        if 'artworkUrl512' in json:
            self.artwork_url_512 = json['artworkUrl512']

        if 'collectionName' in json:
            self.collection_name = json['collectionName']

        if 'collectionId' in json:
            self.collection_id = json['collectionId']

        if 'collectionCensoredName' in json:
            self.collection_censored_name = json['collectionCensoredName']

        if 'collectionViewUrl' in json:
            self.collection_view_url = json['collectionViewUrl']

        if 'collectionPrice' in json:
            self.collection_price = json['collectionPrice']

        if 'trackPrice' in json:
            self.track_price = json['trackPrice']

        if 'collectionHdPrice' in json:
            self.collection_hd_price = json['collectionHdPrice']

        if 'trackHdPrice' in json:
            self.track_hd_price = json['trackHdPrice']

        if 'releaseDate' in json:
            self.release_date = json['releaseDate']

        if 'collectionExplicitness' in json:
            self.collection_explicitness = json['collectionExplicitness']

        if 'trackExplicitness' in json:
            self.track_explicitness = json['trackExplicitness']

        if 'trackTimeMillis' in json:
            self.track_time = json['trackTimeMillis']

        if 'discCount' in json:
            self.disc_count = json['discCount']

        if 'discNumber' in json:
            self.disc_number = json['discNumber']

        if 'country' in json:
            self.country = json['country']

        if 'currency' in json:
            self.currency = json['currency']

        if 'copyright' in json:
            self.copyright = json['copyright']

        if 'contentAdvisoryRating' in json:
            self.content_advisory_rating = json['contentAdvisoryRating']

        if 'shortDescription' in json:
            self.short_description = json['shortDescription']

        if 'longDescription' in json:
            self.long_description = json['longDescription']

        if 'isStreamable' in json:
            self.is_streamable = json['isStreamable']

        if 'fileSizeBytes' in json:
            self.file_size = json['fileSizeBytes']

        if 'genres' in json:
            self.genres = json['genres']

        if 'price' in json:
            self.price = json['price']

        if 'description' in json:
            self.description = json['description']

        if 'genreIds' in json:
            self.genre_ids = json['genreIds']

        if 'artistIds' in json:
            self.artist_ids = json['artistIds']

        if 'formattedPrice' in json:
            self.formatted_price = json['formattedPrice']

        if 'averageUserRating' in json:
            self.average_user_rating = json['averageUserRating']

        if 'userRatingCount' in json:
            self.user_rating_count = json['userRatingCount']

        if 'screenshotUrls' in json:
            self.screenshot_urls = json['screenshotUrls']

        if 'ipadScreenshotUrls' in json:
            self.ipad_screenshot_urls = json['ipadScreenshotUrls']

        if 'features' in json:
            self.features = json['features']

        if 'supportedDevices' in json:
            self.supported_devices = json['supportedDevices']

        if 'advisories' in json:
            self.advisories = json['advisories']

        if 'isGameCenterEnabled' in json:
            self.is_game_center_enabled = json['isGameCenterEnabled']

        if 'userRatingCountForCurrentVersion' in json:
            self.user_rating_count_for_current_version = json['userRatingCountForCurrentVersion']

        if 'languageCodesISO2A' in json:
            self.language_codes = json['languageCodesISO2A']

        if 'averageUserRatingForCurrentVersion' in json:
            self.average_user_rating_for_current_version = json['averageUserRatingForCurrentVersion']

        if 'trackContentRating' in json:
            self.track_content_rating = json['trackContentRating']

        if 'version' in json:
            self.version = json['version']

        if 'sellerName' in json:
            self.seller_name = json['sellerName']

        if 'sellerUrl' in json:
            self.seller_url = json['sellerUrl']

        if 'bundleId' in json:
            self.bundle_id = json['bundleId']

        if 'releaseNotes' in json:
            self.release_notes = json['releaseNotes']

        if 'isVppDeviceBasedLicensingEnabled' in json:
            self.is_vpp_device_based_licensing_enabled = json['isVppDeviceBasedLicensingEnabled']

        if 'minimumOsVersion' in json:
            self.minimum_os_version = json['minimumOsVersion']

    def __repr__(self):
        """
        Retrieves all keys in the class as a String
        :return: String. All the keys available in the class
        """
        string = ''

        for key, value in self.__dict__.items():
            if not key.startswith('__'):
                string += '\n' + key + ':' + str(value)

        return string
<commit_msg>Add self.track_rental_price and self.track_hd_rental_price<commit_after>#!/usr/bin/python
# Made with <3 by Fran González (@spaceisstrange)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a
#  copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>

class ResultItem(object):
    """
    Defines a general result item
    """
    def __init__(self, json):
        """
        Initializes the ResultItem class from the JSON provided
        :param json: String. Raw JSON data to fetch information from
        """
        self.artist_name = json['artistName']
        self.type = None

        if 'wrapperType' in json:
            self.type = json['wrapperType']

            if 'collectionType' in json:
                self.collection_type = json['collectionType']
            elif 'artistType' in json:
                self.artist_type = json['artistType']
            elif 'kind' in json:
                self.track_type = json['kind']
        elif 'kind' in json:
            self.type = json['kind']

        if 'primaryGenreName' in json:
            self.primary_genre_name = json['primaryGenreName']

        if 'artistId' in json:
            self.artist_id = json['artistId']

        if 'artistLinkUrl' in json:
            self.artist_link_url = json['artistLinkUrl']

        if 'radioStationUrl' in json:
            self.artist_radio_url = json['radioStationUrl']

        if 'primaryGenreId' in json:
            self.primary_genre_id = json['primaryGenreId']

        if 'amgArtistId' in json:
            self.artist_amg_id = json['amgArtistId']

        if 'artistViewUrl' in json:
            self.artist_view_url = json['artistViewUrl']

        if 'trackName' in json:
            self.track_name = json['trackName']

        if 'trackId' in json:
            self.track_id = json['trackId']

        if 'trackCensoredName' in json:
            self.track_censored_name = json['trackCensoredName']

        if 'trackViewUrl' in json:
            self.track_view_url = json['trackViewUrl']

        if 'trackCount' in json:
            self.track_count = json['trackCount']

        if 'trackNumber' in json:
            self.track_number = json['trackNumber']

        if 'previewUrl' in json:
            self.preview_url = json['previewUrl']

        if 'artworkUrl30' in json:
            self.artwork_url_30 = json['artworkUrl30']

        if 'artworkUrl60' in json:
            self.artwork_url_60 = json['artworkUrl60']

        if 'artworkUrl100' in json:
            self.artwork_url_100 = json['artworkUrl100']

        if 'artworkUrl512' in json:
            self.artwork_url_512 = json['artworkUrl512']

        if 'collectionName' in json:
            self.collection_name = json['collectionName']

        if 'collectionId' in json:
            self.collection_id = json['collectionId']

        if 'collectionCensoredName' in json:
            self.collection_censored_name = json['collectionCensoredName']

        if 'collectionViewUrl' in json:
            self.collection_view_url = json['collectionViewUrl']

        if 'collectionPrice' in json:
            self.collection_price = json['collectionPrice']

        if 'trackPrice' in json:
            self.track_price = json['trackPrice']

        if 'trackRentalPrice' in json:
            self.track_rental_price = json['trackRentalPrice']

        if 'collectionHdPrice' in json:
            self.collection_hd_price = json['collectionHdPrice']

        if 'trackHdPrice' in json:
            self.track_hd_price = json['trackHdPrice']

        if 'trackHdRentalPrice' in json:
            self.track_hd_rental_price = json['trackHdRentalPrice']

        if 'releaseDate' in json:
            self.release_date = json['releaseDate']

        if 'collectionExplicitness' in json:
            self.collection_explicitness = json['collectionExplicitness']

        if 'trackExplicitness' in json:
            self.track_explicitness = json['trackExplicitness']

        if 'trackTimeMillis' in json:
            self.track_time = json['trackTimeMillis']

        if 'discCount' in json:
            self.disc_count = json['discCount']

        if 'discNumber' in json:
            self.disc_number = json['discNumber']

        if 'country' in json:
            self.country = json['country']

        if 'currency' in json:
            self.currency = json['currency']

        if 'copyright' in json:
            self.copyright = json['copyright']

        if 'contentAdvisoryRating' in json:
            self.content_advisory_rating = json['contentAdvisoryRating']

        if 'shortDescription' in json:
            self.short_description = json['shortDescription']

        if 'longDescription' in json:
            self.long_description = json['longDescription']

        if 'isStreamable' in json:
            self.is_streamable = json['isStreamable']

        if 'fileSizeBytes' in json:
            self.file_size = json['fileSizeBytes']

        if 'genres' in json:
            self.genres = json['genres']

        if 'price' in json:
            self.price = json['price']

        if 'description' in json:
            self.description = json['description']

        if 'genreIds' in json:
            self.genre_ids = json['genreIds']

        if 'artistIds' in json:
            self.artist_ids = json['artistIds']

        if 'formattedPrice' in json:
            self.formatted_price = json['formattedPrice']

        if 'averageUserRating' in json:
            self.average_user_rating = json['averageUserRating']

        if 'userRatingCount' in json:
            self.user_rating_count = json['userRatingCount']

        if 'screenshotUrls' in json:
            self.screenshot_urls = json['screenshotUrls']

        if 'ipadScreenshotUrls' in json:
            self.ipad_screenshot_urls = json['ipadScreenshotUrls']

        if 'features' in json:
            self.features = json['features']

        if 'supportedDevices' in json:
            self.supported_devices = json['supportedDevices']

        if 'advisories' in json:
            self.advisories = json['advisories']

        if 'isGameCenterEnabled' in json:
            self.is_game_center_enabled = json['isGameCenterEnabled']

        if 'userRatingCountForCurrentVersion' in json:
            self.user_rating_count_for_current_version = json['userRatingCountForCurrentVersion']

        if 'languageCodesISO2A' in json:
            self.language_codes = json['languageCodesISO2A']

        if 'averageUserRatingForCurrentVersion' in json:
            self.average_user_rating_for_current_version = json['averageUserRatingForCurrentVersion']

        if 'trackContentRating' in json:
            self.track_content_rating = json['trackContentRating']

        if 'version' in json:
            self.version = json['version']

        if 'sellerName' in json:
            self.seller_name = json['sellerName']

        if 'sellerUrl' in json:
            self.seller_url = json['sellerUrl']

        if 'bundleId' in json:
            self.bundle_id = json['bundleId']

        if 'releaseNotes' in json:
            self.release_notes = json['releaseNotes']

        if 'isVppDeviceBasedLicensingEnabled' in json:
            self.is_vpp_device_based_licensing_enabled = json['isVppDeviceBasedLicensingEnabled']

        if 'minimumOsVersion' in json:
            self.minimum_os_version = json['minimumOsVersion']

    def __repr__(self):
        """
        Retrieves all keys in the class as a String
        :return: String. All the keys available in the class
        """
        string = ''

        for key, value in self.__dict__.items():
            if not key.startswith('__'):
                string += '\n' + key + ':' + str(value)

        return string
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
import os
import argparse
import socket
import sys
from jasmine.config import Config

from jasmine.standalone import JasmineApp
from jasmine.ci import CIRunner

def standalone():
    parser = argparse.ArgumentParser(description='Jasmine Standalone')
    parser.add_argument('-p', '--port', type=int, default=8888,
                        help='The port of the Jasmine html runner')
    parser.add_argument('-o', '--host', type=str, default='127.0.0.1',
                        help='The host of the Jasmine html runner')
    args = parser.parse_args()

    if _check_for_config():
        jasmine_config = _load_config()
        try:
            jasmine_app = JasmineApp(jasmine_config=jasmine_config)
            jasmine_app.app.run(host=args.host, port=args.port, debug=True)
        except socket.error:
            sys.stdout.write('Socket unavailable')

def continuous_integration():
    parser = argparse.ArgumentParser(description='Jasmine-CI')
    parser.add_argument('-b', '--browser', type=str,
                        help='The selenium driver to utilize')
    parser.add_argument('-l', '--logs', action='store_true',
                        help='Displays browser logs')
    parser.add_argument('-s', '--seed', type=str,
                        help='Seed for random spec order')
    args = parser.parse_args()

    if _check_for_config():
        jasmine_config = _load_config()
        jasmine_app = JasmineApp(jasmine_config=jasmine_config)
        CIRunner(jasmine_config=jasmine_config).run(
            browser=args.browser,
            show_logs=args.logs,
            seed=args.seed,
            app=jasmine_app.app,
        )

def _config_paths():
    project_path = os.path.realpath(os.path.dirname(__name__))
    config_file = os.path.join(
        project_path,
        "spec/javascripts/support/jasmine.yml"
    )
    return config_file, project_path

def _check_for_config():
    config_file, _ = _config_paths()

    config_exists = os.path.exists(config_file)
    if not config_exists:
        print("Could not find your config file at {0}".format(config_file))
    return config_exists

def _load_config():
    config_file, project_path = _config_paths()
    return Config(config_file, project_path=project_path)

def query(question):
    valid = {"yes": True, "y": True, "ye": True,
             "no": False, "n": False}
    prompt = " [Y/n] "

    while True:
        sys.stdout.write(question + prompt)

        try:
            choice = raw_input().lower()
        except NameError:
            choice = input().lower()

        if choice == '':
            return True
        elif choice in valid:
            return valid[choice]

def install():
    from jasmine.console_formatter import ConsoleFormatter

    spec_dir = os.path.join(os.getcwd(), 'spec/javascripts/')

    print(ConsoleFormatter.JASMINE_HEADER)

    print('Spec directory')

    msg = "About to create {0}... is this okay?".format(spec_dir)
    if query(msg):
        print('making spec/javascripts')
        mkdir_p(spec_dir)

    yaml_dir = os.path.join(spec_dir, 'support')
    yaml_file_path = os.path.join(yaml_dir, 'jasmine.yml')

    print(("*" * 80) + '\n\nConfig yaml')

    if os.path.exists(yaml_file_path):
        print('found existing {0}, not overwriting'.format(yaml_file_path))
    else:
        msg = "About to create {0}... is this okay?".format(yaml_file_path)
        if query(msg):
            print('making {0}'.format(yaml_dir))

            mkdir_p(yaml_dir)

            print('making {0}'.format(yaml_file_path))
            try:
                with open(yaml_file_path, 'w') as f:
                    f.write(YAML_TEMPLATE)
                    f.flush()
            except IOError:
                pass

def mkdir_p(path):
    import os
    import errno

    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

YAML_TEMPLATE = """
# src_files
#
# Return an array of filepaths relative to src_dir to include before jasmine specs.
# Default: []
#
# EXAMPLE:
#
# src_files:
#   - lib/source1.js
#   - lib/source2.js
#   - dist/**/*.js
#
src_files:

# stylesheets
#
# Return an array of stylesheet filepaths relative to src_dir to include before jasmine specs.
# Default: []
#
# EXAMPLE:
#
# stylesheets:
#   - css/style.css
#   - stylesheets/*.css
#
stylesheets:

# helpers
#
# Return an array of filepaths relative to spec_dir to include before jasmine specs.
# Default: ["helpers/**/*.js"]
#
# EXAMPLE:
#
# helpers:
#   - helpers/**/*.js
#
helpers:
  - "helpers/**/*.js"

# spec_files
#
# Return an array of filepaths relative to spec_dir to include.
# Default: ["**/*[sS]pec.js"]
#
# EXAMPLE:
#
# spec_files:
#   - **/*[sS]pec.js
#
spec_files:
  - "**/*[Ss]pec.js"

# src_dir
#
# Source directory path. Your src_files must be returned relative to this path. Will use root if left blank.
# Default: project root
#
# EXAMPLE:
#
# src_dir: public
#
src_dir:

# spec_dir
#
# Spec directory path. Your spec_files must be returned relative to this path.
# Default: spec/javascripts
#
# EXAMPLE:
#
# spec_dir: spec/javascripts
#
spec_dir: spec/javascripts
"""
<commit_msg>Update default jasmine config with random and stop spec on failure<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
import os
import argparse
import socket
import sys
from jasmine.config import Config

from jasmine.standalone import JasmineApp
from jasmine.ci import CIRunner

def standalone():
    parser = argparse.ArgumentParser(description='Jasmine Standalone')
    parser.add_argument('-p', '--port', type=int, default=8888,
                        help='The port of the Jasmine html runner')
    parser.add_argument('-o', '--host', type=str, default='127.0.0.1',
                        help='The host of the Jasmine html runner')
    args = parser.parse_args()

    if _check_for_config():
        jasmine_config = _load_config()
        try:
            jasmine_app = JasmineApp(jasmine_config=jasmine_config)
            jasmine_app.app.run(host=args.host, port=args.port, debug=True)
        except socket.error:
            sys.stdout.write('Socket unavailable')

def continuous_integration():
    parser = argparse.ArgumentParser(description='Jasmine-CI')
    parser.add_argument('-b', '--browser', type=str,
                        help='The selenium driver to utilize')
    parser.add_argument('-l', '--logs', action='store_true',
                        help='Displays browser logs')
    parser.add_argument('-s', '--seed', type=str,
                        help='Seed for random spec order')
    args = parser.parse_args()

    if _check_for_config():
        jasmine_config = _load_config()
        jasmine_app = JasmineApp(jasmine_config=jasmine_config)
        CIRunner(jasmine_config=jasmine_config).run(
            browser=args.browser,
            show_logs=args.logs,
            seed=args.seed,
            app=jasmine_app.app,
        )

def _config_paths():
    project_path = os.path.realpath(os.path.dirname(__name__))
    config_file = os.path.join(
        project_path,
        "spec/javascripts/support/jasmine.yml"
    )
    return config_file, project_path

def _check_for_config():
    config_file, _ = _config_paths()

    config_exists = os.path.exists(config_file)
    if not config_exists:
        print("Could not find your config file at {0}".format(config_file))
    return config_exists

def _load_config():
    config_file, project_path = _config_paths()
    return Config(config_file, project_path=project_path)

def query(question):
    valid = {"yes": True, "y": True, "ye": True,
             "no": False, "n": False}
    prompt = " [Y/n] "

    while True:
        sys.stdout.write(question + prompt)

        try:
            choice = raw_input().lower()
        except NameError:
            choice = input().lower()

        if choice == '':
            return True
        elif choice in valid:
            return valid[choice]

def install():
    from jasmine.console_formatter import ConsoleFormatter

    spec_dir = os.path.join(os.getcwd(), 'spec/javascripts/')

    print(ConsoleFormatter.JASMINE_HEADER)

    print('Spec directory')

    msg = "About to create {0}... is this okay?".format(spec_dir)
    if query(msg):
        print('making spec/javascripts')
        mkdir_p(spec_dir)

    yaml_dir = os.path.join(spec_dir, 'support')
    yaml_file_path = os.path.join(yaml_dir, 'jasmine.yml')

    print(("*" * 80) + '\n\nConfig yaml')

    if os.path.exists(yaml_file_path):
        print('found existing {0}, not overwriting'.format(yaml_file_path))
    else:
        msg = "About to create {0}... is this okay?".format(yaml_file_path)
        if query(msg):
            print('making {0}'.format(yaml_dir))

            mkdir_p(yaml_dir)

            print('making {0}'.format(yaml_file_path))
            try:
                with open(yaml_file_path, 'w') as f:
                    f.write(YAML_TEMPLATE)
                    f.flush()
            except IOError:
                pass

def mkdir_p(path):
    import os
    import errno

    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

YAML_TEMPLATE = """
# src_files
#
# Return an array of filepaths relative to src_dir to include before jasmine specs.
# Default: []
#
# EXAMPLE:
#
# src_files:
#   - lib/source1.js
#   - lib/source2.js
#   - dist/**/*.js
#
src_files:

# stylesheets
#
# Return an array of stylesheet filepaths relative to src_dir to include before jasmine specs.
# Default: []
#
# EXAMPLE:
#
# stylesheets:
#   - css/style.css
#   - stylesheets/*.css
#
stylesheets:

# helpers
#
# Return an array of filepaths relative to spec_dir to include before jasmine specs.
# Default: ["helpers/**/*.js"]
#
# EXAMPLE:
#
# helpers:
#   - helpers/**/*.js
#
helpers:
  - "helpers/**/*.js"

# spec_files
#
# Return an array of filepaths relative to spec_dir to include.
# Default: ["**/*[sS]pec.js"]
#
# EXAMPLE:
#
# spec_files:
#   - **/*[sS]pec.js
#
spec_files:
  - "**/*[Ss]pec.js"

# src_dir
#
# Source directory path. Your src_files must be returned relative to this path. Will use root if left blank.
# Default: project root
#
# EXAMPLE:
#
# src_dir: public
#
src_dir:

# spec_dir
#
# Spec directory path. Your spec_files must be returned relative to this path.
# Default: spec/javascripts
#
# EXAMPLE:
#
# spec_dir: spec/javascripts
#
spec_dir: spec/javascripts

# stop_spec_on_expectation_failure
#
# Stop executing each spec on the first expectation failure.
# Default: false
#
# EXAMPLE:
#
# stop_spec_on_expectation_failure: true
#
stop_spec_on_expectation_failure:

# random
#
# Run specs in semi-random order.
# Default: false
#
# EXAMPLE:
#
# random: true
#
random:
"""
<|endoftext|>


<commit_before>import datetime
import gzip
import json
import time
import urllib2
import os
from sys import stdout, argv
from bs4 import BeautifulSoup
import xml.etree.ElementTree as ET
from xml.dom import minidom

channels = []

def parse_arguments():
    if len(argv) == 2:
        file = open(argv[1], "r")
        for line in file.readlines():
            try:
                line_array = line.replace("\n", "").split("=")
                if line_array[0] == "channel":
                    print line_array[1]
                    channels.append(line_array[1])
            except IndexError:
                pass

def download_json_files():
    if not os.path.exists('/tmp/xmltv_convert/json'):
        os.makedirs('/tmp/xmltv_convert/json')

    page = urllib2.urlopen('http://xmltv.tvtab.la/json/')
    soup = BeautifulSoup(page)
    soup.prettify()

    for anchor in soup.findAll('a', href=True):
        if anchor['href'] != '../':
            try:
                anchor_list = anchor['href'].split("_")
                channel = anchor_list[0]
                filedate = datetime.datetime.strptime(anchor_list[1][0:10], "%Y-%m-%d").date()
            except IndexError:
                filedate = datetime.datetime.today().date()

            if filedate >= datetime.datetime.today().date():
                if len(channels) == 0 or channel in channels or channel == "channels.js.gz":
                    stdout.write("Downloading http://xmltv.tvtab.la/json/%s " % anchor['href'])
                    f = urllib2.urlopen('http://xmltv.tvtab.la/json/%s' % anchor['href'])
                    data = f.read()
                    with open('/tmp/xmltv_convert/json/%s' % anchor['href'].replace('.gz', ''), 'w+ ') as outfile:
                        outfile.write(data)
                    stdout.write("Done!\n")
                    stdout.flush()

def create_xml():
    if not os.path.exists('/tmp/xmltv_convert/xml'):
        os.makedirs('/tmp/xmltv_convert/xml')

    for filename in os.listdir('/tmp/xmltv_convert/json/'):
        if filename != '.' and filename != '..':
            stdout.write("Opening %s, creating XML " % filename)
            json_data = open('/tmp/xmltv_convert/json/%s' % filename).read()
            data = json.loads(json_data)

            root = ET.Element("tv", { "generator-info-name": "nonametv" })

            if filename != "channels.js":
                for programme in data['jsontv']['programme']:
                    starttime = time.localtime(float(programme['start']))
                    stoptime = time.localtime(float(programme['stop']))

                    starttime_offset = "+0200" if starttime.tm_isdst else "+0100"
                    stoptime_offset = "+0200" if stoptime.tm_isdst else "+0100"

                    # Programme should always contain something
                    xml_programme = ET.SubElement(root, "programme", {"start": "%s %s" % (time.strftime("%Y%m%d%H%M%S", starttime),
                                                                                starttime_offset),
                                                                      "stop": "%s %s" % (time.strftime("%Y%m%d%H%M%S", stoptime),
                                                                                stoptime_offset),
                                                                      "channel": programme['channel'] })

                    # A title should be there
                    if programme.has_key("title"):
                        for key in programme['title'].keys():
                            title = programme['title'][key]
                            parsed_title = ''.join(c for c in title if ord(c) >= 32)
                            xml_desc = ET.SubElement(xml_programme, "title", { "lang": key })
                            xml_desc.text = parsed_title

                    # A subtitle COULD be there
                    if programme.has_key("subTitle"):
                        for key in programme['subTitle'].keys():
                            subtitle = programme['subTitle'][key]
                            parsed_subtitle = ''.join(c for c in subtitle if ord(c) >= 32)
                            xml_subtitle = ET.SubElement(xml_programme, "sub-title", { "lang": key })
                            xml_subtitle.text = parsed_subtitle

                    # A description COULD be there
                    if programme.has_key("desc"):
                        for key in programme['desc'].keys():
                            description = programme['desc'][key]
                            parsed_description = ''.join(c for c in description if ord(c) >= 32)
                            xml_desc = ET.SubElement(xml_programme, "desc", { "lang": key })
                            xml_desc.text = parsed_description

                    # A date COULD be there
                    if programme.has_key("date"):
                        xml_date = ET.SubElement(xml_programme, "date")
                        xml_date.text = programme['date']

                    # A category COULD be there
                    if programme.has_key("category"):
                        for key in programme['category'].keys():
                            for value in programme['category'][key]:
                                xml_category = ET.SubElement(xml_programme, "category", { "lang": key })
                                xml_category.text = value.replace("\n", "")

                    # An episode number sequence COULD be present
                    if programme.has_key("episodeNum"):
                        for key in programme['episodeNum'].keys():
                            episode_num = ET.SubElement(xml_programme, "episode-num", { "system": key })
                            episode_num.text = programme['episodeNum'][key].replace("\n", "")

                    # An url COULD be present
                    if programme.has_key("url"):
                        url = ET.SubElement(xml_programme, "url")
                        url.text = programme['url'][0]

                    # A rating COULD be present
                    if programme.has_key("rating"):
                        if programme['rating'].has_key("stars"):
                            star_rating = ET.SubElement(xml_programme, "star-rating")
                            star_rating_value = ET.SubElement(star_rating, "value")
                            star_rating_value.text = programme['rating']['stars']
                        if programme['rating'].has_key("mpaa"):
                            rating = ET.SubElement(xml_programme, "rating", { "system": "MPAA" })
                            rating_value = ET.SubElement(rating, "value")
                            rating_value.text = programme['rating']['mpaa']

                    # Credits COULD be present
                    if programme.has_key("credits"):
                        xml_credits = ET.SubElement(xml_programme, "credits")
                        for key in programme['credits'].keys():
                            for value in programme['credits'][key]:
                                xml_credit = ET.SubElement(xml_credits, key)
                                xml_credit.text = value

                    # Video COULD be present
                    if programme.has_key("video"):
                        if programme['video'].has_key("aspect"):
                            xml_video = ET.SubElement(xml_programme, "video")
                            xml_video_aspect = ET.SubElement(xml_video, "aspect")
                            xml_video_aspect.text = programme['video']['aspect']

            else:
                for key in data['jsontv']['channels']:
                    xml_channel = ET.SubElement(root, "channel", { "id": key })

                    for lang in data['jsontv']['channels'][key]['displayName'].keys():
                        xml_display_name = ET.SubElement(xml_channel, "display-name", { "lang": lang })
                        xml_display_name.text = data['jsontv']['channels'][key]['displayName'][lang]

                    xml_base_url = ET.SubElement(xml_channel, "base-url")
                    xml_base_url.text = "http://xmltv.tvtab.la/xmltv/"

                    if data['jsontv']['channels'][key].has_key('icon'):
                        xml_icon = ET.SubElement(xml_channel, "icon", { "src": data['jsontv']['channels'][key]['icon'] })

            outfile = gzip.open('/tmp/xmltv_convert/xml/%s' % filename.replace('.js', '.xml.gz'), 'w+')
            text = ET.tostring(root, encoding="utf-8")
            doc = minidom.parseString(text)
            dt = minidom.getDOMImplementation('').createDocumentType('tv', None, 'xmltv.dtd')
            doc.insertBefore(dt, doc.documentElement)
            outfile.write(doc.toprettyxml(encoding="utf-8"))
            outfile.close()

            stdout.write("Done!\n")

os.environ['TZ'] = 'Europe/Stockholm'

print "Deleting old files..."
if os.path.exists('/tmp/xmltv_convert/json'):
    for filename in os.listdir('/tmp/xmltv_convert/json/'):
        if filename != '.' and filename != '..':
            os.remove('/tmp/xmltv_convert/json/%s' % filename)

if os.path.exists('/tmp/xmltv_convert/xml'):
    for filename in os.listdir('/tmp/xmltv_convert/xml/'):
        if filename != '.' and filename != '..':
            os.remove('/tmp/xmltv_convert/xml/%s' % filename)

print "Check which files should be downloaded..."
parse_arguments()

print "Downloading files..."
download_json_files()

print "Reformat json to xml..."
create_xml()
<commit_msg>Fixed GZIP thing by andreas. And changed the url to the new one<commit_after>import datetime
import gzip
import json
import time
import urllib2
import os
from sys import stdout, argv
from bs4 import BeautifulSoup
import xml.etree.ElementTree as ET
from xml.dom import minidom

channels = []

def parse_arguments():
    if len(argv) == 2:
        file = open(argv[1], "r")
        for line in file.readlines():
            try:
                line_array = line.replace("\n", "").split("=")
                if line_array[0] == "channel":
                    print line_array[1]
                    channels.append(line_array[1])
            except IndexError:
                pass

def download_json_files():
    if not os.path.exists('/tmp/xmltv_convert/json'):
        os.makedirs('/tmp/xmltv_convert/json')

    page = urllib2.urlopen('http://json.xmltv.se/')
    soup = BeautifulSoup(page)
    soup.prettify()

    for anchor in soup.findAll('a', href=True):
        if anchor['href'] != '../':
            try:
                anchor_list = anchor['href'].split("_")
                channel = anchor_list[0]
                filedate = datetime.datetime.strptime(anchor_list[1][0:10], "%Y-%m-%d").date()
            except IndexError:
                filedate = datetime.datetime.today().date()

            if filedate >= datetime.datetime.today().date():
                if len(channels) == 0 or channel in channels or channel == "channels.js.gz":
                    stdout.write("Downloading http://xmltv.tvtab.la/json/%s " % anchor['href'])
                    f = urllib2.urlopen('http://xmltv.tvtab.la/json/%s' % anchor['href'])
                    data = f.read()
                    with open('/tmp/xmltv_convert/json/%s' % anchor['href'].replace('.gz', ''), 'w+ ') as outfile:
                        outfile.write(data)
                    stdout.write("Done!\n")
                    stdout.flush()

def create_xml():
    if not os.path.exists('/tmp/xmltv_convert/xml'):
        os.makedirs('/tmp/xmltv_convert/xml')

    for filename in os.listdir('/tmp/xmltv_convert/json/'):
        if filename != '.' and filename != '..':
            stdout.write("Opening %s, creating XML " % filename)
            try:
                json_data = gzip.open('/tmp/xmltv_convert/json/%s' % filename, 'rb').read()
            except IOError:
                json_data = open('/tmp/xmltv_convert/json/%s' % filename).read()
            data = json.loads(json_data)

            root = ET.Element("tv", { "generator-info-name": "nonametv" })

            if filename != "channels.js":
                for programme in data['jsontv']['programme']:
                    starttime = time.localtime(float(programme['start']))
                    stoptime = time.localtime(float(programme['stop']))

                    starttime_offset = "+0200" if starttime.tm_isdst else "+0100"
                    stoptime_offset = "+0200" if stoptime.tm_isdst else "+0100"

                    # Programme should always contain something
                    xml_programme = ET.SubElement(root, "programme", {"start": "%s %s" % (time.strftime("%Y%m%d%H%M%S", starttime),
                                                                                starttime_offset),
                                                                      "stop": "%s %s" % (time.strftime("%Y%m%d%H%M%S", stoptime),
                                                                                stoptime_offset),
                                                                      "channel": programme['channel'] })

                    # A title should be there
                    if programme.has_key("title"):
                        for key in programme['title'].keys():
                            title = programme['title'][key]
                            parsed_title = ''.join(c for c in title if ord(c) >= 32)
                            xml_desc = ET.SubElement(xml_programme, "title", { "lang": key })
                            xml_desc.text = parsed_title

                    # A subtitle COULD be there
                    if programme.has_key("subTitle"):
                        for key in programme['subTitle'].keys():
                            subtitle = programme['subTitle'][key]
                            parsed_subtitle = ''.join(c for c in subtitle if ord(c) >= 32)
                            xml_subtitle = ET.SubElement(xml_programme, "sub-title", { "lang": key })
                            xml_subtitle.text = parsed_subtitle

                    # A description COULD be there
                    if programme.has_key("desc"):
                        for key in programme['desc'].keys():
                            description = programme['desc'][key]
                            parsed_description = ''.join(c for c in description if ord(c) >= 32)
                            xml_desc = ET.SubElement(xml_programme, "desc", { "lang": key })
                            xml_desc.text = parsed_description

                    # A date COULD be there
                    if programme.has_key("date"):
                        xml_date = ET.SubElement(xml_programme, "date")
                        xml_date.text = programme['date']

                    # A category COULD be there
                    if programme.has_key("category"):
                        for key in programme['category'].keys():
                            for value in programme['category'][key]:
                                xml_category = ET.SubElement(xml_programme, "category", { "lang": key })
                                xml_category.text = value.replace("\n", "")

                    # An episode number sequence COULD be present
                    if programme.has_key("episodeNum"):
                        for key in programme['episodeNum'].keys():
                            episode_num = ET.SubElement(xml_programme, "episode-num", { "system": key })
                            episode_num.text = programme['episodeNum'][key].replace("\n", "")

                    # An url COULD be present
                    if programme.has_key("url"):
                        url = ET.SubElement(xml_programme, "url")
                        url.text = programme['url'][0]

                    # A rating COULD be present
                    if programme.has_key("rating"):
                        if programme['rating'].has_key("stars"):
                            star_rating = ET.SubElement(xml_programme, "star-rating")
                            star_rating_value = ET.SubElement(star_rating, "value")
                            star_rating_value.text = programme['rating']['stars']
                        if programme['rating'].has_key("mpaa"):
                            rating = ET.SubElement(xml_programme, "rating", { "system": "MPAA" })
                            rating_value = ET.SubElement(rating, "value")
                            rating_value.text = programme['rating']['mpaa']

                    # Credits COULD be present
                    if programme.has_key("credits"):
                        xml_credits = ET.SubElement(xml_programme, "credits")
                        for key in programme['credits'].keys():
                            for value in programme['credits'][key]:
                                xml_credit = ET.SubElement(xml_credits, key)
                                xml_credit.text = value

                    # Video COULD be present
                    if programme.has_key("video"):
                        if programme['video'].has_key("aspect"):
                            xml_video = ET.SubElement(xml_programme, "video")
                            xml_video_aspect = ET.SubElement(xml_video, "aspect")
                            xml_video_aspect.text = programme['video']['aspect']

            else:
                for key in data['jsontv']['channels']:
                    xml_channel = ET.SubElement(root, "channel", { "id": key })

                    for lang in data['jsontv']['channels'][key]['displayName'].keys():
                        xml_display_name = ET.SubElement(xml_channel, "display-name", { "lang": lang })
                        xml_display_name.text = data['jsontv']['channels'][key]['displayName'][lang]

                    xml_base_url = ET.SubElement(xml_channel, "base-url")
                    xml_base_url.text = "http://xmltv.tvtab.la/xmltv/"

                    if data['jsontv']['channels'][key].has_key('icon'):
                        xml_icon = ET.SubElement(xml_channel, "icon", { "src": data['jsontv']['channels'][key]['icon'] })

            outfile = gzip.open('/tmp/xmltv_convert/xml/%s' % filename.replace('.js', '.xml.gz'), 'w+')
            text = ET.tostring(root, encoding="utf-8")
            doc = minidom.parseString(text)
            dt = minidom.getDOMImplementation('').createDocumentType('tv', None, 'xmltv.dtd')
            doc.insertBefore(dt, doc.documentElement)
            outfile.write(doc.toprettyxml(encoding="utf-8"))
            outfile.close()

            stdout.write("Done!\n")

os.environ['TZ'] = 'Europe/Stockholm'

print "Deleting old files..."
if os.path.exists('/tmp/xmltv_convert/json'):
    for filename in os.listdir('/tmp/xmltv_convert/json/'):
        if filename != '.' and filename != '..':
            os.remove('/tmp/xmltv_convert/json/%s' % filename)

if os.path.exists('/tmp/xmltv_convert/xml'):
    for filename in os.listdir('/tmp/xmltv_convert/xml/'):
        if filename != '.' and filename != '..':
            os.remove('/tmp/xmltv_convert/xml/%s' % filename)

print "Check which files should be downloaded..."
parse_arguments()

print "Downloading files..."
download_json_files()

print "Reformat json to xml..."
create_xml()
<|endoftext|>


<commit_before>#    Copyright 2015 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import six

from fuel_plugin.testing.tests import base

from fuel_plugin.ostf_adapter import mixins

from fuel_plugin.ostf_adapter.storage import models

class TestModelTestMethods(base.BaseIntegrationTest):

    def setUp(self):
        super(TestModelTestMethods, self).setUp()

        self.discovery()

        self.test_set_id = 'general_test'
        self.cluster_id = 1

        self.mock_api_for_cluster(self.cluster_id)

        mixins.discovery_check(self.session, self.cluster_id)
        self.session.flush()

        self.test_obj = self.session.query(models.Test)\
            .filter_by(test_set_id=self.test_set_id)\
            .first()

        self.test_run = models.TestRun.add_test_run(
            self.session,
            self.test_obj.test_set_id,
            self.cluster_id,
            status='running',
            tests=[self.test_obj.name]
        )
        self.session.flush()

    @property
    def test_to_check(self):
        return self.session.query(models.Test)\
            .filter_by(test_run_id=self.test_run.id)\
            .filter_by(name=self.test_obj.name)\
            .first()

    def check_model_obj_attrs(self, obj, attrs):
        for attr_name, attr_val in six.iteritems(attrs):
            self.assertEqual(attr_val, getattr(obj, attr_name))

    def test_add_result(self):
        expected_data = {
            'message': 'test_message',
            'status': 'error',
            'time_taken': 10.4
        }

        models.Test.add_result(self.session,
                               self.test_run.id,
                               self.test_obj.name,
                               expected_data)

        self.check_model_obj_attrs(self.test_to_check, expected_data)

    def test_update_running_tests_default_status(self):
        models.Test.update_running_tests(self.session,
                                         self.test_run.id)

        self.assertEqual(self.test_to_check.status, 'stopped')

    def test_update_running_tests_with_status(self):
        expected_status = 'success'

        models.Test.update_running_tests(self.session,
                                         self.test_run.id,
                                         status=expected_status)

        self.assertEqual(self.test_to_check.status, expected_status)

    def test_update_only_running_tests(self):
        # the method should update only running tests
        expected_status = 'error'
        models.Test.add_result(self.session, self.test_run.id,
                               self.test_obj.name,
                               {'status': expected_status})

        models.Test.update_running_tests(self.session, self.test_run.id)

        # check that status of test is not updated to 'stopped'
        self.assertEqual(self.test_to_check.status, expected_status)

    def test_update_test_run_tests_default_status(self):
        models.Test.add_result(self.session, self.test_run.id,
                               self.test_obj.name,
                               {'time_taken': 10.4})

        models.Test.update_test_run_tests(self.session, self.test_run.id,
                                          [self.test_obj.name])

        expected_attrs = {
            'status': 'wait_running',
            'time_taken': None
        }

        self.check_model_obj_attrs(self.test_to_check, expected_attrs)

    def test_properly_copied_test(self):
        new_test = self.test_obj.copy_test(self.test_run, predefined_tests=[])

        copied_attrs_list = [
            'name',
            'title',
            'description',
            'duration',
            'message',
            'traceback',
            'step',
            'time_taken',
            'meta',
            'deployment_tags',
            'available_since_release',
            'test_set_id',
        ]
        attrs = {}
        for attr_name in copied_attrs_list:
            attrs[attr_name] = getattr(self.test_obj, attr_name)

        self.check_model_obj_attrs(new_test, attrs)

        self.assertEqual(new_test.test_run_id, self.test_run.id)
        self.assertEqual(new_test.status, 'wait_running')

    def test_copy_test_with_predefined_list(self):
        predefined_tests_names = ['some_other_test']
        new_test = self.test_obj.copy_test(self.test_run,
                                           predefined_tests_names)

        self.assertEqual(new_test.status, 'disabled')
<commit_msg>Add tests for methods of TestSet and TestRun models<commit_after>#    Copyright 2015 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import six

from fuel_plugin.testing.tests import base

from fuel_plugin.ostf_adapter import mixins

from fuel_plugin.ostf_adapter.storage import models

class TestModelTestMethods(base.BaseIntegrationTest):

    test_set_id = 'general_test'
    cluster_id = 1

    def setUp(self):
        super(TestModelTestMethods, self).setUp()

        self.discovery()

        self.mock_api_for_cluster(self.cluster_id)

        mixins.discovery_check(self.session, self.cluster_id)
        self.session.flush()

        self.test_obj = self.session.query(models.Test)\
            .filter_by(test_set_id=self.test_set_id)\
            .first()

        self.test_run = models.TestRun.add_test_run(
            self.session,
            self.test_obj.test_set_id,
            self.cluster_id,
            status='running',
            tests=[self.test_obj.name]
        )
        self.session.flush()

    @property
    def test_to_check(self):
        return self.session.query(models.Test)\
            .filter_by(test_run_id=self.test_run.id)\
            .filter_by(name=self.test_obj.name)\
            .first()

    def check_model_obj_attrs(self, obj, attrs):
        for attr_name, attr_val in six.iteritems(attrs):
            self.assertEqual(attr_val, getattr(obj, attr_name))

    def test_add_result(self):
        expected_data = {
            'message': 'test_message',
            'status': 'error',
            'time_taken': 10.4
        }

        models.Test.add_result(self.session,
                               self.test_run.id,
                               self.test_obj.name,
                               expected_data)

        self.check_model_obj_attrs(self.test_to_check, expected_data)

    def test_update_running_tests_default_status(self):
        models.Test.update_running_tests(self.session,
                                         self.test_run.id)

        self.assertEqual(self.test_to_check.status, 'stopped')

    def test_update_running_tests_with_status(self):
        expected_status = 'success'

        models.Test.update_running_tests(self.session,
                                         self.test_run.id,
                                         status=expected_status)

        self.assertEqual(self.test_to_check.status, expected_status)

    def test_update_only_running_tests(self):
        # the method should update only running tests
        expected_status = 'error'
        models.Test.add_result(self.session, self.test_run.id,
                               self.test_obj.name,
                               {'status': expected_status})

        models.Test.update_running_tests(self.session, self.test_run.id)

        # check that status of test is not updated to 'stopped'
        self.assertEqual(self.test_to_check.status, expected_status)

    def test_update_test_run_tests_default_status(self):
        models.Test.add_result(self.session, self.test_run.id,
                               self.test_obj.name,
                               {'time_taken': 10.4})

        models.Test.update_test_run_tests(self.session, self.test_run.id,
                                          [self.test_obj.name])

        expected_attrs = {
            'status': 'wait_running',
            'time_taken': None
        }

        self.check_model_obj_attrs(self.test_to_check, expected_attrs)

    def test_properly_copied_test(self):
        new_test = self.test_obj.copy_test(self.test_run, predefined_tests=[])

        copied_attrs_list = [
            'name',
            'title',
            'description',
            'duration',
            'message',
            'traceback',
            'step',
            'time_taken',
            'meta',
            'deployment_tags',
            'available_since_release',
            'test_set_id',
        ]
        attrs = {}
        for attr_name in copied_attrs_list:
            attrs[attr_name] = getattr(self.test_obj, attr_name)

        self.check_model_obj_attrs(new_test, attrs)

        self.assertEqual(new_test.test_run_id, self.test_run.id)
        self.assertEqual(new_test.status, 'wait_running')

    def test_copy_test_with_predefined_list(self):
        predefined_tests_names = ['some_other_test']
        new_test = self.test_obj.copy_test(self.test_run,
                                           predefined_tests_names)

        self.assertEqual(new_test.status, 'disabled')

class TestModelTestSetMethods(base.BaseIntegrationTest):

    test_set_id = 'general_test'

    def setUp(self):
        super(TestModelTestSetMethods, self).setUp()
        self.discovery()

    def test_get_test_set(self):
        self.assertIsNotNone(
            models.TestSet.get_test_set(
                self.session,
                self.test_set_id
            )
        )
        self.assertIsNone(
            models.TestSet.get_test_set(
                self.session,
                'fake_test'
            )
        )

    def test_frontend_property(self):
        test_set = self.session.query(models.TestSet)\
            .filter_by(id=self.test_set_id)\
            .first()
        expected = {'id': test_set.id, 'name': test_set.description}
        self.assertEqual(expected, test_set.frontend)

class TestModelTestRunMethods(base.BaseIntegrationTest):

    test_set_id = 'general_test'
    cluster_id = 1

    def setUp(self):
        super(TestModelTestRunMethods, self).setUp()
        self.discovery()

        self.mock_api_for_cluster(self.cluster_id)
        mixins.discovery_check(self.session, self.cluster_id)
        self.session.flush()

    def check_enabled(self, expected_test_names, test_run_tests):
        enabled_tests = [
            test.name for test in test_run_tests
            if test.status == 'wait_running'
        ]

        self.assertItemsEqual(expected_test_names, enabled_tests)

    def test_add_test_run(self):
        test_run = models.TestRun.add_test_run(
            self.session, self.test_set_id,
            self.cluster_id
        )

        for attr in ('test_set_id', 'cluster_id'):
            self.assertEqual(getattr(self, attr), getattr(test_run, attr))

        # default status for newly created test_run is 'running'
        self.assertEqual(test_run.status, 'running')

        unassigned_tests = self.session.query(models.Test)\
            .filter_by(test_set_id=self.test_set_id)\
            .filter_by(test_run_id=None)

        test_names_from_test_set = [
            test.name for test in unassigned_tests
        ]
        test_names_from_test_run = [
            test.name for test in test_run.tests
        ]
        self.assertItemsEqual(test_names_from_test_run,
                              test_names_from_test_set)

    def test_add_test_run_non_default_status(self):
        expected_status = 'finished'
        test_run = models.TestRun.add_test_run(
            self.session, self.test_set_id,
            self.cluster_id, status=expected_status
        )
        self.assertEqual(test_run.status, expected_status)

    def test_add_test_run_with_predefined_tests(self):
        expected_test_names = [
            test.name for test in
            self.session.query(models.Test)
                .filter_by(test_set_id=self.test_set_id)
                .filter_by(test_run_id=None)
        ][:3]

        test_run = models.TestRun.add_test_run(
            self.session, self.test_set_id,
            self.cluster_id, tests=expected_test_names
        )
        self.check_enabled(expected_test_names, test_run.tests)

    def test_add_test_run_tests_from_another_test_set_in_predefined(self):
        expected_test_names = [
            test.name for test in
            self.session.query(models.Test)
                .filter_by(test_set_id=self.test_set_id)
                .filter_by(test_run_id=None)
        ][:3]

        additional_test = self.session.query(models.Test)\
            .filter_by(test_set_id='stopped_test')\
            .first()

        tests = expected_test_names + [additional_test]
        test_run = models.TestRun.add_test_run(
            self.session, self.test_set_id,
            self.cluster_id,
            tests=tests
        )

        self.assertNotIn(
            additional_test.name,
            [test.name for test in test_run.tests]
        )
        self.check_enabled(expected_test_names, test_run.tests)

    def test_update_testrun_not_finished_status(self):
        test_run = models.TestRun.add_test_run(
            self.session, self.test_set_id,
            self.cluster_id
        )

        expected_status = 'stopped'
        test_run.update(expected_status)

        self.assertEqual(test_run.status, expected_status)
        self.assertIsNone(test_run.ended_at)
<|endoftext|>


<commit_before>import json
from datetime import datetime, timedelta

import pywikibot

import toolforge
from config import config_page_name  # pylint: disable=E0611,W0614

site = pywikibot.Site()
site.login()

config_page = pywikibot.Page(site, config_page_name)
cfg = config_page.text
cfg = json.loads(cfg)
print(json.dumps(cfg, indent=4, ensure_ascii=False))

if not cfg['longtime_enable']:
    exit('disabled\n')

conn = toolforge.connect('zhwiki')

with conn.cursor() as cur:
    cur.execute('use zhwiki_p')
    cur.execute("""
        SELECT pr_page, page_namespace, page_title
        FROM page_restrictions
        LEFT JOIN page ON pr_page = page_id
        WHERE pr_type = "edit"
            AND pr_level = "autoconfirmed"
            AND pr_expiry = "infinity"
            AND page_namespace = 0
        ORDER BY page_title ASC
    """)
    res = cur.fetchall()

text = cfg['indef_header_text']
text += """{| class="wikitable sortable"
!頁面!!保護日誌"""

for row in res:
    pid = str(row[0])
    ns = str(row[1])
    title = row[2].decode()
    text += "\n|-\n"
    text += "|[[Special:Redirect/page/" + pid + "|{{subst:#ifeq:" + ns + "|0||{{subst:ns:" + ns + "}}:}}" + title + "]]"
    text += "||[{{fullurl:Special:日志/protect|page={{subst:#ifeq:" + ns + "|0||{{subst:ns:" + ns + "}}:}}" + title + "}} 保護日誌]"
text += "\n|}"

page = pywikibot.Page(site, cfg['indef_output_page'])
page.text = text
page.save(summary=cfg['indef_summary'], minor=False)
<commit_msg>unused-import<commit_after>import json

import pywikibot

import toolforge
from config import config_page_name  # pylint: disable=E0611,W0614

site = pywikibot.Site()
site.login()

config_page = pywikibot.Page(site, config_page_name)
cfg = config_page.text
cfg = json.loads(cfg)
print(json.dumps(cfg, indent=4, ensure_ascii=False))

if not cfg['longtime_enable']:
    exit('disabled\n')

conn = toolforge.connect('zhwiki')

with conn.cursor() as cur:
    cur.execute('use zhwiki_p')
    cur.execute("""
        SELECT pr_page, page_namespace, page_title
        FROM page_restrictions
        LEFT JOIN page ON pr_page = page_id
        WHERE pr_type = "edit"
            AND pr_level = "autoconfirmed"
            AND pr_expiry = "infinity"
            AND page_namespace = 0
        ORDER BY page_title ASC
    """)
    res = cur.fetchall()

text = cfg['indef_header_text']
text += """{| class="wikitable sortable"
!頁面!!保護日誌"""

for row in res:
    pid = str(row[0])
    ns = str(row[1])
    title = row[2].decode()
    text += "\n|-\n"
    text += "|[[Special:Redirect/page/" + pid + "|{{subst:#ifeq:" + ns + "|0||{{subst:ns:" + ns + "}}:}}" + title + "]]"
    text += "||[{{fullurl:Special:日志/protect|page={{subst:#ifeq:" + ns + "|0||{{subst:ns:" + ns + "}}:}}" + title + "}} 保護日誌]"
text += "\n|}"

page = pywikibot.Page(site, cfg['indef_output_page'])
page.text = text
page.save(summary=cfg['indef_summary'], minor=False)
<|endoftext|>


<commit_before>from lxml import etree

catoxml_ns = "{http://namespaces.cato.org/catoxml}"

def url_for_us_code(citation):
	def is_range(segment):
		# XXX: This is a quick-and-dirty range finder.
		return (".." in segment)

	def get_range_start(segment):
		return segment[:segment.index("..")]

	# Citations of proposed documents won't follow the same URL format.
	if citation["proposed"]:
		# XXX: We should probably support same-page URLs for these.
		return None

	if "title" not in citation:
		return None

	subpath = citation["title"]

	if citation["subtype"] == "usc-appendix":
		subpath += "a"

	if citation["subtype"] == "usc-chapter":
		if "chapter" in citation:
			subpath += "/chapter-%s" % ( citation["chapter"] )

			if "subchapter" in citation:
				subpath += "/subchapter-%s" % ( citation["subchapter"] )
	else:
		if "section" in citation:
			if is_range(citation["section"]):
				subpath += "/%s" % ( get_range_start(citation["section"]) )
			else:
				subpath += "/%s" % ( citation["section"] )

				if "subsection" in citation:
					if is_range(citation["subsection"]):
						fragment = get_range_start(citation["subsection"])
					else:
						fragment = citation["subsection"]

						for segment in [ "paragraph", "subparagraph", "clause", "subclause", "item", "subitem" ]:
							if segment in citation:
								# Sometimes a segment is empty, so we just skip it.
								if citation[segment] != "":
									if is_range(citation[segment]):
										segment_range_start = get_range_start(citation[segment])
										fragment += segment_range_start if fragment == "" else "_%s" % ( segment_range_start )
										break
									else:
										fragment += citation[segment] if fragment == "" else "_%s" % ( citation[segment] )
							else:
								break

					if fragment != "":
						subpath += "#%s" % ( fragment )

	# convert en-dashes to hyphens
	subpath = subpath.replace(u"\u2013", "-")

	return "http://www.law.cornell.edu/uscode/text/%s" % ( subpath )

def url_for_statute_at_large(citation):
	# Citations of proposed documents won't follow the same URL format.
	if citation["proposed"]:
		# XXX: We should probably support same-page URLs for these.
		return None

	try:
		# An en-dash indicates a page range, but link to just the first page in the range.
		page = citation["page"].split(u"\u2013")[0]
		url = "http://www.gpo.gov/fdsys/search/citation2.result.STATUTE.action?publication=STATUTE&statute.volume=%d&statute.pageNumber=%s" % ( int(citation["volume"]), page)
	except KeyError:
		url = None

	return url

def url_for_public_law(citation):
	# Citations of proposed documents won't follow the same URL format.
	if citation["proposed"]:
		# XXX: We should probably support same-page URLs for these.
		return None

	try:
		url = "https://www.govtrack.us/search?q=P.L.+%d-%d" % ( int(citation["congress"]), int(citation["law"]) )
	except KeyError:
		url = None

	return url

def url_for_act(citation):
	# Citations of proposed documents won't follow the same URL format.
	if citation["proposed"]:
		# XXX: We should probably support same-page URLs for these.
		return None

	try:
		url = "https://www.govtrack.us/congress/bills/browse?congress=__ALL__&sort=relevance&text=%s" % ( citation["act"] )
	except KeyError:
		url = None

	return url

def create_link_url(xml_element):
	import citations

	link_url = None

	xml_tag = xml_element.tag

	if xml_tag.startswith(catoxml_ns):
		xml_tag_name = xml_tag[len(catoxml_ns):]

		if xml_tag_name in [ "entity-ref" ]:
			entity_type = xml_element.get("entity-type")
			entity_value = xml_element.get("value")
			entity_proposed = True if ( xml_element.get("proposed", "false") == "true" ) else False

			if entity_value is not None:
				citation = citations.deepbills_citation_for(entity_type, entity_value, xml_element.text, entity_proposed)

				if citation["type"] == "uscode":
					link_url = url_for_us_code(citation)
				elif citation["type"] == "statute-at-large":
					link_url = url_for_statute_at_large(citation)
				elif citation["type"] == "public-law":
					link_url = url_for_public_law(citation)
				elif citation["type"] == "act":
					link_url = url_for_act(citation)
				else:
					# Unexpected citation type.
					pass
	else:
		if xml_tag == "external-xref":
			legal_doc = xml_element.get("legal-doc")
			parsable_cite = xml_element.get("parsable-cite")

			# XXX: Workaround for citations.deepbills_citation_for().
			if legal_doc == "usc":
				legal_doc = "uscode"

			citation = citations.deepbills_citation_for(legal_doc, parsable_cite, xml_element.text)

#			if citation["type"] == "usc":
			if citation["type"] == "uscode":
				link_url = url_for_us_code(citation)
			elif citation["type"] == "statute-at-large":
				link_url = url_for_statute_at_large(citation)
			elif citation["type"] == "public-law":
				link_url = url_for_public_law(citation)
			elif citation["type"] == "act":
				link_url = url_for_act(citation)
			elif citation["type"] == "executive-order":
				pass
			elif citation["type"] == "regulation":
				pass
			elif citation["type"] == "bill":
				pass
			elif citation["type"] == "senate-rule":
				pass
			elif citation["type"] == "treaty-ust":
				pass
			elif citation["type"] == "treaty-tias":
				pass
			else:
				# Unexpected document type
				pass
		elif xml_tag == "internal-xref":
			idref = xml_element.get("idref")
			legis_path = xml_element.get("legis-path")
		elif xml_tag == "footnote-ref":
			idref = xml_element.get("idref")
		else:
			# Unexpected XML tag.
			pass

	return link_url

def can_be_link(xml_element):
	xml_tag = xml_element.tag

	if xml_tag.startswith(catoxml_ns):
		pass
	else:
		# Favor CatoXML references, because they tend to be more specific.
		# XXX: This is probably enough, but we might want to check for other nested link elements.
		entity_ref_tag_name = "{%s}entity-ref" % ( catoxml_ns )
		if xml_element.iterdescendants(entity_ref_tag_name) or xml_element.iterancestors(entity_ref_tag_name):
			return False

	return True

def convert_element(xml_element, url_fn=create_link_url):
	xml_tag = xml_element.tag

	html_attributes = { "class": [ xml_tag ] }

	for ( name, value ) in xml_element.items():
		html_attributes["data-%s" % ( name )] = value

	if xml_tag.startswith(catoxml_ns):
		xml_tag_name = xml_tag[len(catoxml_ns):]
		html_attributes["class"][html_attributes["class"].index(xml_tag)] = xml_tag_name

		html_tag = "span"
		if xml_tag_name in [ "entity-ref" ]:
			# We aren't allowed to have nested links.
			if can_be_link(xml_element):

				href = url_fn(xml_element)
				if href:
					html_tag = "a"
					html_attributes["href"] = href

	else:
		# Sections
		if xml_tag in [ "bill", "resolution", "amendment-doc" ]:
			html_tag = "article"
		elif xml_tag in [ "form", "action", "legis-body", "resolution-body", "division", "subdivision", "title", "subtitle", "chapter", "subchapter", "part", "subpart", "section", "subsection", "paragraph", "subparagraph", "clause", "subclause", "item", "subitem", "quoted-block", "attestation", "attestation-group", "endorsement", "amendment-form", "amendment-body", "amendment", "amendment-block", "non-statutory-material", "toc", "account", "subaccount", "subsubaccount", "subsubsubaccount", "committee-appointment-paragraph", "preamble", "whereas", "constitution-article", "rule", "rules-clause", "rules-paragraph", "rules-subparagraph", "rules-subdivision", "rules-item", "rules-subitem", "table" ]:
			html_tag = "section"
		elif xml_tag in [ "header", "ttitle" ]:
			html_tag = "h1"
		elif xml_tag in [ "subheader", "rules-clause-header" ]:
			html_tag = "h2"

		# Grouping content
		elif xml_tag in [ "distribution-code", "amend-num", "calendar", "purpose", "current-chamber", "congress", "session", "legis-num", "official-title", "enrolled-dateline", "associated-doc", "action-date", "action-desc", "action-instruction", "legis-type", "official-title-amendment", "text", "attestation-date", "attestor", "proxy", "role", "amendment-instruction", "para", "instructive-para", "graphic", "formula", "toc-entry", "quoted-block-continuation-text", "after-quoted-block", "tdesc" ]:
			html_tag = "p"
		elif xml_tag in [ "pagebreak" ]:
			html_tag = "hr"
		elif xml_tag in [ "list" ]:
			list_type = xml_element.get("list-type")

			# Determine appropriate HTML tag name
			if list_type in [ "numbered", "lettered" ]:
				html_tag = "ol"
			else: # "none"
				html_tag = "ul"

			# Determine appropriate list style type.
			if list_type == "numbered":
				html_attributes["type"] = "1"
			elif list_type == "lettered":
				html_attributes["type"] = "a"
		elif xml_tag in [ "list-item" ]:
			html_tag = "li"

		# Text-level semantics
		elif xml_tag in [ "external-xref", "internal-xref", "footnone-ref" ]:
			html_tag = "span"
			# We aren't allowed to have nested links.
			if can_be_link(xml_element):

				href = url_fn(xml_element)
				if href:
					html_tag = "a"
					html_attributes["href"] = href

				if xml_tag in [ "external-xref" ]:
					if "rel" not in html_attributes:
						html_attributes["rel"] = []

					html_attributes["rel"].append("external")
			else:
				html_tag = "span"
		elif xml_tag in [ "quote" ]:
			html_tag = "q"
		elif xml_tag in [ "term" ]:
			html_tag = "dfn"
		elif xml_tag in [ "subscript" ]:
			html_tag = "sub"
		elif xml_tag in [ "superscript" ]:
			html_tag = "sup"
		elif xml_tag in [ "italic" ]:
			html_tag = "i"
		elif xml_tag in [ "bold" ]:
			html_tag = "b"
		elif xml_tag in [ "linebreak" ]:
			html_tag = "br"

		# Edits
		elif xml_tag in [ "added-phrase" ]:
			html_tag = "ins"
		elif xml_tag in [ "deleted-phrase" ]:
			html_tag = "del"

		# Tabular data
		elif xml_tag in [ "tgroup" ]:
			html_tag = "table"
		elif xml_tag in [ "colspec" ]:
			html_tag = "colgroup"

			# TODO: Process <colspec> attributes into <colgroup> attributes and <col> elements.
		elif xml_tag in [ "thead" ]:
			html_tag = "thead"
		elif xml_tag in [ "tbody" ]:
			html_tag = "tbody"
		elif xml_tag in [ "row" ]:
			html_tag = "tr"
		elif xml_tag in [ "entry" ]:
			html_tag = "td"

			# TODO: Process <entry> attributes.

		# Fallback (everything else)
		# XXX: These appropriations elements are strange and are not documented outside the XSD.
		#      I think their end tags are in the wrong place.
		elif xml_tag in [ "appropriations-major", "appropriations-intermediate", "appropriations-small" ]:
			html_tag = "div"
		else:
			html_tag = "span"

	# Collapse token sets
	for html_attribute in html_attributes:
		# Space-separated tokens
		if html_attribute.lower() in [ "accept-charset", "accesskey", "class", "dropzone", "for", "headers", "itemprop", "itemref", "itemtype", "ping", "rel", "sandbox", "sizes", "sorted" ]:
			html_attributes[html_attribute] = " ".join(html_attributes[html_attribute])
		# Comma-separated tokens
		elif html_attribute.lower() in [ "accept", "srcset" ]:
			html_attributes[html_attribute] = ",".join(html_attributes[html_attribute])

	html_element = etree.Element(html_tag, attrib=html_attributes)

	empty_elements = [ "area", "base", "br", "col", "embed", "hr", "img", "input", "keygen", "link", "menuitem", "meta", "param", "source", "track", "wbr" ]
	html_element.text = "" if xml_element.text is None and html_tag.lower() not in empty_elements else xml_element.text

	html_element.tail = "" if xml_element.tail is None else xml_element.tail

	return html_element

def build_html_tree(node, url_fn=create_link_url):
	html_tree = convert_element(node, url_fn)

	for xml_element in node.getchildren():
		# Ignore certain subtrees and processing instructions
		if xml_element.tag in [ "metadata" ] or not isinstance(xml_element.tag, basestring):
			continue

		html_tree.append(build_html_tree(xml_element))

	return html_tree

def convert_xml(xml_file_path, url_fn=create_link_url):
	xml_tree = etree.parse(xml_file_path, etree.XMLParser(recover=True))

	return etree.ElementTree(build_html_tree(xml_tree.getroot(), url_fn))

# XXX: Is this even necessary? You can just call the write() method on the output of convert_xml()...
def write_html(html_tree, html_file_path):
	return html_tree.write(html_file_path)
<commit_msg>handle bad data in Public Law xrefs<commit_after>from lxml import etree

catoxml_ns = "{http://namespaces.cato.org/catoxml}"

def url_for_us_code(citation):
	def is_range(segment):
		# XXX: This is a quick-and-dirty range finder.
		return (".." in segment)

	def get_range_start(segment):
		return segment[:segment.index("..")]

	# Citations of proposed documents won't follow the same URL format.
	if citation["proposed"]:
		# XXX: We should probably support same-page URLs for these.
		return None

	if "title" not in citation:
		return None

	subpath = citation["title"]

	if citation["subtype"] == "usc-appendix":
		subpath += "a"

	if citation["subtype"] == "usc-chapter":
		if "chapter" in citation:
			subpath += "/chapter-%s" % ( citation["chapter"] )

			if "subchapter" in citation:
				subpath += "/subchapter-%s" % ( citation["subchapter"] )
	else:
		if "section" in citation:
			if is_range(citation["section"]):
				subpath += "/%s" % ( get_range_start(citation["section"]) )
			else:
				subpath += "/%s" % ( citation["section"] )

				if "subsection" in citation:
					if is_range(citation["subsection"]):
						fragment = get_range_start(citation["subsection"])
					else:
						fragment = citation["subsection"]

						for segment in [ "paragraph", "subparagraph", "clause", "subclause", "item", "subitem" ]:
							if segment in citation:
								# Sometimes a segment is empty, so we just skip it.
								if citation[segment] != "":
									if is_range(citation[segment]):
										segment_range_start = get_range_start(citation[segment])
										fragment += segment_range_start if fragment == "" else "_%s" % ( segment_range_start )
										break
									else:
										fragment += citation[segment] if fragment == "" else "_%s" % ( citation[segment] )
							else:
								break

					if fragment != "":
						subpath += "#%s" % ( fragment )

	# convert en-dashes to hyphens
	subpath = subpath.replace(u"\u2013", "-")

	return "http://www.law.cornell.edu/uscode/text/%s" % ( subpath )

def url_for_statute_at_large(citation):
	# Citations of proposed documents won't follow the same URL format.
	if citation["proposed"]:
		# XXX: We should probably support same-page URLs for these.
		return None

	try:
		# An en-dash indicates a page range, but link to just the first page in the range.
		page = citation["page"].split(u"\u2013")[0]
		url = "http://www.gpo.gov/fdsys/search/citation2.result.STATUTE.action?publication=STATUTE&statute.volume=%d&statute.pageNumber=%s" % ( int(citation["volume"]), page)
	except KeyError:
		url = None

	return url

def url_for_public_law(citation):
	# Citations of proposed documents won't follow the same URL format.
	if citation["proposed"]:
		# XXX: We should probably support same-page URLs for these.
		return None

	try:
		url = "https://www.govtrack.us/search?q=P.L.+%d-%d" % ( int(citation["congress"]), int(citation["law"]) )
	except KeyError:
		url = None
	except ValueError:
		# either the congress or the law number was not an integer (maybe blank), not sure why it occurrs but we'll just silently ignore
		url = None

	return url

def url_for_act(citation):
	# Citations of proposed documents won't follow the same URL format.
	if citation["proposed"]:
		# XXX: We should probably support same-page URLs for these.
		return None

	try:
		url = "https://www.govtrack.us/congress/bills/browse?congress=__ALL__&sort=relevance&text=%s" % ( citation["act"] )
	except KeyError:
		url = None

	return url

def create_link_url(xml_element):
	import citations

	link_url = None

	xml_tag = xml_element.tag

	if xml_tag.startswith(catoxml_ns):
		xml_tag_name = xml_tag[len(catoxml_ns):]

		if xml_tag_name in [ "entity-ref" ]:
			entity_type = xml_element.get("entity-type")
			entity_value = xml_element.get("value")
			entity_proposed = True if ( xml_element.get("proposed", "false") == "true" ) else False

			if entity_value is not None:
				citation = citations.deepbills_citation_for(entity_type, entity_value, xml_element.text, entity_proposed)

				if citation["type"] == "uscode":
					link_url = url_for_us_code(citation)
				elif citation["type"] == "statute-at-large":
					link_url = url_for_statute_at_large(citation)
				elif citation["type"] == "public-law":
					link_url = url_for_public_law(citation)
				elif citation["type"] == "act":
					link_url = url_for_act(citation)
				else:
					# Unexpected citation type.
					pass
	else:
		if xml_tag == "external-xref":
			legal_doc = xml_element.get("legal-doc")
			parsable_cite = xml_element.get("parsable-cite")

			# XXX: Workaround for citations.deepbills_citation_for().
			if legal_doc == "usc":
				legal_doc = "uscode"

			citation = citations.deepbills_citation_for(legal_doc, parsable_cite, xml_element.text)

#			if citation["type"] == "usc":
			if citation["type"] == "uscode":
				link_url = url_for_us_code(citation)
			elif citation["type"] == "statute-at-large":
				link_url = url_for_statute_at_large(citation)
			elif citation["type"] == "public-law":
				link_url = url_for_public_law(citation)
			elif citation["type"] == "act":
				link_url = url_for_act(citation)
			elif citation["type"] == "executive-order":
				pass
			elif citation["type"] == "regulation":
				pass
			elif citation["type"] == "bill":
				pass
			elif citation["type"] == "senate-rule":
				pass
			elif citation["type"] == "treaty-ust":
				pass
			elif citation["type"] == "treaty-tias":
				pass
			else:
				# Unexpected document type
				pass
		elif xml_tag == "internal-xref":
			idref = xml_element.get("idref")
			legis_path = xml_element.get("legis-path")
		elif xml_tag == "footnote-ref":
			idref = xml_element.get("idref")
		else:
			# Unexpected XML tag.
			pass

	return link_url

def can_be_link(xml_element):
	xml_tag = xml_element.tag

	if xml_tag.startswith(catoxml_ns):
		pass
	else:
		# Favor CatoXML references, because they tend to be more specific.
		# XXX: This is probably enough, but we might want to check for other nested link elements.
		entity_ref_tag_name = "{%s}entity-ref" % ( catoxml_ns )
		if xml_element.iterdescendants(entity_ref_tag_name) or xml_element.iterancestors(entity_ref_tag_name):
			return False

	return True

def convert_element(xml_element, url_fn=create_link_url):
	xml_tag = xml_element.tag

	html_attributes = { "class": [ xml_tag ] }

	for ( name, value ) in xml_element.items():
		html_attributes["data-%s" % ( name )] = value

	if xml_tag.startswith(catoxml_ns):
		xml_tag_name = xml_tag[len(catoxml_ns):]
		html_attributes["class"][html_attributes["class"].index(xml_tag)] = xml_tag_name

		html_tag = "span"
		if xml_tag_name in [ "entity-ref" ]:
			# We aren't allowed to have nested links.
			if can_be_link(xml_element):

				href = url_fn(xml_element)
				if href:
					html_tag = "a"
					html_attributes["href"] = href

	else:
		# Sections
		if xml_tag in [ "bill", "resolution", "amendment-doc" ]:
			html_tag = "article"
		elif xml_tag in [ "form", "action", "legis-body", "resolution-body", "division", "subdivision", "title", "subtitle", "chapter", "subchapter", "part", "subpart", "section", "subsection", "paragraph", "subparagraph", "clause", "subclause", "item", "subitem", "quoted-block", "attestation", "attestation-group", "endorsement", "amendment-form", "amendment-body", "amendment", "amendment-block", "non-statutory-material", "toc", "account", "subaccount", "subsubaccount", "subsubsubaccount", "committee-appointment-paragraph", "preamble", "whereas", "constitution-article", "rule", "rules-clause", "rules-paragraph", "rules-subparagraph", "rules-subdivision", "rules-item", "rules-subitem", "table" ]:
			html_tag = "section"
		elif xml_tag in [ "header", "ttitle" ]:
			html_tag = "h1"
		elif xml_tag in [ "subheader", "rules-clause-header" ]:
			html_tag = "h2"

		# Grouping content
		elif xml_tag in [ "distribution-code", "amend-num", "calendar", "purpose", "current-chamber", "congress", "session", "legis-num", "official-title", "enrolled-dateline", "associated-doc", "action-date", "action-desc", "action-instruction", "legis-type", "official-title-amendment", "text", "attestation-date", "attestor", "proxy", "role", "amendment-instruction", "para", "instructive-para", "graphic", "formula", "toc-entry", "quoted-block-continuation-text", "after-quoted-block", "tdesc" ]:
			html_tag = "p"
		elif xml_tag in [ "pagebreak" ]:
			html_tag = "hr"
		elif xml_tag in [ "list" ]:
			list_type = xml_element.get("list-type")

			# Determine appropriate HTML tag name
			if list_type in [ "numbered", "lettered" ]:
				html_tag = "ol"
			else: # "none"
				html_tag = "ul"

			# Determine appropriate list style type.
			if list_type == "numbered":
				html_attributes["type"] = "1"
			elif list_type == "lettered":
				html_attributes["type"] = "a"
		elif xml_tag in [ "list-item" ]:
			html_tag = "li"

		# Text-level semantics
		elif xml_tag in [ "external-xref", "internal-xref", "footnone-ref" ]:
			html_tag = "span"
			# We aren't allowed to have nested links.
			if can_be_link(xml_element):

				href = url_fn(xml_element)
				if href:
					html_tag = "a"
					html_attributes["href"] = href

				if xml_tag in [ "external-xref" ]:
					if "rel" not in html_attributes:
						html_attributes["rel"] = []

					html_attributes["rel"].append("external")
			else:
				html_tag = "span"
		elif xml_tag in [ "quote" ]:
			html_tag = "q"
		elif xml_tag in [ "term" ]:
			html_tag = "dfn"
		elif xml_tag in [ "subscript" ]:
			html_tag = "sub"
		elif xml_tag in [ "superscript" ]:
			html_tag = "sup"
		elif xml_tag in [ "italic" ]:
			html_tag = "i"
		elif xml_tag in [ "bold" ]:
			html_tag = "b"
		elif xml_tag in [ "linebreak" ]:
			html_tag = "br"

		# Edits
		elif xml_tag in [ "added-phrase" ]:
			html_tag = "ins"
		elif xml_tag in [ "deleted-phrase" ]:
			html_tag = "del"

		# Tabular data
		elif xml_tag in [ "tgroup" ]:
			html_tag = "table"
		elif xml_tag in [ "colspec" ]:
			html_tag = "colgroup"

			# TODO: Process <colspec> attributes into <colgroup> attributes and <col> elements.
		elif xml_tag in [ "thead" ]:
			html_tag = "thead"
		elif xml_tag in [ "tbody" ]:
			html_tag = "tbody"
		elif xml_tag in [ "row" ]:
			html_tag = "tr"
		elif xml_tag in [ "entry" ]:
			html_tag = "td"

			# TODO: Process <entry> attributes.

		# Fallback (everything else)
		# XXX: These appropriations elements are strange and are not documented outside the XSD.
		#      I think their end tags are in the wrong place.
		elif xml_tag in [ "appropriations-major", "appropriations-intermediate", "appropriations-small" ]:
			html_tag = "div"
		else:
			html_tag = "span"

	# Collapse token sets
	for html_attribute in html_attributes:
		# Space-separated tokens
		if html_attribute.lower() in [ "accept-charset", "accesskey", "class", "dropzone", "for", "headers", "itemprop", "itemref", "itemtype", "ping", "rel", "sandbox", "sizes", "sorted" ]:
			html_attributes[html_attribute] = " ".join(html_attributes[html_attribute])
		# Comma-separated tokens
		elif html_attribute.lower() in [ "accept", "srcset" ]:
			html_attributes[html_attribute] = ",".join(html_attributes[html_attribute])

	html_element = etree.Element(html_tag, attrib=html_attributes)

	empty_elements = [ "area", "base", "br", "col", "embed", "hr", "img", "input", "keygen", "link", "menuitem", "meta", "param", "source", "track", "wbr" ]
	html_element.text = "" if xml_element.text is None and html_tag.lower() not in empty_elements else xml_element.text

	html_element.tail = "" if xml_element.tail is None else xml_element.tail

	return html_element

def build_html_tree(node, url_fn=create_link_url):
	html_tree = convert_element(node, url_fn)

	for xml_element in node.getchildren():
		# Ignore certain subtrees and processing instructions
		if xml_element.tag in [ "metadata" ] or not isinstance(xml_element.tag, basestring):
			continue

		html_tree.append(build_html_tree(xml_element))

	return html_tree

def convert_xml(xml_file_path, url_fn=create_link_url):
	xml_tree = etree.parse(xml_file_path, etree.XMLParser(recover=True))

	return etree.ElementTree(build_html_tree(xml_tree.getroot(), url_fn))

# XXX: Is this even necessary? You can just call the write() method on the output of convert_xml()...
def write_html(html_tree, html_file_path):
	return html_tree.write(html_file_path)
<|endoftext|>


<commit_before>"""
Django settings for liberator project.

Generated by 'django-admin startproject' using Django 1.11.1.

For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = "$SEC_KEY"

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'liberator.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'liberator.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
        },
]

# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/

STATIC_URL = '/static/'
<commit_msg>Bug fix<commit_after>"""
Django settings for liberator project.

Generated by 'django-admin startproject' using Django 1.11.1.

For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = "$SEC_KEY"

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = [*]

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'liberator.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'liberator.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
        },
]

# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/

STATIC_URL = '/static/'
<|endoftext|>