<commit_before># -*- coding: utf-8 -*-

"""
legit.cli
~~~~~~~~~

This module povides the CLI interface to legit.
"""

import sys
from time import sleep

import clint.resources
from clint import args
from clint.eng import join as eng_join
from clint.textui import colored, indent, puts, columns

from .core import __version__
from .settings import settings
from .helpers import is_lin, is_osx, is_win
from .scm import *

def black(s):
    if settings.allow_black_foreground:
        return colored.black(s)
    else:
        return s.encode('utf-8')

# --------
# Dispatch
# --------

def main():
    """Primary Legit command dispatch."""

    if (args.get(0) in cmd_map) or (args.get(0) in short_map):
        arg = args.get(0)
        args.remove(arg)

        if arg in short_map:
            arg = short_map.get(arg)

        cmd_map.get(arg).__call__(args)
        sys.exit()

    elif args.contains(('-h', '--help')):
        display_help()
        sys.exit(1)

    elif args.contains(('-v', '--version')):
        display_version()
        sys.exit(1)

    else:
        if settings.git_transparency:
            # Send everything to git
            git_args = list(sys.argv)
            if settings.git_transparency is True:
                settings.git_transparency = 'git'

            git_args[0] = settings.git_transparency
            os.system(' '.join(git_args))
        else:
            display_info()
            sys.exit(1)

# -------
# Helpers
# -------

def status_log(func, message, *args, **kwargs):
    """Executes a callable with a header message."""

    print message
    log = func(*args, **kwargs)

    if log:
        out = []

        for line in log.split('\n'):
            if not line.startswith('#'):
                out.append(line)
        print black('\n'.join(out))

def switch_to(branch):
    """Runs the cmd_switch command with given branch arg."""

    switch_args = args.copy
    switch_args._args = [branch]

    return cmd_switch(switch_args)

# --------
# Commands
# --------

def cmd_switch(args):
    """Legit Switch command."""

    to_branch = args.get(0)

    if not to_branch:
        display_available_branches()
        sys.exit()

    if to_branch not in get_branch_names():
        print 'Branch not found.'
        sys.exit(1)
    else:
        if repo.is_dirty():
            status_log(stash_it, 'Saving local changes.')

        status_log(checkout_branch, 'Switching to {0}.'.format(
            colored.yellow(to_branch)), to_branch)

        if unstash_index():
            status_log(unstash_it, 'Restoring local changes.')

def cmd_sync(args):
    """Legit Sync command."""

    if args.get(0):
        # Optional branch specifier.
        if args.get(0) in get_branch_names():
            branch = args.get(0)
            is_external = True
            original_branch = repo.head.ref.name
        else:
            print "{0} doesn't exist. Use a branch that does.".format(
                colored.yellow(args.get(0)))
            sys.exit(1)
    else:
        # Sync current branch.
        branch = repo.head.ref.name
        is_external = False

    if branch in get_branch_names(local=False):

        if is_external:
            switch_to(branch)

        if repo.is_dirty():
            status_log(stash_it, 'Saving local changes.', sync=True)

        status_log(smart_pull, 'Pulling commits from the server.')
        status_log(push, 'Pushing commits to the server.', branch)

        if unstash_index(sync=True):
            status_log(unstash_it, 'Restoring local changes.', sync=True)

        if is_external:
            switch_to(original_branch)

    else:
        print '{0} has not been published yet.'.format(
            colored.yellow(branch))
        sys.exit(1)

def cmd_sprout(args):
    """Legit Sprout command."""

    off_branch = args.get(0)
    new_branch = args.get(1)

    if new_branch is None:
        new_branch = off_branch
        off_branch = repo.head.ref.name

    if not off_branch:
        display_available_branches()
        sys.exit()

    branch_names = get_branch_names()

    if off_branch not in branch_names:
        print "{0} doesn't exist. Use a branch that does.".format(
            colored.yellow(off_branch))
        sys.exit(1)

    if new_branch in branch_names:
        print "{0} already exists. Use a unique name.".format(
            colored.yellow(new_branch))
        sys.exit(1)

    if repo.is_dirty():
        status_log(stash_it, 'Saving local changes.')

    status_log(sprout_branch, 'Branching {0} to {1}.'.format(
        colored.yellow(off_branch), colored.yellow(new_branch)),
        off_branch, new_branch)

def cmd_graft(args):

    branch = args.get(0)
    into_branch = args.get(1)

    if not branch:
        display_available_branches()
        sys.exit()

    branch_names = get_branch_names()
    remote_branch_names = get_branch_names(local=False)

    if branch not in branch_names:
        print "{0} doesn't exist. Use a branch that does.".format(
            colored.yellow(branch))
        sys.exit(1)

    if branch in remote_branch_names:
        print "{0} is published. To graft it, unpublish it first.".format(
            colored.yellow(branch))
        sys.exit(1)

    if into_branch not in branch_names:
        print "{0} doesn't exist. Use a branch that does.".format(
            colored.yellow(into_branch))
        sys.exit(1)

    # Go to new branch.
    switch_to(into_branch)

    status_log(graft_branch, 'Grafting {0} into {1}.'.format(
        colored.yellow(branch), colored.yellow(into_branch)), branch)

def cmd_publish(args):

    branch = args.get(0)

    if not branch:
        display_available_branches()
        sys.exit()

    branch_names = get_branch_names(local=False)

    if branch in branch_names:
        print "{0} is already published. Use a branch that isn't.".format(
            colored.yellow(branch))
        sys.exit(1)

    status_log(publish_branch, 'Publishing {0}.'.format(
        colored.yellow(branch)), branch)

def cmd_unpublish(args):

    branch = args.get(0)

    if not branch:
        display_available_branches()
        sys.exit()

    branch_names = get_branch_names(local=False)

    if branch not in branch_names:
        print "{0} isn't published. Use a branch that is.".format(
            colored.yellow(branch))
        sys.exit(1)

    status_log(unpublish_branch, 'Unpublishing {0}.'.format(
        colored.yellow(branch)), branch)

def cmd_add(args):
    commands = ['git', 'add']
    commands.extend(args._args)

    status_log(repo.git.execute, '', commands)

def cmd_commit(args):
    commands = ['git', 'commit']
    commands.extend(args._args)

    status_log(repo.git.execute, '', commands)

def cmd_branches(args):
    """d"""
    display_available_branches()

def cmd_settings(args):
    """Legit settings."""

    path = clint.resources.user.open('config.ini').name

    print 'Legit Settings:\n'

    for (option, _, description) in settings.config_defaults:
        print columns([colored.yellow(option), 25], [description, None])

    print '\nSee {0} for more details.'.format(settings.config_url)

    sleep(0.35)

    if is_osx:
        editor = os.environ.get('EDITOR') or os.environ.get('VISUAL') or 'open'
        os.system("{0} '{1}'".format(editor, path))
    elif is_linux:
        editor = os.environ.get('EDITOR') or os.environ.get('VISUAL') or 'pico'
        os.system("{0} '{1}'".format(editor, path))
    elif is_windows:
        os.system("'{0}'".format(path))
    else:
        print "Edit '{0}' to manage Legit settings.\n".format(path)

    sys.exit()

# -----
# Views
# -----

def display_available_branches():
    """Displays available branches."""

    branches = get_branches()

    branch_col = len(max([b.name for b in branches], key=len)) + 1

    for branch in branches:

        marker = '*' if (branch.name == repo.head.ref.name) else ' '
        color = colored.green if (branch.name == repo.head.ref.name) else colored.yellow
        pub = '(published)' if branch.is_published else '(unpublished)'

        print columns(
            [colored.red(marker), 2],
            [color(branch.name), branch_col],
            [black(pub), 14]
        )

def display_info():
    """Displays Legit informatics."""

    puts('{0}. {1}\n'.format(
        colored.red('legit'),
        black(u'A Kenneth Reitz Project™')
    ))

    puts('Usage: {0}'.format(colored.blue('legit <command>')))
    puts('Commands: {0}.\n'.format(
        eng_join(
            [str(colored.green(c)) for c in sorted(cmd_map.keys())]
        )
    ))

def display_help():
    """Displays Legit help."""

    display_info()

def display_version():
    """Displays Legit version/release."""

    puts('{0} v{1}'.format(
        colored.yellow('legit'),
        __version__
    ))

cmd_map = dict(
    switch=cmd_switch,
    sync=cmd_sync,
    sprout=cmd_sprout,
    graft=cmd_graft,
    publish=cmd_publish,
    unpublish=cmd_unpublish,
    branches=cmd_branches,
    settings=cmd_settings
)

short_map = dict(
    sw='switch',
    sy='sync',
    sp='sprout',
    gr='graft',
    pub='publish',
    unp='unpublish',
    br='branches'
)<commit_msg>git transparency exit status<commit_after># -*- coding: utf-8 -*-

"""
legit.cli
~~~~~~~~~

This module povides the CLI interface to legit.
"""

import sys
from subprocess import call
from time import sleep

import clint.resources
from clint import args
from clint.eng import join as eng_join
from clint.textui import colored, indent, puts, columns

from .core import __version__
from .settings import settings
from .helpers import is_lin, is_osx, is_win
from .scm import *

def black(s):
    if settings.allow_black_foreground:
        return colored.black(s)
    else:
        return s.encode('utf-8')

# --------
# Dispatch
# --------

def main():
    """Primary Legit command dispatch."""

    if (args.get(0) in cmd_map) or (args.get(0) in short_map):
        arg = args.get(0)
        args.remove(arg)

        if arg in short_map:
            arg = short_map.get(arg)

        cmd_map.get(arg).__call__(args)
        sys.exit()

    elif args.contains(('-h', '--help')):
        display_help()
        sys.exit(1)

    elif args.contains(('-v', '--version')):
        display_version()
        sys.exit(1)

    else:
        if settings.git_transparency:
            # Send everything to git
            git_args = list(sys.argv)
            if settings.git_transparency is True:
                settings.git_transparency = 'git'

            git_args[0] = settings.git_transparency

            sys.exit(call(' '.join(git_args), shell=True))

        else:
            display_info()
            sys.exit(1)

# -------
# Helpers
# -------

def status_log(func, message, *args, **kwargs):
    """Executes a callable with a header message."""

    print message
    log = func(*args, **kwargs)

    if log:
        out = []

        for line in log.split('\n'):
            if not line.startswith('#'):
                out.append(line)
        print black('\n'.join(out))

def switch_to(branch):
    """Runs the cmd_switch command with given branch arg."""

    switch_args = args.copy
    switch_args._args = [branch]

    return cmd_switch(switch_args)

# --------
# Commands
# --------

def cmd_switch(args):
    """Legit Switch command."""

    to_branch = args.get(0)

    if not to_branch:
        display_available_branches()
        sys.exit()

    if to_branch not in get_branch_names():
        print 'Branch not found.'
        sys.exit(1)
    else:
        if repo.is_dirty():
            status_log(stash_it, 'Saving local changes.')

        status_log(checkout_branch, 'Switching to {0}.'.format(
            colored.yellow(to_branch)), to_branch)

        if unstash_index():
            status_log(unstash_it, 'Restoring local changes.')

def cmd_sync(args):
    """Legit Sync command."""

    if args.get(0):
        # Optional branch specifier.
        if args.get(0) in get_branch_names():
            branch = args.get(0)
            is_external = True
            original_branch = repo.head.ref.name
        else:
            print "{0} doesn't exist. Use a branch that does.".format(
                colored.yellow(args.get(0)))
            sys.exit(1)
    else:
        # Sync current branch.
        branch = repo.head.ref.name
        is_external = False

    if branch in get_branch_names(local=False):

        if is_external:
            switch_to(branch)

        if repo.is_dirty():
            status_log(stash_it, 'Saving local changes.', sync=True)

        status_log(smart_pull, 'Pulling commits from the server.')
        status_log(push, 'Pushing commits to the server.', branch)

        if unstash_index(sync=True):
            status_log(unstash_it, 'Restoring local changes.', sync=True)

        if is_external:
            switch_to(original_branch)

    else:
        print '{0} has not been published yet.'.format(
            colored.yellow(branch))
        sys.exit(1)

def cmd_sprout(args):
    """Legit Sprout command."""

    off_branch = args.get(0)
    new_branch = args.get(1)

    if new_branch is None:
        new_branch = off_branch
        off_branch = repo.head.ref.name

    if not off_branch:
        display_available_branches()
        sys.exit()

    branch_names = get_branch_names()

    if off_branch not in branch_names:
        print "{0} doesn't exist. Use a branch that does.".format(
            colored.yellow(off_branch))
        sys.exit(1)

    if new_branch in branch_names:
        print "{0} already exists. Use a unique name.".format(
            colored.yellow(new_branch))
        sys.exit(1)

    if repo.is_dirty():
        status_log(stash_it, 'Saving local changes.')

    status_log(sprout_branch, 'Branching {0} to {1}.'.format(
        colored.yellow(off_branch), colored.yellow(new_branch)),
        off_branch, new_branch)

def cmd_graft(args):

    branch = args.get(0)
    into_branch = args.get(1)

    if not branch:
        display_available_branches()
        sys.exit()

    branch_names = get_branch_names()
    remote_branch_names = get_branch_names(local=False)

    if branch not in branch_names:
        print "{0} doesn't exist. Use a branch that does.".format(
            colored.yellow(branch))
        sys.exit(1)

    if branch in remote_branch_names:
        print "{0} is published. To graft it, unpublish it first.".format(
            colored.yellow(branch))
        sys.exit(1)

    if into_branch not in branch_names:
        print "{0} doesn't exist. Use a branch that does.".format(
            colored.yellow(into_branch))
        sys.exit(1)

    # Go to new branch.
    switch_to(into_branch)

    status_log(graft_branch, 'Grafting {0} into {1}.'.format(
        colored.yellow(branch), colored.yellow(into_branch)), branch)

def cmd_publish(args):

    branch = args.get(0)

    if not branch:
        display_available_branches()
        sys.exit()

    branch_names = get_branch_names(local=False)

    if branch in branch_names:
        print "{0} is already published. Use a branch that isn't.".format(
            colored.yellow(branch))
        sys.exit(1)

    status_log(publish_branch, 'Publishing {0}.'.format(
        colored.yellow(branch)), branch)

def cmd_unpublish(args):

    branch = args.get(0)

    if not branch:
        display_available_branches()
        sys.exit()

    branch_names = get_branch_names(local=False)

    if branch not in branch_names:
        print "{0} isn't published. Use a branch that is.".format(
            colored.yellow(branch))
        sys.exit(1)

    status_log(unpublish_branch, 'Unpublishing {0}.'.format(
        colored.yellow(branch)), branch)

def cmd_add(args):
    commands = ['git', 'add']
    commands.extend(args._args)

    status_log(repo.git.execute, '', commands)

def cmd_commit(args):
    commands = ['git', 'commit']
    commands.extend(args._args)

    status_log(repo.git.execute, '', commands)

def cmd_branches(args):
    """d"""
    display_available_branches()

def cmd_settings(args):
    """Legit settings."""

    path = clint.resources.user.open('config.ini').name

    print 'Legit Settings:\n'

    for (option, _, description) in settings.config_defaults:
        print columns([colored.yellow(option), 25], [description, None])

    print '\nSee {0} for more details.'.format(settings.config_url)

    sleep(0.35)

    if is_osx:
        editor = os.environ.get('EDITOR') or os.environ.get('VISUAL') or 'open'
        os.system("{0} '{1}'".format(editor, path))
    elif is_linux:
        editor = os.environ.get('EDITOR') or os.environ.get('VISUAL') or 'pico'
        os.system("{0} '{1}'".format(editor, path))
    elif is_windows:
        os.system("'{0}'".format(path))
    else:
        print "Edit '{0}' to manage Legit settings.\n".format(path)

    sys.exit()

# -----
# Views
# -----

def display_available_branches():
    """Displays available branches."""

    branches = get_branches()

    branch_col = len(max([b.name for b in branches], key=len)) + 1

    for branch in branches:

        marker = '*' if (branch.name == repo.head.ref.name) else ' '
        color = colored.green if (branch.name == repo.head.ref.name) else colored.yellow
        pub = '(published)' if branch.is_published else '(unpublished)'

        print columns(
            [colored.red(marker), 2],
            [color(branch.name), branch_col],
            [black(pub), 14]
        )

def display_info():
    """Displays Legit informatics."""

    puts('{0}. {1}\n'.format(
        colored.red('legit'),
        black(u'A Kenneth Reitz Project™')
    ))

    puts('Usage: {0}'.format(colored.blue('legit <command>')))
    puts('Commands: {0}.\n'.format(
        eng_join(
            [str(colored.green(c)) for c in sorted(cmd_map.keys())]
        )
    ))

def display_help():
    """Displays Legit help."""

    display_info()

def display_version():
    """Displays Legit version/release."""

    puts('{0} v{1}'.format(
        colored.yellow('legit'),
        __version__
    ))

cmd_map = dict(
    switch=cmd_switch,
    sync=cmd_sync,
    sprout=cmd_sprout,
    graft=cmd_graft,
    publish=cmd_publish,
    unpublish=cmd_unpublish,
    branches=cmd_branches,
    settings=cmd_settings
)

short_map = dict(
    sw='switch',
    sy='sync',
    sp='sprout',
    gr='graft',
    pub='publish',
    unp='unpublish',
    br='branches'
)<|endoftext|>


<commit_before>"""
Configure Flask app instance
"""

import logging
import os

from flask import Flask

app = Flask(__name__)

# Running on heroku
if 'HEROKU' in os.environ:
    import example_config

    # example_config.py provides a blueprint for which variables to look for in
    # the environment and set in our app config.
    for var in example_config.HEROKU_ENV_REQUIREMENTS:
        app.config.setdefault(var, os.environ[var])

        if var == 'SECRET_KEY':
            app.secret_key = os.environ[var]

    if 'DEBUG' in os.environ:
        app.debug = True

        print 'Config values'
        for key, value in app.config.iteritems():
            print key, value

else:
    try:
        app.config.from_object(os.environ['APP_SETTINGS'])
    except KeyError:
        import example_config
        print 'Unable to find configuration, using example'
        app.config.from_object(example_config.DevelopmentConfig)

    app.secret_key = app.config['SECRET_KEY']

# Force flask to log to a file otherwise if we're not in DEBUG mode the logs
# won't show up in the console (on heroku)
if not app.debug:
    app.logger.setLevel(logging.INFO)
    app.logger.addHandler(logging.StreamHandler())

import pskb_website.views
import pskb_website.filters

app.jinja_env.filters['date_string'] = filters.date_string
app.jinja_env.filters['url_for_article'] = filters.url_for_article
app.jinja_env.filters['url_for_user'] = filters.url_for_user
app.jinja_env.filters['author_name'] = filters.author_name
<commit_msg>Default missing optional environment variables on heroku deployment<commit_after>"""
Configure Flask app instance
"""

import logging
import os

from flask import Flask

app = Flask(__name__)

# Running on heroku
if 'HEROKU' in os.environ:
    import example_config

    # example_config.py provides a blueprint for which variables to look for in
    # the environment and set in our app config.
    for var in example_config.HEROKU_ENV_REQUIREMENTS:
        try:
            value = os.environ[var]
        except KeyError:
            print 'Unable to find environment variable %s, defaulting to None'
            value = None

            app.config.setdefault(var, value)

        if var == 'SECRET_KEY':
            app.secret_key = os.environ[var]

    if 'DEBUG' in os.environ:
        app.debug = True

        print 'Config values'
        for key, value in app.config.iteritems():
            print key, value

else:
    try:
        app.config.from_object(os.environ['APP_SETTINGS'])
    except KeyError:
        import example_config
        print 'Unable to find configuration, using example'
        app.config.from_object(example_config.DevelopmentConfig)

    app.secret_key = app.config['SECRET_KEY']

# Force flask to log to a file otherwise if we're not in DEBUG mode the logs
# won't show up in the console (on heroku)
if not app.debug:
    app.logger.setLevel(logging.INFO)
    app.logger.addHandler(logging.StreamHandler())

import pskb_website.views
import pskb_website.filters

app.jinja_env.filters['date_string'] = filters.date_string
app.jinja_env.filters['url_for_article'] = filters.url_for_article
app.jinja_env.filters['url_for_user'] = filters.url_for_user
app.jinja_env.filters['author_name'] = filters.author_name
<|endoftext|>


<commit_before>#!/bin/env python3

import requests
import logging
import os
import json
import click

class OpenFigi:
    id_types = {'ID_ISIN': 'ISIN',
                'ID_BB_UNIQUE': 'Unique Bloomberg Identifier',
                'ID_SEDOL': 'Sedol Number',
                'ID_COMMON': 'Common Code',
                'ID_WERTPAPIER': 'Wertpapierkennnummer/WKN',
                'ID_CUSIP': 'CUSIP',
                'ID_BB': 'ID BB',
                'ID_ITALY': 'Italian Identifier Number',
                'ID_EXCH_SYMBOL': 'Local Exchange Security Symbol',
                'ID_FULL_EXCHANGE_SYMBOL': 'Full Exchange Symbol',
                'COMPOSITE_ID_BB_GLOBAL': 'Composite Financial Instrument Global Identifier',
                'ID_BB_GLOBAL_SHARE_CLASS_LEVEL': 'Share Class Financial Instrument Global Identifier',
                'ID_BB_SEC_NUM_DES': 'Security ID Number Description',
                'ID_BB_GLOBAL': 'Financial Instrument Global Identifier (FIGI)',
                'TICKER': 'Ticker',
                'ID_CUSIP_8_CHR': 'CUSIP (8 Characters Only)',
                'OCC_SYMBOL': 'OCC Symbol',
                'UNIQUE_ID_FUT_OPT': 'Unique Identifier for Future Option',
                'OPRA_SYMBOL': 'OPRA Symbol',
                'TRADING_SYSTEM_IDENTIFIER': 'Trading System Identifier'}

    def __init__(self, key=None):
        self.logger = logging.getLogger(__name__)
        self.url = 'https://api.openfigi.com/v1/mapping'
        self.api_key = key
        self.headers = {'Content-Type': 'text/json', 'X-OPENFIGI-APIKEY': key}
        self.data = []

    def enqueue_request(self, id_type, id_value, exchange_code='', mic_code='', currency=''):
        query = {'idType': id_type, 'idValue': id_value}
        if id_type not in self.id_types.keys():
            self.logger.error('Bad id_type.')
            return
        if len(exchange_code) > 0:
            query['exchCode'] = exchange_code
        if len(mic_code) > 0:
            query['micCode'] = mic_code
        if len(currency) > 0:
            query['currency'] = currency

        self.data.append(query)

    def fetch_response(self):
        self.logger.debug(self.data)
        response = requests.post(self.url, json=self.data, headers=self.headers)
        try:
            response.raise_for_status()
        except requests.HTTPError:
            if response.status_code == 400:
                self.logger.error('The request body is not an array.')
            if response.status_code == 401:
                self.logger.error('The API_KEY is invalid.')
            if response.status_code == 404:
                self.logger.error('The requested path is invalid.')
            if response.status_code == 405:
                self.logger.error('The HTTP verb is not POST.')
            if response.status_code == 406:
                self.logger.error('The server does not support the requested Accept type.')
            if response.status_code == 413:
                self.logger.error('The request exceeds the max number of identifiers support in one request.')
            if response.status_code == 429:
                self.logger.error('Too Many Requests.')
            if response.status_code == 500:
                self.logger.error('Internal Server Error.')
            return None
        return response.json()

@click.command()
@click.argument('id_type')
@click.argument('id_value')
@click.option('--exchange_code', default='', help='An optional exchange code if it applies(cannot use with mic_code).')
@click.option('--mic_code', default='',
              help='An optional ISO market identification code(MIC) if it applies(cannot use with exchange_code).')
@click.option('--currency', default='', help='An optional currency if it applies.')
def call_figi(id_type, id_value, exchange_code, mic_code, currency):
    key = None
    if 'openfigi_key' in os.environ:
        key = os.environ['openfigi_key']
        logger.info('openfigi_key variable not present in env. Using anonymous access.')
    figi = OpenFigi(key)
    figi.enqueue_request(id_type, id_value, exchange_code, mic_code, currency)
    text = figi.fetch_response()
    click.echo(json.dumps(text, sort_keys=True, indent=4))

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger(__name__)

    call_figi()
<commit_msg>Return results merged with request ids<commit_after>#!/bin/env python3

import requests
import logging
import os
import json
import click

class OpenFigi:
    id_types = {'ID_ISIN': 'ISIN',
                'ID_BB_UNIQUE': 'Unique Bloomberg Identifier',
                'ID_SEDOL': 'Sedol Number',
                'ID_COMMON': 'Common Code',
                'ID_WERTPAPIER': 'Wertpapierkennnummer/WKN',
                'ID_CUSIP': 'CUSIP',
                'ID_BB': 'ID BB',
                'ID_ITALY': 'Italian Identifier Number',
                'ID_EXCH_SYMBOL': 'Local Exchange Security Symbol',
                'ID_FULL_EXCHANGE_SYMBOL': 'Full Exchange Symbol',
                'COMPOSITE_ID_BB_GLOBAL': 'Composite Financial Instrument Global Identifier',
                'ID_BB_GLOBAL_SHARE_CLASS_LEVEL': 'Share Class Financial Instrument Global Identifier',
                'ID_BB_SEC_NUM_DES': 'Security ID Number Description',
                'ID_BB_GLOBAL': 'Financial Instrument Global Identifier (FIGI)',
                'TICKER': 'Ticker',
                'ID_CUSIP_8_CHR': 'CUSIP (8 Characters Only)',
                'OCC_SYMBOL': 'OCC Symbol',
                'UNIQUE_ID_FUT_OPT': 'Unique Identifier for Future Option',
                'OPRA_SYMBOL': 'OPRA Symbol',
                'TRADING_SYSTEM_IDENTIFIER': 'Trading System Identifier'}

    def __init__(self, key=None):
        self.logger = logging.getLogger(__name__)
        self.url = 'https://api.openfigi.com/v1/mapping'
        self.api_key = key
        self.headers = {'Content-Type': 'text/json', 'X-OPENFIGI-APIKEY': key}
        self.data = []
        self.max_tickers_per_request = 100

    def enqueue_request(self, id_type, id_value, exchange_code='', mic_code='', currency=''):
        query = {'idType': id_type, 'idValue': id_value}
        if id_type not in self.id_types.keys():
            self.logger.error('Bad id_type.')
            return
        if len(exchange_code) > 0:
            query['exchCode'] = exchange_code
        if len(mic_code) > 0:
            query['micCode'] = mic_code
        if len(currency) > 0:
            query['currency'] = currency

        self.data.append(query)

    def fetch_response(self):
        response = requests.post(self.url, json=self.data, headers=self.headers)
        try:
            response.raise_for_status()
        except requests.HTTPError:
            if response.status_code == 400:
                self.logger.error('The request body is not an array.')
            if response.status_code == 401:
                self.logger.error('The API_KEY is invalid.')
            if response.status_code == 404:
                self.logger.error('The requested path is invalid.')
            if response.status_code == 405:
                self.logger.error('The HTTP verb is not POST.')
            if response.status_code == 406:
                self.logger.error('The server does not support the requested Accept type.')
            if response.status_code == 413:
                self.logger.error('The request exceeds the max number of identifiers support in one request.')
            if response.status_code == 429:
                self.logger.error('Too Many Requests.')
            if response.status_code == 500:
                self.logger.error('Internal Server Error.')
            return None

        response_list = response.json()
        if len(response_list) == len(self.data):
            for (i, item) in enumerate(response_list):
                item.update(self.data[i])
        else:
            self.logger.warning('Number of request and response items do not match. Dumping results only.')

        self.data.clear()
        return response_list

@click.command()
@click.argument('id_type')
@click.argument('id_value')
@click.option('--exchange_code', default='', help='An optional exchange code if it applies(cannot use with mic_code).')
@click.option('--mic_code', default='',
              help='An optional ISO market identification code(MIC) if it applies(cannot use with exchange_code).')
@click.option('--currency', default='', help='An optional currency if it applies.')
def call_figi(id_type, id_value, exchange_code, mic_code, currency):
    """
    Calls OpenFIGI API with the specified arguments

    ID_TYPE must be one of the following:
    ID_ISIN, ID_BB_UNIQUE, ID_SEDOL, ID_COMMON, ID_WERTPAPIER, ID_CUSIP,
    ID_BB, ID_ITALY, ID_EXCH_SYMBOL, ID_FULL_EXCHANGE_SYMBOL, COMPOSITE_ID_BB_GLOBAL,
    ID_BB_GLOBAL_SHARE_CLASS_LEVEL, ID_BB_SEC_NUM_DES, ID_BB_GLOBAL, TICKER,
    ID_CUSIP_8_CHR, OCC_SYMBOL, UNIQUE_ID_FUT_OPT, OPRA_SYMBOL, TRADING_SYSTEM_IDENTIFIER
    """
    key = None
    if 'openfigi_key' in os.environ:
        key = os.environ['openfigi_key']
    else:
        logger.info('openfigi_key variable not present in env. Using anonymous access.')
    figi = OpenFigi(key)
    figi.enqueue_request(id_type, id_value, exchange_code, mic_code, currency)
    text = figi.fetch_response()
    click.echo(json.dumps(text, sort_keys=True, indent=4))

def mass_fetch():
    with open('wiki-tickers.txt', 'r') as f:
        tickers = [ticker.strip() for ticker in f.readlines()]
    key = None
    if 'openfigi_key' in os.environ:
        key = os.environ['openfigi_key']
    figi = OpenFigi(key)
    for ticker in tickers[:100]:
        figi.enqueue_request(id_type='TICKER', id_value=ticker, mic_code='XNYS')

    resp = figi.fetch_response()
    click.echo(json.dumps(resp, sort_keys=True, indent=4))

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger(__name__)

    mass_fetch()
#    call_figi()
<|endoftext|>


<commit_before>"""Supporting definitions for the Python regression test."""

import sys

class Error(Exception):
    """Base class for regression test exceptions."""

class TestFailed(Error):
    """Test failed."""

class TestSkipped(Error):
    """Test skipped.

    This can be raised to indicate that a test was deliberatly
    skipped, but not because a feature wasn't available.  For
    example, if some resource can't be used, such as the network
    appears to be unavailable, this should be raised instead of
    TestFailed.
    """

verbose = 1              # Flag set to 0 by regrtest.py
use_resources = None       # Flag set to [] by regrtest.py

# _output_comparison controls whether regrtest will try to compare stdout
# with an expected-output file.  For straight regrtests, it should.
# The doctest driver resets this flag by calling deny_output_comparison().
# Note that this control is in addition to verbose mode:  output will be
# compared if and only if _output_comparison is true and verbose mode is
# not in effect.
_output_comparison = 1

def deny_output_comparison():
    global _output_comparison
    _output_comparison = 0
    sys.stdout = sys.save_stdout

# regrtest's interface to _output_comparison.
def output_comparison_denied():
    global _output_comparison
    denied = not _output_comparison
    _output_comparison = 1
    return denied

def unload(name):
    try:
        del sys.modules[name]
    except KeyError:
        pass

def forget(modname):
    unload(modname)
    import os
    for dirname in sys.path:
        try:
            os.unlink(os.path.join(dirname, modname + '.pyc'))
        except os.error:
            pass

def requires(resource, msg=None):
    if use_resources is not None and resource not in use_resources:
        if msg is None:
            msg = "Use of the `%s' resource not enabled" % resource
        raise TestSkipped(msg)

FUZZ = 1e-6

def fcmp(x, y): # fuzzy comparison function
    if type(x) == type(0.0) or type(y) == type(0.0):
        try:
            x, y = coerce(x, y)
            fuzz = (abs(x) + abs(y)) * FUZZ
            if abs(x-y) <= fuzz:
                return 0
        except:
            pass
    elif type(x) == type(y) and type(x) in (type(()), type([])):
        for i in range(min(len(x), len(y))):
            outcome = fcmp(x[i], y[i])
            if outcome != 0:
                return outcome
        return cmp(len(x), len(y))
    return cmp(x, y)

try:
    unicode
    have_unicode = 1
except NameError:
    have_unicode = 0

import os
# Filename used for testing
if os.name == 'java':
    # Jython disallows @ in module names
    TESTFN = '$test'
elif os.name != 'riscos':
    TESTFN = '@test'
    # Unicode name only used if TEST_FN_ENCODING exists for the platform.
    if have_unicode:
        TESTFN_UNICODE=unicode("@test-\xe0\xf2", "latin-1") # 2 latin characters.
        if os.name=="nt":
            TESTFN_ENCODING="mbcs"
else:
    TESTFN = 'test'
del os

from os import unlink

def findfile(file, here=__file__):
    import os
    if os.path.isabs(file):
        return file
    path = sys.path
    path = [os.path.dirname(here)] + path
    for dn in path:
        fn = os.path.join(dn, file)
        if os.path.exists(fn): return fn
    return file

def verify(condition, reason='test failed'):
    """Verify that condition is true. If not, raise TestFailed.

       The optional argument reason can be given to provide
       a better error text.
    """

    if not condition:
        raise TestFailed(reason)

def sortdict(dict):
    "Like repr(dict), but in sorted order."
    items = dict.items()
    items.sort()
    reprpairs = ["%r: %r" % pair for pair in items]
    withcommas = ", ".join(reprpairs)
    return "{%s}" % withcommas

def check_syntax(statement):
    try:
        compile(statement, '<string>', 'exec')
    except SyntaxError:
        pass
    else:
        print 'Missing SyntaxError: "%s"' % statement

#=======================================================================
# Preliminary PyUNIT integration.

import unittest

class BasicTestRunner:
    def run(self, test):
        result = unittest.TestResult()
        test(result)
        return result

def run_suite(suite):
    """Run tests from a unittest.TestSuite-derived class."""
    if verbose:
        runner = unittest.TextTestRunner(sys.stdout, verbosity=2)
    else:
        runner = BasicTestRunner()

    result = runner.run(suite)
    if not result.wasSuccessful():
        if len(result.errors) == 1 and not result.failures:
            err = result.errors[0][1]
        elif len(result.failures) == 1 and not result.errors:
            err = result.failures[0][1]
        else:
            raise TestFailed("errors occurred in %s.%s"
                             % (testclass.__module__, testclass.__name__))
        raise TestFailed(err)

def run_unittest(testclass):
    """Run tests from a unittest.TestCase-derived class."""
    run_suite(unittest.makeSuite(testclass))

#=======================================================================
# doctest driver.

def run_doctest(module, verbosity=None):
    """Run doctest on the given module.

    If optional argument verbosity is not specified (or is None), pass
    test_support's belief about verbosity on to doctest.  Else doctest's
    usual behavior is used (it searches sys.argv for -v).
    """

    import doctest

    if verbosity is None:
        verbosity = verbose
    else:
        verbosity = None

    deny_output_comparison()
    f, t = doctest.testmod(module, verbose=verbosity)
    if f:
        raise TestFailed("%d of %d doctests failed" % (f, t))
<commit_msg>Set sys.save_stdout (to sys.stdout), so doctest-using tests can be run standalone.<commit_after>"""Supporting definitions for the Python regression test."""

import sys

sys.save_stdout = sys.stdout

class Error(Exception):
    """Base class for regression test exceptions."""

class TestFailed(Error):
    """Test failed."""

class TestSkipped(Error):
    """Test skipped.

    This can be raised to indicate that a test was deliberatly
    skipped, but not because a feature wasn't available.  For
    example, if some resource can't be used, such as the network
    appears to be unavailable, this should be raised instead of
    TestFailed.
    """

verbose = 1              # Flag set to 0 by regrtest.py
use_resources = None       # Flag set to [] by regrtest.py

# _output_comparison controls whether regrtest will try to compare stdout
# with an expected-output file.  For straight regrtests, it should.
# The doctest driver resets this flag by calling deny_output_comparison().
# Note that this control is in addition to verbose mode:  output will be
# compared if and only if _output_comparison is true and verbose mode is
# not in effect.
_output_comparison = 1

def deny_output_comparison():
    global _output_comparison
    _output_comparison = 0
    sys.stdout = sys.save_stdout

# regrtest's interface to _output_comparison.
def output_comparison_denied():
    global _output_comparison
    denied = not _output_comparison
    _output_comparison = 1
    return denied

def unload(name):
    try:
        del sys.modules[name]
    except KeyError:
        pass

def forget(modname):
    unload(modname)
    import os
    for dirname in sys.path:
        try:
            os.unlink(os.path.join(dirname, modname + '.pyc'))
        except os.error:
            pass

def requires(resource, msg=None):
    if use_resources is not None and resource not in use_resources:
        if msg is None:
            msg = "Use of the `%s' resource not enabled" % resource
        raise TestSkipped(msg)

FUZZ = 1e-6

def fcmp(x, y): # fuzzy comparison function
    if type(x) == type(0.0) or type(y) == type(0.0):
        try:
            x, y = coerce(x, y)
            fuzz = (abs(x) + abs(y)) * FUZZ
            if abs(x-y) <= fuzz:
                return 0
        except:
            pass
    elif type(x) == type(y) and type(x) in (type(()), type([])):
        for i in range(min(len(x), len(y))):
            outcome = fcmp(x[i], y[i])
            if outcome != 0:
                return outcome
        return cmp(len(x), len(y))
    return cmp(x, y)

try:
    unicode
    have_unicode = 1
except NameError:
    have_unicode = 0

import os
# Filename used for testing
if os.name == 'java':
    # Jython disallows @ in module names
    TESTFN = '$test'
elif os.name != 'riscos':
    TESTFN = '@test'
    # Unicode name only used if TEST_FN_ENCODING exists for the platform.
    if have_unicode:
        TESTFN_UNICODE=unicode("@test-\xe0\xf2", "latin-1") # 2 latin characters.
        if os.name=="nt":
            TESTFN_ENCODING="mbcs"
else:
    TESTFN = 'test'
del os

from os import unlink

def findfile(file, here=__file__):
    import os
    if os.path.isabs(file):
        return file
    path = sys.path
    path = [os.path.dirname(here)] + path
    for dn in path:
        fn = os.path.join(dn, file)
        if os.path.exists(fn): return fn
    return file

def verify(condition, reason='test failed'):
    """Verify that condition is true. If not, raise TestFailed.

       The optional argument reason can be given to provide
       a better error text.
    """

    if not condition:
        raise TestFailed(reason)

def sortdict(dict):
    "Like repr(dict), but in sorted order."
    items = dict.items()
    items.sort()
    reprpairs = ["%r: %r" % pair for pair in items]
    withcommas = ", ".join(reprpairs)
    return "{%s}" % withcommas

def check_syntax(statement):
    try:
        compile(statement, '<string>', 'exec')
    except SyntaxError:
        pass
    else:
        print 'Missing SyntaxError: "%s"' % statement

#=======================================================================
# Preliminary PyUNIT integration.

import unittest

class BasicTestRunner:
    def run(self, test):
        result = unittest.TestResult()
        test(result)
        return result

def run_suite(suite):
    """Run tests from a unittest.TestSuite-derived class."""
    if verbose:
        runner = unittest.TextTestRunner(sys.stdout, verbosity=2)
    else:
        runner = BasicTestRunner()

    result = runner.run(suite)
    if not result.wasSuccessful():
        if len(result.errors) == 1 and not result.failures:
            err = result.errors[0][1]
        elif len(result.failures) == 1 and not result.errors:
            err = result.failures[0][1]
        else:
            raise TestFailed("errors occurred in %s.%s"
                             % (testclass.__module__, testclass.__name__))
        raise TestFailed(err)

def run_unittest(testclass):
    """Run tests from a unittest.TestCase-derived class."""
    run_suite(unittest.makeSuite(testclass))

#=======================================================================
# doctest driver.

def run_doctest(module, verbosity=None):
    """Run doctest on the given module.

    If optional argument verbosity is not specified (or is None), pass
    test_support's belief about verbosity on to doctest.  Else doctest's
    usual behavior is used (it searches sys.argv for -v).
    """

    import doctest

    if verbosity is None:
        verbosity = verbose
    else:
        verbosity = None

    deny_output_comparison()
    f, t = doctest.testmod(module, verbose=verbosity)
    if f:
        raise TestFailed("%d of %d doctests failed" % (f, t))
<|endoftext|>


<commit_before># coding: utf-8
import os
import sys
include_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'include')
sys.path.append(include_dir)

import base64
import json
import logging
import urlparse
import common

from lyric_base import LyricBase

try:
    import requests_toolbelt.adapters.appengine
    # Use the App Engine Requests adapter. This makes sure that Requests uses
    # URLFetch.
    requests_toolbelt.adapters.appengine.monkeypatch()
except:
    pass

site_class = 'PetitLyrics'
site_index = 'petitlyrics'
site_keyword = 'petitlyrics'
site_url = 'https://petitlyrics.com/'
test_url = 'https://petitlyrics.com/lyrics/914675'
test_expect_length = 1275

'''
    Request URL
        http://petitlyrics.com/com/get_lyrics.ajax
    Method
        POST
    Data
        lyrics_id=914675

    Response
'''
class PetitLyrics(LyricBase):
    def parse_page(self):
        url = self.url

        # 1. get song id from url
        id = self.get_song_id(url)
        if not id:
            logging.error('Failed to get id of url [%s]', url)
            return False

        # 2. get html
        html = self.get_content_by_url(url)

        if not html:
            logging.info('Failed to get html of url [%s]' % (url))
            return False

        # 3. get CSRF token
        token = self.get_csrf_token(html)
        if not token:
            logging.info('Failed to get CSRF token of url [%s]' % (url))
            return False
        logging.debug('CSRF token is %s' % (token, ))

        # 5. get lyric
        lyric = self.get_lyric(id, token)
        if not lyric:
            logging.info('Failed to get lyric of url [%s]' % (url))
            return False

        self.lyric = common.htmlspecialchars_decode(lyric.strip())

        self.parse_artist_title(html)
        self.parse_lyricist(html)
        self.parse_composer(html)

        return True

    def get_song_id(self, url):
        if not url:
            return None

        pattern = '/lyrics/([0-9]+)'
        id = common.get_first_group_by_pattern(url, pattern)
        if not id:
            pattern = '/kashi/([0-9]+)'
            id = common.get_first_group_by_pattern(url, pattern)

        return id

    def get_csrf_token(self, html):
        pattern = '(/lib/pl-lib.js[^"]+)'

        pl_lib_js = common.get_first_group_by_pattern(html, pattern)
        if not pl_lib_js:
            return None

        url = urlparse.urljoin(site_url, pl_lib_js)

        content = self.get_content_by_url(url)

        pattern = "'X-CSRF-Token', '(.*?)'"
        token = common.get_first_group_by_pattern(content, pattern)

        return token

    def get_lyric_raw_json(self, id, token):
        if not id:
            return None

        actionUrl = 'https://petitlyrics.com/com/get_lyrics.ajax'
        headers = {
            'X-CSRF-Token': token,
            'X-Requested-With': 'XMLHttpRequest'
        }
        payload = {
            'lyrics_id': id,
        }

        content = self.get_content_by_url(actionUrl, payload, headers)
        try:
            obj = json.loads(content)
        except:
            return None

        return obj

    def get_lyric(self, id, token):
        if not id:
            return None

        obj = self.get_lyric_raw_json(id, token)
        if not obj:
            return None

        lyric_list = []
        for item in obj:
            lyric_list.append(base64.b64decode(item['lyrics']))

        return '\n'.join(lyric_list).decode('utf8', 'ignore')

    def parse_artist_title(self, html):
        startStr = '"description" content="'
        endStr = u'の歌詞ページです'

        infoStr = common.get_string_by_start_end_string(startStr, endStr, html)
        if not infoStr:
            return None

        infoStr = infoStr.replace(startStr, '')
        infoStr = infoStr.replace(endStr, '')
        infoStr = infoStr.strip()

        items = infoStr.split(' / ')

        if len(items) == 2:
            self.title = common.unicode2string(items[0])
            self.artist = common.unicode2string(items[1])

    def parse_lyricist(self, html):
        prefix = '作詞：'
        suffix = '\t'

        logging.debug('find me LYRICIST')

        raw_string = common.find_string_by_prefix_suffix(html, prefix, suffix, False)
        if not raw_string:
            logging.debug('Failed to find lyricist')
            return False

        self.lyricist = common.htmlspecialchars_decode(common.unicode2string(raw_string)).strip()

    def parse_composer(self, html):
        prefix = '作曲：'
        suffix = '\t'

        raw_string = common.find_string_by_prefix_suffix(html, prefix, suffix, False)
        if not raw_string:
            logging.debug('Failed to find composer')
            return False

        self.composer = common.htmlspecialchars_decode(common.unicode2string(raw_string)).strip()

    def get_content_by_url(self, url, payload=None, headers=None):
        return self.get_content_by_url_by_requests(url, payload, headers)
        #return self.get_content_by_url_by_urllib3(url, payload, headers)

    def get_content_by_url_by_requests(self, url, payload=None, headers=None):
        r = self.request_by_url_requests(url, payload, headers)
        if not r:
            return False
        return r.text

    def get_content_by_url_by_urllib3(self, url, payload=None, headers=None):
        r = self.request_by_url_urllib3(url, payload, headers)
        if not r:
            return False
        return r.data.decode('utf-8', 'ignore')

    def request_by_url_requests(self, url, payload=None, headers=None):
        import urllib3
        urllib3.disable_warnings()
        import requests

        proxies = {
            #'http': '192.168.1.34:8001',
            #'https': '192.168.1.34:8001'
        }

        if not hasattr(self, 'session'):
            self.session = requests.Session()

        if payload:
            r = self.session.post(url, data = payload, headers = headers, proxies=proxies, verify=False)
        else:
            r = self.session.get(url, proxies=proxies, verify=False)

        return r

    def request_by_url_urllib3(self, url, payload=None, headers=None):
        import urllib3
        urllib3.disable_warnings()

        if not hasattr(self, 'http'):
            self.http = urllib3.PoolManager()
            #self.http = urllib3.ProxyManager('http://192.168.1.34:8001')

        if hasattr(self, 'cookie'):
            if not headers:
                headers = {'Cookie': self.cookie}
                logging.debug('no headers in params, set cookie to %s' % (self.cookie))
            elif not hasattr(headers, 'Cookie'):
                headers['Cookie'] = self.cookie
                logging.debug('has headers params, set cookie to %s' % (self.cookie))
            else:
                logging.debug('## cookie in request headers')
        else:
            logging.debug('## no saved cookie')

        if payload:
            r = self.http.request_encode_body('POST', url, payload, headers, encode_multipart=False)
        else:
            r = self.http.request_encode_url('GET', url, payload, headers)

        logging.debug('== == == == ==')
        logging.debug('== request to url [%s]' % (url))

        cookie = r.getheader('Set-Cookie')
        if cookie:
            self.cookie = cookie
            logging.debug('Got Set-Cookie: %s from request to %s' % (cookie, url))

        return r

def get_lyric(url):
    obj = PetitLyrics(url)

    return obj.get()

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    url = test_url
    url = 'https://petitlyrics.com/lyrics/1175487'
    url = 'https://petitlyrics.com/lyrics/1015689'
    url = 'https://petitlyrics.com/lyrics/34690'

    full = get_lyric(url)
    if not full:
        print('Cannot get lyric')
        exit()
    print(full.encode('utf-8', 'ignore'))
<commit_msg>let petitlyrics get from azure<commit_after># coding: utf-8
import os
import sys
include_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'include')
sys.path.append(include_dir)

import base64
import json
import logging
import urlparse
import common

from lyric_base import LyricBase

try:
    import requests_toolbelt.adapters.appengine
    # Use the App Engine Requests adapter. This makes sure that Requests uses
    # URLFetch.
    requests_toolbelt.adapters.appengine.monkeypatch()
except:
    pass

site_class = 'PetitLyrics'
site_index = 'petitlyrics'
site_keyword = 'petitlyrics'
site_url = 'https://petitlyrics.com/'
test_url = 'https://petitlyrics.com/lyrics/914675'
test_expect_length = 1275

'''
    Request URL
        http://petitlyrics.com/com/get_lyrics.ajax
    Method
        POST
    Data
        lyrics_id=914675

    Response
'''
class PetitLyrics(LyricBase):
    def parse_page(self):
        url = self.url

        if not self.get_from_azure(url):
            logging.info('Failed to get from azure, url [%s]', url)
            return False

        return True

    def parse_page_local(self):
        url = self.url

        # 1. get song id from url
        id = self.get_song_id(url)
        if not id:
            logging.error('Failed to get id of url [%s]', url)
            return False

        # 2. get html
        html = self.get_content_by_url(url)

        if not html:
            logging.info('Failed to get html of url [%s]' % (url))
            return False

        # 3. get CSRF token
        token = self.get_csrf_token(html)
        if not token:
            logging.info('Failed to get CSRF token of url [%s]' % (url))
            return False
        logging.debug('CSRF token is %s' % (token, ))

        # 5. get lyric
        lyric = self.get_lyric(id, token)
        if not lyric:
            logging.info('Failed to get lyric of url [%s]' % (url))
            return False

        self.lyric = common.htmlspecialchars_decode(lyric.strip())

        self.parse_artist_title(html)
        self.parse_lyricist(html)
        self.parse_composer(html)

        return True

    def get_song_id(self, url):
        if not url:
            return None

        pattern = '/lyrics/([0-9]+)'
        id = common.get_first_group_by_pattern(url, pattern)
        if not id:
            pattern = '/kashi/([0-9]+)'
            id = common.get_first_group_by_pattern(url, pattern)

        return id

    def get_csrf_token(self, html):
        pattern = '(/lib/pl-lib.js[^"]+)'

        pl_lib_js = common.get_first_group_by_pattern(html, pattern)
        if not pl_lib_js:
            return None

        url = urlparse.urljoin(site_url, pl_lib_js)

        content = self.get_content_by_url(url)

        pattern = "'X-CSRF-Token', '(.*?)'"
        token = common.get_first_group_by_pattern(content, pattern)

        return token

    def get_lyric_raw_json(self, id, token):
        if not id:
            return None

        actionUrl = 'https://petitlyrics.com/com/get_lyrics.ajax'
        headers = {
            'X-CSRF-Token': token,
            'X-Requested-With': 'XMLHttpRequest'
        }
        payload = {
            'lyrics_id': id,
        }

        content = self.get_content_by_url(actionUrl, payload, headers)
        try:
            obj = json.loads(content)
        except:
            return None

        return obj

    def get_lyric(self, id, token):
        if not id:
            return None

        obj = self.get_lyric_raw_json(id, token)
        if not obj:
            return None

        lyric_list = []
        for item in obj:
            lyric_list.append(base64.b64decode(item['lyrics']))

        return '\n'.join(lyric_list).decode('utf8', 'ignore')

    def parse_artist_title(self, html):
        startStr = '"description" content="'
        endStr = u'の歌詞ページです'

        infoStr = common.get_string_by_start_end_string(startStr, endStr, html)
        if not infoStr:
            return None

        infoStr = infoStr.replace(startStr, '')
        infoStr = infoStr.replace(endStr, '')
        infoStr = infoStr.strip()

        items = infoStr.split(' / ')

        if len(items) == 2:
            self.title = common.unicode2string(items[0])
            self.artist = common.unicode2string(items[1])

    def parse_lyricist(self, html):
        prefix = '作詞：'
        suffix = '\t'

        logging.debug('find me LYRICIST')

        raw_string = common.find_string_by_prefix_suffix(html, prefix, suffix, False)
        if not raw_string:
            logging.debug('Failed to find lyricist')
            return False

        self.lyricist = common.htmlspecialchars_decode(common.unicode2string(raw_string)).strip()

    def parse_composer(self, html):
        prefix = '作曲：'
        suffix = '\t'

        raw_string = common.find_string_by_prefix_suffix(html, prefix, suffix, False)
        if not raw_string:
            logging.debug('Failed to find composer')
            return False

        self.composer = common.htmlspecialchars_decode(common.unicode2string(raw_string)).strip()

    def get_content_by_url(self, url, payload=None, headers=None):
        return self.get_content_by_url_by_requests(url, payload, headers)
        #return self.get_content_by_url_by_urllib3(url, payload, headers)

    def get_content_by_url_by_requests(self, url, payload=None, headers=None):
        r = self.request_by_url_requests(url, payload, headers)
        if not r:
            return False
        return r.text

    def get_content_by_url_by_urllib3(self, url, payload=None, headers=None):
        r = self.request_by_url_urllib3(url, payload, headers)
        if not r:
            return False
        return r.data.decode('utf-8', 'ignore')

    def request_by_url_requests(self, url, payload=None, headers=None):
        import urllib3
        urllib3.disable_warnings()
        import requests

        proxies = {
            #'http': '192.168.1.34:8001',
            #'https': '192.168.1.34:8001'
        }

        if not hasattr(self, 'session'):
            self.session = requests.Session()

        if payload:
            r = self.session.post(url, data = payload, headers = headers, proxies=proxies, verify=False)
        else:
            r = self.session.get(url, proxies=proxies, verify=False)

        return r

    def request_by_url_urllib3(self, url, payload=None, headers=None):
        import urllib3
        urllib3.disable_warnings()

        if not hasattr(self, 'http'):
            self.http = urllib3.PoolManager()
            #self.http = urllib3.ProxyManager('http://192.168.1.34:8001')

        if hasattr(self, 'cookie'):
            if not headers:
                headers = {'Cookie': self.cookie}
                logging.debug('no headers in params, set cookie to %s' % (self.cookie))
            elif not hasattr(headers, 'Cookie'):
                headers['Cookie'] = self.cookie
                logging.debug('has headers params, set cookie to %s' % (self.cookie))
            else:
                logging.debug('## cookie in request headers')
        else:
            logging.debug('## no saved cookie')

        if payload:
            r = self.http.request_encode_body('POST', url, payload, headers, encode_multipart=False)
        else:
            r = self.http.request_encode_url('GET', url, payload, headers)

        logging.debug('== == == == ==')
        logging.debug('== request to url [%s]' % (url))

        cookie = r.getheader('Set-Cookie')
        if cookie:
            self.cookie = cookie
            logging.debug('Got Set-Cookie: %s from request to %s' % (cookie, url))

        return r

def get_lyric(url):
    obj = PetitLyrics(url)

    return obj.get()

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    url = test_url
    url = 'https://petitlyrics.com/lyrics/1175487'
    url = 'https://petitlyrics.com/lyrics/1015689'
    url = 'https://petitlyrics.com/lyrics/34690'

    full = get_lyric(url)
    if not full:
        print('Cannot get lyric')
        exit()
    print(full.encode('utf-8', 'ignore'))
<|endoftext|>


<commit_before># Copyright 2016 Google Inc. All Rights Reserved.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Unit test class for OpenHTF.

  Provides ability to test for outcomes, exceptions and measurement values.
  Provides ability to pass DUT serial number into unit test.

  Example:

  import unittest
  import mock
  import example_plug
  import example_phase
  from openhtf.util import test

  class Test(test.TestCase):
    def setUp(self):
      self.plug = example_plug.ExamplePlug
      self.mock_plug = mock.Mock()

    def testExamplePhase(self):
      self.mock_plug.return_value.GetReading.return_value = 1.0
      self.RunPhasee(examle_phase.ExamplePhase, [self.plug], [self.mock_plug])
      self.assertEqual(self.outcome, self.PASS)
      self.assertAlmostEqual(self.values['foo'], 1.0)
      self.assertEqual(self.exception, None)

  if __name__ == '__main__':
    unittest.main()
"""

import collections
import functools
import inspect
import logging
import sys
import types
import unittest

import mock

import openhtf
from openhtf import plugs
from openhtf import util
from openhtf.exe import phase_data
from openhtf.exe import phase_executor
from openhtf.io import test_record
from openhtf.util import measurements

class InvalidTestError(Exception):
  """Raised when there's something invalid about a test."""

class PhaseOrTestIterator(collections.Iterator):
  def __init__(self, iterator, mock_plugs):
    """Create an iterator for iterating over Tests or phases to run.

    Args:
      iterator: Child iterator to use for obtaining Tests or test phases, must
          be a generator.
      mock_plugs: Dict mapping plug types to mock objects to use instead of
          actually instantiating that type.
    """
    if not isinstance(iterator, types.GeneratorType):
      raise InvalidTestError(
          'methods decorated with patch_plugs must yield a test phase',
          iterator)

    # Since we want to run single phases, we instantiate our own PlugManager.
    # Don't do this sort of thing outside OpenHTF unless you really know what
    # you're doing (http://imgur.com/iwBCmQe).
    self.plug_manager = plugs.PlugManager()
    self.iterator = iterator
    self.mock_plugs = mock_plugs
    self.last_result = None

  def _handle_phase(self, phase):
    """Handle execution of a single test phase."""
    # Make sure we initialize any plugs, this will ignore any that have already
    # been initialized.
    self.plug_manager.InitializePlugs(plug.cls for plug in phase.plugs if
                                      plug.cls not in self.mock_plugs)
    for plug_type, plug_value in self.mock_plugs.iteritems():
      self.plug_manager.OverridePlug(plug_type, plug_value)

    # Cobble together a fake phase data to pass to the test phase.  We use the
    # root logger as a logger, our stub plug manager, and a dummy test record
    # that has None for dut_id and station_id.
    phasedata = phase_data.PhaseData(logging.getLogger(), self.plug_manager,
                                     test_record.TestRecord(None, None))
    phase_record = test_record.PhaseRecord(phase.name, phase.code_info)

    with phasedata.RecordPhaseTiming(phase, phase_record):
      try:
        phase_record.result = phase_executor.PhaseOutcome(phase(phasedata))
      except Exception as exc:
        logging.exception('Exception executing phase %s', phase.name)
        phase_record.result = phase_executor.PhaseOutcome(exc)
    return phase_record

  def _handle_test(self, test):
    pass

  def next(self):
    phase_or_test = self.iterator.send(self.last_result)
    if isinstance(phase_or_test, openhtf.Test):
      self.last_result = self._handle_test(phase_or_test)
    elif not callable(phase_or_test):
      raise InvalidTestError(
          'methods decorated with patch_plugs must yield Test instances or '
          'individual test phases', phase_or_test)
    self.last_result = self._handle_phase(
      openhtf.PhaseInfo.WrapOrCopy(phase_or_test))
    return phase_or_test, self.last_result

def patch_plugs(**mock_plugs):
  """Decorator for mocking plugs for a test phase.

  Args:
    **mock_plugs: kwargs mapping argument name to be passed to the test case to
        a string describing the plug type to mock.  The corresponding mock will
        be passed to the decorated test case as a keyword argument.

  Usage:

    @plugs(my_plug=my_plug_module.MyPlug)
    def my_phase_that_uses_my_plug(test, my_plug):
      test.logger.info('Something: %s', my_plug.do_something(10))

    @test.patch_plugs(my_plug_mock='my_plug_module.MyPlug')
    def test_my_phase(self, my_plug_mock):
      # Set up return value for the do_something method on our plug.
      my_plug_mock.do_something.return_value = 'mocked_value'

      # Yield the phase you wish to test. Typically it wouldn't be in the same
      # module like this, but this works for example purposes.
      yield my_phase_that_uses_my_plug

      # Do some assertions to make sure our plug was used correctly.
      my_plug_mock.do_something.assert_called_with(10)
  """
  def test_wrapper(test_func):
    plug_argspec = inspect.getargspec(test_func)
    num_defaults = len(plug_argspec.defaults or ())
    plug_args = set(plug_argspec.args[1:-num_defaults or None])

    # Some sanity checks to make sure the mock arg names match.
    for arg in plug_args:
      if arg not in mock_plugs:
        raise InvalidTestError(
            'Test method %s expected arg %s, but it was not provided in '
            'patch_plugs kwargs: ' % (test_func.__name__, arg), mock_plugs)
    for mock_name in mock_plugs:
      if mock_name not in plug_args:
        raise InvalidTestError(
            'patch_plugs got kwarg %s, but test method %s does not expect '
            'it.' % (mock_name, test_func.__name__), plug_args)

    # Make MagicMock instances for the plugs.
    plug_kwargs = {}  # kwargs to pass to test func.
    plug_typemap = {}  # typemap for PlugManager, maps type to instance.
    for plug_arg_name, plug_typename in mock_plugs.iteritems():
      plug_module, plug_typename = plug_typename.rsplit('.', 1)
      try:
        plug_type = getattr(sys.modules[plug_module], plug_typename)
      except:
        logging.error("Invalid plug type specification %s='%s.%s'",
                      plug_arg_name, plug_module, plug_typename)
        raise
      plug_mock = mock.create_autospec(plug_type, spec_set=True, instance=True)
      plug_typemap[plug_type] = plug_mock
      plug_kwargs[plug_arg_name] = plug_mock

    # functools.wraps is more than just aesthetic here, we need the original
    # name to match so we don't mess with unittest's TestLoader mechanism.
    @functools.wraps(test_func)
    def wrapped_test(self):
      for phase_or_test, result in PhaseOrTestIterator(
          test_func(self, **plug_kwargs), plug_typemap):
        logging.info('Ran %s, result: %s', phase_or_test, result)
    return wrapped_test
  return test_wrapper

class TestCase(unittest.TestCase):

  def assertNotMeasured(self, phase_record, measurement):
    self.assertNotIn(measurement, phase_record.measured_values)
    self.assertIs(measurements.Outcome.UNSET,
                  phase_record.measurements[measurement].outcome)

  def assertMeasured(self, phase_record, measurement, value=mock.ANY):
    self.assertIn(measurement, phase_record.measured_values)
    if value is not mock.ANY:
      self.assertEquals(value, phase_record.measured_values[measurement])

  def assertMeasurementPassed(self, phase_record, measurement):
    self.assertMeasured(phase_record, measurement)
    self.assertIs(measurements.Outcome.PASS,
                  phase_record.measurements[measurement].outcome)

  def assertMeasurementFailed(self, phase_record, measurement):
    self.assertMeasured(phase_record, measurement)
    self.assertIs(measurements.Outcome.FAIL,
                  phase_record.measurements[measurement].outcome)
<commit_msg>change bare except to except Exception<commit_after># Copyright 2016 Google Inc. All Rights Reserved.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Unit test class for OpenHTF.

  Provides ability to test for outcomes, exceptions and measurement values.
  Provides ability to pass DUT serial number into unit test.

  Example:

  import unittest
  import mock
  import example_plug
  import example_phase
  from openhtf.util import test

  class Test(test.TestCase):
    def setUp(self):
      self.plug = example_plug.ExamplePlug
      self.mock_plug = mock.Mock()

    def testExamplePhase(self):
      self.mock_plug.return_value.GetReading.return_value = 1.0
      self.RunPhasee(examle_phase.ExamplePhase, [self.plug], [self.mock_plug])
      self.assertEqual(self.outcome, self.PASS)
      self.assertAlmostEqual(self.values['foo'], 1.0)
      self.assertEqual(self.exception, None)

  if __name__ == '__main__':
    unittest.main()
"""

import collections
import functools
import inspect
import logging
import sys
import types
import unittest

import mock

import openhtf
from openhtf import plugs
from openhtf import util
from openhtf.exe import phase_data
from openhtf.exe import phase_executor
from openhtf.io import test_record
from openhtf.util import measurements

class InvalidTestError(Exception):
  """Raised when there's something invalid about a test."""

class PhaseOrTestIterator(collections.Iterator):
  def __init__(self, iterator, mock_plugs):
    """Create an iterator for iterating over Tests or phases to run.

    Args:
      iterator: Child iterator to use for obtaining Tests or test phases, must
          be a generator.
      mock_plugs: Dict mapping plug types to mock objects to use instead of
          actually instantiating that type.
    """
    if not isinstance(iterator, types.GeneratorType):
      raise InvalidTestError(
          'methods decorated with patch_plugs must yield a test phase',
          iterator)

    # Since we want to run single phases, we instantiate our own PlugManager.
    # Don't do this sort of thing outside OpenHTF unless you really know what
    # you're doing (http://imgur.com/iwBCmQe).
    self.plug_manager = plugs.PlugManager()
    self.iterator = iterator
    self.mock_plugs = mock_plugs
    self.last_result = None

  def _handle_phase(self, phase):
    """Handle execution of a single test phase."""
    # Make sure we initialize any plugs, this will ignore any that have already
    # been initialized.
    self.plug_manager.InitializePlugs(plug.cls for plug in phase.plugs if
                                      plug.cls not in self.mock_plugs)
    for plug_type, plug_value in self.mock_plugs.iteritems():
      self.plug_manager.OverridePlug(plug_type, plug_value)

    # Cobble together a fake phase data to pass to the test phase.  We use the
    # root logger as a logger, our stub plug manager, and a dummy test record
    # that has None for dut_id and station_id.
    phasedata = phase_data.PhaseData(logging.getLogger(), self.plug_manager,
                                     test_record.TestRecord(None, None))
    phase_record = test_record.PhaseRecord(phase.name, phase.code_info)

    with phasedata.RecordPhaseTiming(phase, phase_record):
      try:
        phase_record.result = phase_executor.PhaseOutcome(phase(phasedata))
      except Exception as exc:
        logging.exception('Exception executing phase %s', phase.name)
        phase_record.result = phase_executor.PhaseOutcome(exc)
    return phase_record

  def _handle_test(self, test):
    pass

  def next(self):
    phase_or_test = self.iterator.send(self.last_result)
    if isinstance(phase_or_test, openhtf.Test):
      self.last_result = self._handle_test(phase_or_test)
    elif not callable(phase_or_test):
      raise InvalidTestError(
          'methods decorated with patch_plugs must yield Test instances or '
          'individual test phases', phase_or_test)
    self.last_result = self._handle_phase(
      openhtf.PhaseInfo.WrapOrCopy(phase_or_test))
    return phase_or_test, self.last_result

def patch_plugs(**mock_plugs):
  """Decorator for mocking plugs for a test phase.

  Args:
    **mock_plugs: kwargs mapping argument name to be passed to the test case to
        a string describing the plug type to mock.  The corresponding mock will
        be passed to the decorated test case as a keyword argument.

  Usage:

    @plugs(my_plug=my_plug_module.MyPlug)
    def my_phase_that_uses_my_plug(test, my_plug):
      test.logger.info('Something: %s', my_plug.do_something(10))

    @test.patch_plugs(my_plug_mock='my_plug_module.MyPlug')
    def test_my_phase(self, my_plug_mock):
      # Set up return value for the do_something method on our plug.
      my_plug_mock.do_something.return_value = 'mocked_value'

      # Yield the phase you wish to test. Typically it wouldn't be in the same
      # module like this, but this works for example purposes.
      yield my_phase_that_uses_my_plug

      # Do some assertions to make sure our plug was used correctly.
      my_plug_mock.do_something.assert_called_with(10)
  """
  def test_wrapper(test_func):
    plug_argspec = inspect.getargspec(test_func)
    num_defaults = len(plug_argspec.defaults or ())
    plug_args = set(plug_argspec.args[1:-num_defaults or None])

    # Some sanity checks to make sure the mock arg names match.
    for arg in plug_args:
      if arg not in mock_plugs:
        raise InvalidTestError(
            'Test method %s expected arg %s, but it was not provided in '
            'patch_plugs kwargs: ' % (test_func.__name__, arg), mock_plugs)
    for mock_name in mock_plugs:
      if mock_name not in plug_args:
        raise InvalidTestError(
            'patch_plugs got kwarg %s, but test method %s does not expect '
            'it.' % (mock_name, test_func.__name__), plug_args)

    # Make MagicMock instances for the plugs.
    plug_kwargs = {}  # kwargs to pass to test func.
    plug_typemap = {}  # typemap for PlugManager, maps type to instance.
    for plug_arg_name, plug_typename in mock_plugs.iteritems():
      plug_module, plug_typename = plug_typename.rsplit('.', 1)
      try:
        plug_type = getattr(sys.modules[plug_module], plug_typename)
      except Exception:
        logging.error("Invalid plug type specification %s='%s.%s'",
                      plug_arg_name, plug_module, plug_typename)
        raise
      plug_mock = mock.create_autospec(plug_type, spec_set=True, instance=True)
      plug_typemap[plug_type] = plug_mock
      plug_kwargs[plug_arg_name] = plug_mock

    # functools.wraps is more than just aesthetic here, we need the original
    # name to match so we don't mess with unittest's TestLoader mechanism.
    @functools.wraps(test_func)
    def wrapped_test(self):
      for phase_or_test, result in PhaseOrTestIterator(
          test_func(self, **plug_kwargs), plug_typemap):
        logging.info('Ran %s, result: %s', phase_or_test, result)
    return wrapped_test
  return test_wrapper

class TestCase(unittest.TestCase):

  def assertNotMeasured(self, phase_record, measurement):
    self.assertNotIn(measurement, phase_record.measured_values)
    self.assertIs(measurements.Outcome.UNSET,
                  phase_record.measurements[measurement].outcome)

  def assertMeasured(self, phase_record, measurement, value=mock.ANY):
    self.assertIn(measurement, phase_record.measured_values)
    if value is not mock.ANY:
      self.assertEquals(value, phase_record.measured_values[measurement])

  def assertMeasurementPassed(self, phase_record, measurement):
    self.assertMeasured(phase_record, measurement)
    self.assertIs(measurements.Outcome.PASS,
                  phase_record.measurements[measurement].outcome)

  def assertMeasurementFailed(self, phase_record, measurement):
    self.assertMeasured(phase_record, measurement)
    self.assertIs(measurements.Outcome.FAIL,
                  phase_record.measurements[measurement].outcome)
<|endoftext|>