<commit_before>#! /usr/bin/env python3
import os
import hashlib
import argparse
import itertools
import re
import logging
from collections import defaultdict, namedtuple

logger = logging.getLogger('')

Filename = namedtuple('Filename', ['name', 'base', 'ext', 'path'])

def create_filenames(filenames, root):
    '''
    Makes a generator that yields Filename objects

    Filename objects are a helper to allow multiple representations
    of the same file to be transfered cleanly.
    '''
    logger.info("Creating Filename objects")
    for filename in filenames:
        yield Filename(filename, *os.path.splitext(filename),
                       path=os.path.join(root, filename))

def generate_checksum(filename):
    '''
    A helper function that will generate the
    check sum of a file.
    '''
    logger.info("Generating checksum for {0}".format(filename))
    md5 = hashlib.md5()
    with open(filename, 'rb') as f:
        for chunk in iter(lambda: f.read(128 * md5.block_size), b''):
            md5.update(chunk)
    return md5.digest()

def is_substring(string1, string2):
    '''
    Returns a match if one string is a substring of the other

    For example::

        is_substring('this', 'this1') -> True
        is_substring('that1', 'that') -> True

    But::

        is_substring('that', 'this')  -> False

    '''
    return string1 in string2 or string2 in string1

def compare_filename_name(file1, file2):
    '''
    Uses is_substring to determine if both base and extension match.
    '''
    logger.info("Comparing {0} and {1} by substring".format(file1.path,
                                                            file2.path))
    return is_substring(file1.base, file2.base) and is_substring(file1.ext,
                                                                 file2.ext)

def compare_filename_checksum(file1, file2):
    '''
    Uses the generate_checksum function to compare two files
    '''
    logger.info("Comparing {0} and {1} by checksum".format(file1.path,
                                                           file2.path))
    return generate_checksum(file1.path) == generate_checksum(file2.path)

def pick_basename(file1, file2):
    '''
    This convenience function will help to find the shorter (often better)
    filename

    It picks "file.txt" over "file (1).txt", but beware it also picks
    "f.txt" over "file.txt".
    '''
    logger.debug("Finding the shortest of {0} and {1}".format(file1.name,
                                                              file2.name))
    if len(file1.name) > len(file2.name):
        return file2, file1
    else:
        return file1, file2

def generate_checksum_dict(filenames):
    '''
    This function will create a dictionary of checksums mapped to
    a list of filenames.
    '''
    logger.info("Generating dictionary based on checksum")
    checksum_dict = defaultdict(list)

    for filename in filenames:
        checksum_dict[generate_checksum(filename.path)].append(filename)

    return checksum_dict

def generate_filename_dict(filenames):
    '''
    This function will create a dictionary of filename parts mapped to a list
    of the real filenames.
    '''
    logger.info("Generating dictionary based on regular expression")
    filename_dict = defaultdict(list)

    regex = re.compile(r'(^.+?)( \((\d)\))*(\..+)$')

    for filename in filenames:
        match = regex.match(filename.name)
        if match:
            logger.debug('Regex groups for {0}: {1}'.format(
                filename.name, str(match.groups())))
            logger.info("Found a match for {0} adding to key {1}".format(
                filename.name, ''.join(match.group(1, 4))))
            filename_dict[''.join(match.group(1, 4))].append(filename)

    return filename_dict

def main(path, no_action, recursive, generate_dict, compare_filename):
    '''
    This function handles all options and steps through the directory
    '''
    for root, dirs, filenames in os.walk(path):
        if not recursive and root != path:
            logger.debug("Skipping child directory {0}".format(root))
            continue
        hashes = generate_dict(create_filenames(filenames, root))

        for hash in hashes:
            if len(hashes[hash]) > 1:
                logger.info("Investigating duplicate hash {0}".format(hash))
                logger.debug("Keys for {0} are {1}".format(
                    hash, ', '.join([item.name for item in hashes[hash]])))
                for pair in itertools.combinations(hashes[hash], 2):
                    if compare_filename(*pair):
                        orig, dup = pick_basename(*pair)
                        if no_action:
                            print('{1} to be deleted'.format(orig.name,
                                                             dup.name))
                            logger.info('{1} would have been deleted'.format(
                                orig.name, dup.name))
                        else:
                            logger.info('{1} was deleted'.format(orig.name,
                                                                 dup.name))
                            os.remove(dup.path)
                    else:
                        logger.info('Files {0} and {1} did not match'.format(
                            pair[0].name, pair[1].name))
            else:
                logger.debug(
                    'Skipping non duplicate hash {0} for key {1}'.format(
                        hash, ', '.join([item.name for item in hashes[hash]])))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'path',
        help='This is the path you want to run the checker against')
    parser.add_argument(
        '-n', '--no-action',
        default=False,
        action='store_true',
        help='This will print the output without changing anyfiles')
    parser.add_argument(
        '-r', '--recursive',
        default=False,
        action='store_true',
        help='This option toggles whether the program should search recursively')
    parser.add_argument(
        '-c', '--checksum',
        default=False,
        action='store_true',
        help=
        'This option toggles whether the program searchs first by checksum rather than name')
    parser.add_argument('--verbosity',
                        type=int,
                        default=0,
                        help='Set debug level')
    parser.add_argument('--log-file',
                        help='This option sets a log file to write.')
    parser.add_argument('--log-level',
                        type=int,
                        default=3,
                        help='Set debug level in log file')
    args = parser.parse_args()

    if args.log_level != 3 and not args.log_file:
        parser.error('Log level set without log file')

    stream = logging.StreamHandler()
    stream.setLevel((5 - args.verbosity) * 10)
    formatter_simple = logging.Formatter(
        '%(asctime)s - %(levelname)s - %(message)s')
    stream.setFormatter(formatter_simple)
    logger.addHandler(stream)
    logger.setLevel(logging.DEBUG)

    if args.log_file:
        log_file = logging.FileHandler(args.log_file)
        log_file.setFormatter(formatter_simple)
        log_file.setLevel((5 - args.log_level) * 10)
        logger.addHandler(log_file)

    logger.debug("Args: {0}".format(args))

    if args.checksum:
        main(args.path, args.no_action, args.recursive, generate_checksum_dict,
             compare_filename_name)
    else:
        main(args.path, args.no_action, args.recursive, generate_filename_dict,
             compare_filename_checksum)
<commit_msg>Remove extra option for now<commit_after>#! /usr/bin/env python3
import os
import hashlib
import argparse
import itertools
import re
import logging
from collections import defaultdict, namedtuple

logger = logging.getLogger('')

Filename = namedtuple('Filename', ['name', 'base', 'ext', 'path'])

def create_filenames(filenames, root):
    '''
    Makes a generator that yields Filename objects

    Filename objects are a helper to allow multiple representations
    of the same file to be transfered cleanly.
    '''
    logger.info("Creating Filename objects")
    for filename in filenames:
        yield Filename(filename, *os.path.splitext(filename),
                       path=os.path.join(root, filename))

def generate_checksum(filename):
    '''
    A helper function that will generate the
    check sum of a file.
    '''
    logger.info("Generating checksum for {0}".format(filename))
    md5 = hashlib.md5()
    with open(filename, 'rb') as f:
        for chunk in iter(lambda: f.read(128 * md5.block_size), b''):
            md5.update(chunk)
    return md5.digest()

def is_substring(string1, string2):
    '''
    Returns a match if one string is a substring of the other

    For example::

        is_substring('this', 'this1') -> True
        is_substring('that1', 'that') -> True

    But::

        is_substring('that', 'this')  -> False

    '''
    return string1 in string2 or string2 in string1

def compare_filename_name(file1, file2):
    '''
    Uses is_substring to determine if both base and extension match.
    '''
    logger.info("Comparing {0} and {1} by substring".format(file1.path,
                                                            file2.path))
    return is_substring(file1.base, file2.base) and is_substring(file1.ext,
                                                                 file2.ext)

def compare_filename_checksum(file1, file2):
    '''
    Uses the generate_checksum function to compare two files
    '''
    logger.info("Comparing {0} and {1} by checksum".format(file1.path,
                                                           file2.path))
    return generate_checksum(file1.path) == generate_checksum(file2.path)

def pick_basename(file1, file2):
    '''
    This convenience function will help to find the shorter (often better)
    filename

    It picks "file.txt" over "file (1).txt", but beware it also picks
    "f.txt" over "file.txt".
    '''
    logger.debug("Finding the shortest of {0} and {1}".format(file1.name,
                                                              file2.name))
    if len(file1.name) > len(file2.name):
        return file2, file1
    else:
        return file1, file2

def generate_checksum_dict(filenames):
    '''
    This function will create a dictionary of checksums mapped to
    a list of filenames.
    '''
    logger.info("Generating dictionary based on checksum")
    checksum_dict = defaultdict(list)

    for filename in filenames:
        checksum_dict[generate_checksum(filename.path)].append(filename)

    return checksum_dict

def generate_filename_dict(filenames):
    '''
    This function will create a dictionary of filename parts mapped to a list
    of the real filenames.
    '''
    logger.info("Generating dictionary based on regular expression")
    filename_dict = defaultdict(list)

    regex = re.compile(r'(^.+?)( \((\d)\))*(\..+)$')

    for filename in filenames:
        match = regex.match(filename.name)
        if match:
            logger.debug('Regex groups for {0}: {1}'.format(
                filename.name, str(match.groups())))
            logger.info("Found a match for {0} adding to key {1}".format(
                filename.name, ''.join(match.group(1, 4))))
            filename_dict[''.join(match.group(1, 4))].append(filename)

    return filename_dict

def main(path, no_action, recursive):
    '''
    This function handles all options and steps through the directory
    '''
    for root, dirs, filenames in os.walk(path):
        if not recursive and root != path:
            logger.debug("Skipping child directory {0}".format(root))
            continue
        names = generate_filename_dict(create_filenames(filenames, root))

        for hash in hashes:
            if len(hashes[hash]) > 1:
                logger.info("Investigating duplicate hash {0}".format(hash))
                logger.debug("Keys for {0} are {1}".format(
                    hash, ', '.join([item.name for item in hashes[hash]])))
                for pair in itertools.combinations(hashes[hash], 2):
                    if compare_filename(*pair):
                        orig, dup = pick_basename(*pair)
                        if no_action:
                            print('{1} to be deleted'.format(orig.name,
                                                             dup.name))
                            logger.info('{1} would have been deleted'.format(
                                orig.name, dup.name))
                        else:
                            logger.info('{1} was deleted'.format(orig.name,
                                                                 dup.name))
                            os.remove(dup.path)
                    else:
                        logger.info('Files {0} and {1} did not match'.format(
                            pair[0].name, pair[1].name))
            else:
                logger.debug(
                    'Skipping non duplicate hash {0} for key {1}'.format(
                        hash, ', '.join([item.name for item in hashes[hash]])))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'path',
        help='This is the path you want to run the checker against')
    parser.add_argument(
        '-n', '--no-action',
        default=False,
        action='store_true',
        help='This will print the output without changing anyfiles')
    parser.add_argument(
        '-r', '--recursive',
        default=False,
        action='store_true',
        help='This option toggles whether the program should search recursively')
    parser.add_argument('--verbosity',
                        type=int,
                        default=0,
                        help='Set debug level')
    parser.add_argument('--log-file',
                        help='This option sets a log file to write.')
    parser.add_argument('--log-level',
                        type=int,
                        default=3,
                        help='Set debug level in log file')
    args = parser.parse_args()

    if args.log_level != 3 and not args.log_file:
        parser.error('Log level set without log file')

    stream = logging.StreamHandler()
    stream.setLevel((5 - args.verbosity) * 10)
    formatter_simple = logging.Formatter(
        '%(asctime)s - %(levelname)s - %(message)s')
    stream.setFormatter(formatter_simple)
    logger.addHandler(stream)
    logger.setLevel(logging.DEBUG)

    if args.log_file:
        log_file = logging.FileHandler(args.log_file)
        log_file.setFormatter(formatter_simple)
        log_file.setLevel((5 - args.log_level) * 10)
        logger.addHandler(log_file)

    logger.debug("Args: {0}".format(args))

    main(args.path, args.no_action, args.recursive)
<|endoftext|>


<commit_before>try:
    import urllib.request as urllib_request
    import urllib.error as urllib_error
except ImportError:
    import urllib2 as urllib_request
    import urllib2 as urllib_error

try:
    from cStringIO import StringIO
except ImportError:
    from io import BytesIO as StringIO

from twitter.twitter_globals import POST_ACTIONS
from twitter.auth import NoAuth

import re
import gzip

try:
    import json
except ImportError:
    import simplejson as json

class _DEFAULT(object):
    pass

class TwitterError(Exception):
    """
    Base Exception thrown by the Twitter object when there is a
    general error interacting with the API.
    """
    pass

class TwitterHTTPError(TwitterError):
    """
    Exception thrown by the Twitter object when there is an
    HTTP error interacting with twitter.com.
    """
    def __init__(self, e, uri, format, uriparts):
        self.e = e
        self.uri = uri
        self.format = format
        self.uriparts = uriparts
        if self.e.headers['Content-Encoding'] == 'gzip':
            buf = StringIO(self.e.fp.read())
            f = gzip.GzipFile(fileobj=buf)
            self.response_data = f.read()
        else:
            self.response_data = self.e.fp.read()

    def __str__(self):
        fmt = ("." + self.format) if self.format else ""
        return (
            "Twitter sent status %i for URL: %s%s using parameters: "
            "(%s)\ndetails: %s" %(
                self.e.code, self.uri, fmt, self.uriparts,
                self.response_data))

class TwitterResponse(object):
    """
    Response from a twitter request. Behaves like a list or a string
    (depending on requested format) but it has a few other interesting
    attributes.

    `headers` gives you access to the response headers as an
    httplib.HTTPHeaders instance. You can do
    `response.headers.get('h')` to retrieve a header.
    """
    def __init__(self, headers):
        self.headers = headers

    @property
    def rate_limit_remaining(self):
        """
        Remaining requests in the current rate-limit.
        """
        return int(self.headers.get('X-RateLimit-Remaining', "0"))

    @property
    def rate_limit_reset(self):
        """
        Time in UTC epoch seconds when the rate limit will reset.
        """
        return int(self.headers.get('X-RateLimit-Reset', "0"))

def wrap_response(response, headers):
    response_typ = type(response)
    if response_typ is bool:
        # HURF DURF MY NAME IS PYTHON AND I CAN'T SUBCLASS bool.
        response_typ = int

    class WrappedTwitterResponse(response_typ, TwitterResponse):
        __doc__ = TwitterResponse.__doc__

        def __init__(self, response, headers):
            response_typ.__init__(self, response)
            TwitterResponse.__init__(self, headers)
        def __new__(cls, response, headers):
            return response_typ.__new__(cls, response)

    return WrappedTwitterResponse(response, headers)

class TwitterCall(object):

    def __init__(
        self, auth, format, domain, callable_cls, uri="",
        uriparts=None, secure=True):
        self.auth = auth
        self.format = format
        self.domain = domain
        self.callable_cls = callable_cls
        self.uri = uri
        self.uriparts = uriparts
        self.secure = secure

    def __getattr__(self, k):
        try:
            return object.__getattr__(self, k)
        except AttributeError:
            def extend_call(arg):
                return self.callable_cls(
                    auth=self.auth, format=self.format, domain=self.domain,
                    callable_cls=self.callable_cls, uriparts=self.uriparts \
                        + (arg,),
                    secure=self.secure)
            if k == "_":
                return extend_call
            else:
                return extend_call(k)

    def __call__(self, **kwargs):
        # Build the uri.
        uriparts = []
        for uripart in self.uriparts:
            # If this part matches a keyword argument, use the
            # supplied value otherwise, just use the part.
            uriparts.append(str(kwargs.pop(uripart, uripart)))
        uri = '/'.join(uriparts)

        method = kwargs.pop('_method', None)
        if not method:
            method = "GET"
            for action in POST_ACTIONS:
                if re.search("%s(/\d+)?$" % action, uri):
                    method = "POST"
                    break

        # If an id kwarg is present and there is no id to fill in in
        # the list of uriparts, assume the id goes at the end.
        id = kwargs.pop('id', None)
        if id:
            uri += "/%s" %(id)

        # If an _id kwarg is present, this is treated as id as a CGI
        # param.
        _id = kwargs.pop('_id', None)
        if _id:
            kwargs['id'] = _id

        secure_str = ''
        if self.secure:
            secure_str = 's'
        dot = ""
        if self.format:
            dot = "."
        uriBase = "http%s://%s/%s%s%s" %(
                    secure_str, self.domain, uri, dot, self.format)

        headers = {'Accept-Encoding': 'gzip'}
        if self.auth:
            headers.update(self.auth.generate_headers())
            arg_data = self.auth.encode_params(uriBase, method, kwargs)
            if method == 'GET':
                uriBase += '?' + arg_data
                body = None
            else:
                body = arg_data.encode('utf8')

        req = urllib_request.Request(uriBase, body, headers)
        return self._handle_response(req, uri, arg_data)

    def _handle_response(self, req, uri, arg_data):
        try:
            handle = urllib_request.urlopen(req)
            if handle.headers['Content-Type'] in ['image/jpeg', 'image/png']:
                return handle
            elif handle.info().get('Content-Encoding') == 'gzip':
                # Handle gzip decompression
                buf = StringIO(handle.read())
                f = gzip.GzipFile(fileobj=buf)
                data = f.read()
            else:
                data = handle.read()

            if "json" == self.format:
                res = json.loads(data.decode('utf8'))
                return wrap_response(res, handle.headers)
            else:
                return wrap_response(
                    data.decode('utf8'), handle.headers)
        except urllib_error.HTTPError as e:
            if (e.code == 304):
                return []
            else:
                raise TwitterHTTPError(e, uri, self.format, arg_data)

class Twitter(TwitterCall):
    """
    The minimalist yet fully featured Twitter API class.

    Get RESTful data by accessing members of this class. The result
    is decoded python objects (lists and dicts).

    The Twitter API is documented at:

      http://dev.twitter.com/doc

    Examples::

        t = Twitter(
            auth=OAuth(token, token_key, con_secret, con_secret_key)))

        # Get the public timeline
        t.statuses.public_timeline()

        # Get a particular friend's timeline
        t.statuses.friends_timeline(id="billybob")

        # Also supported (but totally weird)
        t.statuses.friends_timeline.billybob()

        # Update your status
        t.statuses.update(
            status="Using @sixohsix's sweet Python Twitter Tools.")

        # Send a direct message
        t.direct_messages.new(
            user="billybob",
            text="I think yer swell!")

        # Get the members of tamtar's list "Things That Are Rad"
        t._("tamtar")._("things-that-are-rad").members()

        # Note how the magic `_` method can be used to insert data
        # into the middle of a call. You can also use replacement:
        t.user.list.members(user="tamtar", list="things-that-are-rad")

    Searching Twitter::

        twitter_search = Twitter(domain="search.twitter.com")

        # Find the latest search trends
        twitter_search.trends()

        # Search for the latest News on #gaza
        twitter_search.search(q="#gaza")

    Using the data returned
    -----------------------

    Twitter API calls return decoded JSON. This is converted into
    a bunch of Python lists, dicts, ints, and strings. For example::

        x = twitter.statuses.public_timeline()

        # The first 'tweet' in the timeline
        x[0]

        # The screen name of the user who wrote the first 'tweet'
        x[0]['user']['screen_name']

    Getting raw XML data
    --------------------

    If you prefer to get your Twitter data in XML format, pass
    format="xml" to the Twitter object when you instantiate it::

        twitter = Twitter(format="xml")

    The output will not be parsed in any way. It will be a raw string
    of XML.

    """
    def __init__(
        self, format="json",
        domain="api.twitter.com", secure=True, auth=None,
        api_version=_DEFAULT):
        """
        Create a new twitter API connector.

        Pass an `auth` parameter to use the credentials of a specific
        user. Generally you'll want to pass an `OAuth`
        instance::

            twitter = Twitter(auth=OAuth(
                    token, token_secret, consumer_key, consumer_secret))

        `domain` lets you change the domain you are connecting. By
        default it's `api.twitter.com` but `search.twitter.com` may be
        useful too.

        If `secure` is False you will connect with HTTP instead of
        HTTPS.

        `api_version` is used to set the base uri. By default it's
        '1'. If you are using "search.twitter.com" set this to None.
        """
        if not auth:
            auth = NoAuth()

        if (format not in ("json", "xml", "")):
            raise ValueError("Unknown data format '%s'" %(format))

        if api_version is _DEFAULT:
            if domain == 'api.twitter.com':
                api_version = '1.1'
            else:
                api_version = None

        uriparts = ()
        if api_version:
            uriparts += (str(api_version),)

        TwitterCall.__init__(
            self, auth=auth, format=format, domain=domain,
            callable_cls=TwitterCall,
            secure=secure, uriparts=uriparts)

__all__ = ["Twitter", "TwitterError", "TwitterHTTPError", "TwitterResponse"]
<commit_msg>Fixes 91. Also added the property rate_limit_limit to get rate limit ceiling for current request.<commit_after>try:
    import urllib.request as urllib_request
    import urllib.error as urllib_error
except ImportError:
    import urllib2 as urllib_request
    import urllib2 as urllib_error

try:
    from cStringIO import StringIO
except ImportError:
    from io import BytesIO as StringIO

from twitter.twitter_globals import POST_ACTIONS
from twitter.auth import NoAuth

import re
import gzip

try:
    import json
except ImportError:
    import simplejson as json

class _DEFAULT(object):
    pass

class TwitterError(Exception):
    """
    Base Exception thrown by the Twitter object when there is a
    general error interacting with the API.
    """
    pass

class TwitterHTTPError(TwitterError):
    """
    Exception thrown by the Twitter object when there is an
    HTTP error interacting with twitter.com.
    """
    def __init__(self, e, uri, format, uriparts):
        self.e = e
        self.uri = uri
        self.format = format
        self.uriparts = uriparts
        if self.e.headers['Content-Encoding'] == 'gzip':
            buf = StringIO(self.e.fp.read())
            f = gzip.GzipFile(fileobj=buf)
            self.response_data = f.read()
        else:
            self.response_data = self.e.fp.read()

    def __str__(self):
        fmt = ("." + self.format) if self.format else ""
        return (
            "Twitter sent status %i for URL: %s%s using parameters: "
            "(%s)\ndetails: %s" %(
                self.e.code, self.uri, fmt, self.uriparts,
                self.response_data))

class TwitterResponse(object):
    """
    Response from a twitter request. Behaves like a list or a string
    (depending on requested format) but it has a few other interesting
    attributes.

    `headers` gives you access to the response headers as an
    httplib.HTTPHeaders instance. You can do
    `response.headers.get('h')` to retrieve a header.
    """
    def __init__(self, headers):
        self.headers = headers

    @property
    def rate_limit_remaining(self):
        """
        Remaining requests in the current rate-limit.
        """
        return int(self.headers.get('X-Rate-Limit-Remaining', "0"))

    @property
    def rate_limit_limit(self):
        """
        Remaining requests in the current rate-limit.
        """
        return int(self.headers.get('X-Rate-Limit-Limit', "0"))

    @property
    def rate_limit_reset(self):
        """
        Time in UTC epoch seconds when the rate limit will reset.
        """
        return int(self.headers.get('X-Rate-Limit-Reset', "0"))

def wrap_response(response, headers):
    response_typ = type(response)
    if response_typ is bool:
        # HURF DURF MY NAME IS PYTHON AND I CAN'T SUBCLASS bool.
        response_typ = int

    class WrappedTwitterResponse(response_typ, TwitterResponse):
        __doc__ = TwitterResponse.__doc__

        def __init__(self, response, headers):
            response_typ.__init__(self, response)
            TwitterResponse.__init__(self, headers)
        def __new__(cls, response, headers):
            return response_typ.__new__(cls, response)

    return WrappedTwitterResponse(response, headers)

class TwitterCall(object):

    def __init__(
        self, auth, format, domain, callable_cls, uri="",
        uriparts=None, secure=True):
        self.auth = auth
        self.format = format
        self.domain = domain
        self.callable_cls = callable_cls
        self.uri = uri
        self.uriparts = uriparts
        self.secure = secure

    def __getattr__(self, k):
        try:
            return object.__getattr__(self, k)
        except AttributeError:
            def extend_call(arg):
                return self.callable_cls(
                    auth=self.auth, format=self.format, domain=self.domain,
                    callable_cls=self.callable_cls, uriparts=self.uriparts \
                        + (arg,),
                    secure=self.secure)
            if k == "_":
                return extend_call
            else:
                return extend_call(k)

    def __call__(self, **kwargs):
        # Build the uri.
        uriparts = []
        for uripart in self.uriparts:
            # If this part matches a keyword argument, use the
            # supplied value otherwise, just use the part.
            uriparts.append(str(kwargs.pop(uripart, uripart)))
        uri = '/'.join(uriparts)

        method = kwargs.pop('_method', None)
        if not method:
            method = "GET"
            for action in POST_ACTIONS:
                if re.search("%s(/\d+)?$" % action, uri):
                    method = "POST"
                    break

        # If an id kwarg is present and there is no id to fill in in
        # the list of uriparts, assume the id goes at the end.
        id = kwargs.pop('id', None)
        if id:
            uri += "/%s" %(id)

        # If an _id kwarg is present, this is treated as id as a CGI
        # param.
        _id = kwargs.pop('_id', None)
        if _id:
            kwargs['id'] = _id

        secure_str = ''
        if self.secure:
            secure_str = 's'
        dot = ""
        if self.format:
            dot = "."
        uriBase = "http%s://%s/%s%s%s" %(
                    secure_str, self.domain, uri, dot, self.format)

        headers = {'Accept-Encoding': 'gzip'}
        if self.auth:
            headers.update(self.auth.generate_headers())
            arg_data = self.auth.encode_params(uriBase, method, kwargs)
            if method == 'GET':
                uriBase += '?' + arg_data
                body = None
            else:
                body = arg_data.encode('utf8')
        req = urllib_request.Request(uriBase, body, headers)
        return self._handle_response(req, uri, arg_data)

    def _handle_response(self, req, uri, arg_data):
        try:
            handle = urllib_request.urlopen(req)
            if handle.headers['Content-Type'] in ['image/jpeg', 'image/png']:
                return handle
            elif handle.info().get('Content-Encoding') == 'gzip':
                # Handle gzip decompression
                buf = StringIO(handle.read())
                f = gzip.GzipFile(fileobj=buf)
                data = f.read()
            else:
                data = handle.read()

            if "json" == self.format:
                res = json.loads(data.decode('utf8'))
                return wrap_response(res, handle.headers)
            else:
                return wrap_response(
                    data.decode('utf8'), handle.headers)
        except urllib_error.HTTPError as e:
            if (e.code == 304):
                return []
            else:
                raise TwitterHTTPError(e, uri, self.format, arg_data)

class Twitter(TwitterCall):
    """
    The minimalist yet fully featured Twitter API class.

    Get RESTful data by accessing members of this class. The result
    is decoded python objects (lists and dicts).

    The Twitter API is documented at:

      http://dev.twitter.com/doc

    Examples::

        t = Twitter(
            auth=OAuth(token, token_key, con_secret, con_secret_key)))

        # Get the public timeline
        t.statuses.public_timeline()

        # Get a particular friend's timeline
        t.statuses.friends_timeline(id="billybob")

        # Also supported (but totally weird)
        t.statuses.friends_timeline.billybob()

        # Update your status
        t.statuses.update(
            status="Using @sixohsix's sweet Python Twitter Tools.")

        # Send a direct message
        t.direct_messages.new(
            user="billybob",
            text="I think yer swell!")

        # Get the members of tamtar's list "Things That Are Rad"
        t._("tamtar")._("things-that-are-rad").members()

        # Note how the magic `_` method can be used to insert data
        # into the middle of a call. You can also use replacement:
        t.user.list.members(user="tamtar", list="things-that-are-rad")

    Searching Twitter::

        twitter_search = Twitter(domain="search.twitter.com")

        # Find the latest search trends
        twitter_search.trends()

        # Search for the latest News on #gaza
        twitter_search.search(q="#gaza")

    Using the data returned
    -----------------------

    Twitter API calls return decoded JSON. This is converted into
    a bunch of Python lists, dicts, ints, and strings. For example::

        x = twitter.statuses.public_timeline()

        # The first 'tweet' in the timeline
        x[0]

        # The screen name of the user who wrote the first 'tweet'
        x[0]['user']['screen_name']

    Getting raw XML data
    --------------------

    If you prefer to get your Twitter data in XML format, pass
    format="xml" to the Twitter object when you instantiate it::

        twitter = Twitter(format="xml")

    The output will not be parsed in any way. It will be a raw string
    of XML.

    """
    def __init__(
        self, format="json",
        domain="api.twitter.com", secure=True, auth=None,
        api_version=_DEFAULT):
        """
        Create a new twitter API connector.

        Pass an `auth` parameter to use the credentials of a specific
        user. Generally you'll want to pass an `OAuth`
        instance::

            twitter = Twitter(auth=OAuth(
                    token, token_secret, consumer_key, consumer_secret))

        `domain` lets you change the domain you are connecting. By
        default it's `api.twitter.com` but `search.twitter.com` may be
        useful too.

        If `secure` is False you will connect with HTTP instead of
        HTTPS.

        `api_version` is used to set the base uri. By default it's
        '1'. If you are using "search.twitter.com" set this to None.
        """
        if not auth:
            auth = NoAuth()

        if (format not in ("json", "xml", "")):
            raise ValueError("Unknown data format '%s'" %(format))

        if api_version is _DEFAULT:
            if domain == 'api.twitter.com':
                api_version = '1.1'
            else:
                api_version = None

        uriparts = ()
        if api_version:
            uriparts += (str(api_version),)

        TwitterCall.__init__(
            self, auth=auth, format=format, domain=domain,
            callable_cls=TwitterCall,
            secure=secure, uriparts=uriparts)

__all__ = ["Twitter", "TwitterError", "TwitterHTTPError", "TwitterResponse"]
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys
import os
import json
import datetime

import pymongo

import tornado.options

# Set up all the options. (Generally found in the config file.)

# Clever hack to parse a config file off the command line.
tornado.options.define(
    'config', type=str,
    help='configuration file',
    callback=lambda path: tornado.options.parse_config_file(path, final=False))

tornado.options.define(
    'python_path', type=str,
    help='Python modules directory (optional)')

tornado.options.define(
    'mongo_database', type=str, default='tworld',
    help='name of mongodb database')

tornado.options.define(
    'removeworld', type=bool,
    help='remove world completely')

tornado.options.define(
    'remove', type=bool,
    help='only remove the named room or room.prop')

tornado.options.define(
    'display', type=bool,
    help='only display the named room or room.prop')

# Parse 'em up.
args = tornado.options.parse_command_line()
opts = tornado.options.options

# But tornado options don't support options after arguments. Hack to work
# around this.
if '--display' in args:
    args.remove('--display')
    opts.display = True
if '--remove' in args:
    args.remove('--remove')
    opts.remove = True
if '--removeworld' in args:
    args.remove('--removeworld')
    opts.removeworld = True

if opts.python_path:
    sys.path.insert(0, opts.python_path)

from two.interp import sluggify

if not args:
    print('usage: twloadworld.py worldfile [ room ... or room.prop ... ]')
    sys.exit(-1)

class World(object):
    def __init__(self):
        self.creator = 'Admin'
        self.wid = None
        self.name = None
        self.copyable = True
        self.instancing = 'standard'
        self.props = {}
        self.proplist = []
        self.locations = {}
        self.locationlist = []

class Location(object):
    def __init__(self, name, key=None):
        self.name = name
        if key is None:
            self.key = sluggify(name)
        else:
            self.key = key
        self.props = {}
        self.proplist = []
    def __repr__(self):
        return '<Location %s: "%s">' % (self.key, self.name)

def parse_world(filename):
    world = World()
    curloc = None
    curprop = None

    fl = open(filename)
    while True:
        ln = fl.readline()
        if not ln:
            break
        ln = ln.rstrip()
        isindent = False
        val = ln.lstrip()
        if len(val) < len(ln):
            ln = val
            isindent = True

        if not ln or ln.startswith('#'):
            continue

        if ln.startswith('*'):
            # New location.
            curprop = None
            lockey, dummy, locname = ln[1:].partition(':')
            lockey = lockey.strip()
            locname = locname.strip()
            if not locname:
                locname = lockey
                lockey = sluggify(locname)
            if lockey in world.locations:
                error('Location defined twice: %s' % (lockey,))
            curloc = Location(locname, lockey)
            world.locations[lockey] = curloc
            world.locationlist.append(lockey)
            curprop = 'desc'
            continue

        if isindent and curprop is not None:
            if not curloc:
                append_to_prop(world.props, curprop, ln)
            else:
                append_to_prop(curloc.props, curprop, ln)
            continue

        key, dummy, val = ln.partition(':')
        if not dummy:
            error('Line does not define a property: %s' % (ln[:36],))
            continue

        key = key.strip()
        val = val.strip()

        if not curloc and key.startswith('$'):
            curprop = None
            if key == '$wid':
                world.wid = val
            elif key == '$name':
                world.name = val
            elif key == '$creator':
                world.creator = val
            elif key == '$instancing':
                world.instancing = val
                if val not in ('shared', 'solo', 'standard'):
                    error('$instancing value must be shared, solo, or standard')
            else:
                error('Unknown key: %s' % (key,))
            continue

        if not key.isidentifier():
            error('Property key is not valid: %s' % (key,))

        propval = parse_prop(val)

        if not curloc:
            if key in world.props:
                error('World key defined twice: %s' % (key,))
            world.props[key] = propval
            world.proplist.append(key)
            curprop = key
        else:
            if key in curloc.props:
                error('Location key defined twice in %s: %s' % (curloc.key, key,))
            curloc.props[key] = propval
            curloc.proplist.append(key)
            curprop = key

    fl.close()
    return world

def parse_prop(prop):
    if prop.startswith('*'):
        key, dummy, val = prop[1:].partition(' ')
        if not val:
            error('%s must be followed by a value' % (key,))
            return None
        val = sluggify(val.strip())
        if key == 'move':
            return {'type':'move', 'loc':val}
        elif key == 'focus':
            return {'type':'focus', 'key':val}
        elif key == 'event':
            return {'type':'event', 'text':val}
        elif key == 'text':
            return {'type':'text', 'text':val}
        elif key == 'code':
            return {'type':'code', 'text':val}
        else:
            error('Unknown special property type: *%s' % (key,))
            return None

    try:
        return json.loads(prop)
    except:
        pass

    return {'type':'text', 'text':prop}

def append_to_prop(dic, key, ln):
    val = dic.get(key, None)
    if not val:
        val = {'type':'text', 'text':ln}
        dic[key] = val
    elif type(val) is str:
        val += ('\n\n' + ln)
        dic[key] = val
    elif type(val) is dict and 'text' in val:
        val['text'] += ('\n\n' + ln)
    else:
        error('Cannot append to property %s' % (key,))

def prop_to_string(val):
    if type(val) is not dict:
        return json.dumps(val)
    key = val.get('type', None)
    if key == 'move':
        return '*move %s' % (val['loc'],)
    if key == 'focus':
        return '*focus %s' % (val['key'],)
    if key == 'event':
        return '*event %s' % (val['text'],)
    if key == 'text':
        val = val['text']
        if '\n\n' in val:
            return val.replace('\n\n', '\n\t')
        return val
    if key == 'code':
        return '*code %s' % (val['text'],)
    return json.dumps(val)

errorcount = 0

def error(msg):
    global errorcount
    errorcount = errorcount + 1
    print('Error: %s' % (msg,))

filename = args.pop(0)

world = parse_world(filename)

if errorcount:
    print('%d errors; stopping here.' % (errorcount,))
    sys.exit(1)

if opts.display:
    if not args:
        args = ['.'] + world.locationlist
    for val in args:
        if '.' in val:
            lockey, dummy, key = val.partition('.')
        else:
            lockey, key = (val, None)
        if not key:
            key = None

        if not lockey:
            print('* (world properties)')
            print()
            if key is None:
                for key in world.proplist:
                    print('%s: %s' % (key, prop_to_string(world.props[key])))
                    print()
            else:
                if key not in world.props:
                    error('Property not found in %s: %s' % ('*', key))
                    continue
                print('%s: %s' % (key, prop_to_string(world.props[key])))
                print()
            continue

        loc = world.locations.get(lockey, None)
        if loc is None:
            error('Location not found: %s' % (lockey,))
            continue

        print('* %s: %s' % (loc.key, loc.name))
        print()
        if key is None:
            for key in loc.proplist:
                print('%s: %s' % (key, prop_to_string(loc.props[key])))
                print()
        else:
            if key not in loc.props:
                error('Property not found in %s: %s' % (loc.key, key))
                continue
            print('%s: %s' % (key, prop_to_string(loc.props[key])))
            print()

#client = pymongo.MongoClient()
#db = client[opts.mongo_database]

<commit_msg>The loader seems basically good.<commit_after>#!/usr/bin/env python3

import sys
import os
import json
import datetime

from bson.objectid import ObjectId
import pymongo

import tornado.options

# Set up all the options. (Generally found in the config file.)

# Clever hack to parse a config file off the command line.
tornado.options.define(
    'config', type=str,
    help='configuration file',
    callback=lambda path: tornado.options.parse_config_file(path, final=False))

tornado.options.define(
    'python_path', type=str,
    help='Python modules directory (optional)')

tornado.options.define(
    'mongo_database', type=str, default='tworld',
    help='name of mongodb database')

tornado.options.define(
    'removeworld', type=bool,
    help='remove world completely')

tornado.options.define(
    'remove', type=bool,
    help='only remove the named room or room.prop')

tornado.options.define(
    'display', type=bool,
    help='only display the named room or room.prop')

# Parse 'em up.
args = tornado.options.parse_command_line()
opts = tornado.options.options

# But tornado options don't support options after arguments. Hack to work
# around this.
if '--display' in args:
    args.remove('--display')
    opts.display = True
if '--remove' in args:
    args.remove('--remove')
    opts.remove = True
if '--removeworld' in args:
    args.remove('--removeworld')
    opts.removeworld = True

if opts.python_path:
    sys.path.insert(0, opts.python_path)

from two.interp import sluggify

if not args:
    print('usage: twloadworld.py worldfile [ room ... or room.prop ... ]')
    sys.exit(-1)

class World(object):
    def __init__(self):
        self.creator = 'Admin'
        self.wid = None
        self.name = None
        self.copyable = True
        self.instancing = 'standard'
        self.props = {}
        self.proplist = []
        self.locations = {}
        self.locationlist = []

class Location(object):
    def __init__(self, name, key=None):
        self.name = name
        if key is None:
            self.key = sluggify(name)
        else:
            self.key = key
        self.locid = None
        self.props = {}
        self.proplist = []
    def __repr__(self):
        return '<Location %s: "%s">' % (self.key, self.name)

def parse_world(filename):
    world = World()
    curloc = None
    curprop = None

    fl = open(filename)
    while True:
        ln = fl.readline()
        if not ln:
            break
        ln = ln.rstrip()
        isindent = False
        val = ln.lstrip()
        if len(val) < len(ln):
            ln = val
            isindent = True

        if not ln or ln.startswith('#'):
            continue

        if ln.startswith('*'):
            # New location.
            curprop = None
            lockey, dummy, locname = ln[1:].partition(':')
            lockey = lockey.strip()
            locname = locname.strip()
            if not locname:
                locname = lockey
                lockey = sluggify(locname)
            if lockey in world.locations:
                error('Location defined twice: %s' % (lockey,))
            curloc = Location(locname, lockey)
            world.locations[lockey] = curloc
            world.locationlist.append(lockey)
            curprop = 'desc'
            continue

        if isindent and curprop is not None:
            if not curloc:
                append_to_prop(world.props, curprop, ln)
            else:
                append_to_prop(curloc.props, curprop, ln)
            continue

        key, dummy, val = ln.partition(':')
        if not dummy:
            error('Line does not define a property: %s' % (ln[:36],))
            continue

        key = key.strip()
        val = val.strip()

        if not curloc and key.startswith('$'):
            curprop = None
            if key == '$wid':
                world.wid = val
            elif key == '$name':
                world.name = val
            elif key == '$creator':
                world.creator = val
            elif key == '$instancing':
                world.instancing = val
                if val not in ('shared', 'solo', 'standard'):
                    error('$instancing value must be shared, solo, or standard')
            else:
                error('Unknown key: %s' % (key,))
            continue

        if not key.isidentifier():
            error('Property key is not valid: %s' % (key,))

        propval = parse_prop(val)

        if not curloc:
            if key in world.props:
                error('World key defined twice: %s' % (key,))
            world.props[key] = propval
            world.proplist.append(key)
            curprop = key
        else:
            if key in curloc.props:
                error('Location key defined twice in %s: %s' % (curloc.key, key,))
            curloc.props[key] = propval
            curloc.proplist.append(key)
            curprop = key

    fl.close()
    return world

def parse_prop(prop):
    if prop.startswith('*'):
        key, dummy, val = prop[1:].partition(' ')
        if not val:
            error('%s must be followed by a value' % (key,))
            return None
        val = sluggify(val.strip())
        if key == 'move':
            return {'type':'move', 'loc':val}
        elif key == 'focus':
            return {'type':'focus', 'key':val}
        elif key == 'event':
            return {'type':'event', 'text':val}
        elif key == 'text':
            return {'type':'text', 'text':val}
        elif key == 'code':
            return {'type':'code', 'text':val}
        else:
            error('Unknown special property type: *%s' % (key,))
            return None

    try:
        return json.loads(prop)
    except:
        pass

    return {'type':'text', 'text':prop}

def append_to_prop(dic, key, ln):
    val = dic.get(key, None)
    if not val:
        val = {'type':'text', 'text':ln}
        dic[key] = val
    elif type(val) is str:
        val += ('\n\n' + ln)
        dic[key] = val
    elif type(val) is dict and 'text' in val:
        val['text'] += ('\n\n' + ln)
    else:
        error('Cannot append to property %s' % (key,))

def prop_to_string(val):
    if type(val) is not dict:
        return json.dumps(val)
    key = val.get('type', None)
    if key == 'move':
        return '*move %s' % (val['loc'],)
    if key == 'focus':
        return '*focus %s' % (val['key'],)
    if key == 'event':
        return '*event %s' % (val['text'],)
    if key == 'text':
        val = val['text']
        if '\n\n' in val:
            return val.replace('\n\n', '\n\t')
        return val
    if key == 'code':
        return '*code %s' % (val['text'],)
    return json.dumps(val)

errorcount = 0

def error(msg):
    global errorcount
    errorcount = errorcount + 1
    print('Error: %s' % (msg,))

filename = args.pop(0)

world = parse_world(filename)

if errorcount:
    print('%d errors; stopping here.' % (errorcount,))
    sys.exit(1)

if opts.display:
    if not args:
        args = ['.'] + world.locationlist
    for val in args:
        if '.' in val:
            lockey, dummy, key = val.partition('.')
        else:
            lockey, key = (val, None)
        if not key:
            key = None

        if not lockey:
            print('* (world properties)')
            print()
            if key is None:
                for key in world.proplist:
                    print('%s: %s' % (key, prop_to_string(world.props[key])))
                    print()
            else:
                if key not in world.props:
                    error('Property not found in %s: %s' % ('*', key))
                    continue
                print('%s: %s' % (key, prop_to_string(world.props[key])))
                print()
            continue

        loc = world.locations.get(lockey, None)
        if loc is None:
            error('Location not found: %s' % (lockey,))
            continue

        print('* %s: %s' % (loc.key, loc.name))
        print()
        if key is None:
            for key in loc.proplist:
                print('%s: %s' % (key, prop_to_string(loc.props[key])))
                print()
        else:
            if key not in loc.props:
                error('Property not found in %s: %s' % (loc.key, key))
                continue
            print('%s: %s' % (key, prop_to_string(loc.props[key])))
            print()

    sys.exit(0)

client = pymongo.MongoClient()
db = client[opts.mongo_database]

dbcreator = db.players.find_one({'name':world.creator})
if not dbcreator:
    error('Creator %s not found in database.' % (world.creator,))

world.creatoruid = dbcreator['_id']

if not world.wid:
    # Look for a world with this name. If not found, create it. If found,
    # use it (if the creator matches)
    dbworld = db.worlds.find_one({'name':world.name})
    if dbworld and dbworld.get('creator') != world.creatoruid:
        error('Found world "%s", but it was not created by %s.' % (world.name, world.creator))
        sys.exit(1)
else:
    # If the world with this wid does not exist, we'll have to create it.
    dbworld = db.worlds.find_one({'_id':ObjectId(world.wid)})

if dbworld:
    wid = dbworld['_id']
    print('Found world "%s" (%s)' % (dbworld['name'], wid))
else:
    dbworld = {
        'creator': world.creatoruid,
        'name': world.name,
        'copyable': True,
        'instancing': world.instancing,
        }
    wid = db.worlds.insert(dbworld)
    dbworld = db.worlds.find_one({'_id':wid})
    if not dbworld:
        error('Unable to create world!')
        sys.exit(1)
    print('Created world "%s" (%s)' % (dbworld['name'], wid))

if opts.remove:
    if not args:
        error('Use --removeworld to remove the entire world.')
        sys.exit(1)
    for val in args:
        if '.' in val:
            lockey, dummy, key = val.partition('.')
        else:
            lockey, key = (val, None)
        if not key:
            key = None

        if not lockey:
            if key is None:
                db.worldprop.remove({'wid':wid, 'locid':None})
                print('removing all world properties')
            else:
                db.worldprop.remove({'wid':wid, 'locid':None, 'key':key})
                print('removing world property: %s' % (key,))
            continue

        loc = world.locations.get(lockey, None)
        if loc is None:
            error('Location not found: %s' % (lockey,))
            continue

        if not loc.locid:
            dbloc = db.locations.find_one({'wid':wid, 'key':lockey})
            if dbloc:
                loc.locid = dbloc['_id']
            else:
                error('Location does not exist in database: %s' % (lockey,))
                continue

        if key is None:
            db.worldprop.remove({'wid':wid, 'locid':loc.locid})
            print('removing all properties in %s' % (lockey,))
        else:
            db.worldprop.remove({'wid':wid, 'locid':loc.locid, 'key':key})
            print('removing property in %s: %s' % (lockey, key,))

    sys.exit(0)

# The adding-stuff-to-the-database case.
if not args:
    args = ['.'] + world.locationlist
for val in args:
    if '.' in val:
        lockey, dummy, key = val.partition('.')
    else:
        lockey, key = (val, None)
    if not key:
        key = None

    if not lockey:
        # World properties
        if key is None:
            # All world properties
            for key in world.props:
                val = world.props[key]
                print('Writing world property: %s' % (key,))
                db.worldprop.update({'wid':wid, 'locid':None, 'key':key},
                                    {'wid':wid, 'locid':None, 'key':key, 'val':val},
                                    upsert=True)
        else:
            if key not in world.props:
                error('Property not found in %s: %s' % ('*', key))
                continue
            val = world.props[key]
            print('Writing world property: %s' % (key,))
            db.worldprop.update({'wid':wid, 'locid':None, 'key':key},
                                {'wid':wid, 'locid':None, 'key':key, 'val':val},
                                upsert=True)
        continue

    loc = world.locations.get(lockey, None)
    if loc is None:
        error('Location not found: %s' % (lockey,))
        continue

    if not loc.locid:
        dbloc = db.locations.find_one({'wid':wid, 'key':lockey})
        if dbloc:
            loc.locid = dbloc['_id']
        else:
            print('Creating location: %s' % (loc.key,))
            dbloc = {
                'wid': wid,
                'key': loc.key,
                'name': loc.name,
                }
            loc.locid = db.locations.insert(dbloc)

    if key is None:
        for key in loc.props:
            val = loc.props[key]
            print('Writing property in %s: %s' % (loc.key, key,))
            db.worldprop.update({'wid':wid, 'locid':loc.locid, 'key':key},
                                {'wid':wid, 'locid':loc.locid, 'key':key, 'val':val},
                                upsert=True)
    else:
        if key not in loc.props:
            error('Property not found in %s: %s' % (loc.key, key))
            continue
        val = loc.props[key]
        print('Writing property in %s: %s' % (loc.key, key,))
        db.worldprop.update({'wid':wid, 'locid':loc.locid, 'key':key},
                            {'wid':wid, 'locid':loc.locid, 'key':key, 'val':val},
                            upsert=True)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
twython.api
~~~~~~~~~~~

This module contains functionality for access to core Twitter API calls,
Twitter Authentication, and miscellaneous methods that are useful when
dealing with the Twitter API
"""

import sys

import requests
from requests.auth import HTTPBasicAuth
from requests_oauthlib import OAuth1, OAuth2

from . import __version__
from .compat import json, urlencode, parse_qsl, quote_plus, str, is_py2
from .endpoints import EndpointsMixin
from .exceptions import TwythonError, TwythonAuthError, TwythonRateLimitError
from .helpers import _transparent_params

class Twython(EndpointsMixin, object):
    def __init__(self, app_key=None, app_secret=None, oauth_token=None,
                 oauth_token_secret=None, access_token=None, token_type='bearer',
                 oauth_version=1, api_version='1.1', client_args=None):
        """Instantiates an instance of Twython. Takes optional parameters for authentication and such (see below).

        :param app_key: (optional) Your applications key
        :param app_secret: (optional) Your applications secret key
        :param oauth_token: (optional) When using **OAuth 1**, combined with oauth_token_secret to make authenticated calls
        :param oauth_token_secret: (optional) When using **OAuth 1** combined with oauth_token to make authenticated calls
        :param access_token: (optional) When using **OAuth 2**, provide a valid access token if you have one
        :param token_type: (optional) When using **OAuth 2**, provide your token type. Default: bearer
        :param oauth_version: (optional) Choose which OAuth version to use. Default: 1
        :param api_version: (optional) Choose which Twitter API version to use. Default: 1.1

        :param client_args: (optional) Accepts some requests Session parameters and some requests Request parameters.
                            See http://docs.python-requests.org/en/latest/api/#sessionapi and requests section below it for details.
                            [ex. headers, proxies, verify(SSL verification)]

        """

        # API urls, OAuth urls and API version; needed for hitting that there API.
        self.api_version = api_version
        self.api_url = 'https://api.twitter.com/%s'

        self.app_key = app_key
        self.app_secret = app_secret
        self.oauth_token = oauth_token
        self.oauth_token_secret = oauth_token_secret
        self.access_token = access_token

        # OAuth 1
        self.request_token_url = self.api_url % 'oauth/request_token'
        self.access_token_url = self.api_url % 'oauth/access_token'
        self.authenticate_url = self.api_url % 'oauth/authenticate'

        if self.access_token:  # If they pass an access token, force OAuth 2
            oauth_version = 2

        self.oauth_version = oauth_version

        # OAuth 2
        if oauth_version == 2:
            self.request_token_url = self.api_url % 'oauth2/token'

        self.client_args = client_args or {}
        default_headers = {'User-Agent': 'Twython v' + __version__}
        if not 'headers' in self.client_args:
            # If they didn't set any headers, set our defaults for them
            self.client_args['headers'] = default_headers
        elif 'User-Agent' not in self.client_args['headers']:
            # If they set headers, but didn't include User-Agent.. set it for them
            self.client_args['headers'].update(default_headers)

        # Generate OAuth authentication object for the request
        # If no keys/tokens are passed to __init__, auth=None allows for
        # unauthenticated requests, although I think all v1.1 requests need auth
        auth = None
        if oauth_version == 1:
            # User Authentication is through OAuth 1
            if self.app_key is not None and self.app_secret is not None and \
               self.oauth_token is None and self.oauth_token_secret is None:
                auth = OAuth1(self.app_key, self.app_secret)

            if self.app_key is not None and self.app_secret is not None and \
               self.oauth_token is not None and self.oauth_token_secret is not None:
                auth = OAuth1(self.app_key, self.app_secret,
                              self.oauth_token, self.oauth_token_secret)
        elif oauth_version == 2 and self.access_token:
            # Application Authentication is through OAuth 2
            token = {'token_type': token_type, 'access_token': self.access_token}
            auth = OAuth2(self.app_key, token=token)

        self.client = requests.Session()
        self.client.auth = auth

        # Make a copy of the client args and iterate over them
        # Pop out all the acceptable args at this point because they will
        # Never be used again.
        client_args_copy = self.client_args.copy()
        for k, v in client_args_copy.items():
            if k in ('cert', 'headers', 'hooks', 'max_redirects', 'proxies'):
                setattr(self.client, k, v)
                self.client_args.pop(k)  # Pop, pop!

        self._last_call = None

    def __repr__(self):
        return '<Twython: %s>' % (self.app_key)

    def _request(self, url, method='GET', params=None, api_call=None):
        """Internal request method"""
        method = method.lower()
        params = params or {}

        func = getattr(self.client, method)
        params, files = _transparent_params(params)

        requests_args = {}
        for k, v in self.client_args.items():
            # Maybe this should be set as a class variable and only done once?
            if k in ('timeout', 'allow_redirects', 'stream', 'verify'):
                requests_args[k] = v

        if method == 'get':
            requests_args['params'] = params
        else:
            requests_args.update({
                'data': params,
                'files': files,
            })
        try:
            response = func(url, **requests_args)
        except requests.RequestException as e:
            raise TwythonError, str(e), sys.exc_info()[2]
        content = response.content.decode('utf-8')

        # create stash for last function intel
        self._last_call = {
            'api_call': api_call,
            'api_error': None,
            'cookies': response.cookies,
            'headers': response.headers,
            'status_code': response.status_code,
            'url': response.url,
            'content': content,
        }

        #  Wrap the json loads in a try, and defer an error
        #  Twitter will return invalid json with an error code in the headers
        json_error = False
        try:
            try:
                # try to get json
                content = content.json()
            except AttributeError:
                # if unicode detected
                content = json.loads(content)
        except ValueError:
            json_error = True
            content = {}

        if response.status_code > 304:
            # If there is no error message, use a default.
            errors = content.get('errors',
                                 [{'message': 'An error occurred processing your request.'}])
            if errors and isinstance(errors, list):
                error_message = errors[0]['message']
            else:
                error_message = errors
            self._last_call['api_error'] = error_message

            ExceptionType = TwythonError
            if response.status_code == 429:
                # Twitter API 1.1, always return 429 when rate limit is exceeded
                ExceptionType = TwythonRateLimitError
            elif response.status_code == 401 or 'Bad Authentication data' in error_message:
                # Twitter API 1.1, returns a 401 Unauthorized or
                # a 400 "Bad Authentication data" for invalid/expired app keys/user tokens
                ExceptionType = TwythonAuthError

            raise ExceptionType(error_message,
                                error_code=response.status_code,
                                retry_after=response.headers.get('retry-after'))

        # if we have a json error here, then it's not an official Twitter API error
        if json_error and not response.status_code in (200, 201, 202):
            raise TwythonError('Response was not valid JSON, unable to decode.')

        return content

    def request(self, endpoint, method='GET', params=None, version='1.1'):
        """Return dict of response received from Twitter's API

        :param endpoint: (required) Full url or Twitter API endpoint (e.g. search/tweets)
        :type endpoint: string
        :param method: (optional) Method of accessing data, either GET or POST. (default GET)
        :type method: string
        :param params: (optional) Dict of parameters (if any) accepted the by Twitter API endpoint you are trying to access (default None)
        :type params: dict or None
        :param version: (optional) Twitter API version to access (default 1.1)
        :type version: string

        :rtype: dict
        """

        # In case they want to pass a full Twitter URL
        # i.e. https://api.twitter.com/1.1/search/tweets.json
        if endpoint.startswith('http://') or endpoint.startswith('https://'):
            url = endpoint
        else:
            url = '%s/%s.json' % (self.api_url % version, endpoint)

        content = self._request(url, method=method, params=params, api_call=url)

        return content

    def get(self, endpoint, params=None, version='1.1'):
        """Shortcut for GET requests via :class:`request`"""
        return self.request(endpoint, params=params, version=version)

    def post(self, endpoint, params=None, version='1.1'):
        """Shortcut for POST requests via :class:`request`"""
        return self.request(endpoint, 'POST', params=params, version=version)

    def get_lastfunction_header(self, header, default_return_value=None):
        """Returns a specific header from the last API call
        This will return None if the header is not present

        :param header: (required) The name of the header you want to get the value of

        Most useful for the following header information:
            x-rate-limit-limit,
            x-rate-limit-remaining,
            x-rate-limit-class,
            x-rate-limit-reset

        """
        if self._last_call is None:
            raise TwythonError('This function must be called after an API call.  It delivers header information.')

        return self._last_call['headers'].get(header, default_return_value)

    def get_authentication_tokens(self, callback_url=None, force_login=False, screen_name=''):
        """Returns a dict including an authorization URL, ``auth_url``, to direct a user to

        :param callback_url: (optional) Url the user is returned to after they authorize your app (web clients only)
        :param force_login: (optional) Forces the user to enter their credentials to ensure the correct users account is authorized.
        :param app_secret: (optional) If forced_login is set OR user is not currently logged in, Prefills the username input box of the OAuth login screen with the given value
        :rtype: dict
        """
        if self.oauth_version != 1:
            raise TwythonError('This method can only be called when your OAuth version is 1.0.')

        request_args = {}
        if callback_url:
            request_args['oauth_callback'] = callback_url
        response = self.client.get(self.request_token_url, params=request_args)

        if response.status_code == 401:
            raise TwythonAuthError(response.content, error_code=response.status_code)
        elif response.status_code != 200:
            raise TwythonError(response.content, error_code=response.status_code)

        request_tokens = dict(parse_qsl(response.content.decode('utf-8')))
        if not request_tokens:
            raise TwythonError('Unable to decode request tokens.')

        oauth_callback_confirmed = request_tokens.get('oauth_callback_confirmed') == 'true'

        auth_url_params = {
            'oauth_token': request_tokens['oauth_token'],
        }

        if force_login:
            auth_url_params.update({
                'force_login': force_login,
                'screen_name': screen_name
            })

        # Use old-style callback argument if server didn't accept new-style
        if callback_url and not oauth_callback_confirmed:
            auth_url_params['oauth_callback'] = self.callback_url

        request_tokens['auth_url'] = self.authenticate_url + '?' + urlencode(auth_url_params)

        return request_tokens

    def get_authorized_tokens(self, oauth_verifier):
        """Returns a dict of authorized tokens after they go through the :class:`get_authentication_tokens` phase.

        :param oauth_verifier: (required) The oauth_verifier (or a.k.a PIN for non web apps) retrieved from the callback url querystring
        :rtype: dict

        """
        if self.oauth_version != 1:
            raise TwythonError('This method can only be called when your OAuth version is 1.0.')

        response = self.client.get(self.access_token_url, params={'oauth_verifier': oauth_verifier})
        authorized_tokens = dict(parse_qsl(response.content.decode('utf-8')))
        if not authorized_tokens:
            raise TwythonError('Unable to decode authorized tokens.')

        return authorized_tokens

    def obtain_access_token(self):
        """Returns an OAuth 2 access token to make OAuth 2 authenticated read-only calls.

        :rtype: string
        """
        if self.oauth_version != 2:
            raise TwythonError('This method can only be called when your OAuth version is 2.0.')

        data = {'grant_type': 'client_credentials'}
        basic_auth = HTTPBasicAuth(self.app_key, self.app_secret)
        try:
            response = self.client.post(self.request_token_url,
                                        data=data, auth=basic_auth)
            content = response.content.decode('utf-8')
            try:
                content = content.json()
            except AttributeError:
                content = json.loads(content)
                access_token = content['access_token']
        except (ValueError, requests.exceptions.RequestException):
            raise TwythonAuthError('Unable to obtain OAuth 2 access token.')
        else:
            return access_token

    @staticmethod
    def construct_api_url(api_url, **params):
        """Construct a Twitter API url, encoded, with parameters

        :param api_url: URL of the Twitter API endpoint you are attempting to construct
        :param \*\*params: Parameters that are accepted by Twitter for the endpoint you're requesting
        :rtype: string

        Usage::

          >>> from twython import Twython
          >>> twitter = Twython()

          >>> api_url = 'https://api.twitter.com/1.1/search/tweets.json'
          >>> constructed_url = twitter.construct_api_url(api_url, q='python', result_type='popular')
          >>> print constructed_url
          https://api.twitter.com/1.1/search/tweets.json?q=python&result_type=popular

        """
        querystring = []
        params, _ = _transparent_params(params or {})
        params = requests.utils.to_key_val_list(params)
        for (k, v) in params:
            querystring.append(
                '%s=%s' % (Twython.encode(k), quote_plus(Twython.encode(v)))
            )
        return '%s?%s' % (api_url, '&'.join(querystring))

    def search_gen(self, search_query, **params):
        """Returns a generator of tweets that match a specified query.

        Documentation: https://dev.twitter.com/docs/api/1.1/get/search/tweets

        :param search_query: Query you intend to search Twitter for
        :param \*\*params: Extra parameters to send with your search request
        :rtype: generator

        Usage::

          >>> from twython import Twython
          >>> twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)

          >>> search = twitter.search_gen('python')
          >>> for result in search:
          >>>   print result

        """
        content = self.search(q=search_query, **params)

        if not content.get('statuses'):
            raise StopIteration

        for tweet in content['statuses']:
            yield tweet

        try:
            if not 'since_id' in params:
                params['since_id'] = (int(content['statuses'][0]['id_str']) + 1)
        except (TypeError, ValueError):
            raise TwythonError('Unable to generate next page of search results, `page` is not a number.')

        for tweet in self.search_gen(search_query, **params):
            yield tweet

    @staticmethod
    def unicode2utf8(text):
        try:
            if is_py2 and isinstance(text, str):
                text = text.encode('utf-8')
        except:
            pass
        return text

    @staticmethod
    def encode(text):
        if is_py2 and isinstance(text, (str)):
            return Twython.unicode2utf8(text)
        return str(text)
<commit_msg>Fix Python 3 compat<commit_after># -*- coding: utf-8 -*-

"""
twython.api
~~~~~~~~~~~

This module contains functionality for access to core Twitter API calls,
Twitter Authentication, and miscellaneous methods that are useful when
dealing with the Twitter API
"""

import requests
from requests.auth import HTTPBasicAuth
from requests_oauthlib import OAuth1, OAuth2

from . import __version__
from .compat import json, urlencode, parse_qsl, quote_plus, str, is_py2
from .endpoints import EndpointsMixin
from .exceptions import TwythonError, TwythonAuthError, TwythonRateLimitError
from .helpers import _transparent_params

class Twython(EndpointsMixin, object):
    def __init__(self, app_key=None, app_secret=None, oauth_token=None,
                 oauth_token_secret=None, access_token=None, token_type='bearer',
                 oauth_version=1, api_version='1.1', client_args=None):
        """Instantiates an instance of Twython. Takes optional parameters for authentication and such (see below).

        :param app_key: (optional) Your applications key
        :param app_secret: (optional) Your applications secret key
        :param oauth_token: (optional) When using **OAuth 1**, combined with oauth_token_secret to make authenticated calls
        :param oauth_token_secret: (optional) When using **OAuth 1** combined with oauth_token to make authenticated calls
        :param access_token: (optional) When using **OAuth 2**, provide a valid access token if you have one
        :param token_type: (optional) When using **OAuth 2**, provide your token type. Default: bearer
        :param oauth_version: (optional) Choose which OAuth version to use. Default: 1
        :param api_version: (optional) Choose which Twitter API version to use. Default: 1.1

        :param client_args: (optional) Accepts some requests Session parameters and some requests Request parameters.
                            See http://docs.python-requests.org/en/latest/api/#sessionapi and requests section below it for details.
                            [ex. headers, proxies, verify(SSL verification)]

        """

        # API urls, OAuth urls and API version; needed for hitting that there API.
        self.api_version = api_version
        self.api_url = 'https://api.twitter.com/%s'

        self.app_key = app_key
        self.app_secret = app_secret
        self.oauth_token = oauth_token
        self.oauth_token_secret = oauth_token_secret
        self.access_token = access_token

        # OAuth 1
        self.request_token_url = self.api_url % 'oauth/request_token'
        self.access_token_url = self.api_url % 'oauth/access_token'
        self.authenticate_url = self.api_url % 'oauth/authenticate'

        if self.access_token:  # If they pass an access token, force OAuth 2
            oauth_version = 2

        self.oauth_version = oauth_version

        # OAuth 2
        if oauth_version == 2:
            self.request_token_url = self.api_url % 'oauth2/token'

        self.client_args = client_args or {}
        default_headers = {'User-Agent': 'Twython v' + __version__}
        if not 'headers' in self.client_args:
            # If they didn't set any headers, set our defaults for them
            self.client_args['headers'] = default_headers
        elif 'User-Agent' not in self.client_args['headers']:
            # If they set headers, but didn't include User-Agent.. set it for them
            self.client_args['headers'].update(default_headers)

        # Generate OAuth authentication object for the request
        # If no keys/tokens are passed to __init__, auth=None allows for
        # unauthenticated requests, although I think all v1.1 requests need auth
        auth = None
        if oauth_version == 1:
            # User Authentication is through OAuth 1
            if self.app_key is not None and self.app_secret is not None and \
               self.oauth_token is None and self.oauth_token_secret is None:
                auth = OAuth1(self.app_key, self.app_secret)

            if self.app_key is not None and self.app_secret is not None and \
               self.oauth_token is not None and self.oauth_token_secret is not None:
                auth = OAuth1(self.app_key, self.app_secret,
                              self.oauth_token, self.oauth_token_secret)
        elif oauth_version == 2 and self.access_token:
            # Application Authentication is through OAuth 2
            token = {'token_type': token_type, 'access_token': self.access_token}
            auth = OAuth2(self.app_key, token=token)

        self.client = requests.Session()
        self.client.auth = auth

        # Make a copy of the client args and iterate over them
        # Pop out all the acceptable args at this point because they will
        # Never be used again.
        client_args_copy = self.client_args.copy()
        for k, v in client_args_copy.items():
            if k in ('cert', 'headers', 'hooks', 'max_redirects', 'proxies'):
                setattr(self.client, k, v)
                self.client_args.pop(k)  # Pop, pop!

        self._last_call = None

    def __repr__(self):
        return '<Twython: %s>' % (self.app_key)

    def _request(self, url, method='GET', params=None, api_call=None):
        """Internal request method"""
        method = method.lower()
        params = params or {}

        func = getattr(self.client, method)
        params, files = _transparent_params(params)

        requests_args = {}
        for k, v in self.client_args.items():
            # Maybe this should be set as a class variable and only done once?
            if k in ('timeout', 'allow_redirects', 'stream', 'verify'):
                requests_args[k] = v

        if method == 'get':
            requests_args['params'] = params
        else:
            requests_args.update({
                'data': params,
                'files': files,
            })
        try:
            response = func(url, **requests_args)
        except requests.RequestException as e:
            raise TwythonError(str(e))
        content = response.content.decode('utf-8')

        # create stash for last function intel
        self._last_call = {
            'api_call': api_call,
            'api_error': None,
            'cookies': response.cookies,
            'headers': response.headers,
            'status_code': response.status_code,
            'url': response.url,
            'content': content,
        }

        #  Wrap the json loads in a try, and defer an error
        #  Twitter will return invalid json with an error code in the headers
        json_error = False
        try:
            try:
                # try to get json
                content = content.json()
            except AttributeError:
                # if unicode detected
                content = json.loads(content)
        except ValueError:
            json_error = True
            content = {}

        if response.status_code > 304:
            # If there is no error message, use a default.
            errors = content.get('errors',
                                 [{'message': 'An error occurred processing your request.'}])
            if errors and isinstance(errors, list):
                error_message = errors[0]['message']
            else:
                error_message = errors
            self._last_call['api_error'] = error_message

            ExceptionType = TwythonError
            if response.status_code == 429:
                # Twitter API 1.1, always return 429 when rate limit is exceeded
                ExceptionType = TwythonRateLimitError
            elif response.status_code == 401 or 'Bad Authentication data' in error_message:
                # Twitter API 1.1, returns a 401 Unauthorized or
                # a 400 "Bad Authentication data" for invalid/expired app keys/user tokens
                ExceptionType = TwythonAuthError

            raise ExceptionType(error_message,
                                error_code=response.status_code,
                                retry_after=response.headers.get('retry-after'))

        # if we have a json error here, then it's not an official Twitter API error
        if json_error and not response.status_code in (200, 201, 202):
            raise TwythonError('Response was not valid JSON, unable to decode.')

        return content

    def request(self, endpoint, method='GET', params=None, version='1.1'):
        """Return dict of response received from Twitter's API

        :param endpoint: (required) Full url or Twitter API endpoint (e.g. search/tweets)
        :type endpoint: string
        :param method: (optional) Method of accessing data, either GET or POST. (default GET)
        :type method: string
        :param params: (optional) Dict of parameters (if any) accepted the by Twitter API endpoint you are trying to access (default None)
        :type params: dict or None
        :param version: (optional) Twitter API version to access (default 1.1)
        :type version: string

        :rtype: dict
        """

        # In case they want to pass a full Twitter URL
        # i.e. https://api.twitter.com/1.1/search/tweets.json
        if endpoint.startswith('http://') or endpoint.startswith('https://'):
            url = endpoint
        else:
            url = '%s/%s.json' % (self.api_url % version, endpoint)

        content = self._request(url, method=method, params=params, api_call=url)

        return content

    def get(self, endpoint, params=None, version='1.1'):
        """Shortcut for GET requests via :class:`request`"""
        return self.request(endpoint, params=params, version=version)

    def post(self, endpoint, params=None, version='1.1'):
        """Shortcut for POST requests via :class:`request`"""
        return self.request(endpoint, 'POST', params=params, version=version)

    def get_lastfunction_header(self, header, default_return_value=None):
        """Returns a specific header from the last API call
        This will return None if the header is not present

        :param header: (required) The name of the header you want to get the value of

        Most useful for the following header information:
            x-rate-limit-limit,
            x-rate-limit-remaining,
            x-rate-limit-class,
            x-rate-limit-reset

        """
        if self._last_call is None:
            raise TwythonError('This function must be called after an API call.  It delivers header information.')

        return self._last_call['headers'].get(header, default_return_value)

    def get_authentication_tokens(self, callback_url=None, force_login=False, screen_name=''):
        """Returns a dict including an authorization URL, ``auth_url``, to direct a user to

        :param callback_url: (optional) Url the user is returned to after they authorize your app (web clients only)
        :param force_login: (optional) Forces the user to enter their credentials to ensure the correct users account is authorized.
        :param app_secret: (optional) If forced_login is set OR user is not currently logged in, Prefills the username input box of the OAuth login screen with the given value
        :rtype: dict
        """
        if self.oauth_version != 1:
            raise TwythonError('This method can only be called when your OAuth version is 1.0.')

        request_args = {}
        if callback_url:
            request_args['oauth_callback'] = callback_url
        response = self.client.get(self.request_token_url, params=request_args)

        if response.status_code == 401:
            raise TwythonAuthError(response.content, error_code=response.status_code)
        elif response.status_code != 200:
            raise TwythonError(response.content, error_code=response.status_code)

        request_tokens = dict(parse_qsl(response.content.decode('utf-8')))
        if not request_tokens:
            raise TwythonError('Unable to decode request tokens.')

        oauth_callback_confirmed = request_tokens.get('oauth_callback_confirmed') == 'true'

        auth_url_params = {
            'oauth_token': request_tokens['oauth_token'],
        }

        if force_login:
            auth_url_params.update({
                'force_login': force_login,
                'screen_name': screen_name
            })

        # Use old-style callback argument if server didn't accept new-style
        if callback_url and not oauth_callback_confirmed:
            auth_url_params['oauth_callback'] = self.callback_url

        request_tokens['auth_url'] = self.authenticate_url + '?' + urlencode(auth_url_params)

        return request_tokens

    def get_authorized_tokens(self, oauth_verifier):
        """Returns a dict of authorized tokens after they go through the :class:`get_authentication_tokens` phase.

        :param oauth_verifier: (required) The oauth_verifier (or a.k.a PIN for non web apps) retrieved from the callback url querystring
        :rtype: dict

        """
        if self.oauth_version != 1:
            raise TwythonError('This method can only be called when your OAuth version is 1.0.')

        response = self.client.get(self.access_token_url, params={'oauth_verifier': oauth_verifier})
        authorized_tokens = dict(parse_qsl(response.content.decode('utf-8')))
        if not authorized_tokens:
            raise TwythonError('Unable to decode authorized tokens.')

        return authorized_tokens

    def obtain_access_token(self):
        """Returns an OAuth 2 access token to make OAuth 2 authenticated read-only calls.

        :rtype: string
        """
        if self.oauth_version != 2:
            raise TwythonError('This method can only be called when your OAuth version is 2.0.')

        data = {'grant_type': 'client_credentials'}
        basic_auth = HTTPBasicAuth(self.app_key, self.app_secret)
        try:
            response = self.client.post(self.request_token_url,
                                        data=data, auth=basic_auth)
            content = response.content.decode('utf-8')
            try:
                content = content.json()
            except AttributeError:
                content = json.loads(content)
                access_token = content['access_token']
        except (ValueError, requests.exceptions.RequestException):
            raise TwythonAuthError('Unable to obtain OAuth 2 access token.')
        else:
            return access_token

    @staticmethod
    def construct_api_url(api_url, **params):
        """Construct a Twitter API url, encoded, with parameters

        :param api_url: URL of the Twitter API endpoint you are attempting to construct
        :param \*\*params: Parameters that are accepted by Twitter for the endpoint you're requesting
        :rtype: string

        Usage::

          >>> from twython import Twython
          >>> twitter = Twython()

          >>> api_url = 'https://api.twitter.com/1.1/search/tweets.json'
          >>> constructed_url = twitter.construct_api_url(api_url, q='python', result_type='popular')
          >>> print constructed_url
          https://api.twitter.com/1.1/search/tweets.json?q=python&result_type=popular

        """
        querystring = []
        params, _ = _transparent_params(params or {})
        params = requests.utils.to_key_val_list(params)
        for (k, v) in params:
            querystring.append(
                '%s=%s' % (Twython.encode(k), quote_plus(Twython.encode(v)))
            )
        return '%s?%s' % (api_url, '&'.join(querystring))

    def search_gen(self, search_query, **params):
        """Returns a generator of tweets that match a specified query.

        Documentation: https://dev.twitter.com/docs/api/1.1/get/search/tweets

        :param search_query: Query you intend to search Twitter for
        :param \*\*params: Extra parameters to send with your search request
        :rtype: generator

        Usage::

          >>> from twython import Twython
          >>> twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)

          >>> search = twitter.search_gen('python')
          >>> for result in search:
          >>>   print result

        """
        content = self.search(q=search_query, **params)

        if not content.get('statuses'):
            raise StopIteration

        for tweet in content['statuses']:
            yield tweet

        try:
            if not 'since_id' in params:
                params['since_id'] = (int(content['statuses'][0]['id_str']) + 1)
        except (TypeError, ValueError):
            raise TwythonError('Unable to generate next page of search results, `page` is not a number.')

        for tweet in self.search_gen(search_query, **params):
            yield tweet

    @staticmethod
    def unicode2utf8(text):
        try:
            if is_py2 and isinstance(text, str):
                text = text.encode('utf-8')
        except:
            pass
        return text

    @staticmethod
    def encode(text):
        if is_py2 and isinstance(text, (str)):
            return Twython.unicode2utf8(text)
        return str(text)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
TXFMTrackService
(C) 2015
David Rieger
v1.1
"""

import urllib.request
import json
from storagemanager import StorageManager

def now_playing():

    # New TXFM website has a nicer API that makes things much easier
    URL = "http://www.txfm.ie/assets/includes/ajax/player_info.php?type=On+Air&currentstationID=11"

    # It returns a JSON which has a key called "data" which contains - get this - HTML
    jsonstr = urllib.request.urlopen(URL).readall().decode("UTF-8")
    onairinfo = json.loads(jsonstr)

    # Try to obtain current song info from JSON response
    try:
        artist = onairinfo["currentTitle"]
        title = onairinfo["currentArtist"]
    except IndexError:
        pass

    # If song info was not available - set show title as title
    if not (artist or title):
        artist = ""
        try:
            title = onairinfo["title"]
        except IndexError:
            title = "ERROR RETRIEVING SONG INFO"

    return {"title": title, "artist": artist}

if __name__ == "__main__":
    print(now_playing())
    StorageManager().add_song(now_playing())
<commit_msg>bugfix<commit_after>#!/usr/bin/env python3

"""
TXFMTrackService
(C) 2015
David Rieger
v1.1
"""

import urllib.request
import json
from storagemanager import StorageManager

def now_playing():

    # New TXFM website has a nicer API that makes things much easier
    URL = "http://www.txfm.ie/assets/includes/ajax/player_info.php?type=On+Air&currentstationID=11"

    # It returns a JSON which has a key called "data" which contains - get this - HTML
    jsonstr = urllib.request.urlopen(URL).readall().decode("UTF-8")
    onairinfo = json.loads(jsonstr)

    # Try to obtain current song info from JSON response
    try:
        title = onairinfo["currentTitle"]
        artist = onairinfo["currentArtist"]
    except IndexError:
        pass

    # If either song or title info not available - set show title as title
    if not (artist and title):
        artist = ""
        try:
            title = onairinfo["title"]
        except IndexError:
            title = "ERROR RETRIEVING SONG INFO"

    return {"title": title, "artist": artist}

if __name__ == "__main__":
    print(now_playing())
    StorageManager().add_song(now_playing())
<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

VERSION = '0.9.9'
<commit_msg>Bump to version 0.9.10.<commit_after># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

VERSION = '0.9.10'
<|endoftext|>