<commit_before>#!/usr/bin/env python

#    Copyright (C) 2001  Jeff Epler  <jepler@unpythonic.dhs.org>
#    Copyright (C) 2006  Csaba Henk  <csaba.henk@creo.hu>
#
#    This program can be distributed under the terms of the GNU LGPL.
#    See the file COPYING.
#

import os
import sys
import errno
import stat
import fcntl

import fuse

if not hasattr(fuse, '__version__'):
    raise RuntimeError, \
        "your fuse-py doesn't know of fuse.__version__, probably it's too old."

fuse.fuse_python_api = (0, 2)

# We use a custom file class
fuse.feature_assert('stateful_files', 'has_init')

def flag2mode(flags):
    md = {os.O_RDONLY: 'r', os.O_WRONLY: 'w', os.O_RDWR: 'w+'}
    m = md[flags & (os.O_RDONLY | os.O_WRONLY | os.O_RDWR)]

    if flags | os.O_APPEND:
        m = m.replace('w', 'a', 1)

    return m

class Xmp(fuse.Fuse):

    def __init__(self, *args, **kw):

        fuse.Fuse.__init__(self, *args, **kw)

        # do stuff to set up your filesystem here, if you want
        #import thread
        #thread.start_new_thread(self.mythread, ())
        self.root = '/'
        self.file_class = self.XmpFile

#    def mythread(self):
#
#        """
#        The beauty of the FUSE python implementation is that with the python interp
#        running in foreground, you can have threads
#        """
#        print "mythread: started"
#        while 1:
#            time.sleep(120)
#            print "mythread: ticking"

    def getattr(self, path):
        return os.lstat("." + path)

    def readlink(self, path):
        return os.readlink("." + path)

    def readdir(self, path, offset):
        for e in os.listdir("." + path):
            yield fuse.Direntry(e)

    def unlink(self, path):
        os.unlink("." + path)

    def rmdir(self, path):
        os.rmdir("." + path)

    def symlink(self, path, path1):
        os.symlink(path, "." + path1)

    def rename(self, path, path1):
        os.rename("." + path, "." + path1)

    def link(self, path, path1):
        os.link("." + path, "." + path1)

    def chmod(self, path, mode):
        os.chmod("." + path, mode)

    def chown(self, path, user, group):
        os.chown("." + path, user, group)

    def truncate(self, path, len):
        f = open("." + path, "a")
        f.truncate(len)
        f.close()

    def mknod(self, path, mode, dev):
        os.mknod("." + path, mode, dev)

    def mkdir(self, path, mode):
        os.mkdir("." + path, mode)

    def utime(self, path, times):
        os.utime("." + path, times)

#    The following utimens method would do the same as the above utime method.
#    We can't make it better though as the Python stdlib doesn't know of
#    subsecond preciseness in acces/modify times.
#
#    def utimens(self, path, ts_acc, ts_mod):
#      os.utime("." + path, (ts_acc.tv_sec, ts_mod.tv_sec))

    def access(self, path, mode):
        if not os.access("." + path, mode):
            return -errno.EACCES

#    This is how we could add stub extended attribute handlers...
#    (We can't have ones which aptly delegate requests to the underlying fs
#    because Python lacks a standard xattr interface.)
#
#    def getxattr(self, path, name, size):
#        val = name.swapcase() + '@' + path
#        if size == 0:
#            # We are asked for size of the value.
#            return len(val)
#        return val
#
#    def listxattr(self, path, size):
#        # We use the "user" namespace to please XFS utils
#        aa = ["user." + a for a in ("foo", "bar")]
#        if size == 0:
#            # We are asked for size of the attr list, ie. joint size of attrs
#            # plus null separators.
#            return len("".join(aa)) + len(aa)
#        return aa

    def statfs(self):
        """
        Should return an object with statvfs attributes (f_bsize, f_frsize...).
        Eg., the return value of os.statvfs() is such a thing (since py 2.2).
        If you are not reusing an existing statvfs object, start with
        fuse.StatVFS(), and define the attributes.

        To provide usable information (ie., you want sensible df(1)
        output, you are suggested to specify the following attributes:

            - f_bsize - preferred size of file blocks, in bytes
            - f_frsize - fundamental size of file blcoks, in bytes
                [if you have no idea, use the same as blocksize]
            - f_blocks - total number of blocks in the filesystem
            - f_bfree - number of free blocks
            - f_files - total number of file inodes
            - f_ffree - nunber of free file inodes
        """

        return os.statvfs(".")

    def fsinit(self):
        os.chdir(self.root)

    class XmpFile(object):

        def __init__(self, path, flags, *mode):
            self.file = os.fdopen(os.open("." + path, flags, *mode),
                                  flag2mode(flags))
            self.fd = self.file.fileno()

        def read(self, length, offset):
            self.file.seek(offset)
            return self.file.read(length)

        def write(self, buf, offset):
            self.file.seek(offset)
            self.file.write(buf)
            return len(buf)

        def release(self, flags):
            self.file.close()

        def _fflush(self):
            if 'w' in self.file.mode or 'a' in self.file.mode:
                self.file.flush()

        def fsync(self, isfsyncfile):
            self._fflush()
            if isfsyncfile and hasattr(os, 'fdatasync'):
                os.fdatasync(self.fd)
            else:
                os.fsync(self.fd)

        def flush(self):
            self._fflush()
            # cf. xmp_flush() in fusexmp_fh.c
            os.close(os.dup(self.fd))

        def fgetattr(self):
            return os.fstat(self.fd)

        def ftruncate(self, len):
            self.file.truncate(len)

        def lock(self, cmd, owner, **kw):
            # The code here is much rather just a demonstration of the locking
            # API than something which actually was seen to be useful.

            # Advisory file locking is pretty messy in Unix, and the Python
            # interface to this doesn't make it better.
            # We can't do fcntl(2)/F_GETLK from Python in a platfrom independent
            # way. The following implementation *might* work under Linux.
            #
            # if cmd == fcntl.F_GETLK:
            #     import struct
            #
            #     lockdata = struct.pack('hhQQi', kw['l_type'], os.SEEK_SET,
            #                            kw['l_start'], kw['l_len'], kw['l_pid'])
            #     ld2 = fcntl.fcntl(self.fd, fcntl.F_GETLK, lockdata)
            #     flockfields = ('l_type', 'l_whence', 'l_start', 'l_len', 'l_pid')
            #     uld2 = struct.unpack('hhQQi', ld2)
            #     res = {}
            #     for i in xrange(len(uld2)):
            #          res[flockfields[i]] = uld2[i]
            #
            #     return fuse.Flock(**res)

            # Convert fcntl-ish lock parameters to Python's weird
            # lockf(3)/flock(2) medley locking API...
            op = { fcntl.F_UNLCK : fcntl.LOCK_UN,
                   fcntl.F_RDLCK : fcntl.LOCK_SH,
                   fcntl.F_WRLCK : fcntl.LOCK_EX }[kw['l_type']]
            if cmd == fcntl.F_GETLK:
                return -errno.EOPNOTSUPP
            elif cmd == fcntl.F_SETLK:
                if op != fcntl.LOCK_UN:
                    op |= fcntl.LOCK_NB
            elif cmd == fcntl.F_SETLKW:
                pass
            else:
                return -errno.EINVAL

            fcntl.lockf(self.fd, op, kw['l_start'], kw['l_len'])

def main():

    usage = """
Userspace nullfs-alike: mirror the filesystem tree from some point on.

""" + fuse.Fuse.fusage

    server = Xmp(version="%prog " + fuse.__version__,
                 usage=usage)

    server.parser.add_option(mountopt="root", metavar="PATH", default='/',
                             help="mirror filesystem from under PATH [default: %default]")
    server.parse(values=server, errex=1)

    try:
        if server.fuse_args.mount_expected():
            os.chdir(server.root)
    except OSError:
        print >> sys.stderr, "can't enter root of underlying filesystem"
        sys.exit(1)

    server.main()

if __name__ == '__main__':
    main()
<commit_msg>Add logging module<commit_after>#!/usr/bin/env python

#    Copyright (C) 2001  Jeff Epler  <jepler@unpythonic.dhs.org>
#    Copyright (C) 2006  Csaba Henk  <csaba.henk@creo.hu>
#
#    This program can be distributed under the terms of the GNU LGPL.
#    See the file COPYING.
#

import os
import sys
import errno
import stat
import fcntl
import logging

import fuse

_logger = logging.getLogger(os.path.basename(__file__))

if not hasattr(fuse, '__version__'):
    raise RuntimeError, \
        "your fuse-py doesn't know of fuse.__version__, probably it's too old."

fuse.fuse_python_api = (0, 2)

# We use a custom file class
fuse.feature_assert('stateful_files', 'has_init')

def flag2mode(flags):
    md = {os.O_RDONLY: 'r', os.O_WRONLY: 'w', os.O_RDWR: 'w+'}
    m = md[flags & (os.O_RDONLY | os.O_WRONLY | os.O_RDWR)]

    if flags | os.O_APPEND:
        m = m.replace('w', 'a', 1)

    return m

class Xmp(fuse.Fuse):

    def __init__(self, *args, **kw):

        fuse.Fuse.__init__(self, *args, **kw)

        # do stuff to set up your filesystem here, if you want
        #import thread
        #thread.start_new_thread(self.mythread, ())
        self.root = '/'
        self.file_class = self.XmpFile

#    def mythread(self):
#
#        """
#        The beauty of the FUSE python implementation is that with the python interp
#        running in foreground, you can have threads
#        """
#        print "mythread: started"
#        while 1:
#            time.sleep(120)
#            print "mythread: ticking"

    def getattr(self, path):
        return os.lstat("." + path)

    def readlink(self, path):
        return os.readlink("." + path)

    def readdir(self, path, offset):
        for e in os.listdir("." + path):
            yield fuse.Direntry(e)

    def unlink(self, path):
        os.unlink("." + path)

    def rmdir(self, path):
        os.rmdir("." + path)

    def symlink(self, path, path1):
        os.symlink(path, "." + path1)

    def rename(self, path, path1):
        os.rename("." + path, "." + path1)

    def link(self, path, path1):
        os.link("." + path, "." + path1)

    def chmod(self, path, mode):
        os.chmod("." + path, mode)

    def chown(self, path, user, group):
        os.chown("." + path, user, group)

    def truncate(self, path, len):
        f = open("." + path, "a")
        f.truncate(len)
        f.close()

    def mknod(self, path, mode, dev):
        os.mknod("." + path, mode, dev)

    def mkdir(self, path, mode):
        os.mkdir("." + path, mode)

    def utime(self, path, times):
        os.utime("." + path, times)

#    The following utimens method would do the same as the above utime method.
#    We can't make it better though as the Python stdlib doesn't know of
#    subsecond preciseness in acces/modify times.
#
#    def utimens(self, path, ts_acc, ts_mod):
#      os.utime("." + path, (ts_acc.tv_sec, ts_mod.tv_sec))

    def access(self, path, mode):
        if not os.access("." + path, mode):
            return -errno.EACCES

#    This is how we could add stub extended attribute handlers...
#    (We can't have ones which aptly delegate requests to the underlying fs
#    because Python lacks a standard xattr interface.)
#
#    def getxattr(self, path, name, size):
#        val = name.swapcase() + '@' + path
#        if size == 0:
#            # We are asked for size of the value.
#            return len(val)
#        return val
#
#    def listxattr(self, path, size):
#        # We use the "user" namespace to please XFS utils
#        aa = ["user." + a for a in ("foo", "bar")]
#        if size == 0:
#            # We are asked for size of the attr list, ie. joint size of attrs
#            # plus null separators.
#            return len("".join(aa)) + len(aa)
#        return aa

    def statfs(self):
        """
        Should return an object with statvfs attributes (f_bsize, f_frsize...).
        Eg., the return value of os.statvfs() is such a thing (since py 2.2).
        If you are not reusing an existing statvfs object, start with
        fuse.StatVFS(), and define the attributes.

        To provide usable information (ie., you want sensible df(1)
        output, you are suggested to specify the following attributes:

            - f_bsize - preferred size of file blocks, in bytes
            - f_frsize - fundamental size of file blcoks, in bytes
                [if you have no idea, use the same as blocksize]
            - f_blocks - total number of blocks in the filesystem
            - f_bfree - number of free blocks
            - f_files - total number of file inodes
            - f_ffree - nunber of free file inodes
        """

        return os.statvfs(".")

    def fsinit(self):
        os.chdir(self.root)

    class XmpFile(object):

        def __init__(self, path, flags, *mode):
            self.file = os.fdopen(os.open("." + path, flags, *mode),
                                  flag2mode(flags))
            self.fd = self.file.fileno()

        def read(self, length, offset):
            self.file.seek(offset)
            return self.file.read(length)

        def write(self, buf, offset):
            self.file.seek(offset)
            self.file.write(buf)
            return len(buf)

        def release(self, flags):
            self.file.close()

        def _fflush(self):
            if 'w' in self.file.mode or 'a' in self.file.mode:
                self.file.flush()

        def fsync(self, isfsyncfile):
            self._fflush()
            if isfsyncfile and hasattr(os, 'fdatasync'):
                os.fdatasync(self.fd)
            else:
                os.fsync(self.fd)

        def flush(self):
            self._fflush()
            # cf. xmp_flush() in fusexmp_fh.c
            os.close(os.dup(self.fd))

        def fgetattr(self):
            return os.fstat(self.fd)

        def ftruncate(self, len):
            self.file.truncate(len)

        def lock(self, cmd, owner, **kw):
            # The code here is much rather just a demonstration of the locking
            # API than something which actually was seen to be useful.

            # Advisory file locking is pretty messy in Unix, and the Python
            # interface to this doesn't make it better.
            # We can't do fcntl(2)/F_GETLK from Python in a platfrom independent
            # way. The following implementation *might* work under Linux.
            #
            # if cmd == fcntl.F_GETLK:
            #     import struct
            #
            #     lockdata = struct.pack('hhQQi', kw['l_type'], os.SEEK_SET,
            #                            kw['l_start'], kw['l_len'], kw['l_pid'])
            #     ld2 = fcntl.fcntl(self.fd, fcntl.F_GETLK, lockdata)
            #     flockfields = ('l_type', 'l_whence', 'l_start', 'l_len', 'l_pid')
            #     uld2 = struct.unpack('hhQQi', ld2)
            #     res = {}
            #     for i in xrange(len(uld2)):
            #          res[flockfields[i]] = uld2[i]
            #
            #     return fuse.Flock(**res)

            # Convert fcntl-ish lock parameters to Python's weird
            # lockf(3)/flock(2) medley locking API...
            op = { fcntl.F_UNLCK : fcntl.LOCK_UN,
                   fcntl.F_RDLCK : fcntl.LOCK_SH,
                   fcntl.F_WRLCK : fcntl.LOCK_EX }[kw['l_type']]
            if cmd == fcntl.F_GETLK:
                return -errno.EOPNOTSUPP
            elif cmd == fcntl.F_SETLK:
                if op != fcntl.LOCK_UN:
                    op |= fcntl.LOCK_NB
            elif cmd == fcntl.F_SETLKW:
                pass
            else:
                return -errno.EINVAL

            fcntl.lockf(self.fd, op, kw['l_start'], kw['l_len'])

def main():

    usage = """
Userspace nullfs-alike: mirror the filesystem tree from some point on.

""" + fuse.Fuse.fusage

    server = Xmp(version="%prog " + fuse.__version__,
                 usage=usage)

    server.parser.add_option(mountopt="root", metavar="PATH", default='/',
                             help="mirror filesystem from under PATH [default: %default]")
    server.parse(values=server, errex=1)

    try:
        if server.fuse_args.mount_expected():
            os.chdir(server.root)
    except OSError:
        print >> sys.stderr, "can't enter root of underlying filesystem"
        sys.exit(1)

    logging.basicConfig(level=logging.DEBUG if "debug" in server.fuse_args.optlist else logging.INFO)

    server.main()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
from BeautifulSoup import BeautifulSoup
import re
import urllib2

def fetch_data():
    return urllib2.urlopen("http://www.yle.fi/tekstitv/txt/P100_01.html").read()

def print_http_headers(content_length):
    print "\n".join((
                     "Expires: -1",
                     "Cache-Control: private, max-age=0",
                     "Content-Type: text/html; charset=UTF-8",
                     "Content-Length: %d" % content_length,
                   ))
    print "\n"

response_text = "";
soup = BeautifulSoup(fetch_data())
bigs = soup.findAll('big')
for big in bigs:
    m = re.search('<big>(\d+)(.+)</big>', str(big))
    if m:
        href = m.group(1)
        title = m.group(3)
        response_text += '
%s
\n' % (href, title)

print_http_headers(len(response_text))
print response_text
<commit_msg>News item titles may no longer contain html tags. Also, now producing proper utf-8 output.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
from BeautifulSoup import BeautifulSoup
import re
import urllib2

def fetch_data():
    return urllib2.urlopen("http://www.yle.fi/tekstitv/txt/P100_01.html").read()

def print_http_headers(content_length):
    print "\n".join((
                     "Expires: -1",
                     "Cache-Control: private, max-age=0",
                     "Content-Type: text/html; charset=UTF-8",
                     "Content-Length: %d" % content_length,
                   ))
    print "\n"

response_text = u'';
soup = BeautifulSoup(fetch_data())
bigs = soup.findAll('big')
for big in bigs:
    m = re.search('<big>(\d+)(.+)</big>', str(big))
    if m:
        href = m.group(1)
        inner = m.group(3)
	title = ''.join(BeautifulSoup(inner).findAll(text=True))
        response_text += u'
%s
\n' % (href, title)

print_http_headers(len(response_text))
print response_text.encode("utf-8")
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import argparse
import httplib2
from config import *
from apiclient.discovery import build
from apiclient.errors import HttpError
from oauth2client.client import flow_from_clientsecrets
from oauth2client.file import Storage
from oauth2client.tools import argparser, run_flow

class Options():
    def __init__(self):
        self.id_config = False
        self.username_config = False
        self.id = None
        self.username = None
        self.related = None

    def process_options(self):
        # Check for mutual exclusion of config options
        try:
            if (CHANNELID and USERNAME):
                sys.exit("May only specify either CHANNELID or USERNAME")
        except NameError:
            pass

        # Process config options
        try:
            if CHANNELID:
                self.id_config = True
        except NameError:
            self.id_config = False

        try:
            if USERNAME:
                self.username_config = True
        except NameError:
            self.username_config = False

        # Process commandline arguments
        # Check args first, then config options if specified
        try:
            self.id = args.id if (args.id) else CHANNELID
        except NameError:
            self.id = None

        try:
            self.username = args.username if (args.username) else USERNAME
        except NameError:
            self.username = None

        try:
            self.related = args.related if (args.related) else RELATED
        except NameError:
            self.related = None

def process_arguments():
    parser = argparse.ArgumentParser(description="YouTube Playlist Backup script",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        parents=[argparser])

    retrieval_method = parser.add_mutually_exclusive_group()
    retrieval_method.add_argument("-i", "--id", help="Retrieve playlists using channel ID")
    retrieval_method.add_argument("-u", "--username", help="Retrieve playlists using legacy YouTube username")
    parser.add_argument("-r", "--related", help="Also retrieve related playlists (likes, history, etc.",
                        action="store_true")

    return parser.parse_args()

# Creates the resource object for interacting with the YouTube API
# Sets up OAuth 2.0 for authorized requests if necessary
def create_resource_object():
    if (opt.id or opt.username):
        youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
            developerKey=DEVELOPER_KEY)
    else:
        flow = flow_from_clientsecrets(CLIENT_SECRETS_FILE,
            message=MISSING_CLIENT_SECRETS_MESSAGE,
            scope=YOUTUBE_READONLY_SCOPE)

        storage = Storage("%s-oauth2.json" % sys.argv[0])
        credentials = storage.get()

        if credentials is None or credentials.invalid:
            credentials = run_flow(flow, storage, args)

        youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
            http=credentials.authorize(httplib2.Http()))

    return youtube

# Creates a request for the user's playlists using their channel ID
def setup_id_request():
   request = youtube.playlists().list(
       part="id,snippet",
       fields="items(id,snippet/title),nextPageToken",
       channelId=opt.id,
       maxResults=MAX_RESULTS
   )

   return request

# Creates a request for the user's playlists using their username
# First uses a channel request to obtain channel ID from username
def setup_username_request():
   channel_request = youtube.channels().list(
       part="id",
       forUsername=opt.username,
       maxResults=MAX_RESULTS
   )

   channel_response = channel_request.execute()

   try:
       request = youtube.playlists().list(
           part="id,snippet",
           fields="items(id,snippet/title),nextPageToken",
           channelId=channel_response["items"][0]["id"],
           maxResults=MAX_RESULTS
       )
   except IndexError:
       sys.exit("No channel found for {}".format(opt.username))

   return request

# Creates an authenticated request for accessing the user's private playlists
def setup_private_request():
    request = youtube.playlists().list(
        part="id,snippet",
        fields="items(id,snippet/title),nextPageToken",
        mine="true",
        maxResults=MAX_RESULTS
    )

    return request

# Creates a channel request necessary for obtaining the channel's
# related playlists, via channel ID
def create_id_channel_request():
    channel_request = youtube.channels().list(
        part="contentDetails",
        fields="items(contentDetails/relatedPlaylists)",
        id=opt.id
    )

    return channel_request

# Creates a channel request necessary for obtaining the channel's
# related playlists, via username
def create_username_channel_request():
    channel_request = youtube.channels().list(
        part="contentDetails",
        fields="items(contentDetails/relatedPlaylists)",
        forUsername=opt.username
    )

    return channel_request

# Creates a channel request necessary for obtaining the channel's
# related playlists, via authentication
def create_private_channel_request():
    channel_request = youtube.channels().list(
        part="contentDetails",
        fields="items(contentDetails/relatedPlaylists)",
        mine="true"
    )

    return channel_request

# Create request for obtaining the user's related playlists
def setup_related_request(channel_request):
    playlist_id_list = []

    channel_response = channel_request.execute()

    # Traverse channel_response to create list of related playlist IDs
    for channel in channel_response["items"]:
        for playlist, playlist_id in channel["contentDetails"]["relatedPlaylists"].items():
            playlist_id_list.append(playlist_id)

    request = youtube.playlists().list(
        part="id,snippet",
        fields="items(id,snippet/title),nextPageToken",
        id=",".join(playlist_id_list),
        maxResults=MAX_RESULTS
    )

    return request

# Backs-up playlists using the provided request
def backup_playlists(playlists_request):
    # Fetch pages of playlists until end
    while playlists_request:
        playlists_response = playlists_request.execute()

        for playlist in playlists_response["items"]:
            video_count = 0

            # Assemble request for videos in each playlist
            playlist_items_request = youtube.playlistItems().list(
                part="id,snippet",
                fields="items(id,snippet/title),nextPageToken",
                playlistId=playlist["id"],
                maxResults=MAX_RESULTS
            )

            print "{}".format(playlist["snippet"]["title"].encode("utf-8"))

            # Fetch pages of videos until end
            while playlist_items_request:
                playlist_items_response = playlist_items_request.execute()

                # Print videos in each playlist
                for i, video in enumerate(playlist_items_response["items"], start=1):
                    print "{}. {}".format((i + video_count), video["snippet"]["title"].encode("utf-8"))

                # Request next page of videos
                playlist_items_request = youtube.playlistItems().list_next(
                    playlist_items_request, playlist_items_response)

                if playlist_items_request is not None:
                    video_count += MAX_RESULTS
                else:
                    print

        # Request next page of playlists
        playlists_request = youtube.playlists().list_next(
            playlists_request, playlists_response)

if __name__ == "__main__":
    args = process_arguments()

    opt = Options()
    opt.process_options()

    try:
        youtube = create_resource_object()

        if (args.id or opt.id_config and args.username is None):
            req = setup_id_request()
            ch_req = create_id_channel_request()
        elif (args.username or opt.username_config and args.id is None):
            req = setup_username_request()
            ch_req = create_username_channel_request()
        else:
            req = setup_private_request()
            ch_req = create_private_channel_request()

        backup_playlists(req)

        if (opt.related):
            req = setup_related_request(ch_req)
            backup_playlists(req)

    except HttpError as e:
        print "An HTTP error %d occurred:\n%s" % (e.resp.status, e.content)

<commit_msg>Renamed setup functions to create<commit_after>#!/usr/bin/env python

import sys
import argparse
import httplib2
from config import *
from apiclient.discovery import build
from apiclient.errors import HttpError
from oauth2client.client import flow_from_clientsecrets
from oauth2client.file import Storage
from oauth2client.tools import argparser, run_flow

class Options():
    def __init__(self):
        self.id_config = False
        self.username_config = False
        self.id = None
        self.username = None
        self.related = None

    def process_options(self):
        # Check for mutual exclusion of config options
        try:
            if (CHANNELID and USERNAME):
                sys.exit("May only specify either CHANNELID or USERNAME")
        except NameError:
            pass

        # Process config options
        try:
            if CHANNELID:
                self.id_config = True
        except NameError:
            self.id_config = False

        try:
            if USERNAME:
                self.username_config = True
        except NameError:
            self.username_config = False

        # Process commandline arguments
        # Check args first, then config options if specified
        try:
            self.id = args.id if (args.id) else CHANNELID
        except NameError:
            self.id = None

        try:
            self.username = args.username if (args.username) else USERNAME
        except NameError:
            self.username = None

        try:
            self.related = args.related if (args.related) else RELATED
        except NameError:
            self.related = None

def process_arguments():
    parser = argparse.ArgumentParser(description="YouTube Playlist Backup script",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        parents=[argparser])

    retrieval_method = parser.add_mutually_exclusive_group()
    retrieval_method.add_argument("-i", "--id", help="Retrieve playlists using channel ID")
    retrieval_method.add_argument("-u", "--username", help="Retrieve playlists using legacy YouTube username")
    parser.add_argument("-r", "--related", help="Also retrieve related playlists (likes, history, etc.",
                        action="store_true")

    return parser.parse_args()

# Creates the resource object for interacting with the YouTube API
# Sets up OAuth 2.0 for authorized requests if necessary
def create_resource_object():
    if (opt.id or opt.username):
        youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
            developerKey=DEVELOPER_KEY)
    else:
        flow = flow_from_clientsecrets(CLIENT_SECRETS_FILE,
            message=MISSING_CLIENT_SECRETS_MESSAGE,
            scope=YOUTUBE_READONLY_SCOPE)

        storage = Storage("%s-oauth2.json" % sys.argv[0])
        credentials = storage.get()

        if credentials is None or credentials.invalid:
            credentials = run_flow(flow, storage, args)

        youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
            http=credentials.authorize(httplib2.Http()))

    return youtube

# Creates a request for the user's playlists using their channel ID
def create_id_request():
   request = youtube.playlists().list(
       part="id,snippet",
       fields="items(id,snippet/title),nextPageToken",
       channelId=opt.id,
       maxResults=MAX_RESULTS
   )

   return request

# Creates a request for the user's playlists using their username
# First uses a channel request to obtain channel ID from username
def create_username_request():
   channel_request = youtube.channels().list(
       part="id",
       forUsername=opt.username,
       maxResults=MAX_RESULTS
   )

   channel_response = channel_request.execute()

   try:
       request = youtube.playlists().list(
           part="id,snippet",
           fields="items(id,snippet/title),nextPageToken",
           channelId=channel_response["items"][0]["id"],
           maxResults=MAX_RESULTS
       )
   except IndexError:
       sys.exit("No channel found for {}".format(opt.username))

   return request

# Creates an authenticated request for accessing the user's private playlists
def create_private_request():
    request = youtube.playlists().list(
        part="id,snippet",
        fields="items(id,snippet/title),nextPageToken",
        mine="true",
        maxResults=MAX_RESULTS
    )

    return request

# Creates a channel request necessary for obtaining the channel's
# related playlists, via channel ID
def create_id_channel_request():
    channel_request = youtube.channels().list(
        part="contentDetails",
        fields="items(contentDetails/relatedPlaylists)",
        id=opt.id
    )

    return channel_request

# Creates a channel request necessary for obtaining the channel's
# related playlists, via username
def create_username_channel_request():
    channel_request = youtube.channels().list(
        part="contentDetails",
        fields="items(contentDetails/relatedPlaylists)",
        forUsername=opt.username
    )

    return channel_request

# Creates a channel request necessary for obtaining the channel's
# related playlists, via authentication
def create_private_channel_request():
    channel_request = youtube.channels().list(
        part="contentDetails",
        fields="items(contentDetails/relatedPlaylists)",
        mine="true"
    )

    return channel_request

# Create request for obtaining the user's related playlists
def create_related_request(channel_request):
    playlist_id_list = []

    channel_response = channel_request.execute()

    # Traverse channel_response to create list of related playlist IDs
    for channel in channel_response["items"]:
        for playlist, playlist_id in channel["contentDetails"]["relatedPlaylists"].items():
            playlist_id_list.append(playlist_id)

    request = youtube.playlists().list(
        part="id,snippet",
        fields="items(id,snippet/title),nextPageToken",
        id=",".join(playlist_id_list),
        maxResults=MAX_RESULTS
    )

    return request

# Backs-up playlists using the provided request
def backup_playlists(playlists_request):
    # Fetch pages of playlists until end
    while playlists_request:
        playlists_response = playlists_request.execute()

        for playlist in playlists_response["items"]:
            video_count = 0

            # Assemble request for videos in each playlist
            playlist_items_request = youtube.playlistItems().list(
                part="id,snippet",
                fields="items(id,snippet/title),nextPageToken",
                playlistId=playlist["id"],
                maxResults=MAX_RESULTS
            )

            print "{}".format(playlist["snippet"]["title"].encode("utf-8"))

            # Fetch pages of videos until end
            while playlist_items_request:
                playlist_items_response = playlist_items_request.execute()

                # Print videos in each playlist
                for i, video in enumerate(playlist_items_response["items"], start=1):
                    print "{}. {}".format((i + video_count), video["snippet"]["title"].encode("utf-8"))

                # Request next page of videos
                playlist_items_request = youtube.playlistItems().list_next(
                    playlist_items_request, playlist_items_response)

                if playlist_items_request is not None:
                    video_count += MAX_RESULTS
                else:
                    print

        # Request next page of playlists
        playlists_request = youtube.playlists().list_next(
            playlists_request, playlists_response)

if __name__ == "__main__":
    args = process_arguments()

    opt = Options()
    opt.process_options()

    try:
        youtube = create_resource_object()

        if (args.id or opt.id_config and args.username is None):
            req = create_id_request()
            ch_req = create_id_channel_request()
        elif (args.username or opt.username_config and args.id is None):
            req = create_username_request()
            ch_req = create_username_channel_request()
        else:
            req = create_private_request()
            ch_req = create_private_channel_request()

        backup_playlists(req)

        if (opt.related):
            req = create_related_request(ch_req)
            backup_playlists(req)

    except HttpError as e:
        print "An HTTP error %d occurred:\n%s" % (e.resp.status, e.content)

<|endoftext|>


<commit_before>#!/bin/env python
# -*- coding: utf8 -*-
from sqlibrist import main

if __name__ == '__main__':
    main()
<commit_msg>run script fix<commit_after>#!/usr/bin/env python
# -*- coding: utf8 -*-
from sqlibrist import main

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
from pkg_resources import load_entry_point
import sys
from squad.version import __version__
from squad.manage import main as manage

__usage__ = """usage: squad [OPTIONS]

Options:

  -h, --help            show this help message and exit
  -v, --version         show program's version number and exit

  ALL other options are passed as-is to gunicorn. See gunicorn(1), gunicorn3(1)
  `gunicorn --help`, or `gunicorn3 --help` for details.
"""

def usage():
    print(__usage__)

def main():
    gunicorn = load_entry_point('gunicorn', 'console_scripts', 'gunicorn')

    argv = sys.argv

    if '--help' in argv or '-h' in argv:
        usage()
        return

    if '--version' in argv or '-v' in argv:
        print('squad (version %s)' % __version__)
        return

    os.environ.setdefault("ENV", "production")

    sys.argv = ['squad-admin', 'migrate']
    manage()

    sys.argv = ['squad-admin', 'collectstatic', '--no-input', '-v', '0']
    manage()

    sys.argv = ['gunicorn', 'squad.wsgi'] + argv[1:]
    gunicorn()

if __name__ == "__main__":
    main()
<commit_msg>squad.run: exec an actual gunicorn process<commit_after>import os
from pkg_resources import load_entry_point
import sys
from squad.version import __version__
from squad.manage import main as manage

__usage__ = """usage: squad [OPTIONS]

Options:

  -h, --help            show this help message and exit
  -v, --version         show program's version number and exit

  ALL other options are passed as-is to gunicorn. See gunicorn(1), gunicorn3(1)
  `gunicorn --help`, or `gunicorn3 --help` for details.
"""

def usage():
    print(__usage__)

def main():
    argv = sys.argv

    if '--help' in argv or '-h' in argv:
        usage()
        return

    if '--version' in argv or '-v' in argv:
        print('squad (version %s)' % __version__)
        return

    os.environ.setdefault("ENV", "production")

    sys.argv = ['squad-admin', 'migrate']
    manage()

    sys.argv = ['squad-admin', 'collectstatic', '--no-input', '-v', '0']
    manage()

    os.system('sh -c "echo $PATH"')
    os.system('rm -f /tmp/foo')

    argv = ['python3', '-m', 'gunicorn.app.wsgiapp', 'squad.wsgi'] + argv[1:]
    os.execvp('python3', argv)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2018 The Open AI Team Authors and The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tokenization classes for OpenAI GPT."""

import json
import os
import re
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging
from ..bert.tokenization_bert import BasicTokenizer

logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "openai-gpt": 512,
}

def get_pairs(word):
    """
    Return set of symbol pairs in a word. word is represented as tuple of symbols (symbols being variable-length
    strings)
    """
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs

def text_standardize(text):
    """
    fixes some issues the spacy tokenizer had on books corpus also does some whitespace standardization
    """
    text = text.replace("—", "-")
    text = text.replace("–", "-")
    text = text.replace("―", "-")
    text = text.replace("…", "...")
    text = text.replace("´", "'")
    text = re.sub(r"""(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)""", r" \1 ", text)
    text = re.sub(r"\s*\n\s*", " \n ", text)
    text = re.sub(r"[^\S\n]+", " ", text)
    return text.strip()

class OpenAIGPTTokenizer(PreTrainedTokenizer):
    """
    Construct a GPT Tokenizer. Based on Byte-Pair-Encoding with the following peculiarities:

    - lowercases all inputs,
    - uses `SpaCy` tokenizer and `ftfy` for pre-BPE tokenization if they are installed, fallback to BERT's
      `BasicTokenizer` if not.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        merges_file (`str`):
            Path to the merges file.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
    """

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
        super().__init__(unk_token=unk_token, **kwargs)

        try:
            import ftfy
            from spacy.lang.en import English

            _nlp = English()
            self.nlp = _nlp.tokenizer
            self.fix_text = ftfy.fix_text
        except ImportError:
            logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.")
            self.nlp = BasicTokenizer(do_lower_case=True)
            self.fix_text = None

        with open(vocab_file, encoding="utf-8") as vocab_handle:
            self.encoder = json.load(vocab_handle)
        self.decoder = {v: k for k, v in self.encoder.items()}
        with open(merges_file, encoding="utf-8") as merges_handle:
            merges = merges_handle.read().split("\n")[1:-1]
        merges = [tuple(merge.split()) for merge in merges]
        self.bpe_ranks = dict(zip(merges, range(len(merges))))
        self.cache = {}

    @property
    def do_lower_case(self):
        return True

    @property
    def vocab_size(self):
        return len(self.encoder)

    def get_vocab(self):
        return dict(self.encoder, **self.added_tokens_encoder)

    def bpe(self, token):
        word = tuple(token[:-1]) + (token[-1] + "</w>",)
        if token in self.cache:
            return self.cache[token]
        pairs = get_pairs(word)

        if not pairs:
            return token + "</w>"

        while True:
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
                except ValueError:
                    new_word.extend(word[i:])
                    break
                else:
                    new_word.extend(word[i:j])
                    i = j

                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                    new_word.append(first + second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            else:
                pairs = get_pairs(word)
        word = " ".join(word)
        if word == "\n  </w>":
            word = "\n</w>"
        self.cache[token] = word
        return word

    def _tokenize(self, text):
        """Tokenize a string."""
        split_tokens = []
        if self.fix_text is None:
            # Using BERT's BasicTokenizer
            text = self.nlp.tokenize(text)
            for token in text:
                split_tokens.extend([t for t in self.bpe(token).split(" ")])
        else:
            # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
            text = self.nlp(text_standardize(self.fix_text(text)))
            for token in text:
                split_tokens.extend([t for t in self.bpe(token.text.lower()).split(" ")])
        return split_tokens

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self.encoder.get(token, self.encoder.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """Converts an id in a token (BPE) using the vocab."""
        return self.decoder.get(index, self.unk_token)

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        out_string = "".join(tokens).replace("</w>", " ").strip()
        return out_string

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )
        merge_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
        )

        with open(vocab_file, "w", encoding="utf-8") as f:
            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")

        index = 0
        with open(merge_file, "w", encoding="utf-8") as writer:
            writer.write("#version: 0.2\n")
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
                if index != token_index:
                    logger.warning(
                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                        " Please check that the tokenizer is not corrupted!"
                    )
                    index = token_index
                writer.write(" ".join(bpe_tokens) + "\n")
                index += 1

        return vocab_file, merge_file
<commit_msg>Remove interdependency from OpenAI tokenizer (#19327)<commit_after># coding=utf-8
# Copyright 2018 The Open AI Team Authors and The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tokenization classes for OpenAI GPT."""

import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging

logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "openai-gpt": 512,
}

# Copied from transformers.models.bert.tokenization_bert.whitespace_tokenize
def whitespace_tokenize(text):
    """Runs basic whitespace cleaning and splitting on a piece of text."""
    text = text.strip()
    if not text:
        return []
    tokens = text.split()
    return tokens

# Copied from transformers.models.bert.tokenization_bert.BasicTokenizer
class BasicTokenizer(object):
    """
    Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).

    Args:
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether or not to lowercase the input when tokenizing.
        never_split (`Iterable`, *optional*):
            Collection of tokens which will never be split during tokenization. Only has an effect when
            `do_basic_tokenize=True`
        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
            Whether or not to tokenize Chinese characters.

            This should likely be deactivated for Japanese (see this
            [issue](https://github.com/huggingface/transformers/issues/328)).
        strip_accents (`bool`, *optional*):
            Whether or not to strip all accents. If this option is not specified, then it will be determined by the
            value for `lowercase` (as in the original BERT).
    """

    def __init__(self, do_lower_case=True, never_split=None, tokenize_chinese_chars=True, strip_accents=None):
        if never_split is None:
            never_split = []
        self.do_lower_case = do_lower_case
        self.never_split = set(never_split)
        self.tokenize_chinese_chars = tokenize_chinese_chars
        self.strip_accents = strip_accents

    def tokenize(self, text, never_split=None):
        """
        Basic Tokenization of a piece of text. Split on "white spaces" only, for sub-word tokenization, see
        WordPieceTokenizer.

        Args:
            never_split (`List[str]`, *optional*)
                Kept for backward compatibility purposes. Now implemented directly at the base class level (see
                [`PreTrainedTokenizer.tokenize`]) List of token not to split.
        """
        # union() returns a new set by concatenating the two sets.
        never_split = self.never_split.union(set(never_split)) if never_split else self.never_split
        text = self._clean_text(text)

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        if self.tokenize_chinese_chars:
            text = self._tokenize_chinese_chars(text)
        orig_tokens = whitespace_tokenize(text)
        split_tokens = []
        for token in orig_tokens:
            if token not in never_split:
                if self.do_lower_case:
                    token = token.lower()
                    if self.strip_accents is not False:
                        token = self._run_strip_accents(token)
                elif self.strip_accents:
                    token = self._run_strip_accents(token)
            split_tokens.extend(self._run_split_on_punc(token, never_split))

        output_tokens = whitespace_tokenize(" ".join(split_tokens))
        return output_tokens

    def _run_strip_accents(self, text):
        """Strips accents from a piece of text."""
        text = unicodedata.normalize("NFD", text)
        output = []
        for char in text:
            cat = unicodedata.category(char)
            if cat == "Mn":
                continue
            output.append(char)
        return "".join(output)

    def _run_split_on_punc(self, text, never_split=None):
        """Splits punctuation on a piece of text."""
        if never_split is not None and text in never_split:
            return [text]
        chars = list(text)
        i = 0
        start_new_word = True
        output = []
        while i < len(chars):
            char = chars[i]
            if _is_punctuation(char):
                output.append([char])
                start_new_word = True
            else:
                if start_new_word:
                    output.append([])
                start_new_word = False
                output[-1].append(char)
            i += 1

        return ["".join(x) for x in output]

    def _tokenize_chinese_chars(self, text):
        """Adds whitespace around any CJK character."""
        output = []
        for char in text:
            cp = ord(char)
            if self._is_chinese_char(cp):
                output.append(" ")
                output.append(char)
                output.append(" ")
            else:
                output.append(char)
        return "".join(output)

    def _is_chinese_char(self, cp):
        """Checks whether CP is the codepoint of a CJK character."""
        # This defines a "chinese character" as anything in the CJK Unicode block:
        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
        #
        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
        # despite its name. The modern Korean Hangul alphabet is a different block,
        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
        # space-separated words, so they are not treated specially and handled
        # like the all of the other languages.
        if (
            (cp >= 0x4E00 and cp <= 0x9FFF)
            or (cp >= 0x3400 and cp <= 0x4DBF)  #
            or (cp >= 0x20000 and cp <= 0x2A6DF)  #
            or (cp >= 0x2A700 and cp <= 0x2B73F)  #
            or (cp >= 0x2B740 and cp <= 0x2B81F)  #
            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
            or (cp >= 0xF900 and cp <= 0xFAFF)
            or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
        ):  #
            return True

        return False

    def _clean_text(self, text):
        """Performs invalid character removal and whitespace cleanup on text."""
        output = []
        for char in text:
            cp = ord(char)
            if cp == 0 or cp == 0xFFFD or _is_control(char):
                continue
            if _is_whitespace(char):
                output.append(" ")
            else:
                output.append(char)
        return "".join(output)

def get_pairs(word):
    """
    Return set of symbol pairs in a word. word is represented as tuple of symbols (symbols being variable-length
    strings)
    """
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs

def text_standardize(text):
    """
    fixes some issues the spacy tokenizer had on books corpus also does some whitespace standardization
    """
    text = text.replace("—", "-")
    text = text.replace("–", "-")
    text = text.replace("―", "-")
    text = text.replace("…", "...")
    text = text.replace("´", "'")
    text = re.sub(r"""(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)""", r" \1 ", text)
    text = re.sub(r"\s*\n\s*", " \n ", text)
    text = re.sub(r"[^\S\n]+", " ", text)
    return text.strip()

class OpenAIGPTTokenizer(PreTrainedTokenizer):
    """
    Construct a GPT Tokenizer. Based on Byte-Pair-Encoding with the following peculiarities:

    - lowercases all inputs,
    - uses `SpaCy` tokenizer and `ftfy` for pre-BPE tokenization if they are installed, fallback to BERT's
      `BasicTokenizer` if not.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        merges_file (`str`):
            Path to the merges file.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
    """

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
        super().__init__(unk_token=unk_token, **kwargs)

        try:
            import ftfy
            from spacy.lang.en import English

            _nlp = English()
            self.nlp = _nlp.tokenizer
            self.fix_text = ftfy.fix_text
        except ImportError:
            logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.")
            self.nlp = BasicTokenizer(do_lower_case=True)
            self.fix_text = None

        with open(vocab_file, encoding="utf-8") as vocab_handle:
            self.encoder = json.load(vocab_handle)
        self.decoder = {v: k for k, v in self.encoder.items()}
        with open(merges_file, encoding="utf-8") as merges_handle:
            merges = merges_handle.read().split("\n")[1:-1]
        merges = [tuple(merge.split()) for merge in merges]
        self.bpe_ranks = dict(zip(merges, range(len(merges))))
        self.cache = {}

    @property
    def do_lower_case(self):
        return True

    @property
    def vocab_size(self):
        return len(self.encoder)

    def get_vocab(self):
        return dict(self.encoder, **self.added_tokens_encoder)

    def bpe(self, token):
        word = tuple(token[:-1]) + (token[-1] + "</w>",)
        if token in self.cache:
            return self.cache[token]
        pairs = get_pairs(word)

        if not pairs:
            return token + "</w>"

        while True:
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
                except ValueError:
                    new_word.extend(word[i:])
                    break
                else:
                    new_word.extend(word[i:j])
                    i = j

                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                    new_word.append(first + second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            else:
                pairs = get_pairs(word)
        word = " ".join(word)
        if word == "\n  </w>":
            word = "\n</w>"
        self.cache[token] = word
        return word

    def _tokenize(self, text):
        """Tokenize a string."""
        split_tokens = []
        if self.fix_text is None:
            # Using BERT's BasicTokenizer
            text = self.nlp.tokenize(text)
            for token in text:
                split_tokens.extend([t for t in self.bpe(token).split(" ")])
        else:
            # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
            text = self.nlp(text_standardize(self.fix_text(text)))
            for token in text:
                split_tokens.extend([t for t in self.bpe(token.text.lower()).split(" ")])
        return split_tokens

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self.encoder.get(token, self.encoder.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """Converts an id in a token (BPE) using the vocab."""
        return self.decoder.get(index, self.unk_token)

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        out_string = "".join(tokens).replace("</w>", " ").strip()
        return out_string

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )
        merge_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
        )

        with open(vocab_file, "w", encoding="utf-8") as f:
            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")

        index = 0
        with open(merge_file, "w", encoding="utf-8") as writer:
            writer.write("#version: 0.2\n")
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
                if index != token_index:
                    logger.warning(
                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                        " Please check that the tokenizer is not corrupted!"
                    )
                    index = token_index
                writer.write(" ".join(bpe_tokens) + "\n")
                index += 1

        return vocab_file, merge_file
<|endoftext|>


<commit_before>import asyncio
import discord
import time
from   operator import itemgetter
from   discord.ext import commands
from   Cogs import Settings
from   Cogs import ReadableTime
from   Cogs import DisplayName
from   Cogs import Nullify

# This is the profiles module.

class Profile:

	# Init with the bot reference, and a reference to the settings var
	def __init__(self, bot, settings):
		self.bot = bot
		self.settings = settings

	@commands.command(pass_context=True)
	async def addprofile(self, ctx, name : str = None, *, link : str = None):
		"""Add a profile to your profile list."""

		# Check if we're suppressing @here and @everyone mentions
		if self.settings.getServerStat(ctx.message.guild, "SuppressMentions").lower() == "yes":
			suppress = True
		else:
			suppress = False

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.guild

		'''# Check for role requirements
		requiredRole = self.settings.getServerStat(server, "RequiredXPRole")
		if requiredRole == "":
			#admin only
			isAdmin = author.permissions_in(channel).administrator
			if not isAdmin:
				await channel.send('You do not have sufficient privileges to access this command.')
				return
		else:
			#role requirement
			hasPerms = False
			for role in author.roles:
				if str(role.id) == str(requiredRole):
					hasPerms = True
			if not hasPerms:
				await channel.send('You do not have sufficient privileges to access this command.')
				return'''

		# Passed role requirements!
		if name == None or link == None:
			msg = 'Usage: `{}addprofile "[profile name]" [link]`'.format(ctx.prefix)
			await channel.send(msg)
			return

		linkList = self.settings.getUserStat(author, server, "Profiles")
		if not linkList:
			linkList = []

		found = False
		currentTime = int(time.time())
		for alink in linkList:
			if alink['Name'].lower() == name.lower():
				# The link exists!
				msg = '*{}\'s* *{}* profile was updated!'.format(DisplayName.name(author), name)
				# Check for suppress
				if suppress:
					msg = Nullify.clean(msg)
				alink['URL'] = link
				alink['Updated'] = currentTime
				found = True
		if not found:
			linkList.append({"Name" : name, "URL" : link, "Created" : currentTime})
			msg = '*{}* added to *{}\'s* profile list!'.format(name, DisplayName.name(author))
			# Check for suppress
			if suppress:
				msg = Nullify.clean(msg)

		self.settings.setUserStat(author, server, "Profiles", linkList)
		await channel.send(msg)

	@commands.command(pass_context=True)
	async def removeprofile(self, ctx, *, name : str = None):
		"""Remove a profile from your profile list."""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.guild

		# Check if we're suppressing @here and @everyone mentions
		if self.settings.getServerStat(ctx.message.guild, "SuppressMentions").lower() == "yes":
			suppress = True
		else:
			suppress = False

		# Why did I do this?  There shouldn't be role requirements for your own profiles...
		'''# Check for role requirements
		requiredRole = self.settings.getServerStat(server, "RequiredXPRole")
		if requiredRole == "":
			#admin only
			isAdmin = author.permissions_in(channel).administrator
			if not isAdmin:
				await channel.send('You do not have sufficient privileges to access this command.')
				return
		else:
			#role requirement
			hasPerms = False
			for role in author.roles:
				if str(role.id) == str(requiredRole):
					hasPerms = True
			if not hasPerms:
				await channel.send('You do not have sufficient privileges to access this command.')
				return'''

		if name == None:
			msg = 'Usage: `{}removeprofile "[profile name]"`'.format(ctx.prefix)
			await channel.send(msg)
			return

		linkList = self.settings.getUserStat(author, server, "Profiles")
		if not linkList or linkList == []:
			msg = '*{}* has no profiles set!  They can add some with the `{}addprofile "[profile name]" [url]` command!'.format(DisplayName.name(author), ctx.prefix)
			await channel.send(msg)
			return

		for alink in linkList:
			if alink['Name'].lower() == name.lower():
				linkList.remove(alink)
				self.settings.setUserStat(author, server, "Profiles", linkList)
				msg = '*{}* removed from *{}\'s* profile list!'.format(alink['Name'], DisplayName.name(author))
				# Check for suppress
				if suppress:
					msg = Nullify.clean(msg)
				await channel.send(msg)
				return

		msg = '*{}* not found in *{}\'s* profile list!'.format(name, DisplayName.name(author))
		# Check for suppress
		if suppress:
			msg = Nullify.clean(msg)
		await channel.send(msg)

	@commands.command(pass_context=True)
	async def profile(self, ctx, *, member : str = None, name : str = None):
		"""Retrieve a profile from the passed user's profile list."""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.guild

		# Check if we're suppressing @here and @everyone mentions
		if self.settings.getServerStat(ctx.message.guild, "SuppressMentions").lower() == "yes":
			suppress = True
		else:
			suppress = False

		if not member:
			msg = 'Usage: `{}profile [member] [profile name]`'.format(ctx.prefix)
			await channel.send(msg)
			return

		# name is likely to be empty unless quotes are used
		if name == None:
			# Either a name wasn't set - or it's the last section
			if type(member) is str:
				# It' a string - the hope continues
				# Let's search for a name at the beginning - and a profile at the end
				parts = member.split()
				for j in range(len(parts)):
					# Reverse search direction
					i = len(parts)-1-j
					memFromName = None
					foundProf   = False
					# Name = 0 up to i joined by space
					nameStr    = ' '.join(parts[0:i+1])
					# Profile = end of name -> end of parts joined by space
					profileStr = ' '.join(parts[i+1:])
					memFromName = DisplayName.memberForName(nameStr, ctx.message.guild)
					if memFromName:
						# We got a member - let's check for a profile
						linkList = self.settings.getUserStat(memFromName, server, "Profiles")
						if not linkList or linkList == []:
							pass

						for alink in linkList:
							if alink['Name'].lower() == profileStr.lower():
								# Found the link - return it.
								msg = '*{}\'s {} Profile:*\n\n{}'.format(DisplayName.name(memFromName), alink['Name'], alink['URL'])
								# Check for suppress
								if suppress:
									msg = Nullify.clean(msg)
								await channel.send(msg)
								return
		# Check if there is no member specified
		linkList = self.settings.getUserStat(author, server, "Profiles")
		if not linkList or linkList == []:
			pass

		for alink in linkList:
			if alink['Name'].lower() == member.lower():
				# Found the link - return it.
				msg = '*{}\'s {} Profile:*\n\n{}'.format(DisplayName.name(author), alink['Name'], alink['URL'])
				# Check for suppress
				if suppress:
					msg = Nullify.clean(msg)
				await channel.send(msg)
				return

		# If we got this far - we didn't find them or somehow they added a name
		msg = 'Sorry, I couldn\'t find that user/profile.'
		await channel.send(msg)

	@commands.command(pass_context=True)
	async def profileinfo(self, ctx, *, member : str = None, name : str = None):
		"""Displays info about a profile from the passed user's profile list."""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.guild

		# Check if we're suppressing @here and @everyone mentions
		if self.settings.getServerStat(ctx.message.guild, "SuppressMentions").lower() == "yes":
			suppress = True
		else:
			suppress = False

		if not member:
			msg = 'Usage: `{}profileinfo [member] [profile name]`'.format(ctx.prefix)
			await channel.send(msg)
			return

		profile = None

		# name is likely to be empty unless quotes are used
		if name == None:
			# Either a name wasn't set - or it's the last section
			if type(member) is str:
				# It' a string - the hope continues
				# Let's search for a name at the beginning - and a profile at the end
				parts = member.split()
				for j in range(len(parts)):
					# Reverse search direction
					i = len(parts)-1-j
					memFromName = None
					foundProf   = False
					# Name = 0 up to i joined by space
					nameStr    = ' '.join(parts[0:i+1])
					# Profile = end of name -> end of parts joined by space
					profileStr = ' '.join(parts[i+1:])
					memFromName = DisplayName.memberForName(nameStr, ctx.message.guild)
					if memFromName:
						# We got a member - let's check for a profile
						linkList = self.settings.getUserStat(memFromName, server, "Profiles")
						if not linkList or linkList == []:
							pass

						for alink in linkList:
							if alink['Name'].lower() == profileStr.lower():
								# Found the link - return it.
								profile = alink
								break

		if not profile:
			# Check if there is no member specified
			linkList = self.settings.getUserStat(author, server, "Profiles")
			if not linkList or linkList == []:
				pass

			for alink in linkList:
				if alink['Name'].lower() == member.lower():
					# Found the link - return it.
					profile = alink

		if not profile:
			# At this point - we've exhausted our search
			msg = 'Sorry, I couldn\'t find that user/profile.'
			await channel.send(msg)
			return

		# We have a profile
		currentTime = int(time.time())
		msg = '**{}:**'.format(profile['Name'])
		try:
			createdTime = int(profile['Created'])
			timeString  = ReadableTime.getReadableTimeBetween(createdTime, currentTime)
			msg = '{}\nCreated : *{}* ago'.format(msg, timeString)
		except KeyError as e:
			msg = '{}\nCreated : `UNKNOWN`'.format(msg)
		try:
			createdTime = profile['Updated']
			createdTime = int(createdTime)
			timeString  = ReadableTime.getReadableTimeBetween(createdTime, currentTime)
			msg = '{}\nUpdated : *{}* ago'.format(msg, timeString)
		except:
			pass
		# Check for suppress
		if suppress:
			msg = Nullify.clean(msg)
		await channel.send(msg)
		return

	@commands.command(pass_context=True)
	async def profiles(self, ctx, *, member : str = None):
		"""List all profiles in the passed user's profile list."""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.guild

		# Check if we're suppressing @here and @everyone mentions
		if self.settings.getServerStat(ctx.message.guild, "SuppressMentions").lower() == "yes":
			suppress = True
		else:
			suppress = False

		if not member:
			member = author
		else:
			newMember = DisplayName.memberForName(member, server)
			if not newMember:
				# no member found by that name
				msg = 'I couldn\'t find *{}* on this server.'.format(member)
				# Check for suppress
				if suppress:
					msg = Nullify.clean(msg)
				await channel.send(msg)
				return
			else:
				member = newMember

		# We have a member here

		linkList = self.settings.getUserStat(member, server, "Profiles")
		if linkList == None or linkList == []:
			msg = '*{}* hasn\'t added any profiles yet!  They can do so with the `{}addprofile "[profile name]" [url]` command!'.format(DisplayName.name(member), ctx.prefix)
			await channel.send(msg)
			return

		# Sort by link name
		linkList = sorted(linkList, key=itemgetter('Name'))
		linkText = "*{}'s* Profiles:\n\n".format(DisplayName.name(member))
		for alink in linkList:
			linkText = '{}*{}*, '.format(linkText, alink['Name'])

		# Check for suppress
		if suppress:
			linkText = Nullify.clean(linkText)

		# Speak the link list while cutting off the end ", "
		await channel.send(linkText[:-2])<commit_msg>Add message overflow handling - just in case<commit_after>import asyncio
import discord
import time
from   operator import itemgetter
from   discord.ext import commands
from   Cogs import Settings
from   Cogs import ReadableTime
from   Cogs import DisplayName
from   Cogs import Nullify
from   Cogs import Message

# This is the profiles module.

class Profile:

	# Init with the bot reference, and a reference to the settings var
	def __init__(self, bot, settings):
		self.bot = bot
		self.settings = settings

	@commands.command(pass_context=True)
	async def addprofile(self, ctx, name : str = None, *, link : str = None):
		"""Add a profile to your profile list."""

		# Check if we're suppressing @here and @everyone mentions
		if self.settings.getServerStat(ctx.message.guild, "SuppressMentions").lower() == "yes":
			suppress = True
		else:
			suppress = False

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.guild

		'''# Check for role requirements
		requiredRole = self.settings.getServerStat(server, "RequiredXPRole")
		if requiredRole == "":
			#admin only
			isAdmin = author.permissions_in(channel).administrator
			if not isAdmin:
				await channel.send('You do not have sufficient privileges to access this command.')
				return
		else:
			#role requirement
			hasPerms = False
			for role in author.roles:
				if str(role.id) == str(requiredRole):
					hasPerms = True
			if not hasPerms:
				await channel.send('You do not have sufficient privileges to access this command.')
				return'''

		# Passed role requirements!
		if name == None or link == None:
			msg = 'Usage: `{}addprofile "[profile name]" [link]`'.format(ctx.prefix)
			await channel.send(msg)
			return

		linkList = self.settings.getUserStat(author, server, "Profiles")
		if not linkList:
			linkList = []

		found = False
		currentTime = int(time.time())
		for alink in linkList:
			if alink['Name'].lower() == name.lower():
				# The link exists!
				msg = '*{}\'s* *{}* profile was updated!'.format(DisplayName.name(author), name)
				# Check for suppress
				if suppress:
					msg = Nullify.clean(msg)
				alink['URL'] = link
				alink['Updated'] = currentTime
				found = True
		if not found:
			linkList.append({"Name" : name, "URL" : link, "Created" : currentTime})
			msg = '*{}* added to *{}\'s* profile list!'.format(name, DisplayName.name(author))
			# Check for suppress
			if suppress:
				msg = Nullify.clean(msg)

		self.settings.setUserStat(author, server, "Profiles", linkList)
		await channel.send(msg)

	@commands.command(pass_context=True)
	async def removeprofile(self, ctx, *, name : str = None):
		"""Remove a profile from your profile list."""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.guild

		# Check if we're suppressing @here and @everyone mentions
		if self.settings.getServerStat(ctx.message.guild, "SuppressMentions").lower() == "yes":
			suppress = True
		else:
			suppress = False

		# Why did I do this?  There shouldn't be role requirements for your own profiles...
		'''# Check for role requirements
		requiredRole = self.settings.getServerStat(server, "RequiredXPRole")
		if requiredRole == "":
			#admin only
			isAdmin = author.permissions_in(channel).administrator
			if not isAdmin:
				await channel.send('You do not have sufficient privileges to access this command.')
				return
		else:
			#role requirement
			hasPerms = False
			for role in author.roles:
				if str(role.id) == str(requiredRole):
					hasPerms = True
			if not hasPerms:
				await channel.send('You do not have sufficient privileges to access this command.')
				return'''

		if name == None:
			msg = 'Usage: `{}removeprofile "[profile name]"`'.format(ctx.prefix)
			await channel.send(msg)
			return

		linkList = self.settings.getUserStat(author, server, "Profiles")
		if not linkList or linkList == []:
			msg = '*{}* has no profiles set!  They can add some with the `{}addprofile "[profile name]" [url]` command!'.format(DisplayName.name(author), ctx.prefix)
			await channel.send(msg)
			return

		for alink in linkList:
			if alink['Name'].lower() == name.lower():
				linkList.remove(alink)
				self.settings.setUserStat(author, server, "Profiles", linkList)
				msg = '*{}* removed from *{}\'s* profile list!'.format(alink['Name'], DisplayName.name(author))
				# Check for suppress
				if suppress:
					msg = Nullify.clean(msg)
				await channel.send(msg)
				return

		msg = '*{}* not found in *{}\'s* profile list!'.format(name, DisplayName.name(author))
		# Check for suppress
		if suppress:
			msg = Nullify.clean(msg)
		await channel.send(msg)

	@commands.command(pass_context=True)
	async def profile(self, ctx, *, member : str = None, name : str = None):
		"""Retrieve a profile from the passed user's profile list."""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.guild

		# Check if we're suppressing @here and @everyone mentions
		if self.settings.getServerStat(ctx.message.guild, "SuppressMentions").lower() == "yes":
			suppress = True
		else:
			suppress = False

		if not member:
			msg = 'Usage: `{}profile [member] [profile name]`'.format(ctx.prefix)
			await channel.send(msg)
			return

		# name is likely to be empty unless quotes are used
		if name == None:
			# Either a name wasn't set - or it's the last section
			if type(member) is str:
				# It' a string - the hope continues
				# Let's search for a name at the beginning - and a profile at the end
				parts = member.split()
				for j in range(len(parts)):
					# Reverse search direction
					i = len(parts)-1-j
					memFromName = None
					foundProf   = False
					# Name = 0 up to i joined by space
					nameStr    = ' '.join(parts[0:i+1])
					# Profile = end of name -> end of parts joined by space
					profileStr = ' '.join(parts[i+1:])
					memFromName = DisplayName.memberForName(nameStr, ctx.message.guild)
					if memFromName:
						# We got a member - let's check for a profile
						linkList = self.settings.getUserStat(memFromName, server, "Profiles")
						if not linkList or linkList == []:
							pass

						for alink in linkList:
							if alink['Name'].lower() == profileStr.lower():
								# Found the link - return it.
								msg = '*{}\'s {} Profile:*\n\n{}'.format(DisplayName.name(memFromName), alink['Name'], alink['URL'])
								# Check for suppress
								if suppress:
									msg = Nullify.clean(msg)
								await channel.send(msg)
								return
		# Check if there is no member specified
		linkList = self.settings.getUserStat(author, server, "Profiles")
		if not linkList or linkList == []:
			pass

		for alink in linkList:
			if alink['Name'].lower() == member.lower():
				# Found the link - return it.
				msg = '*{}\'s {} Profile:*\n\n{}'.format(DisplayName.name(author), alink['Name'], alink['URL'])
				# Check for suppress
				if suppress:
					msg = Nullify.clean(msg)
				await channel.send(msg)
				return

		# If we got this far - we didn't find them or somehow they added a name
		msg = 'Sorry, I couldn\'t find that user/profile.'
		await channel.send(msg)

	@commands.command(pass_context=True)
	async def profileinfo(self, ctx, *, member : str = None, name : str = None):
		"""Displays info about a profile from the passed user's profile list."""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.guild

		# Check if we're suppressing @here and @everyone mentions
		if self.settings.getServerStat(ctx.message.guild, "SuppressMentions").lower() == "yes":
			suppress = True
		else:
			suppress = False

		if not member:
			msg = 'Usage: `{}profileinfo [member] [profile name]`'.format(ctx.prefix)
			await channel.send(msg)
			return

		profile = None

		# name is likely to be empty unless quotes are used
		if name == None:
			# Either a name wasn't set - or it's the last section
			if type(member) is str:
				# It' a string - the hope continues
				# Let's search for a name at the beginning - and a profile at the end
				parts = member.split()
				for j in range(len(parts)):
					# Reverse search direction
					i = len(parts)-1-j
					memFromName = None
					foundProf   = False
					# Name = 0 up to i joined by space
					nameStr    = ' '.join(parts[0:i+1])
					# Profile = end of name -> end of parts joined by space
					profileStr = ' '.join(parts[i+1:])
					memFromName = DisplayName.memberForName(nameStr, ctx.message.guild)
					if memFromName:
						# We got a member - let's check for a profile
						linkList = self.settings.getUserStat(memFromName, server, "Profiles")
						if not linkList or linkList == []:
							pass

						for alink in linkList:
							if alink['Name'].lower() == profileStr.lower():
								# Found the link - return it.
								profile = alink
								break

		if not profile:
			# Check if there is no member specified
			linkList = self.settings.getUserStat(author, server, "Profiles")
			if not linkList or linkList == []:
				pass

			for alink in linkList:
				if alink['Name'].lower() == member.lower():
					# Found the link - return it.
					profile = alink

		if not profile:
			# At this point - we've exhausted our search
			msg = 'Sorry, I couldn\'t find that user/profile.'
			await channel.send(msg)
			return

		# We have a profile
		currentTime = int(time.time())
		msg = '**{}:**'.format(profile['Name'])
		try:
			createdTime = int(profile['Created'])
			timeString  = ReadableTime.getReadableTimeBetween(createdTime, currentTime)
			msg = '{}\nCreated : *{}* ago'.format(msg, timeString)
		except KeyError as e:
			msg = '{}\nCreated : `UNKNOWN`'.format(msg)
		try:
			createdTime = profile['Updated']
			createdTime = int(createdTime)
			timeString  = ReadableTime.getReadableTimeBetween(createdTime, currentTime)
			msg = '{}\nUpdated : *{}* ago'.format(msg, timeString)
		except:
			pass
		# Check for suppress
		if suppress:
			msg = Nullify.clean(msg)
		await channel.send(msg)
		return

	@commands.command(pass_context=True)
	async def profiles(self, ctx, *, member : str = None):
		"""List all profiles in the passed user's profile list."""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.guild

		# Check if we're suppressing @here and @everyone mentions
		if self.settings.getServerStat(ctx.message.guild, "SuppressMentions").lower() == "yes":
			suppress = True
		else:
			suppress = False

		if not member:
			member = author
		else:
			newMember = DisplayName.memberForName(member, server)
			if not newMember:
				# no member found by that name
				msg = 'I couldn\'t find *{}* on this server.'.format(member)
				# Check for suppress
				if suppress:
					msg = Nullify.clean(msg)
				await channel.send(msg)
				return
			else:
				member = newMember

		# We have a member here

		linkList = self.settings.getUserStat(member, server, "Profiles")
		if linkList == None or linkList == []:
			msg = '*{}* hasn\'t added any profiles yet!  They can do so with the `{}addprofile "[profile name]" [url]` command!'.format(DisplayName.name(member), ctx.prefix)
			await channel.send(msg)
			return

		# Sort by link name
		linkList = sorted(linkList, key=itemgetter('Name'))
		linkText = "*{}'s* Profiles:\n\n".format(DisplayName.name(member))
		for alink in linkList:
			linkText = '{}*{}*, '.format(linkText, alink['Name'])

		# Check for suppress
		if suppress:
			linkText = Nullify.clean(linkText)

		# Speak the link list while cutting off the end ", "
		# await channel.send(linkText[:-2])
		await Message.say(self.bot, linkText[:-2], ctx.channel, ctx.author, 1)<|endoftext|>


<commit_before>import curses

from random import randrange, choice

from collections import defaultdict

actions = ['Up', 'Left', 'Down', 'Right', 'Restart', 'Exit']

letter_codes = [ord(ch) for ch in 'WASDRQwasdrq']

actions_dict = dict((zip(letter_codes, actions*2)))

def get_user_action(keyboard):
    char = "N"
    while char not in actions_dict:
       char = keyboard.getch()
    return actions_dict[char]

def transpose(field):
    return [list(row) for row in zip(*field)]

def invert(field):
    return [row[::-1] for row in field]

class GameField(object):
    def __init__(self, height=4, width=4, win=2048):
        self.height = height
        self.width = width
        self.win_value = 2048
        self.score = 0
        self.highscore = 0
        self.reset()

    def spawn(self):
        if randrange(100)>89:
            new_element = 4
        else:
            new_element = 2
        (i,j) = choice([(i,j) for i in range(self.width) for j in range(self.height) if self.field[i][j] == 0])
        self.field[i][j] = new_element

<commit_msg>testing<commit_after>import curses

from random import randrange, choice

from collections import defaultdict

actions = ['Up', 'Left', 'Down', 'Right', 'Restart', 'Exit']

letter_codes = [ord(ch) for ch in 'WASDRQwasdrq']

actions_dict = dict((zip(letter_codes, actions*2)))

def move_row_left(row):
    def tighten(row):
        new_row = [i for i in row if i != 0]
        new_row += [0 for i in range(len(row) - len(new_row))]
        return new_row

    def merge(row):
        pair = False
        new_row = []
        for i in range(len(row)):
            if pair:
                new_row.append(2 * row[i])
                self.score += 2 * row[i]
                pair = False
            else:
                if i + 1 < len(row) and row[i] == row[i + 1]:
                    pair = True
                    new_row.append(0)
                else:
                    new_row.append(row[i])
        assert len(new_row) == len(row)
        return new_row

    return tighten(merge(tighten(row)))

def get_user_action(keyboard):
    char = "N"
    while char not in actions_dict:
       char = keyboard.getch()
    return actions_dict[char]

def transpose(field):
    return [list(row) for row in zip(*field)]

def invert(field):
    return [row[::-1] for row in field]

class GameField(object):
    def __init__(self, height=4, width=4, win=2048):
        self.height = height
        self.width = width
        self.win_value = 2048
        self.score = 0
        self.highscore = 0
        self.reset()

    def spawn(self):
        if randrange(100)>89:
            new_element = 4
        else:
            new_element = 2
        (i,j) = choice([(i,j) for i in range(self.width) for j in range(self.height) if self.field[i][j] == 0])
        self.field[i][j] = new_element

    def reset(self):
        if self.score > self.highscore:
            self.highscore = self.score
        self.score = 0
        self.field = [[0 for i in range(self.width)] for j in range(self.height)]
        self.spawn()

<|endoftext|>