<commit_before>import httpx,os,re

class TurkAnime:

    url="http://www.turkanime.tv/arama"
    cookies = {
        '__cfduid': 'deb9e04ac510c1b707412b1fd7daadec91564216182',
        'yew490': '1',
        '_ga': 'GA1.2.284686093.1564216182',
        '_gid': 'GA1.2.1256976049.1564216182',
        '__PPU_SESSION_1_1683592_false': '1564216202929|1|1564216202929|1|1',
        '_gat': '1',
    }

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:68.0) Gecko/20100101 Firefox/68.0',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'tr-TR,tr;q=0.8,en-US;q=0.5,en;q=0.3',
        'Content-Type': 'application/x-www-form-urlencoded',
        'Alt-Used': 'www.turkanime.tv:443',
        'Connection': 'keep-alive',
        'Referer': 'http://www.turkanime.tv/',
        'Upgrade-Insecure-Requests': '1',
        'TE': 'Trailers',
    }

    def __init__(self):
        pass
    def anime_ara(self,ara):

        data = {
        'arama': ara
        }
        veri=httpx.post(self.url, headers=self.headers, cookies=self.cookies, data=data).content.decode("utf8")
        liste=[]
        r=re.findall('

(.*?)<\/a>',veri)
        for slug,_,title in r:
            liste.append([title,slug])
        return liste
if __name__=="__main__":
    t=TurkAnime()
    t.anime_ara("naruto")
<commit_msg>Bölüm listesi eklendi<commit_after>import httpx,os,re

class TurkAnime:

    url="http://www.turkanime.tv/"
    cookies = {
        '__cfduid': 'deb9e04ac510c1b707412b1fd7daadec91564216182',
        'yew490': '1',
        '_ga': 'GA1.2.284686093.1564216182',
        '_gid': 'GA1.2.1256976049.1564216182',
        '__PPU_SESSION_1_1683592_false': '1564216202929|1|1564216202929|1|1',
        '_gat': '1',
    }

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:68.0) Gecko/20100101 Firefox/68.0',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'tr-TR,tr;q=0.8,en-US;q=0.5,en;q=0.3',
        'Content-Type': 'application/x-www-form-urlencoded',
        'Alt-Used': 'www.turkanime.tv:443',
        'Connection': 'keep-alive',
        'Referer': 'http://www.turkanime.tv/',
        'Upgrade-Insecure-Requests': '1',
        'TE': 'Trailers',
    }

    def __init__(self):
        pass
    def anime_ara(self,ara):

        data = {
        'arama': ara
        }
        veri=httpx.post(self.url+"/arama", headers=self.headers, cookies=self.cookies, data=data).content.decode("utf8")
        liste=[]
        r=re.findall('

(.*?)<\/a>',veri)
        for slug,_,title in r:
            liste.append([title,slug])
        return liste
    def bolumler(self,slug):
        veri=httpx.get(self.url+"/anime/"+slug, headers=self.headers, cookies=self.cookies).content.decode("utf8")
        h=self.headers.copy()
        h.update({"X-Requested-With":"XMLHttpRequest","Accept":"*/*"})
        animeId=veri.split("ajax/bolumler&animeId=")[1].split('"')[0]
        liste=[]
        a=httpx.get(f"http://www.turkanime.tv/ajax/bolumler&animeId={animeId}",headers=h,cookies=self.cookies).content.decode("utf8")
        r=re.findall('(.*?)<\/span><\/a>',a)
        for slug,_,title in r:
            liste.append([title,slug])
        return liste
if __name__=="__main__":
    t=TurkAnime()
    print(t.bolumler("naruto"))
<|endoftext|>


<commit_before># coding: utf-8

import os
import time
import json
import uuid
import webob
import base64
import httplib
from collections import namedtuple

class Error(Exception):
    status_code = 0
    reason = ''

class NotFoundError(Error):
    status_code = httplib.NOT_FOUND
    reason = 'Not Found'

class NotImplementedError(Error):
    status_code = httplib.NOT_IMPLEMENTED
    reason = 'Feature Not Implemented'

class MethodNotAllowedError(Error):
    status_code = httplib.METHOD_NOT_ALLOWED
    reason = 'Method Not Allowed'

class MissingVersionError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Missing Tus-Resumable Header'

class UnsupportedVersionError(Error):
    status_code = httplib.PRECONDITION_FAILED
    reason = 'Precondition Failed'

class MissingUidError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Missing Uid Part In Url'

class MaxSizeExceededError(Error):
    status_code = httplib.REQUEST_ENTITY_TOO_LARGE
    reason = 'Request Entity Too Large'

class InvalidUploadLengthError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Invalid Upload-Length Header'

class ConflictUploadLengthError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Conflict Upload-Length'

class MissingUploadLengthError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Missing Upload-Length Header'

class InvalidUploadOffsetError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Invalid Upload-Offset Header'

class ConflictUploadOffsetError(Error):
    status_code = httplib.CONFLICT
    reason = 'Conflict'

class MissingUploadOffsetError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Missing Upload-Offset Header'

class InvalidUploadMetadataError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Invalid Upload-Metadata Header'

class InvalidContentTypeError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Invalid Content-Type Header'

class FileLockedError(Error):
    status_code = httplib.LOCKED
    reason = 'File Currently Locked'

class UploadNotFinishedError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'One Of The Partial Uploads Is Not Finished'

class InvalidConcatError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Invalid Upload-Concat Header'

class ModifyFinalError(Error):
    status_code = httplib.FORBIDDEN
    reason = 'Modifying A Final Upload Is Not Allowed'

Env = namedtuple('Env', ['req', 'resp', 'values'])

class TusFilter(object):

    versions = ['1.0.0']
    max_size = 2 ** 30   # 1G
    checksum_algorisum = ['sha1']
    extensions = [
        'creation',
        'expiration',
        'termination',
        # 'checksum',                  # todo
        # 'creation-defer-length',     # todo
        # 'checksum-trailer',          # todo
        # 'concatenation',             # todo
        # 'concatenation-unfinished',  # todo
    ]

    def __init__(self, app, upload_path, tmp_dir='/tmp/upload', expire=60*60*60, send_file=False):
        self.app = app
        self.tmp_dir = tmp_dir
        self.upload_path = upload_path
        self.expire = expire
        self.send_file = send_file

        if not os.path.exists(tmp_dir):
            os.makedirs(tmp_dir)

    def __call__(self, environ, start_response):
        req = webob.Request(environ)
        resp = webob.Response()
        values = dict(upload_finished=False)
        env = Env(req=req, resp=resp, values=values)
        if not req.path.startswith(self.upload_path):
            return self.app(environ, start_response)
        try:
            self.handle(env)
        except Error as e:
            self.finish_error(env, e)

        if not env.values['upload_finished']:
            return resp(environ, start_response)

        resp = req.get_response(self.app)
        self.delete_info_file(env)
        return resp(environ, start_response)

    def handle(self, env):
        try:
            x_method = env.req.headers.get('X-HTTP-Method-Override')
        except AttributeError:
            x_method = None
        method = (x_method or env.req.method).upper()

        path_parts = env.req.path.split('/', 2)
        uid = path_parts[2] if 2 < len(path_parts) else None
        if method not in ['POST', 'OPTIONS'] and not uid:
            raise MissingUidError()
        env.values['uid'] = uid

        version = env.req.headers.get('Tus-Resumable')
        if method != 'OPTIONS':
            if not version:
                raise MissingVersionError()
            if version not in self.versions:
                raise UnsupportedVersionError()
        version = version or self.versions[0]   # OPTIONS version maybe None
        env.values['version'] = version        # for multi versions
        env.resp.headers['Tus-Resumable'] = version

        if method == 'POST':
            self.post(env)
        elif method == 'HEAD':
            self.head(env)
        elif method == 'PATCH':
            self.patch(env)
        elif method == 'OPTIONS':
            self.options(env)
        elif method == 'DELETE':
            self.delete(env)
        else:
            raise MethodNotAllowedError()

    def options(self, env):
        env.resp.headers['Tus-Version'] = ','.join(self.versions)
        env.resp.headers['Tus-Max-Size'] = str(self.max_size)
        # todo: waitting for checksum extension
        # self.resp.headers['Tus-Checksum-Algorithm'] = ','.join(self.checksum_algorisum)
        env.resp.headers['Tus-Extension'] = ','.join(self.extensions)
        env.resp.status = httplib.NO_CONTENT

    def post(self, env):
        upload_length = env.req.headers.get('Upload-Length')
        # todo: waitting for creation-defer-length extension
        # upload_defer_length = self.req.headers.get('Upload-Defer-Length')
        if not upload_length:
            raise MissingUploadLengthError()
        try:
            length = int(upload_length)
        except ValueError:
            raise InvalidUploadLengthError()
        if length < 0:
            raise InvalidUploadLengthError()
        if length > self.max_size:
            raise MaxSizeExceededError()
        env.values['upload_length'] = length

        upload_metadata = dict()
        upload_metadata_str = env.req.headers.get('Upload-Metadata')
        if upload_metadata_str:
            upload_metadata_tuples = [v.strip().split() for v in upload_metadata_str.split(',')]
            for t in upload_metadata_tuples:
                if len(t) != 2:
                    raise InvalidUploadMetadataError()
                try:
                    upload_metadata[t[0]] = base64.standard_b64decode(t[1])
                except:
                    raise InvalidUploadMetadataError()
        env.values['upload_metadata'] = upload_metadata

        if env.values['uid']:
            self.check_files(env)
        else:
            self.create_files(env)

        env.resp.headers['Upload-Expires'] = self.get_fexpires(env)
        env.resp.headers['Location'] = os.path.join(self.upload_path, env.values['uid'])
        env.resp.status = httplib.CREATED

    def head(self, env):
        upload_offset = self.get_current_offset(env)
        upload_length = self.get_end_length(env)
        upload_metadata = self.get_metadata(env)
        env.resp.headers['Upload-Offset'] = str(upload_offset)
        env.resp.headers['Upload-Length'] = str(upload_length)
        if upload_metadata:
            env.resp.headers['Upload-Metadata'] = str(','.join(['%s %s' % (t[0], base64.standard_b64encode(t[1]))
                                                                 for t in upload_metadata.items()]))
        env.resp.headers['Cache-Control'] = 'no-store'
        env.resp.status = httplib.OK

    def patch(self, env):
        if env.req.headers.get('Content-Type') != 'application/offset+octet-stream':
            raise InvalidContentTypeError()

        upload_offset = env.req.headers.get('Upload-Offset')
        if not upload_offset:
            raise MissingUploadOffsetError()
        try:
            offset = int(upload_offset)
        except ValueError:
            raise InvalidUploadOffsetError()
        if offset < 0:
            raise InvalidUploadOffsetError()
        if offset != self.get_current_offset(env):
            raise ConflictUploadOffsetError()
        env.values['upload_offset'] = offset

        current_offset = self.write_data(env)
        if current_offset == self.get_end_length(env):
            self.finish_upload(env)

        env.resp.headers['Upload-Offset'] = str(current_offset)
        env.resp.headers['Upload-Expires'] = self.get_fexpires(env)
        env.resp.status = httplib.NO_CONTENT

    def delete(self, env):
        self.delete_files(env)
        env.resp.status = httplib.NO_CONTENT

    def finish_upload(self, env):
        env.values['upload_finished'] = True
        env.req.body = self.get_fpath(env) if not self.send_file else open(self.get_fpath(env), 'rb').read()

    def create_files(self, env):
        self.cleanup()
        config = dict()
        config['upload_length'] = env.values['upload_length']
        config['upload_metadata'] = env.values['upload_metadata']

        uid = uuid.uuid4().hex
        fpath = self.get_fpath(env, uid)
        while os.path.exists(fpath):
            uid = uuid.uuid4().hex
            fpath = self.get_fpath(uid)
        env.values['uid'] = uid
        with open(fpath, 'w') as _:
            pass  # create file
        with open(fpath+'.info', 'w') as f:
            json.dump(config, f, indent=4)
        return uid

    def delete_files(self, env):
        fpath = self.get_fpath(env)
        info_path = fpath + '.info'
        if not os.path.exists(fpath):
            raise NotFoundError()
        os.remove(fpath)
        if not os.path.exists(info_path):
            raise NotFoundError()
        os.remove(info_path)

    def delete_info_file(self, env):
        fpath = self.get_fpath(env)
        info_path = fpath + '.info'
        if not os.path.exists(info_path):
            raise NotFoundError()
        os.remove(info_path)

    def check_files(self, env):
        fpath = self.get_fpath(env)
        info_path = fpath + '.info'

        if not os.path.exists(fpath) or not os.path.exists(fpath+'.info'):
            raise NotFoundError()

        with open(info_path, 'r') as f:
            config = json.load(f)

        length = env.values['upload_length']
        if length != config['upload_length']:
            raise ConflictUploadLengthError()

        if config['upload_metadata'] != env.values['upload_metadata']:
            config['upload_metadata'].update(env.values['upload_metadata'])
            with open(info_path, 'w') as f:
                json.dump(config, f, indent=4)

    def get_fpath(self, env, uid=None):
        uid = uid or env.values['uid']
        return os.path.join(self.tmp_dir, uid)

    def get_current_offset(self, env):
        fpath = self.get_fpath(env)
        if not os.path.exists(fpath):
            raise NotFoundError()
        return os.path.getsize(fpath)

    def get_end_length(self, env):
        fpath = self.get_fpath(env)
        info_path = fpath + '.info'

        if not os.path.exists(info_path):
            raise NotFoundError()
        with open(info_path, 'r') as f:
            config = json.load(f)
        return config['upload_length']

    def get_metadata(self, env):
        fpath = self.get_fpath(env)
        info_path = fpath + '.info'

        if not os.path.exists(info_path):
            raise NotFoundError()
        with open(info_path, 'r') as f:
            config = json.load(f)
        return config['upload_metadata']

    def get_fexpires(self, env):
        fpath = self.get_fpath(env)
        if not os.path.exists(fpath):
            raise NotFoundError()
        seconds = os.path.getmtime(fpath) + self.expire
        # rfc 7231  datetime format
        return time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(seconds))

    def write_data(self, env):
        fpath = self.get_fpath(env)
        info_path = fpath + '.info'
        body = env.req.body
        if not os.path.exists(fpath) or not os.path.exists(info_path):
            raise NotFoundError()
        with open(fpath, 'ab+') as f:
            f.seek(0, os.SEEK_END)
            if hasattr(body, 'read'):
                if hasattr(body, 'seek'):
                    body.seek(0)
                while True:
                    chunk = body.read(2 << 16)
                    if not chunk:
                        break
                    f.write(chunk)
            else:
                f.write(body)
            offset = f.tell()

        os.utime(info_path, None)
        return offset

    def finish_error(self, env, error):
        env.resp.status = '%i %s' % (error.status_code, error.reason)

    def cleanup(self):
        for fname in os.listdir(self.tmp_dir):
            fpath = os.path.join(self.tmp_dir, fname)
            if os.path.isfile(fpath) and time.time() - os.path.getmtime(fpath) > self.expire:
                os.remove(fpath)
<commit_msg>fix #2: implement checksum extension<commit_after># coding: utf-8

import os
import time
import json
import uuid
import webob
import base64
import hashlib
import httplib
from collections import namedtuple

class Error(Exception):
    status_code = 0
    reason = ''

class NotFoundError(Error):
    status_code = httplib.NOT_FOUND
    reason = 'Not Found'

class NotImplementedError(Error):
    status_code = httplib.NOT_IMPLEMENTED
    reason = 'Feature Not Implemented'

class MethodNotAllowedError(Error):
    status_code = httplib.METHOD_NOT_ALLOWED
    reason = 'Method Not Allowed'

class MissingVersionError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Missing Tus-Resumable Header'

class UnsupportedVersionError(Error):
    status_code = httplib.PRECONDITION_FAILED
    reason = 'Precondition Failed'

class MissingUidError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Missing Uid Part In Url'

class MaxSizeExceededError(Error):
    status_code = httplib.REQUEST_ENTITY_TOO_LARGE
    reason = 'Request Entity Too Large'

class InvalidUploadLengthError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Invalid Upload-Length Header'

class ConflictUploadLengthError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Conflict Upload-Length'

class MissingUploadLengthError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Missing Upload-Length Header'

class InvalidUploadOffsetError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Invalid Upload-Offset Header'

class ConflictUploadOffsetError(Error):
    status_code = httplib.CONFLICT
    reason = 'Conflict'

class MissingUploadOffsetError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Missing Upload-Offset Header'

class InvalidUploadMetadataError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Invalid Upload-Metadata Header'

class InvalidContentTypeError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Invalid Content-Type Header'

class ChecksumAlgorisumsNotSuppertedError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Bad Request'

class ChecksumMismatchError(Error):
    status_code = 460
    reason = 'Checksum Mismatch'

class FileLockedError(Error):
    status_code = httplib.LOCKED
    reason = 'File Currently Locked'

class UploadNotFinishedError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'One Of The Partial Uploads Is Not Finished'

class InvalidConcatError(Error):
    status_code = httplib.BAD_REQUEST
    reason = 'Invalid Upload-Concat Header'

class ModifyFinalError(Error):
    status_code = httplib.FORBIDDEN
    reason = 'Modifying A Final Upload Is Not Allowed'

Env = namedtuple('Env', ['req', 'resp', 'values'])

class TusFilter(object):
    versions = ['1.0.0']
    max_size = 2 ** 30   # 1G
    checksum_algorisums = ['sha1']
    extensions = [
        'creation',
        'expiration',
        'termination',
        'checksum',
        # 'creation-defer-length',     # todo
        # 'checksum-trailer',          # todo
        # 'concatenation',             # todo
        # 'concatenation-unfinished',  # todo
    ]

    def __init__(self, app, upload_path, tmp_dir='/tmp/upload', expire=60*60*60, send_file=False):
        self.app = app
        self.tmp_dir = tmp_dir
        self.upload_path = upload_path
        self.expire = expire
        self.send_file = send_file

        if not os.path.exists(tmp_dir):
            os.makedirs(tmp_dir)

    def __call__(self, environ, start_response):
        req = webob.Request(environ)
        resp = webob.Response()
        values = dict(upload_finished=False)
        env = Env(req=req, resp=resp, values=values)
        if not req.path.startswith(self.upload_path):
            return self.app(environ, start_response)
        try:
            self.handle(env)
        except Error as e:
            self.finish_error(env, e)

        if not env.values['upload_finished']:
            return resp(environ, start_response)

        resp = req.get_response(self.app)
        self.delete_info_file(env)
        return resp(environ, start_response)

    def handle(self, env):
        try:
            x_method = env.req.headers.get('X-HTTP-Method-Override')
        except AttributeError:
            x_method = None
        method = (x_method or env.req.method).upper()

        path_parts = env.req.path.split('/', 2)
        uid = path_parts[2] if 2 < len(path_parts) else None
        if method not in ['POST', 'OPTIONS'] and not uid:
            raise MissingUidError()
        env.values['uid'] = uid

        version = env.req.headers.get('Tus-Resumable')
        if method != 'OPTIONS':
            if not version:
                raise MissingVersionError()
            if version not in self.versions:
                raise UnsupportedVersionError()
        version = version or self.versions[0]   # OPTIONS version maybe None
        env.values['version'] = version        # for multi versions
        env.resp.headers['Tus-Resumable'] = version

        if method == 'POST':
            self.post(env)
        elif method == 'HEAD':
            self.head(env)
        elif method == 'PATCH':
            self.patch(env)
        elif method == 'OPTIONS':
            self.options(env)
        elif method == 'DELETE':
            self.delete(env)
        else:
            raise MethodNotAllowedError()

    def options(self, env):
        env.resp.headers['Tus-Version'] = ','.join(self.versions)
        env.resp.headers['Tus-Max-Size'] = str(self.max_size)
        env.resp.headers['Tus-Checksum-Algorithm'] = ','.join(self.checksum_algorisums)
        env.resp.headers['Tus-Extension'] = ','.join(self.extensions)
        env.resp.status = httplib.NO_CONTENT

    def post(self, env):
        upload_length = env.req.headers.get('Upload-Length')
        # todo: waitting for creation-defer-length extension
        # upload_defer_length = self.req.headers.get('Upload-Defer-Length')
        if not upload_length:
            raise MissingUploadLengthError()
        try:
            length = int(upload_length)
        except ValueError:
            raise InvalidUploadLengthError()
        if length < 0:
            raise InvalidUploadLengthError()
        if length > self.max_size:
            raise MaxSizeExceededError()
        env.values['upload_length'] = length

        upload_metadata = dict()
        upload_metadata_str = env.req.headers.get('Upload-Metadata')
        if upload_metadata_str:
            upload_metadata_tuples = [v.strip().split() for v in upload_metadata_str.split(',')]
            for t in upload_metadata_tuples:
                if len(t) != 2:
                    raise InvalidUploadMetadataError()
                try:
                    upload_metadata[t[0]] = base64.standard_b64decode(t[1])
                except:
                    raise InvalidUploadMetadataError()
        env.values['upload_metadata'] = upload_metadata

        if env.values['uid']:
            self.check_files(env)
        else:
            self.create_files(env)

        env.resp.headers['Upload-Expires'] = self.get_fexpires(env)
        env.resp.headers['Location'] = os.path.join(self.upload_path, env.values['uid'])
        env.resp.status = httplib.CREATED

    def head(self, env):
        upload_offset = self.get_current_offset(env)
        upload_length = self.get_end_length(env)
        upload_metadata = self.get_metadata(env)
        env.resp.headers['Upload-Offset'] = str(upload_offset)
        env.resp.headers['Upload-Length'] = str(upload_length)
        if upload_metadata:
            env.resp.headers['Upload-Metadata'] = str(','.join(['%s %s' % (t[0], base64.standard_b64encode(t[1]))
                                                                for t in upload_metadata.items()]))
        env.resp.headers['Cache-Control'] = 'no-store'
        env.resp.status = httplib.OK

    def patch(self, env):
        if env.req.headers.get('Content-Type') != 'application/offset+octet-stream':
            raise InvalidContentTypeError()

        upload_offset = env.req.headers.get('Upload-Offset')
        if not upload_offset:
            raise MissingUploadOffsetError()
        try:
            offset = int(upload_offset)
        except ValueError:
            raise InvalidUploadOffsetError()
        if offset < 0:
            raise InvalidUploadOffsetError()
        if offset != self.get_current_offset(env):
            raise ConflictUploadOffsetError()
        env.values['upload_offset'] = offset

        upload_checksum = env.req.headers.get('Upload-Checksum')
        if upload_checksum:
            algorisum, checksum_base64 = upload_checksum.strip().split(None, 1)
            if algorisum not in self.checksum_algorisums:
                raise ChecksumAlgorisumsNotSuppertedError()
            checksum = base64.standard_b64decode(checksum_base64)
            body = env.req.body
            if checksum != hashlib.sha1(body).digest():
                raise ChecksumMismatchError()
        current_offset = self.write_data(env)
        if current_offset == self.get_end_length(env):
            self.finish_upload(env)

        env.resp.headers['Upload-Offset'] = str(current_offset)
        env.resp.headers['Upload-Expires'] = self.get_fexpires(env)
        env.resp.status = httplib.NO_CONTENT

    def delete(self, env):
        self.delete_files(env)
        env.resp.status = httplib.NO_CONTENT

    def finish_upload(self, env):
        env.values['upload_finished'] = True
        env.req.body = self.get_fpath(env) if not self.send_file else open(self.get_fpath(env), 'rb').read()

    def create_files(self, env):
        self.cleanup()
        config = dict()
        config['upload_length'] = env.values['upload_length']
        config['upload_metadata'] = env.values['upload_metadata']

        uid = uuid.uuid4().hex
        fpath = self.get_fpath(env, uid)
        while os.path.exists(fpath):
            uid = uuid.uuid4().hex
            fpath = self.get_fpath(uid)
        env.values['uid'] = uid
        with open(fpath, 'w') as _:
            pass  # create file
        with open(fpath+'.info', 'w') as f:
            json.dump(config, f, indent=4)
        return uid

    def delete_files(self, env):
        fpath = self.get_fpath(env)
        info_path = fpath + '.info'
        if not os.path.exists(fpath):
            raise NotFoundError()
        os.remove(fpath)
        if not os.path.exists(info_path):
            raise NotFoundError()
        os.remove(info_path)

    def delete_info_file(self, env):
        fpath = self.get_fpath(env)
        info_path = fpath + '.info'
        if not os.path.exists(info_path):
            raise NotFoundError()
        os.remove(info_path)

    def check_files(self, env):
        fpath = self.get_fpath(env)
        info_path = fpath + '.info'

        if not os.path.exists(fpath) or not os.path.exists(fpath+'.info'):
            raise NotFoundError()

        with open(info_path, 'r') as f:
            config = json.load(f)

        length = env.values['upload_length']
        if length != config['upload_length']:
            raise ConflictUploadLengthError()

        if config['upload_metadata'] != env.values['upload_metadata']:
            config['upload_metadata'].update(env.values['upload_metadata'])
            with open(info_path, 'w') as f:
                json.dump(config, f, indent=4)

    def get_fpath(self, env, uid=None):
        uid = uid or env.values['uid']
        return os.path.join(self.tmp_dir, uid)

    def get_current_offset(self, env):
        fpath = self.get_fpath(env)
        if not os.path.exists(fpath):
            raise NotFoundError()
        return os.path.getsize(fpath)

    def get_end_length(self, env):
        fpath = self.get_fpath(env)
        info_path = fpath + '.info'

        if not os.path.exists(info_path):
            raise NotFoundError()
        with open(info_path, 'r') as f:
            config = json.load(f)
        return config['upload_length']

    def get_metadata(self, env):
        fpath = self.get_fpath(env)
        info_path = fpath + '.info'

        if not os.path.exists(info_path):
            raise NotFoundError()
        with open(info_path, 'r') as f:
            config = json.load(f)
        return config['upload_metadata']

    def get_fexpires(self, env):
        fpath = self.get_fpath(env)
        if not os.path.exists(fpath):
            raise NotFoundError()
        seconds = os.path.getmtime(fpath) + self.expire
        # rfc 7231  datetime format
        return time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(seconds))

    def write_data(self, env):
        fpath = self.get_fpath(env)
        info_path = fpath + '.info'
        body = env.req.body_file
        if not os.path.exists(fpath) or not os.path.exists(info_path):
            raise NotFoundError()
        with open(fpath, 'ab+') as f:
            f.seek(0, os.SEEK_END)
            body.seek(0)
            while True:
                chunk = body.read(2 << 16)
                if not chunk:
                    break
                f.write(chunk)
            offset = f.tell()

        os.utime(info_path, None)
        return offset

    def finish_error(self, env, error):
        env.resp.status = '%i %s' % (error.status_code, error.reason)

    def cleanup(self):
        for fname in os.listdir(self.tmp_dir):
            fpath = os.path.join(self.tmp_dir, fname)
            if os.path.isfile(fpath) and time.time() - os.path.getmtime(fpath) > self.expire:
                os.remove(fpath)
<|endoftext|>


<commit_before>from ggame import App
myapp = App()
myapp.run()

from ggame import App, Color, LineStyle, Sprite
from ggame import RectangleAsset, CircleAsset, EllipseAsset, PolygonAsset

# Three primary colors with no transparency (alpha = 1.0)
red = Color(0xff0000, 1.0)
redtransparent = Color(0xff0000, 0.5)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)

# Define a line style that is a thin (1 pixel) wide black line
thinline = LineStyle(1, black)
# A graphics asset that represents a rectangle
brectangle = RectangleAsset(50, 20, thinline, blue)
rrectangle = RectangleAsset(50, 20, thinline, red)
bellipse = EllipseAsset(500,250,thinline, blue)
rpolygon = PolygonAsset([(50,0),(950,0),(1000,50),(1000,450),
       (950,500),(50,500),(0,450),(0,50),(50,0)],thinline, redtransparent)

# Now display a rectangle
Sprite(brectangle,(100,100))
Sprite(rrectangle,(110,110))
Sprite(bellipse,(500,250))
polySprite = Sprite(rpolygon,(0,0))
polySprite.scale = 0.5

myapp = App()
myapp.run()<commit_msg>Updated from Brython Server: 5/8/2016 12:33:30 PM<commit_after>from random import randomfrom ggame import App
myapp = App()
myapp.run()

from ggame import App, Color, LineStyle, Sprite
from ggame import RectangleAsset, CircleAsset, EllipseAsset, PolygonAsset

# Three primary colors with no transparency (alpha = 1.0)
red = Color(0xff0000, 1.0)
redtransparent = Color(0xff0000, 0.5)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)

# Define a line style that is a thin (1 pixel) wide black line
thinline = LineStyle(1, black)
# A graphics asset that represents a rectangle
brectangle = RectangleAsset(50, 20, thinline, blue)
rrectangle = RectangleAsset(50, 20, thinline, red)
bellipse = EllipseAsset(500,250,thinline, blue)
rpolygon = PolygonAsset([(50,0),(950,0),(1000,50),(1000,450),
       (950,500),(50,500),(0,450),(0,50),(50,0)],thinline, redtransparent)

# Now display a rectangle
Sprite(brectangle,(100,100))
Sprite(rrectangle,(110,110))
Sprite(bellipse,(500,250))
polySprite = Sprite(rpolygon,(0,0))
count = 0;

while (count < 10000)
    polySprite.scale = random.random()
    count = count + 1

myapp = App()
myapp.run()<|endoftext|>


<commit_before>
<commit_msg>Updated from Brython Server: 1/29/2016 2:33:46 PM<commit_after>from ggame import App, Color, LineStyle, Sprite
from ggame import CircleAsset

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)

thinline = LineStyle(1, black)
mycircle = CircleAsset(5, thinline, blue)
xcoordinates = range(100, 600, 10)

# Generate a list of sprites that form a line!
sprites = [Sprite(mycircle, (x, x*0.5 + 100)) for x in xcoordinates]

myapp = App()
myapp.run()
<|endoftext|>


<commit_before>import datetime
from django.contrib.auth.models import User
from django.utils import timezone
from django.db import models

# Create your models here.
class Show(models.Model):
    name = models.CharField(max_length=200)
    description = models.TextField(null=True,blank=True)
    date_added = models.DateTimeField('date_added',auto_now_add=True, editable=False,blank=True,null=True)
    date_updated = models.DateTimeField('date_updated',auto_now_add=True, auto_now=True, editable=False,blank=True,null=True)
    date_aired = models.DateTimeField('date_aired',null=True,blank=True)
    date_next_episode = models.DateTimeField('date_next_episode',null=True,blank=True)
    file_path = models.CharField(max_length=200,null=True,blank=True)
    tvdbid = models.CharField(max_length=10,null=True,blank=True)
    genre = models.CharField(max_length=200,null=True,blank=True)
    status = models.CharField(max_length=25,null=True,blank=True)
    airs = models.CharField(max_length=25,null=True,blank=True)
    content_rating = models.CharField(max_length=25,null=True,blank=True)
    network = models.CharField(max_length=25,null=True,blank=True)
    actors = models.CharField(max_length=200,null=True,blank=True)
    runtime = models.CharField(max_length=25,null=True,blank=True)
    use_sickbeard = models.BooleanField(default=True)
    def __unicode__(self):
        return u'%s (%s)' % (self.name, self.tvdbid)
    def updated_recently(self):
        return self.date_updated >= timezone.now() - datetime.timedelta(days=3)
    class Meta:
        verbose_name_plural = "shows"
        ordering = ['name']
        permissions = (("can_view_shows", "Can view shows"),)

class Season(models.Model):
    show = models.ForeignKey(Show)
    number = models.IntegerField(default=0,null=True,blank=True)
    date_added = models.DateTimeField('date_added',auto_now_add=True, editable=False,blank=True,null=True)
    date_updated = models.DateTimeField('date_updated',auto_now_add=True, auto_now=True, editable=False,blank=True,null=True)
    date_aired = models.DateTimeField('date aired',null=True,blank=True)
    tvdbid = models.CharField(max_length=10,null=True,blank=True)
    def __unicode__(self):
        return u'%s - S%s (%s)' % (self.show.name, self.number, self.tvdbid)
    class Meta:
        verbose_name_plural = "seasons"

class Episode(models.Model):
    season = models.ForeignKey(Season)
    show = models.ForeignKey(Show)
    name = models.CharField(max_length=200)
    number = models.IntegerField(default=0,null=True,blank=True)
    description = models.TextField(null=True,blank=True)
    hits = models.IntegerField(default=0,null=True,blank=True,editable=False)
    date_added = models.DateTimeField('date_added',auto_now_add=True, editable=False,blank=True,null=True)
    date_updated = models.DateTimeField('date_updated',auto_now_add=True, auto_now=True, editable=False,blank=True,null=True)
    date_aired = models.DateTimeField('date aired',null=True,blank=True)
    rating = models.CharField(max_length=10,null=True,blank=True)
    tvdbid = models.CharField(max_length=10,null=True,blank=True)
    status = models.CharField(max_length=25,null=True,blank=True,editable=False)
    location = models.CharField(max_length=250,null=True,blank=True)
    file_size = models.CharField(max_length=10,null=True,blank=True,editable=False)
    file_exists = models.BooleanField(default=False)
    info_update = models.BooleanField(default=False)
    def __unicode__(self):
        return u'%s - S%sE%s %s' % (self.show.name, self.season.number, self.number, self.name)
    class Meta:
        verbose_name_plural = "episodes"
        permissions = (("can_play_episodes", "Can play episodes"),)

class Episode_Play(models.Model):
    viewer = models.ForeignKey(User)
    episode = models.ForeignKey(Episode)
    date_played = models.DateTimeField('date_played',auto_now_add=True, editable=False,blank=True,null=True)
    def __unicode__(self):
        return u'%s - %s' % (self.viewer.username, self.date_played)
    class Meta:
        verbose_name_plural = "plays"<commit_msg>model modifications for sickbeard integration<commit_after>import datetime
from django.contrib.auth.models import User
from django.utils import timezone
from django.db import models

# Create your models here.
class Show(models.Model):
	name = models.CharField(max_length=200)
	description = models.TextField(null=True,blank=True)
	date_added = models.DateTimeField('date_added',auto_now_add=True, editable=False,blank=True,null=True)
	date_updated = models.DateTimeField('date_updated',auto_now_add=True, auto_now=True, editable=False,blank=True,null=True)
	date_aired = models.DateTimeField('date_aired',null=True,blank=True)
	date_next_episode = models.DateTimeField('date_next_episode',null=True,blank=True)
	file_path = models.CharField(max_length=200,null=True,blank=True)
	tvdbid = models.CharField(max_length=10,null=True,blank=True)
	genre = models.CharField(max_length=200,null=True,blank=True)
	status = models.CharField(max_length=25,null=True,blank=True)
	airs = models.CharField(max_length=25,null=True,blank=True)
	content_rating = models.CharField(max_length=25,null=True,blank=True)
	network = models.CharField(max_length=25,null=True,blank=True)
	actors = models.CharField(max_length=200,null=True,blank=True)
	runtime = models.CharField(max_length=25,null=True,blank=True)
	use_sickbeard = models.BooleanField(default=True)
	def __unicode__(self):
		return u'%s (%s)' % (self.name, self.tvdbid)
	def updated_recently(self):
		return self.date_updated >= timezone.now() - datetime.timedelta(days=3)
	class Meta:
		verbose_name_plural = "shows"
		ordering = ['name']
		permissions = (("can_view_shows", "Can view shows"),)

class Season(models.Model):
	show = models.ForeignKey(Show)
	number = models.IntegerField(default=0,null=True,blank=True)
	date_added = models.DateTimeField('date_added',auto_now_add=True, editable=False,blank=True,null=True)
	date_updated = models.DateTimeField('date_updated',auto_now_add=True, auto_now=True, editable=False,blank=True,null=True)
	date_aired = models.DateTimeField('date aired',null=True,blank=True)
	tvdbid = models.CharField(max_length=10,null=True,blank=True)
	def __unicode__(self):
		return u'%s - S%s (%s)' % (self.show.name, self.number, self.tvdbid)
	class Meta:
		verbose_name_plural = "seasons"

class Episode(models.Model):
	season = models.ForeignKey(Season)
	show = models.ForeignKey(Show)
	name = models.CharField(max_length=200)
	number = models.IntegerField(default=0,null=True,blank=True)
	description = models.TextField(null=True,blank=True)
	hits = models.IntegerField(default=0,null=True,blank=True,editable=False)
	date_added = models.DateTimeField('date_added',auto_now_add=True, editable=False,blank=True,null=True)
	date_updated = models.DateTimeField('date_updated',auto_now_add=True, auto_now=True, editable=False,blank=True,null=True)
	date_aired = models.DateTimeField('date aired',null=True,blank=True)
	rating = models.CharField(max_length=10,null=True,blank=True)
	tvdbid = models.CharField(max_length=10,null=True,blank=True)
	status = models.CharField(max_length=25,null=True,blank=True,editable=False)
	location = models.CharField(max_length=250,null=True,blank=True)
	file_size = models.CharField(max_length=10,null=True,blank=True,editable=False)
	file_exists = models.BooleanField(default=False)
	info_update = models.BooleanField(default=False)
	def __unicode__(self):
		return u'%s - S%sE%s %s' % (self.show.name, self.season.number, self.number, self.name)
	class Meta:
		verbose_name_plural = "episodes"
		permissions = (("can_play_episodes", "Can play episodes"),)

class Episode_Play(models.Model):
	viewer = models.ForeignKey(User)
	episode = models.ForeignKey(Episode)
	date_played = models.DateTimeField('date_played',auto_now_add=True, editable=False,blank=True,null=True)
	def __unicode__(self):
		return u'%s - %s' % (self.viewer.username, self.date_played)
	class Meta:
		verbose_name_plural = "plays"<|endoftext|>


<commit_before>'''
First Version: Tweets contents from subreddits
'''
import time
import configparser
from collections import namedtuple
import praw
import tweepy

# defines our interface with reddit post model.
Post = namedtuple('Post', ['id', 'title', 'url',
                           'reddit_url', 'domain', 'user', 'subreddit_name', 'is_self', 'stickied'])

def strip_title(title, max_len):
    '''
    shortens title to fit in the tweet
    '''
    if len(title) <= max_len:
        return title
    else:
        return title[:max_len - 3] + "..."

def get_posts(reddit_connection, subreddit_name):
    '''
    grabs posts from reddit and formats them for processing
    '''
    posts = []
    print('[bot] Getting posts from Reddit')
    subreddit = reddit_connection.subreddit(subreddit_name)
    for submission in subreddit.hot(limit=20):
        post = Post(id=submission.id,
                    title=submission.title,
                    url=submission.url,
                    reddit_url=submission.shortlink,
                    domain=submission.domain,
                    user=submission.author.name,
                    subreddit_name=submission.subreddit.display_name,
                    is_self=submission.is_self,
                    stickied=submission.stickied
                    )
        posts.append(post)
    return posts

def setup_reddit_connection():
    '''
    Gets a connection to the subreddit.
    uses the config in praw.ini  needs to be in working directory
    see: http://praw.readthedocs.io/en/latest/getting_started/configuration/prawini.html
    '''
    print('[bot1] setting up connection with Reddit')
    return praw.Reddit('bot1', user_agent='bot1 user agent')

def duplicate_check(post_id):
    '''
    Checks the 'db' to see if this has already been posted
    '''
    with open('posted_posts.txt', 'r') as file:
        for line in file:
            if post_id in line:
                return True
    return False

def add_id_to_file(post_id):
    '''
    adds an entry to the 'db' to track what has been posted
    '''
    with open('posted_posts.txt', 'a') as file:
        file.write(str(post_id) + "\n")

def filter_posts(posts):
    '''
    filters the posts
    '''
    filtered_domains = {'imgur.com'}

    should_be_filtered = \
        lambda post: post.domain in filtered_domains or \
            duplicate_check(post.id) or \
            post.stickied

    return (x for x in posts if not should_be_filtered(x))

def convert_post_to_tweet(post):
    '''
    split the post into two tweets, one primary and one that is the reply.
    keep all the communication here.
    '''
    user_preamble = 'u/{user} says: '.format(user=post.user)
    # use 138 to reserve spaces to split url, hashtags and title
    hash_tag = '#{tag}'.format(tag=post.subreddit_name)
    shortened_title = strip_title(
        post.title, 138 - 23 - len(user_preamble) - len(hash_tag))
    primary_tweet = user_preamble + shortened_title + ' ' + post.url + ' ' + hash_tag
    reply_tweet = ''
    if not post.is_self:
        reply_tweet = 'further discussion to be had here: {url}'.format(
            url=post.reddit_url)
    return primary_tweet, reply_tweet

def tweeter(posts):
    '''
    puts the posts on twitter
    '''
    config = read_config()
    auth = tweepy.OAuthHandler(
        config['consumer_key'], config['consumer_secret'])
    auth.set_access_token(config['access_token'],
                          config['access_token_secret'])
    api = tweepy.API(auth)
    for post in filter_posts(posts):
        print('[bot] Posting these tweets')
        primary_tweet, reply_tweet = convert_post_to_tweet(post)
        print('\t' + primary_tweet)
        print('\t\t' + reply_tweet)
        try:
            first_tweet_status = api.update_status(primary_tweet)
            if '' != reply_tweet:
                api.update_status(reply_tweet, first_tweet_status.id)
        except Exception as e:
            print(e)
        add_id_to_file(post.id)
        time.sleep(30)

def read_config():
    '''
    reads from 'tweepy.ini' which must be in the working directory
    '''
    config = configparser.ConfigParser()
    config.read('tweepy.ini')
    return {key: config['bot1'][key] for key in config['bot1']}

def write_config(config_dict):
    '''
    write 'tweepy.ini' with updated values
    '''
    config = configparser.ConfigParser()
    config.read('tweepy.ini')
    for key in config_dict:
        config['bot1'][key] = config_dict[key]
    with open('tweepy.ini', 'w') as configfile:
        config.write(configfile)

def refresh_access_token():
    '''
    interactive function that gets the access token and access token secret
    '''
    config = read_config()
    auth = tweepy.OAuthHandler(consumer_key=config['consumer_key'],
                               consumer_secret=config['consumer_secret'])
    print('navigate to: ' + auth.get_authorization_url())
    print('return with the pin')
    pin = input('type the pin here:')
    auth.get_access_token(pin)
    config['access_token'] = auth.access_token
    config['access_token_secret'] = auth.access_token_secret
    write_config(config)
    return config

def main():
    '''
    connects the pieces to grab posts from reddit and throw them on twitter
    '''
    reddit_connection = setup_reddit_connection()
    posts = get_posts(reddit_connection, 'ethereum')
    tweeter(posts)

if __name__ == '__main__':
    main()
<commit_msg>still needs some testing but this is the basic idea.  need to fix append<commit_after>'''
First Version: Tweets contents from subreddits
'''
import time
import configparser
from collections import namedtuple
import praw
import tweepy

# defines our interface with reddit post model.
Post = namedtuple('Post', ['id', 'title', 'url',
                           'reddit_url', 'domain', 'user', 'subreddit_name', 'is_self', 'stickied'])

def strip_title(title, max_len):
    '''
    shortens title to fit in the tweet
    '''
    if len(title) <= max_len:
        return title
    else:
        return title[:max_len - 3] + "..."

def get_posts(reddit_connection, subreddit_name):
    '''
    grabs posts from reddit and formats them for processing
    '''
    posts = []
    print('[bot] Getting posts from Reddit')
    subreddit = reddit_connection.subreddit(subreddit_name)
    for submission in subreddit.hot(limit=20):
        post = Post(id=submission.id,
                    title=submission.title,
                    url=submission.url,
                    reddit_url=submission.shortlink,
                    domain=submission.domain,
                    user=submission.author.name,
                    subreddit_name=submission.subreddit.display_name,
                    is_self=submission.is_self,
                    stickied=submission.stickied
                    )
        posts.append(post)
    return posts

def setup_reddit_connection():
    '''
    Gets a connection to the subreddit.
    uses the config in praw.ini  needs to be in working directory
    see: http://praw.readthedocs.io/en/latest/getting_started/configuration/prawini.html
    '''
    print('[bot1] setting up connection with Reddit')
    return praw.Reddit('bot1', user_agent='bot1 user agent')

def duplicate_check(post_id):
    '''
    Checks the 'db' to see if this has already been posted
    '''
    with open('posted_posts.txt', 'r') as file:
        for line in file:
            if post_id in line:
                return True
    return False

def add_id_to_file(post_id):
    '''
    adds an entry to the 'db' to track what has been posted
    '''
    with open('posted_posts.txt', 'a') as file:
        file.write(str(post_id) + "\n")

def filter_posts(posts):
    '''
    filters the posts
    '''
    filtered_domains = {'imgur.com'}

    should_be_filtered = \
        lambda post: post.domain in filtered_domains or \
        duplicate_check(post.id) or \
        post.stickied

    return (x for x in posts if not should_be_filtered(x))

def convert_post_to_tweet(post):
    '''
    split the post into two tweets, one primary and one that is the reply.
    keep all the communication here.
    '''
    user_preamble = 'u/{user} says: '.format(user=post.user)
    # use 138 to reserve spaces to split url, hashtags and title
    hash_tag = '#{tag}'.format(tag=post.subreddit_name)
    shortened_title = strip_title(
        post.title, 138 - 23 - len(user_preamble) - len(hash_tag))
    primary_tweet = user_preamble + shortened_title + ' ' + post.url + ' ' + hash_tag
    reply_tweet = ''
    if not post.is_self:
        reply_tweet = 'further discussion to be had here: {url}'.format(
            url=post.reddit_url)
    return primary_tweet, reply_tweet

def tweeter(posts):
    '''
    puts the posts on twitter
    '''
    config = read_config()
    auth = tweepy.OAuthHandler(
        config['consumer_key'], config['consumer_secret'])
    auth.set_access_token(config['access_token'],
                          config['access_token_secret'])
    api = tweepy.API(auth)
    for post in filter_posts(posts):
        print('[bot] Posting these tweets')
        primary_tweet, reply_tweet = convert_post_to_tweet(post)
        print('\t' + primary_tweet)
        print('\t\t' + reply_tweet)
        try:
            first_tweet_status = api.update_status(primary_tweet)
            if '' != reply_tweet:
                api.update_status(reply_tweet, first_tweet_status.id)
        except Exception as e:
            print(e)
        add_id_to_file(post.id)
        time.sleep(30)

def read_config():
    '''
    reads from 'tweepy.ini' which must be in the working directory
    '''
    config = configparser.ConfigParser()
    config.read('tweepy.ini')
    return {key: config['bot1'][key] for key in config['bot1']}

def write_config(config_dict):
    '''
    write 'tweepy.ini' with updated values
    '''
    config = configparser.ConfigParser()
    config.read('tweepy.ini')
    for key in config_dict:
        config['bot1'][key] = config_dict[key]
    with open('tweepy.ini', 'w') as configfile:
        config.write(configfile)

def refresh_access_token():
    '''
    interactive function that gets the access token and access token secret
    '''
    config = read_config()
    auth = tweepy.OAuthHandler(consumer_key=config['consumer_key'],
                               consumer_secret=config['consumer_secret'])
    print('navigate to: ' + auth.get_authorization_url())
    print('return with the pin')
    pin = input('type the pin here:')
    auth.get_access_token(pin)
    config['access_token'] = auth.access_token
    config['access_token_secret'] = auth.access_token_secret
    write_config(config)
    return config

def main():
    '''
    connects the pieces to grab posts from reddit and throw them on twitter
    '''
    reddit_connection = setup_reddit_connection()
    source_subreddits = {'ethereum', 'btc', 'bitcoin',
                         'monero', 'dailyverse', 'dataisugly', 'ProgrammerHumor'}
    posts = []
    for source_subreddit in source_subreddits:
        posts.append(get_posts(reddit_connection, source_subreddit))
        time.sleep(30)
    # should interleave all the posts since id should be cross site.
    posts.sort(key=lambda x: x.id)
    tweeter(posts)

def bootstrap_source(reddit_connection, source_subreddit):
    for post in get_posts(reddit_connection, source_subreddit):
        add_id_to_file(post.id)
        time.sleep(30)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import sys
import os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))+'/../')
import spaceship.constants as consts
from spaceship.screen_functions import center, longest, colored
from bearlibterminal import terminal as term
from spaceship.setup import setup
from collections import namedtuple
from win32api import GetSystemMetrics as sysize

class Option:
    def __init__(self, title, opt=None, subopts=None):
        self.title = title
        # opt list
        self.opts = [opt] if opt else []
        # opt index
        self.optindex = 0
        # subopt list
        self.subopts = [subopts] if subopts else []
        # subopt pointers
        self.suboptindex = [-1] if subopts else []
        # list of expansions
        self.expand = set()

    def option(self):
        return self.opts[self.optindex]

    def suboption(self):
        return self.subopts[self.optindex][self.suboptindex[self.optindex]]

    def suboptsize(self):
        return len(self.subopts[self.optindex])

    def reset_all(self):
        self.optindex = 0
        self.suboptindex = [-1 for i in range(len(self.opts))]
        self.collapse_all()

    def expansion(self, n: int):
        if n > len(self.opts):
            raise IndexError('invalid index for expansion')

        if n not in self.expand:
            self.expand.add(n)

    def expand_all(self):
        for i in range(len(self.opts)):
            if i not in self.expand:
                self.expand.add(i)

    def collapse(self, n: int):
        if n > len(self.opts):
            raise IndexError('invalid index for collapse')

        if n in self.expand:
            self.expand.remove(n)

    def collapse_all(self):
        self.expand = set()

    def add_opt(self, opt: str, subopts: list):
        self.opts.append(opt)
        self.subopts.append(subopts)
        self.suboptindex.append(-1)

    def move_pointer(self, move: int):
        self.optindex = self.optindex+move

    def correct_pointer(self):
        self.optindex = max(min(self.optindex, len(self.opts)-1), 0)

    def move_subpointer(self, move: int):
        self.suboptindex[self.optindex] += move

    def correct_subpointer(self):
        self.suboptindex[self.optindex] = max(
            min(self.suboptindex[self.optindex], len(self.subopts[self.optindex])-1), -1)

# TODO: expanding list
option = Option("Options Screen")
# 80x25 -> 8x16 | 80x50 -> 8x8 | 160x50 -> 16x16 | FullScreen -> 16x16
option.add_opt("Screen Size", ["80x25", "80x50", "160x50",
                                "Full Screen: {}x{}".format(sysize(0), sysize(1))])
option.add_opt("Cell Size", ["Auto", "8x16", "8x8", "16x16"])
option.add_opt("Font Choice", ["Default", "IBM_CGA", "Andale", "Courier", "Unscii-8", "Unscii-8-thin", "VeraMono"])
option.add_opt("Coloring", ["Dynamic", "Dark", "Light", "Colorblind"])

def options():
    def parse_screensize(p, screensize):
        if "Full Screen" in screensize:
            sx, sy = sysize(0)//term.state(term.TK_CELL_WIDTH), sysize(1)//term.state(term.TK_CELL_HEIGHT)
            print(sx, sy)
        else:
            sx, sy = list(map(lambda x: int(x), screensize.split('x')))
        if (p['gx'], p['gy']) != (sx, sy):
            p['gx'], p['gy'] = sx, sy
            return p, True
        return p, False

    def parse_cellsize(p, cellsize):
        if cellsize == "Auto":
            if p['cx'] != 'Auto':
                p['cx'], p['cy'] = "auto", None
                return p, True
        else:
            print(cellsize, p)
            print(term.font('Courier'))
            cx, cy = list(map(lambda x: int(x), cellsize.split('x')))
            if (p['cx'], p['cy']) != (cx, cy):
                p['cx'], p['cy'] = (cx, cy)
                return p, True
        return p, False

    def parse_fonts(p, font):
        print(p)
        if option == "Default":
            term.set('font: default, size={}{}'.format(
                p['cx'],
                'x'+str(p['cy']) if p['cy'] != p['cx'] else ''))

        else:
            print(font, p['cx'], p['cy'])
            if p['cx'] == "auto":
                cy = 8 if font not in ("Andale, Courier, VeraMono") else 16
                term.set("font: ./fonts/{}.ttf, size={}{}".format(
                    font,
                    8, cy))
            else:
                term.set("font: ./fonts/{}.ttf, size={}{}".format(
                    font,
                    p['cx'],
                    'x'+str(p['cy']) if p['cy'] != p['cx'] else ''))
            # p['cx'], p['cy'] = "auto", None
            reset_screen(p)

    def reset_screen(properties):
        print(properties)
        if properties['cx'] == "auto":
            term.set("window: size={}x{}, cellsize={}".format(
                properties['gx'],
                properties['gy'],
                properties['cx'],
            ))
        else:
            term.set("window: size={}x{}, cellsize={}x{}".format(*(v for _, v in properties.items())))
        term.refresh()

    prop = {
        'gx': term.state(term.TK_WIDTH),
        'gy': term.state(term.TK_HEIGHT),
        'cx': term.state(term.TK_CELL_WIDTH),
        'cy': term.state(term.TK_CELL_HEIGHT),
    }

    while True:

        term.clear()

        # options title
        term.puts(center(option.title, term.state(term.TK_WIDTH)), 1, option.title)

        # options
        height = 3
        print(option.optindex, option.suboptindex)
        for index, opt in enumerate(option.opts):
            selected = index == option.optindex
            expanded = index in option.expand
            if selected:
                opt = ("[[-]] " if expanded else "[[+]] ") + "[c=#00ffff]{}[/c]".format(opt)
            else:
                opt = ("[[-]] " if expanded else "[[+]] ") + opt

            term.puts(term.state(term.TK_WIDTH)//5, height, opt)
            height += term.state(term.TK_HEIGHT)//25
            if expanded:
                for index, subopt in enumerate(option.subopts[index]):
                    if selected:
                        if index == option.suboptindex[option.optindex]:
                            subopt = "[c=#00ffff]{}[/c]".format(subopt)
                        else:
                             subopt = subopt
                    term.puts(term.state(term.TK_WIDTH)//4+3, height, subopt)
                    height += term.state(term.TK_HEIGHT)//25
            height += term.state(term.TK_HEIGHT)//25
        term.puts(term.state(term.TK_WIDTH)//5, height+1, "{}".format(term.state(term.TK_WIDTH)))
        term.puts(term.state(term.TK_WIDTH)//5, height+2, "{}".format(term.state(term.TK_HEIGHT)))
        term.puts(term.state(term.TK_WIDTH)//5, height+3, "{}".format(term.state(term.TK_CELL_WIDTH)))
        term.puts(term.state(term.TK_WIDTH)//5, height+4, "{}".format(term.state(term.TK_CELL_HEIGHT)))

        term.refresh()
        key = term.read()

        if key in (term.TK_CLOSE, term.TK_Q, term.TK_ESCAPE):
            option.reset_all()
            break

        elif key == term.TK_ENTER:
            if option.optindex in option.expand:
                # action stuff
                if option.suboptindex[option.optindex] != -1:
                    print('SELECTED: {}|{}'.format(
                        option.opts[option.optindex],
                        option.subopts[option.optindex][option.suboptindex[option.optindex]]))
                    if option.option() == "Screen Size":
                        print(prop)
                        prop, changed = parse_screensize(prop, option.suboption())
                        if changed:
                            print('changing screen properties')
                            reset_screen(prop)
                        print(prop)
                    elif option.option() == "Cell Size":
                        print(prop)
                        prop, changed = parse_cellsize(prop, option.suboption())
                        if changed:
                            print('changing cell properties')
                            reset_screen(prop)
                        print(prop)
                    elif option.option() == "Font Choice":
                        parse_fonts(prop, option.suboption())
                else:
                    option.collapse(option.optindex)
            else:
                option.expansion(option.optindex)
                # option.move_subpointer(1)

        # Arrow keys (UP | DOWN)
        elif key == term.TK_DOWN:
            if len(option.expand):
                option.move_subpointer(1)
                option.correct_subpointer()
            else:
                option.move_pointer(1)
                option.correct_pointer()
        elif key == term.TK_UP:
            if len(option.expand):
                option.move_subpointer(-1)
                option.correct_subpointer()
            else:
                option.move_pointer(-1)
                option.correct_pointer()

if __name__ == "__main__":
    setup()
    print(options())
<commit_msg>Removed win32 from options.py to test on linux<commit_after>import sys
import os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))+'/../')
import spaceship.constants as consts
from spaceship.screen_functions import center, longest, colored
from bearlibterminal import terminal as term
from spaceship.setup import setup
from collections import namedtuple

# from win32api import GetSystemMetrics as sysize

class Option:
    def __init__(self, title, opt=None, subopts=None):
        self.title = title
        # opt list
        self.opts = [opt] if opt else []
        # opt index
        self.optindex = 0
        # subopt list
        self.subopts = [subopts] if subopts else []
        # subopt pointers
        self.suboptindex = [-1] if subopts else []
        # list of expansions
        self.expand = set()

    def option(self):
        return self.opts[self.optindex]

    def suboption(self):
        return self.subopts[self.optindex][self.suboptindex[self.optindex]]

    def suboptsize(self):
        return len(self.subopts[self.optindex])

    def reset_all(self):
        self.optindex = 0
        self.suboptindex = [-1 for i in range(len(self.opts))]
        self.collapse_all()

    def expansion(self, n: int):
        if n > len(self.opts):
            raise IndexError('invalid index for expansion')

        if n not in self.expand:
            self.expand.add(n)

    def expand_all(self):
        for i in range(len(self.opts)):
            if i not in self.expand:
                self.expand.add(i)

    def collapse(self, n: int):
        if n > len(self.opts):
            raise IndexError('invalid index for collapse')

        if n in self.expand:
            self.expand.remove(n)

    def collapse_all(self):
        self.expand = set()

    def add_opt(self, opt: str, subopts: list):
        self.opts.append(opt)
        self.subopts.append(subopts)
        self.suboptindex.append(-1)

    def move_pointer(self, move: int):
        self.optindex = self.optindex+move

    def correct_pointer(self):
        self.optindex = max(min(self.optindex, len(self.opts)-1), 0)

    def move_subpointer(self, move: int):
        self.suboptindex[self.optindex] += move

    def correct_subpointer(self):
        self.suboptindex[self.optindex] = max(
            min(self.suboptindex[self.optindex], len(self.subopts[self.optindex])-1), -1)

# TODO: expanding list
option = Option("Options Screen")
# 80x25 -> 8x16 | 80x50 -> 8x8 | 160x50 -> 16x16 | FullScreen -> 16x16
option.add_opt("Screen Size", ["80x25", "80x50", "160x50",])
                                # "Full Screen: {}x{}".format(sysize(0), sysize(1))])
option.add_opt("Cell Size", ["Auto", "8x16", "8x8", "16x16"])
option.add_opt("Font Choice", ["Default", "IBM_CGA", "Andale", "Courier", "Unscii-8", "Unscii-8-thin", "VeraMono"])
option.add_opt("Coloring", ["Dynamic", "Dark", "Light", "Colorblind"])

def options():
    def parse_screensize(p, screensize):
        if "Full Screen" in screensize:
            sx, sy = sysize(0)//term.state(term.TK_CELL_WIDTH), sysize(1)//term.state(term.TK_CELL_HEIGHT)
            print(sx, sy)
        else:
            sx, sy = list(map(lambda x: int(x), screensize.split('x')))
        if (p['gx'], p['gy']) != (sx, sy):
            p['gx'], p['gy'] = sx, sy
            return p, True
        return p, False

    def parse_cellsize(p, cellsize):
        if cellsize == "Auto":
            if p['cx'] != 'Auto':
                p['cx'], p['cy'] = "auto", None
                return p, True
        else:
            print(cellsize, p)
            print(term.font('Courier'))
            cx, cy = list(map(lambda x: int(x), cellsize.split('x')))
            if (p['cx'], p['cy']) != (cx, cy):
                p['cx'], p['cy'] = (cx, cy)
                return p, True
        return p, False

    def parse_fonts(p, font):
        print(p)
        if option == "Default":
            term.set('font: default, size={}{}'.format(
                p['cx'],
                'x'+str(p['cy']) if p['cy'] != p['cx'] else ''))

        else:
            print(font, p['cx'], p['cy'])
            if p['cx'] == "auto":
                cy = 8 if font not in ("Andale, Courier, VeraMono") else 16
                term.set("font: ./fonts/{}.ttf, size={}{}".format(
                    font,
                    8, cy))
            else:
                term.set("font: ./fonts/{}.ttf, size={}{}".format(
                    font,
                    p['cx'],
                    'x'+str(p['cy']) if p['cy'] != p['cx'] else ''))
            # p['cx'], p['cy'] = "auto", None
            reset_screen(p)

    def reset_screen(properties):
        print(properties)
        if properties['cx'] == "auto":
            term.set("window: size={}x{}, cellsize={}".format(
                properties['gx'],
                properties['gy'],
                properties['cx'],
            ))
        else:
            term.set("window: size={}x{}, cellsize={}x{}".format(*(v for _, v in properties.items())))
        term.refresh()

    prop = {
        'gx': term.state(term.TK_WIDTH),
        'gy': term.state(term.TK_HEIGHT),
        'cx': term.state(term.TK_CELL_WIDTH),
        'cy': term.state(term.TK_CELL_HEIGHT),
    }

    while True:

        term.clear()

        # options title
        term.puts(center(option.title, term.state(term.TK_WIDTH)), 1, option.title)

        # options
        height = 3
        print(option.optindex, option.suboptindex)
        for index, opt in enumerate(option.opts):
            selected = index == option.optindex
            expanded = index in option.expand
            if selected:
                opt = ("[[-]] " if expanded else "[[+]] ") + "[c=#00ffff]{}[/c]".format(opt)
            else:
                opt = ("[[-]] " if expanded else "[[+]] ") + opt

            term.puts(term.state(term.TK_WIDTH)//5, height, opt)
            height += term.state(term.TK_HEIGHT)//25
            if expanded:
                for index, subopt in enumerate(option.subopts[index]):
                    if selected:
                        if index == option.suboptindex[option.optindex]:
                            subopt = "[c=#00ffff]{}[/c]".format(subopt)
                        else:
                             subopt = subopt
                    term.puts(term.state(term.TK_WIDTH)//4+3, height, subopt)
                    height += term.state(term.TK_HEIGHT)//25
            height += term.state(term.TK_HEIGHT)//25
        term.puts(term.state(term.TK_WIDTH)//5, height+1, "{}".format(term.state(term.TK_WIDTH)))
        term.puts(term.state(term.TK_WIDTH)//5, height+2, "{}".format(term.state(term.TK_HEIGHT)))
        term.puts(term.state(term.TK_WIDTH)//5, height+3, "{}".format(term.state(term.TK_CELL_WIDTH)))
        term.puts(term.state(term.TK_WIDTH)//5, height+4, "{}".format(term.state(term.TK_CELL_HEIGHT)))

        term.refresh()
        key = term.read()

        if key in (term.TK_CLOSE, term.TK_Q, term.TK_ESCAPE):
            option.reset_all()
            break

        elif key == term.TK_ENTER:
            if option.optindex in option.expand:
                # action stuff
                if option.suboptindex[option.optindex] != -1:
                    print('SELECTED: {}|{}'.format(
                        option.opts[option.optindex],
                        option.subopts[option.optindex][option.suboptindex[option.optindex]]))
                    if option.option() == "Screen Size":
                        print(prop)
                        prop, changed = parse_screensize(prop, option.suboption())
                        if changed:
                            print('changing screen properties')
                            reset_screen(prop)
                        print(prop)
                    elif option.option() == "Cell Size":
                        print(prop)
                        prop, changed = parse_cellsize(prop, option.suboption())
                        if changed:
                            print('changing cell properties')
                            reset_screen(prop)
                        print(prop)
                    elif option.option() == "Font Choice":
                        parse_fonts(prop, option.suboption())
                else:
                    option.collapse(option.optindex)
            else:
                option.expansion(option.optindex)
                # option.move_subpointer(1)

        # Arrow keys (UP | DOWN)
        elif key == term.TK_DOWN:
            if len(option.expand):
                option.move_subpointer(1)
                option.correct_subpointer()
            else:
                option.move_pointer(1)
                option.correct_pointer()
        elif key == term.TK_UP:
            if len(option.expand):
                option.move_subpointer(-1)
                option.correct_subpointer()
            else:
                option.move_pointer(-1)
                option.correct_pointer()

if __name__ == "__main__":
    setup()
    print(options())
<|endoftext|>


<commit_before># encoding: utf8
from __future__ import unicode_literals, print_function

from os import path

from ..language import Language, BaseDefaults
from ..tokenizer import Tokenizer
from ..tagger import Tagger
from ..attrs import LANG
from ..tokens import Doc

from .language_data import *

import re
from collections import namedtuple

ShortUnitWord = namedtuple('ShortUnitWord', ['surface', 'base_form', 'part_of_speech'])

def try_mecab_import():
    """Mecab is required for Japanese support, so check for it.

    It it's not available blow up and explain how to fix it."""
    try:
        import MeCab
        return MeCab
    except ImportError:
        raise ImportError("Japanese support requires MeCab: "
                          "https://github.com/SamuraiT/mecab-python3")

class JapaneseTokenizer(object):
    def __init__(self, cls, nlp=None):
        self.vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp)
        MeCab = try_mecab_import()
        self.tokenizer = MeCab.Tagger()

    def __call__(self, text):
        words = [x.surface for x in detailed_tokens(self.tokenizer, text)]
        return Doc(self.vocab, words=words, spaces=[False]*len(words))

def resolve_pos(token):
    """If necessary, add a field to the POS tag for UD mapping.

    Under Universal Dependencies, sometimes the same Unidic POS tag can
    be mapped differently depending on the literal token or its context
    in the sentence. This function adds information to the POS tag to
    resolve ambiguous mappings.
    """

    # NOTE: This is a first take. The rules here are crude approximations.
    # For many of these, full dependencies are needed to properly resolve
    # PoS mappings.

    if token.part_of_speech == '連体詞,*,*,*':
        # determiner-likes get DET, otherwise ADJ
        if re.match('^[こそあど此其彼]の', token.surface):
            return token.part_of_speech + ',DET'
        else:
            return token.part_of_speech + ',ADJ'
    return token.part_of_speech

def detailed_tokens(tokenizer, text):
    """Format Mecab output into a nice data structure, based on Janome."""

    node = tokenizer.parseToNode(text)
    node = node.next # first node is beginning of sentence and empty, skip it
    words = []
    while node.posid != 0:
        parts = node.feature.split(',')
        pos = ','.join(parts[0:4])
        reading = parts[6]
        base = parts[7]
        surface = parts[8]

        words.append( ShortUnitWord(surface, base, pos) )
        node = node.next
    return words

class JapaneseTagger(object):
    def __init__(self, vocab):
        MeCab = try_mecab_import()
        self.tagger = Tagger(vocab)
        self.tokenizer = MeCab.Tagger()

    def __call__(self, tokens):
        # two parts to this:
        # 1. get raw JP tags
        # 2. add features to tags as necessary for UD

        # TODO: if the text has been tokenized, this info is already available
        # How to set the data when tokenizing or save it for the tagger to find?

        dtokens = detailed_tokens(self.tokenizer, tokens.text)
        rawtags = list(map(resolve_pos, dtokens))
        self.tagger.tag_from_strings(tokens, rawtags)

class JapaneseDefaults(BaseDefaults):
    tag_map = TAG_MAP

    @classmethod
    def create_tokenizer(cls, nlp=None):
        return JapaneseTokenizer(cls, nlp)

    @classmethod
    def create_tagger(cls, tokenizer):
        return JapaneseTagger(tokenizer.vocab)

class Japanese(Language):
    lang = 'ja'

    Defaults = JapaneseDefaults

    def make_doc(self, text):
        words = [str(t) for t in self.tokenizer(text)]
        doc = Doc(self.vocab, words=words, spaces=[False]*len(words))
        tagger = JapaneseDefaults.create_tagger(self.tokenizer)
        tagger(doc)
        return doc
<commit_msg>Fix pronoun handling<commit_after># encoding: utf8
from __future__ import unicode_literals, print_function

from os import path

from ..language import Language, BaseDefaults
from ..tokenizer import Tokenizer
from ..tagger import Tagger
from ..attrs import LANG
from ..tokens import Doc

from .language_data import *

import re
from collections import namedtuple

ShortUnitWord = namedtuple('ShortUnitWord', ['surface', 'base_form', 'part_of_speech'])

def try_mecab_import():
    """Mecab is required for Japanese support, so check for it.

    It it's not available blow up and explain how to fix it."""
    try:
        import MeCab
        return MeCab
    except ImportError:
        raise ImportError("Japanese support requires MeCab: "
                          "https://github.com/SamuraiT/mecab-python3")

class JapaneseTokenizer(object):
    def __init__(self, cls, nlp=None):
        self.vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp)
        MeCab = try_mecab_import()
        self.tokenizer = MeCab.Tagger()

    def __call__(self, text):
        words = [x.surface for x in detailed_tokens(self.tokenizer, text)]
        return Doc(self.vocab, words=words, spaces=[False]*len(words))

def resolve_pos(token):
    """If necessary, add a field to the POS tag for UD mapping.

    Under Universal Dependencies, sometimes the same Unidic POS tag can
    be mapped differently depending on the literal token or its context
    in the sentence. This function adds information to the POS tag to
    resolve ambiguous mappings.
    """

    # NOTE: This is a first take. The rules here are crude approximations.
    # For many of these, full dependencies are needed to properly resolve
    # PoS mappings.

    if token.part_of_speech == '連体詞,*,*,*':
        if re.match('^[こそあど此其彼]の', token.surface):
            return token.part_of_speech + ',DET'
        if re.match('^[こそあど此其彼]', token.surface):
            return token.part_of_speech + ',PRON'
        else:
            return token.part_of_speech + ',ADJ'
    return token.part_of_speech

def detailed_tokens(tokenizer, text):
    """Format Mecab output into a nice data structure, based on Janome."""

    node = tokenizer.parseToNode(text)
    node = node.next # first node is beginning of sentence and empty, skip it
    words = []
    while node.posid != 0:
        parts = node.feature.split(',')
        pos = ','.join(parts[0:4])
        reading = parts[6]
        base = parts[7]
        surface = parts[8]

        words.append( ShortUnitWord(surface, base, pos) )
        node = node.next
    return words

class JapaneseTagger(object):
    def __init__(self, vocab):
        MeCab = try_mecab_import()
        self.tagger = Tagger(vocab)
        self.tokenizer = MeCab.Tagger()

    def __call__(self, tokens):
        # two parts to this:
        # 1. get raw JP tags
        # 2. add features to tags as necessary for UD

        # TODO: if the text has been tokenized, this info is already available
        # How to set the data when tokenizing or save it for the tagger to find?

        dtokens = detailed_tokens(self.tokenizer, tokens.text)
        rawtags = list(map(resolve_pos, dtokens))
        self.tagger.tag_from_strings(tokens, rawtags)

class JapaneseDefaults(BaseDefaults):
    tag_map = TAG_MAP

    @classmethod
    def create_tokenizer(cls, nlp=None):
        return JapaneseTokenizer(cls, nlp)

    @classmethod
    def create_tagger(cls, tokenizer):
        return JapaneseTagger(tokenizer.vocab)

class Japanese(Language):
    lang = 'ja'

    Defaults = JapaneseDefaults

    def make_doc(self, text):
        words = [str(t) for t in self.tokenizer(text)]
        doc = Doc(self.vocab, words=words, spaces=[False]*len(words))
        tagger = JapaneseDefaults.create_tagger(self.tokenizer)
        tagger(doc)
        return doc
<|endoftext|>


<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#
import os
import shutil
import util
import string

# Note: when creating templates, be aware that they are evaulated
# twice, in the context of two different dictionaries.
#  o  keys from keywords should have a # single %, as should "args".
#  o  keys passed in through the macros argument will need %% to
#     escape them for delayed evaluation.  This will include at
#     least %%(builddir)s for all, and doInstall will also get
#     %%(destdir)s

class ShellCommand:
    def __init__(self, *args, **keywords):
        # initialize initialize our keywords to the defaults
        self.__dict__.update(self.keywords)
        # check to make sure that we don't get a keyword we don't expect
        for key in keywords.keys():
            if key not in self.keywords.keys():
                raise TypeError, ("%s.__init__() got an unexpected keyword argument "
                                  "'%s'" % (self.__class__.__name__, key))
        # copy the keywords into our dict, overwriting the defaults
        self.__dict__.update(keywords)
        self.args = string.join(args)
        # pre-fill in the preMake and arguments
        self.command = self.template % self.__dict__

class Automake(ShellCommand):
    # note: no use of %(args)s -- which command would it apply to?
    template = ('cd %%(builddir)s; '
                'aclocal %%(m4DirArgs)s %(acLocalArgs)s; '
		'%(preAutoconf)s autoconf %(autoConfArgs)s; '
		'automake %(autoMakeArgs)s')
    keywords = {'autoConfArgs': '--force',
                'autoMakeArgs': '--copy --force',
		'acLocalArgs': '',
		'preAutoconf': '',
                'm4Dir': ''}

    def doBuild(self, macros):
	macros = macros.copy()
        if self.m4Dir:
	    macros.update({'m4DirArgs': '-I %s' %(self.m4Dir)})
        util.execute(self.command %macros)

class Configure(ShellCommand):
    template = ('cd %%(builddir)s; '
                '%%(mkObjdir)s '
		'%(preConfigure)s %%(configure)s '
		'  --prefix=%%(prefix)s '
		'  --bindir=%%(bindir)s '
                '  --sysconfdir=%%(sysconfdir)s '
		'  --datadir=%%(datadir)s '
		'  --mandir=%%(mandir)s --infodir=%%(infodir)s '
		'  %(args)s')
    keywords = {'preConfigure': '',
                'objDir': ''}

    def doBuild(self, macros):
	macros = macros.copy()
        if self.objDir:
            macros['mkObjdir'] = 'mkdir -p %s; cd %s;' \
	                         %(self.objDir, self.objDir)
	    macros['configure'] = '../configure'
        else:
            macros['configure'] = './configure'
        util.execute(self.command %macros)

class ManualConfigure(Configure):
    template = ('cd %%(builddir)s; '
                '%%(mkObjdir)s '
	        '%(preConfigure)s %%(configure)s %(args)s')

class Make(ShellCommand):
    template = 'cd %%(builddir)s; %(preMake)s make %%(mflags)s %%(parallelmflags)s %(args)s'
    keywords = {'preMake': ''}

    def doBuild(self, macros):
	macros = macros.copy()
        util.execute(self.command %macros)

class MakeInstall(ShellCommand):
    template = ('cd %%(builddir)s; '
                '%(preMake)s make %%(mflags)s %(rootVar)s=%%(destdir)s %(installtarget)s %(args)s')
    keywords = {'rootVar': 'DESTDIR',
                'preMake': '',
		'installtarget': 'install'}

    def doInstall(self, macros):
	util.execute(self.command %macros)

class InstallFile:

    # make sure that the decimal value really is unreasonable before
    # adding a new translation to this file.
    permmap = {
	1755: 01755,
	4755: 04755,
	755: 0755,
	750: 0750,
	644: 0644,
	640: 0640,
    }

    def doInstall(self, macros):
	dest = macros['destdir'] + self.toFile %macros
	util.mkdirChain(os.path.dirname(dest))

	for fromFile in self.fromFiles:
	    thisdest = dest
	    if dest[-1:] == '/':
		thisdest = dest + fromFile
	    shutil.copyfile(fromFile, thisdest)
	    os.chmod(thisdest, self.mode)

    def __init__(self, fromFiles, toFile, perms = 0644):
	self.toFile = toFile
	if type(fromFiles) is str:
	    self.fromFiles = (fromFiles,)
	else:
	    self.fromFiles = fromFiles
	# notice obviously broken permissions
	if self.permmap.has_key(perms):
	    print 'odd permission %o, correcting to %o: add initial "0"?' \
	          %(perms, permmap[perms])
	    perms = permmap[perms]
	self.mode = perms

class InstallSymlink:

    def doInstall(self, macros):
	dest = macros['destdir'] + self.toFile %macros
	util.mkdirChain(os.path.dirname(dest))
	os.symlink(self.fromFile, macros['destdir'] + self.toFile %macros)

    def __init__(self, fromFile, toFile):
	self.fromFile = fromFile
	self.toFile = toFile

class RemoveFiles:

    def doInstall(self, macros):
	if self.recursive:
	    util.execute("rm -rf %s/%s" %(macros['destdir'], self.filespec %macros))
	else:
	    util.execute("rm -f %s/%s" %(macros['destdir'], self.filespec %macros))

    def __init__(self, filespec, recursive=0):
	self.filespec = filespec
	self.recursive = recursive

class MoveFile:

    def doInstall(self, macros):
	dest = macros['destdir'] + self.toFile %macros
	util.mkdirChain(os.path.dirname(dest))
	os.rename(macros['destdir'] + self.fromFile %macros,
	          macros['destdir'] + self.toFile %macros)

    def __init__(self, fromFile, toFile):
	self.fromFile = fromFile
	self.toFile = toFile
<commit_msg>support installing a file that is in a subdir of builddir.  Without this it will attempt to install to the wrong path<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#
import os
import shutil
import util
import string

# Note: when creating templates, be aware that they are evaulated
# twice, in the context of two different dictionaries.
#  o  keys from keywords should have a # single %, as should "args".
#  o  keys passed in through the macros argument will need %% to
#     escape them for delayed evaluation.  This will include at
#     least %%(builddir)s for all, and doInstall will also get
#     %%(destdir)s

class ShellCommand:
    def __init__(self, *args, **keywords):
        # initialize initialize our keywords to the defaults
        self.__dict__.update(self.keywords)
        # check to make sure that we don't get a keyword we don't expect
        for key in keywords.keys():
            if key not in self.keywords.keys():
                raise TypeError, ("%s.__init__() got an unexpected keyword argument "
                                  "'%s'" % (self.__class__.__name__, key))
        # copy the keywords into our dict, overwriting the defaults
        self.__dict__.update(keywords)
        self.args = string.join(args)
        # pre-fill in the preMake and arguments
        self.command = self.template % self.__dict__

class Automake(ShellCommand):
    # note: no use of %(args)s -- which command would it apply to?
    template = ('cd %%(builddir)s; '
                'aclocal %%(m4DirArgs)s %(acLocalArgs)s; '
		'%(preAutoconf)s autoconf %(autoConfArgs)s; '
		'automake %(autoMakeArgs)s')
    keywords = {'autoConfArgs': '--force',
                'autoMakeArgs': '--copy --force',
		'acLocalArgs': '',
		'preAutoconf': '',
                'm4Dir': ''}

    def doBuild(self, macros):
	macros = macros.copy()
        if self.m4Dir:
	    macros.update({'m4DirArgs': '-I %s' %(self.m4Dir)})
        util.execute(self.command %macros)

class Configure(ShellCommand):
    template = ('cd %%(builddir)s; '
                '%%(mkObjdir)s '
		'%(preConfigure)s %%(configure)s '
		'  --prefix=%%(prefix)s '
		'  --bindir=%%(bindir)s '
                '  --sysconfdir=%%(sysconfdir)s '
		'  --datadir=%%(datadir)s '
		'  --mandir=%%(mandir)s --infodir=%%(infodir)s '
		'  %(args)s')
    keywords = {'preConfigure': '',
                'objDir': ''}

    def doBuild(self, macros):
	macros = macros.copy()
        if self.objDir:
            macros['mkObjdir'] = 'mkdir -p %s; cd %s;' \
	                         %(self.objDir, self.objDir)
	    macros['configure'] = '../configure'
        else:
            macros['configure'] = './configure'
        util.execute(self.command %macros)

class ManualConfigure(Configure):
    template = ('cd %%(builddir)s; '
                '%%(mkObjdir)s '
	        '%(preConfigure)s %%(configure)s %(args)s')

class Make(ShellCommand):
    template = 'cd %%(builddir)s; %(preMake)s make %%(mflags)s %%(parallelmflags)s %(args)s'
    keywords = {'preMake': ''}

    def doBuild(self, macros):
	macros = macros.copy()
        util.execute(self.command %macros)

class MakeInstall(ShellCommand):
    template = ('cd %%(builddir)s; '
                '%(preMake)s make %%(mflags)s %(rootVar)s=%%(destdir)s %(installtarget)s %(args)s')
    keywords = {'rootVar': 'DESTDIR',
                'preMake': '',
		'installtarget': 'install'}

    def doInstall(self, macros):
	util.execute(self.command %macros)

class InstallFile:

    # make sure that the decimal value really is unreasonable before
    # adding a new translation to this file.
    permmap = {
	1755: 01755,
	4755: 04755,
	755: 0755,
	750: 0750,
	644: 0644,
	640: 0640,
    }

    def doInstall(self, macros):
	dest = macros['destdir'] + self.toFile %macros
	util.mkdirChain(os.path.dirname(dest))

	for fromFile in self.fromFiles:
	    thisdest = dest
	    if dest[-1:] == '/':
		thisdest = dest + os.path.basename(fromFile)
	    shutil.copyfile(fromFile, thisdest)
	    os.chmod(thisdest, self.mode)

    def __init__(self, fromFiles, toFile, perms = 0644):
	self.toFile = toFile
	if type(fromFiles) is str:
	    self.fromFiles = (fromFiles,)
	else:
	    self.fromFiles = fromFiles
	# notice obviously broken permissions
	if self.permmap.has_key(perms):
	    print 'odd permission %o, correcting to %o: add initial "0"?' \
	          %(perms, permmap[perms])
	    perms = permmap[perms]
	self.mode = perms

class InstallSymlink:

    def doInstall(self, macros):
	dest = macros['destdir'] + self.toFile %macros
	util.mkdirChain(os.path.dirname(dest))
	os.symlink(self.fromFile, macros['destdir'] + self.toFile %macros)

    def __init__(self, fromFile, toFile):
	self.fromFile = fromFile
	self.toFile = toFile

class RemoveFiles:

    def doInstall(self, macros):
	if self.recursive:
	    util.execute("rm -rf %s/%s" %(macros['destdir'], self.filespec %macros))
	else:
	    util.execute("rm -f %s/%s" %(macros['destdir'], self.filespec %macros))

    def __init__(self, filespec, recursive=0):
	self.filespec = filespec
	self.recursive = recursive

class MoveFile:

    def doInstall(self, macros):
	dest = macros['destdir'] + self.toFile %macros
	util.mkdirChain(os.path.dirname(dest))
	os.rename(macros['destdir'] + self.fromFile %macros,
	          macros['destdir'] + self.toFile %macros)

    def __init__(self, fromFile, toFile):
	self.fromFile = fromFile
	self.toFile = toFile
<|endoftext|>


<commit_before># encoding: utf8
from __future__ import unicode_literals, print_function

from ..language import Language
from ..attrs import LANG
from ..tokens import Doc
from .language_data import *

class RussianTokenizer(object):
    try:
        from pymorphy2 import MorphAnalyzer
    except ImportError:
        raise ImportError(
            "The Russian tokenizer requires the pymorphy2 library: "
            "try to fix it with "
            "pip install pymorphy2==0.8")

    _morph = MorphAnalyzer()

    def __init__(self, spacy_tokenizer, cls, nlp=None):
        self.vocab = nlp.vocab if nlp else cls.create_vocab(nlp)
        self._spacy_tokenizer = spacy_tokenizer

    def __call__(self, text):
        words = [self._normalize(RussianTokenizer._get_word(token))
                 for token in self._spacy_tokenizer(text)]

        return Doc(self.vocab, words, [False] * len(words))

    @staticmethod
    def _get_word(token):
        return token.lemma_ if len(token.lemma_) > 0 else token.text

    @classmethod
    def _normalize(cls, word):
        return cls._morph.parse(word)[0].normal_form

class RussianDefaults(Language.Defaults):
    lex_attr_getters = dict(Language.Defaults.lex_attr_getters)
    lex_attr_getters[LANG] = lambda text: 'ru'

    tokenizer_exceptions = TOKENIZER_EXCEPTIONS
    stop_words = STOP_WORDS

    @classmethod
    def create_tokenizer(cls, nlp=None):
        tokenizer = super(RussianDefaults, cls).create_tokenizer(nlp)
        return RussianTokenizer(tokenizer, cls, nlp)

class Russian(Language):
    lang = 'ru'

    Defaults = RussianDefaults
<commit_msg>updated Russian tokenizer<commit_after># encoding: utf8
from __future__ import unicode_literals, print_function

from ..language import Language
from ..attrs import LANG
from ..tokens import Doc
from .language_data import *

class RussianTokenizer(object):
    _morph = None

    def __init__(self, spacy_tokenizer, cls, nlp=None):
        try:
            from pymorphy2 import MorphAnalyzer
        except ImportError:
            raise ImportError(
                "The Russian tokenizer requires the pymorphy2 library: "
                "try to fix it with "
                "pip install pymorphy2==0.8")

        RussianTokenizer._morph = RussianTokenizer._create_morph(MorphAnalyzer)

        self.vocab = nlp.vocab if nlp else cls.create_vocab(nlp)
        self._spacy_tokenizer = spacy_tokenizer

    def __call__(self, text):
        words = [self._normalize(RussianTokenizer._get_word(token))
                 for token in self._spacy_tokenizer(text)]

        return Doc(self.vocab, words, [False] * len(words))

    @staticmethod
    def _get_word(token):
        return token.lemma_ if len(token.lemma_) > 0 else token.text

    @classmethod
    def _normalize(cls, word):
        return cls._morph.parse(word)[0].normal_form

    @classmethod
    def _create_morph(cls, morph_analyzer_class):
        if not cls._morph:
            cls._morph = morph_analyzer_class()
        return cls._morph

class RussianDefaults(Language.Defaults):
    lex_attr_getters = dict(Language.Defaults.lex_attr_getters)
    lex_attr_getters[LANG] = lambda text: 'ru'

    tokenizer_exceptions = TOKENIZER_EXCEPTIONS
    stop_words = STOP_WORDS

    @classmethod
    def create_tokenizer(cls, nlp=None):
        tokenizer = super(RussianDefaults, cls).create_tokenizer(nlp)
        return RussianTokenizer(tokenizer, cls, nlp)

class Russian(Language):
    lang = 'ru'

    Defaults = RussianDefaults
<|endoftext|>