<commit_before>#!/usr/bin/env python
import os
import urllib2
import json
import argparse

import pymongo

def get_books(api_host='localhost', api_port='5984'):
    try:
        query1 = urllib2.urlopen('http://{0}:{1}/scielobooks_1a/_design/scielobooks/_view/books'.format(api_host, api_port))
    except urllib2.URLError:
        print "API connection refused, please check the script configurations running ./{0} -h".format(os.path.basename(__file__))
        raise

    jsondocs = json.loads(query1.read())

    books = {}
    for reg in jsondocs['rows']:  # rows e uma array (lista) de registros no JSON
        book = books.setdefault(reg['id'], {})
        book['editor'] = reg['value']['publisher']
        book['title'] = reg['value']['title']

        creators = []
        if 'creators' in reg['value']:
            for creator in reg['value']['creators']:
                creators.append(creator[1][1])
        book['creators'] = '; '.join(creators)

        book['year'] = 'not defined'
        if 'year' in reg['value']:
            book['year'] = reg['value']['year'][0:10]

        book['creation_date'] = 'not defined'
        if 'creation_date' in reg['value']:
            book['creation_date'] = reg['value']['creation_date'][0:10]

    return books

def sumarize_by_month(stats):
    sumarized = {}
    for key, value in stats.items():

        month = key[4:10]
        if month:
            if key[:3] == 'htm':
                html_stats = sumarized.setdefault('html', {})
                html_stats.setdefault(month, 0)
                html_stats[month] += value

            if key[:3] == 'pdf':
                pdf_stats = sumarized.setdefault('pdf', {})
                pdf_stats.setdefault(month, 0)
                pdf_stats[month] += value

            if key[:3] == 'epu':
                pdf_stats = sumarized.setdefault('epu', {})
                pdf_stats.setdefault(month, 0)
                pdf_stats[month] += value

    return sumarized

def get_collection(mongodb_host='localhost',
               mongodb_port=27017,
               mongodb_database='bks',
               mongodb_collection='bks'):

    try:
        conn = pymongo.Connection(mongodb_host, mongodb_port)
    except pymongo.errors.AutoReconnect:
        print "MongoDB connection refused, please check the script configurations running ./{0} -h".format(os.path.basename(__file__))
        raise

    db = conn[mongodb_database + "_accesslog"]
    coll = db[mongodb_collection + "_analytics"]

    return coll

def main(*args, **xargs):

    books = get_books(api_host=xargs['api_host'],
                      api_port=int(xargs['api_port']))

    analytics = get_collection(mongodb_host=xargs['mongodb_host'],
                               mongodb_port=int(xargs['mongodb_port']))

    report = open(xargs['output_file'], 'w')
    books_stats = analytics.find({'context': 'book'})

    stats_dict = {}
    for stat in books_stats:
        book_code = stat['code'].lower()
        stats_dict[book_code] = sumarize_by_month(stat)

    report.write(u"code|title|authors|publicated at|created at|editor|access type|month access|accesses\r\n")

    for book_code, doc_types in stats_dict.items():
        if book_code in books:
            for doc_type, months in doc_types.items():
                for month, accesses in months.items():
                    line = u"{0}|{1}|{2}|{3}|{4}|{5}|{6}|{7}|{8}\r\n".format(book_code,
                                                        books[book_code]['title'],
                                                        books[book_code]['creators'],
                                                        books[book_code]['year'],
                                                        books[book_code]['creation_date'],
                                                        books[book_code]['editor'],
                                                        doc_type,
                                                        month,
                                                        accesses).encode(xargs['charset_standard'], 'replace')
                    report.write(line)

parser = argparse.ArgumentParser(description="Create an access report")
parser.add_argument('--api_host', default='localhost', help='The CouchDB API hostname')
parser.add_argument('--api_port', default='5984', help='The CouchDB API port')
parser.add_argument('--mongodb_host', default='localhost', help='The MongoDB accesslog database hostname')
parser.add_argument('--mongodb_port', default='27017', help='The MongoDB accesslog database port')
parser.add_argument('--output_file', default='report.txt', help='File name where the report will be stored')
parser.add_argument('--charset_standard', default='UTF-8', help='Charset standart for the report')
args = parser.parse_args()

if __name__ == "__main__":
    main(api_host=args.api_host,
         api_port=args.api_port,
         mongodb_host=args.mongodb_host,
         mongodb_port=args.mongodb_port,
         output_file=args.output_file,
         charset_standard=args.charset_standard)
<commit_msg>ajustes na exception<commit_after>#!/usr/bin/env python
import os
import urllib2
import json
import argparse

import pymongo

def get_books(api_host='localhost', api_port='5984'):
    try:
        query1 = urllib2.urlopen('http://{0}:{1}/scielobooks_1a/_design/scielobooks/_view/books'.format(api_host, api_port))
    except urllib2.URLError:
        exit("API connection refused, please check the script configurations running ./{0} -h".format(os.path.basename(__file__)))

    jsondocs = json.loads(query1.read())

    books = {}
    for reg in jsondocs['rows']:  # rows e uma array (lista) de registros no JSON
        book = books.setdefault(reg['id'], {})
        book['editor'] = reg['value']['publisher']
        book['title'] = reg['value']['title']

        creators = []
        if 'creators' in reg['value']:
            for creator in reg['value']['creators']:
                creators.append(creator[1][1])
        book['creators'] = '; '.join(creators)

        book['year'] = 'not defined'
        if 'year' in reg['value']:
            book['year'] = reg['value']['year'][0:10]

        book['creation_date'] = 'not defined'
        if 'creation_date' in reg['value']:
            book['creation_date'] = reg['value']['creation_date'][0:10]

    return books

def sumarize_by_month(stats):
    sumarized = {}
    for key, value in stats.items():

        month = key[4:10]
        if month:
            if key[:3] == 'htm':
                html_stats = sumarized.setdefault('html', {})
                html_stats.setdefault(month, 0)
                html_stats[month] += value

            if key[:3] == 'pdf':
                pdf_stats = sumarized.setdefault('pdf', {})
                pdf_stats.setdefault(month, 0)
                pdf_stats[month] += value

            if key[:3] == 'epu':
                pdf_stats = sumarized.setdefault('epu', {})
                pdf_stats.setdefault(month, 0)
                pdf_stats[month] += value

    return sumarized

def get_collection(mongodb_host='localhost',
               mongodb_port=27017,
               mongodb_database='bks',
               mongodb_collection='bks'):

    try:
        conn = pymongo.Connection(mongodb_host, mongodb_port)
    except pymongo.errors.AutoReconnect:
        exit("MongoDB connection refused, please check the script configurations running ./{0} -h".format(os.path.basename(__file__)))

    db = conn[mongodb_database + "_accesslog"]
    coll = db[mongodb_collection + "_analytics"]

    return coll

def main(*args, **xargs):

    books = get_books(api_host=xargs['api_host'],
                      api_port=int(xargs['api_port']))

    analytics = get_collection(mongodb_host=xargs['mongodb_host'],
                               mongodb_port=int(xargs['mongodb_port']))

    report = open(xargs['output_file'], 'w')
    books_stats = analytics.find({'context': 'book'})

    stats_dict = {}
    for stat in books_stats:
        book_code = stat['code'].lower()
        stats_dict[book_code] = sumarize_by_month(stat)

    report.write(u"code|title|authors|publicated at|created at|editor|access type|month access|accesses\r\n")

    for book_code, doc_types in stats_dict.items():
        if book_code in books:
            for doc_type, months in doc_types.items():
                for month, accesses in months.items():
                    line = u"{0}|{1}|{2}|{3}|{4}|{5}|{6}|{7}|{8}\r\n".format(book_code,
                                                        books[book_code]['title'],
                                                        books[book_code]['creators'],
                                                        books[book_code]['year'],
                                                        books[book_code]['creation_date'],
                                                        books[book_code]['editor'],
                                                        doc_type,
                                                        month,
                                                        accesses).encode(xargs['charset_standard'], 'replace')
                    report.write(line)

parser = argparse.ArgumentParser(description="Create an access report")
parser.add_argument('--api_host', default='localhost', help='The CouchDB API hostname')
parser.add_argument('--api_port', default='5984', help='The CouchDB API port')
parser.add_argument('--mongodb_host', default='localhost', help='The MongoDB accesslog database hostname')
parser.add_argument('--mongodb_port', default='27017', help='The MongoDB accesslog database port')
parser.add_argument('--output_file', default='report.txt', help='File name where the report will be stored')
parser.add_argument('--charset_standard', default='UTF-8', help='Charset standart for the report')
args = parser.parse_args()

if __name__ == "__main__":
    main(api_host=args.api_host,
         api_port=args.api_port,
         mongodb_host=args.mongodb_host,
         mongodb_port=args.mongodb_port,
         output_file=args.output_file,
         charset_standard=args.charset_standard)
<|endoftext|>


<commit_before>import requests
import sys

from .adapters import UnixAdapter

DEFAULT_SCHEME = 'http+unix://'

class Session(requests.Session):
    def __init__(self, url_scheme=DEFAULT_SCHEME, *args, **kwargs):
        super(Session, self).__init__(*args, **kwargs)
        self.mount(url_scheme, UnixAdapter())

class monkeypatch(object):
    def __init__(self, url_scheme=DEFAULT_SCHEME):
        self.session = Session()
        requests = self._get_global_requests_module()

        # Methods to replace
        self.methods = ('request', 'get', 'head', 'post',
                        'patch', 'put', 'delete', 'options')
        # Store the original methods
        self.orig_methods = dict( (m,requests.__dict__[m]) for m in self.methods )
        # Monkey patch
        g = globals()
        for m in self.methods:
            requests.__dict__[m] = g[m]

    def _get_global_requests_module(self):
        return sys.modules['requests']

    def __enter__(self):
        return self

    def __exit__(self, *args):
        requests = self._get_global_requests_module()
        for m in self.methods:
            requests.__dict__[m] = self.orig_methods[m]

# These are the same methods defined for the global requests object
def request(method, url, **kwargs):
    session = Session()
    return session.request(method=method, url=url, **kwargs)

def get(url, **kwargs):
    kwargs.setdefault('allow_redirects', True)
    return request('get', url, **kwargs)

def head(url, **kwargs):
    kwargs.setdefault('allow_redirects', False)
    return request('head', url, **kwargs)

def post(url, **kwargs):
    return request('post', url, data=data, **kwargs)

def patch(url, **kwargs):
    return request('patch', url,  data=data, **kwargs)

def put(url, **kwargs):
    return request('put', url, data=data, **kwargs)

def delete(url, **kwargs):
    return request('delete', url, **kwargs)

def options(url, **kwargs):
    kwargs.setdefault('allow_redirects', True)
    return request('options', url, **kwargs)
<commit_msg>Tweak a few things in PR 12<commit_after>import requests
import sys

from .adapters import UnixAdapter

DEFAULT_SCHEME = 'http+unix://'

class Session(requests.Session):
    def __init__(self, url_scheme=DEFAULT_SCHEME, *args, **kwargs):
        super(Session, self).__init__(*args, **kwargs)
        self.mount(url_scheme, UnixAdapter())

class monkeypatch(object):
    def __init__(self, url_scheme=DEFAULT_SCHEME):
        self.session = Session()
        requests = self._get_global_requests_module()

        # Methods to replace
        self.methods = ('request', 'get', 'head', 'post',
                        'patch', 'put', 'delete', 'options')
        # Store the original methods
        self.orig_methods = dict(
            (m, requests.__dict__[m]) for m in self.methods)
        # Monkey patch
        g = globals()
        for m in self.methods:
            requests.__dict__[m] = g[m]

    def _get_global_requests_module(self):
        return sys.modules['requests']

    def __enter__(self):
        return self

    def __exit__(self, *args):
        requests = self._get_global_requests_module()
        for m in self.methods:
            requests.__dict__[m] = self.orig_methods[m]

# These are the same methods defined for the global requests object
def request(method, url, **kwargs):
    session = Session()
    return session.request(method=method, url=url, **kwargs)

def get(url, **kwargs):
    kwargs.setdefault('allow_redirects', True)
    return request('get', url, **kwargs)

def head(url, **kwargs):
    kwargs.setdefault('allow_redirects', False)
    return request('head', url, **kwargs)

def post(url, **kwargs):
    return request('post', url, **kwargs)

def patch(url, **kwargs):
    return request('patch', url,  **kwargs)

def put(url, **kwargs):
    return request('put', url, **kwargs)

def delete(url, **kwargs):
    return request('delete', url, **kwargs)

def options(url, **kwargs):
    kwargs.setdefault('allow_redirects', True)
    return request('options', url, **kwargs)
<|endoftext|>


<commit_before>import re
import os
from astroid import MANAGER, CallFunc, Name, Assign, Keyword, List, Tuple, Const
from requirements_detector.requirement import DetectedRequirement

__all__ = ['find_dependencies',
           'RequirementsNotFound',
           'CouldNotParseRequirements',
           'get_prospector_profiles']

_PIP_OPTIONS = (
    '-i', '--index-url',
    '--extra-index-url',
    '--no-index',
    '-f', '--find-links',
    '-r'
)

class RequirementsNotFound(Exception):
    pass

class CouldNotParseRequirements(Exception):
    pass

def find_requirements(path):
    """
    This method tries to determine the requirements of a particular project
    by inspecting the possible places that they could be defined.

    It will attempt, in order:

    1) to parse setup.py in the root for an install_requires value
    2) to read a requirements.txt file in the root
    3) to read all .txt files in a folder called 'requirements' in the root
    4) to read files matching "*requirements*.txt" and "*reqs*.txt" in the root,
       excluding any starting or ending with 'test'

    If one of these succeeds, then a list of pkg_resources.Requirement's
    will be returned. If none can be found, then a RequirementsNotFound
    will be raised
    """

    setup_py = os.path.join(path, 'setup.py')
    if os.path.exists(setup_py) and os.path.isfile(setup_py):
        try:
            return from_setup_py(setup_py)
        except CouldNotParseRequirements:
            pass

    requirements_txt = os.path.join(path, 'requirements.txt')
    if os.path.exists(requirements_txt) and os.path.isfile(requirements_txt):
        try:
            return from_requirements_txt(requirements_txt)
        except CouldNotParseRequirements:
            pass

    requirements_dir = os.path.join(path, 'requirements')
    if os.path.exists(requirements_dir) and os.path.isdir(requirements_dir):
        requirements = from_requirements_dir(requirements_dir)
        if requirements:
            return requirements

    requirements = from_requirements_blob(path)
    if requirements:
        return requirements

    raise RequirementsNotFound

class SetupWalker(object):

    def __init__(self, ast):
        self._ast = ast
        self._setup_call = None
        self._top_level_assigns = {}
        self.walk()

    def walk(self, node=None):
        top = node is None
        node = node or self._ast

        # test to see if this is a call to setup()
        if isinstance(node, CallFunc):
            for child_node in node.get_children():
                if isinstance(child_node, Name) and child_node.name == 'setup':
                    # TODO: what if this isn't actually the distutils setup?
                    self._setup_call = node

        for child_node in node.get_children():
            if top and isinstance(child_node, Assign):
                for target in child_node.targets:
                    self._top_level_assigns[target.name] = child_node.value
            self.walk(child_node)

    def _get_list_value(self, list_node):
        values = []
        for child_node in list_node.get_children():
            if not isinstance(child_node, Const):
                # we can't handle anything fancy, only constant values
                raise CouldNotParseRequirements
            values.append(child_node.value)
        return values

    def get_install_requires(self):
        # first, if we have a call to setup, then we can see what its "install_requires" argument is
        if not self._setup_call:
            raise CouldNotParseRequirements

        for child_node in self._setup_call.get_children():
            if not isinstance(child_node, Keyword):
                # do we want to try to handle positional arguments?
                continue

            if child_node.arg != 'install_requires':
                continue

            if isinstance(child_node.value, (List, Tuple)):
                # joy! this is a simple list or tuple of requirements
                # this is a Keyword -> List or.nameword -> Tuple
                return self._get_list_value(child_node.value)

            if isinstance(child_node.value, Name):
                # otherwise, it's referencing a value defined elsewhere
                # this will be a Keyword -> Name
                try:
                    reqs = self._top_level_assigns[child_node.value.name]
                except KeyError:
                    raise CouldNotParseRequirements
                else:
                    if isinstance(reqs, (List, Tuple)):
                        return self._get_list_value(reqs)

            # otherwise it's something funky and we can't handle it
            raise CouldNotParseRequirements

        # if we've fallen off the bottom, we simply didn't find anything useful
        raise CouldNotParseRequirements

def from_setup_py(setup_file):
    ast = MANAGER.ast_from_file(setup_file)
    walker = SetupWalker(ast)

    requirements = []
    for req in walker.get_install_requires():
        requirements.append(DetectedRequirement.parse(req))

    requirements.sort(key=lambda r: r.name)
    return requirements

def from_requirements_txt(requirements_file):
    # see http://www.pip-installer.org/en/latest/logic.html
    requirements = []
    with open(requirements_file) as f:
        for req in f.readlines():
            if req.strip() == '':
                # empty line
                continue
            if req.strip().startswith('#'):
                # this is a comment
                continue
            if req.strip().split()[0] in _PIP_OPTIONS:
                # this is a pip option
                continue
            if req.strip().startswith('-e'):
                # TODO: this should handle local/vcs dependencies too
                continue
            if any([req.strip().startswith(protocol) for protocol in ('http', 'https', 'ftp', 'sftp')]):
                # TODO: this should also deal with archive URLs
                continue
            requirements.append(DetectedRequirement.parse(req))

    requirements.sort(key=lambda r: r.name)
    return requirements

def from_requirements_dir(path):
    requirements = []
    for entry in os.listdir(path):
        filepath = os.path.join(path, entry)
        if os.path.isfile(filepath) and entry.endswith('.txt'):
            # TODO: deal with duplicates
            requirements += from_requirements_txt(filepath)

    requirements.sort(key=lambda r: r.name)
    return requirements

def from_requirements_blob(path):
    requirements = []

    for entry in os.listdir(path):
        filepath = os.path.join(path, entry)
        if not os.path.isfile(filepath):
            continue
        m = re.match(r'^(\w*)req(uirement)?s(\w*)\.txt$', entry)
        if m is None:
            continue
        if m.group(1).startswith('test') or m.group(3).endswith('test'):
            continue
        requirements += from_requirements_txt(filepath)

    requirements.sort(key=lambda r: r.name)
    return requirements<commit_msg>Will now parse URLs and VCS dependencies too<commit_after>import re
import os
from astroid import MANAGER, CallFunc, Name, Assign, Keyword, List, Tuple, Const
from requirements_detector.requirement import DetectedRequirement

__all__ = ['find_dependencies',
           'RequirementsNotFound',
           'CouldNotParseRequirements',
           'get_prospector_profiles']

_PIP_OPTIONS = (
    '-i', '--index-url',
    '--extra-index-url',
    '--no-index',
    '-f', '--find-links',
    '-r'
)

class RequirementsNotFound(Exception):
    pass

class CouldNotParseRequirements(Exception):
    pass

def find_requirements(path):
    """
    This method tries to determine the requirements of a particular project
    by inspecting the possible places that they could be defined.

    It will attempt, in order:

    1) to parse setup.py in the root for an install_requires value
    2) to read a requirements.txt file in the root
    3) to read all .txt files in a folder called 'requirements' in the root
    4) to read files matching "*requirements*.txt" and "*reqs*.txt" in the root,
       excluding any starting or ending with 'test'

    If one of these succeeds, then a list of pkg_resources.Requirement's
    will be returned. If none can be found, then a RequirementsNotFound
    will be raised
    """

    setup_py = os.path.join(path, 'setup.py')
    if os.path.exists(setup_py) and os.path.isfile(setup_py):
        try:
            return from_setup_py(setup_py)
        except CouldNotParseRequirements:
            pass

    requirements_txt = os.path.join(path, 'requirements.txt')
    if os.path.exists(requirements_txt) and os.path.isfile(requirements_txt):
        try:
            return from_requirements_txt(requirements_txt)
        except CouldNotParseRequirements:
            pass

    requirements_dir = os.path.join(path, 'requirements')
    if os.path.exists(requirements_dir) and os.path.isdir(requirements_dir):
        requirements = from_requirements_dir(requirements_dir)
        if requirements:
            return requirements

    requirements = from_requirements_blob(path)
    if requirements:
        return requirements

    raise RequirementsNotFound

class SetupWalker(object):

    def __init__(self, ast):
        self._ast = ast
        self._setup_call = None
        self._top_level_assigns = {}
        self.walk()

    def walk(self, node=None):
        top = node is None
        node = node or self._ast

        # test to see if this is a call to setup()
        if isinstance(node, CallFunc):
            for child_node in node.get_children():
                if isinstance(child_node, Name) and child_node.name == 'setup':
                    # TODO: what if this isn't actually the distutils setup?
                    self._setup_call = node

        for child_node in node.get_children():
            if top and isinstance(child_node, Assign):
                for target in child_node.targets:
                    self._top_level_assigns[target.name] = child_node.value
            self.walk(child_node)

    def _get_list_value(self, list_node):
        values = []
        for child_node in list_node.get_children():
            if not isinstance(child_node, Const):
                # we can't handle anything fancy, only constant values
                raise CouldNotParseRequirements
            values.append(child_node.value)
        return values

    def get_install_requires(self):
        # first, if we have a call to setup, then we can see what its "install_requires" argument is
        if not self._setup_call:
            raise CouldNotParseRequirements

        for child_node in self._setup_call.get_children():
            if not isinstance(child_node, Keyword):
                # do we want to try to handle positional arguments?
                continue

            if child_node.arg != 'install_requires':
                continue

            if isinstance(child_node.value, (List, Tuple)):
                # joy! this is a simple list or tuple of requirements
                # this is a Keyword -> List or Keyword -> Tuple
                return self._get_list_value(child_node.value)

            if isinstance(child_node.value, Name):
                # otherwise, it's referencing a value defined elsewhere
                # this will be a Keyword -> Name
                try:
                    reqs = self._top_level_assigns[child_node.value.name]
                except KeyError:
                    raise CouldNotParseRequirements
                else:
                    if isinstance(reqs, (List, Tuple)):
                        return self._get_list_value(reqs)

            # otherwise it's something funky and we can't handle it
            raise CouldNotParseRequirements

        # if we've fallen off the bottom, we simply didn't find anything useful
        raise CouldNotParseRequirements

def from_setup_py(setup_file):
    ast = MANAGER.ast_from_file(setup_file)
    walker = SetupWalker(ast)

    requirements = []
    for req in walker.get_install_requires():
        requirements.append(DetectedRequirement.parse(req))

    requirements.sort(key=lambda r: r.name)
    return requirements

def from_requirements_txt(requirements_file):
    # see http://www.pip-installer.org/en/latest/logic.html
    requirements = []
    with open(requirements_file) as f:
        for req in f.readlines():
            if req.strip() == '':
                # empty line
                continue
            if req.strip().startswith('#'):
                # this is a comment
                continue
            if req.strip().split()[0] in _PIP_OPTIONS:
                # this is a pip option
                continue
            requirements.append(DetectedRequirement.parse(req))

    requirements.sort(key=lambda r: r.name)
    return requirements

def from_requirements_dir(path):
    requirements = []
    for entry in os.listdir(path):
        filepath = os.path.join(path, entry)
        if os.path.isfile(filepath) and entry.endswith('.txt'):
            # TODO: deal with duplicates
            requirements += from_requirements_txt(filepath)

    requirements.sort(key=lambda r: r.name)
    return requirements

def from_requirements_blob(path):
    requirements = []

    for entry in os.listdir(path):
        filepath = os.path.join(path, entry)
        if not os.path.isfile(filepath):
            continue
        m = re.match(r'^(\w*)req(uirement)?s(\w*)\.txt$', entry)
        if m is None:
            continue
        if m.group(1).startswith('test') or m.group(3).endswith('test'):
            continue
        requirements += from_requirements_txt(filepath)

    requirements.sort(key=lambda r: r.name)
    return requirements<|endoftext|>


<commit_before>from app.brain import twitter
from app.brain.classifier import Classifier
from app.brain.markov import Markov
from app.models import Muse, Tweet, Doc
from app.config import config

from tweepy.error import TweepError
from mongoengine.errors import NotUniqueError, OperationError
from pymongo.errors import DuplicateKeyError

import random

# Logging
from app.logger import logger
logger = logger(__name__)

# Load the classifier and markov.
# Loaded here so we can keep it in memory.
# accessible via app.brain.CLS or app.brain.MKV
CLS = Classifier()
MKV = Markov(ramble=config().ramble, ngram_size=config().ngram_size, spasm=config().spasm)

def ponder():
    """
    Fetch tweets from the Muses
    and memorize them;
    i.e. train classifier or Markov on them.
    """
    logger.info('Pondering new twitter data...')

    # Each of these are just a list
    # of tweets as strings.
    pos = []
    neg = []

    # For the good muses...
    for muse in Muse.objects(negative=False):
        pos += _process_muse(muse)

    # For the evil muses...
    for muse in Muse.objects(negative=True):
        neg += _process_muse(muse)

    # Combine the new tweets.
    new_tweets = pos_txts + neg_txts

    # Extract the tweet contents into lists.
    pos_txts = _get_tweet_texts(pos)
    neg_txts = _get_tweet_texts(neg)

    # Construct training matrix and labels.
    labels = [1 for i in range(len(pos_txts))] + [0 for i in range(len(neg_txts))]

    # See if there's anything to retweet.
    _consider_retweets(pos)

    # Update the classifier and markov.
    logger.info('Collected %s new tweets, training...' % len(new_tweets))
    CLS.train(new_tweets, labels)
    MKV.train(pos_txts)

def consider():
    """
    Decide whether or not to act (tweet).
    """
    logger.info('Considering tweeting...')
    roll = random.random()
    chance = config().chance_to_act
    if roll < chance:
        logger.info('Rolled %s, chance to act is %s, tweeting.' % (roll, chance))
        twitter.tweet(MKV.generate())
    else:
        logger.info('Rolled %s, chance to act is %s, NOT tweeting.' % (roll, chance))

def retrain():
    """
    Retrains the Markov generator on the documents in the database.
    """
    MKV.reset()
    tweets = [tweet.body for tweet in Tweet.objects.all()]
    MKV.train(tweets)

    docs = [doc.body for doc in Doc.objects.all()]
    MKV.train(docs)

def _process_muse(muse):
    """
    Processes a Muse's tweets,
    creating Tweet objects
    and saving them to the db.
    """
    username = muse.username
    logger.info('Collecting tweets for %s...' % username)

    try:
        tweets = twitter.tweets(username=username)
    except TweepError:
        return []

    new_tweets = []
    for tweet in tweets:
        data = {
                'body': tweet['body'],
                'tid': tweet['tid'],
                'username': username
        }
        t = Tweet(**data)
        try:
            t.save()
            new_tweets.append(tweet)
        except (NotUniqueError, DuplicateKeyError, OperationError):
            # Duplicate tweet
            pass
    return new_tweets

def _consider_retweets(tweets):
    """
    Retweets if positive
    classification is above THRESHOLD.
    0 = neg, 1 = pos
    """
    logger.info('Considering retweeting...')
    num_retweeted = 0
    retweet_threshold = config().retweet_threshold

    # Filter out protected tweets.
    candidates = [tweet for tweet in tweets if not tweet['protected'] and not tweet['retweeted']]
    txts = _get_tweet_texts(candidates)
    if txts:
        for idx, doc_probs in enumerate(CLS.classify(txts)):
            if num_retweeted >= config().max_retweets:
                logger.info('Hit maximum retweet limit, stopping for now.')
                break
            if doc_probs[1] > retweet_threshold:
                logger.info('Classified as %s retweetable, above %s threshold, retweeting...' % (doc_probs[1], retweet_threshold))
                twitter.retweet(candidates[idx]['tid'])
                num_retweeted += 1
            else:
                logger.info('Classified as %s retweetable, below %s threshold, not retweeting...' % (doc_probs[1], retweet_threshold))

def _get_tweet_texts(tweets):
    """
    Pulls out the content of a list of Tweets.
    """
    return [tweet['body'] for tweet in tweets]
<commit_msg>weird fix<commit_after>from app.brain import twitter
from app.brain.classifier import Classifier
from app.brain.markov import Markov
from app.models import Muse, Tweet, Doc
from app.config import config

from tweepy.error import TweepError
from mongoengine.errors import NotUniqueError, OperationError
from pymongo.errors import DuplicateKeyError

import random

# Logging
from app.logger import logger
logger = logger(__name__)

# Load the classifier and markov.
# Loaded here so we can keep it in memory.
# accessible via app.brain.CLS or app.brain.MKV
CLS = Classifier()
MKV = Markov(ramble=config().ramble, ngram_size=config().ngram_size, spasm=config().spasm)

def ponder():
    """
    Fetch tweets from the Muses
    and memorize them;
    i.e. train classifier or Markov on them.
    """
    logger.info('Pondering new twitter data...')

    # Each of these are just a list
    # of tweets as strings.
    pos = []
    neg = []

    # For the good muses...
    for muse in Muse.objects(negative=False):
        pos += _process_muse(muse)

    # For the evil muses...
    for muse in Muse.objects(negative=True):
        neg += _process_muse(muse)

    # Extract the tweet contents into lists.
    pos_txts = _get_tweet_texts(pos)
    neg_txts = _get_tweet_texts(neg)

    # Construct training matrix and labels.
    labels = [1 for i in range(len(pos_txts))] + [0 for i in range(len(neg_txts))]

    # See if there's anything to retweet.
    _consider_retweets(pos)

    # Combine the new tweets.
    new_tweets = postxts + neg_txts

    # Update the classifier and markov.
    logger.info('Collected %s new tweets, training...' % len(new_tweets))
    CLS.train(new_tweets, labels)
    MKV.train(pos_txts)

def consider():
    """
    Decide whether or not to act (tweet).
    """
    logger.info('Considering tweeting...')
    roll = random.random()
    chance = config().chance_to_act
    if roll < chance:
        logger.info('Rolled %s, chance to act is %s, tweeting.' % (roll, chance))
        twitter.tweet(MKV.generate())
    else:
        logger.info('Rolled %s, chance to act is %s, NOT tweeting.' % (roll, chance))

def retrain():
    """
    Retrains the Markov generator on the documents in the database.
    """
    MKV.reset()
    tweets = [tweet.body for tweet in Tweet.objects.all()]
    MKV.train(tweets)

    docs = [doc.body for doc in Doc.objects.all()]
    MKV.train(docs)

def _process_muse(muse):
    """
    Processes a Muse's tweets,
    creating Tweet objects
    and saving them to the db.
    """
    username = muse.username
    logger.info('Collecting tweets for %s...' % username)

    try:
        tweets = twitter.tweets(username=username)
    except TweepError:
        return []

    new_tweets = []
    for tweet in tweets:
        data = {
                'body': tweet['body'],
                'tid': tweet['tid'],
                'username': username
        }
        t = Tweet(**data)
        try:
            t.save()
            new_tweets.append(tweet)
        except (NotUniqueError, DuplicateKeyError, OperationError):
            # Duplicate tweet
            pass
    return new_tweets

def _consider_retweets(tweets):
    """
    Retweets if positive
    classification is above THRESHOLD.
    0 = neg, 1 = pos
    """
    logger.info('Considering retweeting...')
    num_retweeted = 0
    retweet_threshold = config().retweet_threshold

    # Filter out protected tweets.
    candidates = [tweet for tweet in tweets if not tweet['protected'] and not tweet['retweeted']]
    txts = _get_tweet_texts(candidates)
    if txts:
        for idx, doc_probs in enumerate(CLS.classify(txts)):
            if num_retweeted >= config().max_retweets:
                logger.info('Hit maximum retweet limit, stopping for now.')
                break
            if doc_probs[1] > retweet_threshold:
                logger.info('Classified as %s retweetable, above %s threshold, retweeting...' % (doc_probs[1], retweet_threshold))
                twitter.retweet(candidates[idx]['tid'])
                num_retweeted += 1
            else:
                logger.info('Classified as %s retweetable, below %s threshold, not retweeting...' % (doc_probs[1], retweet_threshold))

def _get_tweet_texts(tweets):
    """
    Pulls out the content of a list of Tweets.
    """
    return [tweet['body'] for tweet in tweets]
<|endoftext|>


<commit_before>from flask import session, request, jsonify, abort
import tablib
import json
import csv

in_data = tablib.Dataset()
out_data = tablib.Dataset()
base_route = "/api/classification/"
out_data_path = None

def load(opts):
    if opts.input_data != None:
        if opts.debug: print("Loading Data")
        with open(opts.input_data, "r") as f:
            reader = csv.reader(f)
            in_data.headers = next(reader)
            for row in reader:
                in_data.append(row)
        if opts.debug: print("Loaded %s rows of data." % len(in_data) )

        if opts.output_header:
            header = opts.output_header
        else:
            header = "data"

        out_data.headers = [ header, "example", "nonexample", "unknown" ]
        fake_row = [ "", 0, 0, 0]

        for row in range(len(in_data)):
            out_data.append(fake_row)

        global out_data_path
        out_data_path = opts.output_data if opts.output_data else "out.csv"
        with open(out_data_path, "w") as f:
            f.write(out_data.csv)

def register(app, opts):

    @app.route(base_route + "element/count/", methods=["GET"])
    def getNumberOfElements():
        return jsonify({"result": len(in_data)})

    @app.route(base_route + "element/", methods=["GET"])
    def getElementInSessionRange():
        if "next" in session:
            nxt = session["next"]
            session["current"] = nxt
            element = in_data["tweet"][nxt]
            resp = jsonify({"result": element})
            session["next"] += 1
            return resp
        else:
            abort(400)

    @app.route(base_route + "element/classify/<classification>/", methods=["POST"])
    def setElementClassification(classification):
        if "current" in session:
            current = session["current"]
            element = in_data[current]
            out_element = out_data[current]

            if classification == "example":
                out_data[current] = (element[4], out_element[1]+1, out_element[2], out_element[3])
            elif classification == "nonexample":
                out_data[current] = (element[4], out_element[1], out_element[2]+1, out_element[3])
            elif classification == "unknown":
                out_data[current] = (element[4], out_element[1], out_element[2], out_element[3]+1)

            global out_data_path
            with open(out_data_path, "w") as f:
                f.write(out_data.csv)

            return jsonify({"response": True})

    @app.route(base_route + "range/<int:start>/<int:end>/", methods=["POST"])
    def setRange(start, end):
        session["start"] = start - 1
        session["end"] = end
        session["next"] = start - 1
        session["current"] = None
        return jsonify({"result": True})

<commit_msg>Adds the ability to relaod output file to resume classification<commit_after>from flask import session, request, jsonify, abort
import tablib
import json
import csv
from os import path

in_data = tablib.Dataset()
out_data = tablib.Dataset()
base_route = "/api/classification/"
out_data_path = None

def load(opts):
    if opts.input_data != None:
        if opts.debug: print("Loading Data")
        with open(opts.input_data, "r") as f:
            reader = csv.reader(f)
            in_data.headers = next(reader)
            for row in reader:
                in_data.append(row)
        if opts.debug: print("Loaded %s rows of data." % len(in_data) )

        if opts.output_header:
            header = opts.output_header
        else:
            header = "data"

        out_data.headers = [ header, "example", "nonexample", "unknown" ]
        fake_row = [ "", 0, 0, 0]

        global out_data_path
        out_data_path = opts.output_data if opts.output_data else "out.csv"

        if not path.exists(out_data_path):
            if opts.debug: print("Create new output file.")
            for row in range(len(in_data)):
                out_data.append(fake_row)

            with open(out_data_path, "w") as f:
                f.write(out_data.csv)
        else:
            if opts.debug: print("Load existing output file.")
            with open(out_data_path, "r") as f:
                reader = csv.reader(f)
                headers = next(reader)
                out_data.headers = headers
                for row in reader:
                    out_data.append((row[0], int(row[1]), int(row[2]), int(row[3])))

def register(app, opts):

    @app.route(base_route + "element/count/", methods=["GET"])
    def getNumberOfElements():
        return jsonify({"result": len(in_data)})

    @app.route(base_route + "element/", methods=["GET"])
    def getElementInSessionRange():
        if "next" in session:
            nxt = session["next"]
            session["current"] = nxt
            element = in_data["tweet"][nxt]
            resp = jsonify({"result": element})
            session["next"] += 1
            return resp
        else:
            abort(400)

    @app.route(base_route + "element/classify/<classification>/", methods=["POST"])
    def setElementClassification(classification):
        if "current" in session:
            current = session["current"]
            element = in_data[current]
            out_element = out_data[current]

            if classification == "example":
                out_data[current] = (element[4], out_element[1]+1, out_element[2], out_element[3])
            elif classification == "nonexample":
                out_data[current] = (element[4], out_element[1], out_element[2]+1, out_element[3])
            elif classification == "unknown":
                out_data[current] = (element[4], out_element[1], out_element[2], out_element[3]+1)

            global out_data_path
            with open(out_data_path, "w") as f:
                f.write(out_data.csv)

            return jsonify({"response": True})

    @app.route(base_route + "range/<int:start>/<int:end>/", methods=["POST"])
    def setRange(start, end):
        session["start"] = start - 1
        session["end"] = end
        session["next"] = start - 1
        session["current"] = None
        return jsonify({"result": True})

<|endoftext|>


<commit_before># Test of receiving Bluetooth controller commands on Raspberry Pi 3
# Murray Ireland
# 22/12/2016
__author__ = 'Murray Ireland'

# Load library functions
import time
import pygame
import RPi.GPIO

# Initialise
pygame.init()
pygame.joystick.init()

# Number of joysticks available
joystick_count = pygame.joystick.get_count()

if joystick_count == 0 :
    print "No controllers available"
else :
    joystick = pygame.joystick.Joystick(0)
    joystick.init()

    # Get the joystick name
    name = joystick.get_name()
    print("Joystick name: {}".format(name))

    # Get joystick axes
    axes = joystick.get_numaxes()
    print("Number of axes: {}".format(axes))

    # Get joystick buttons
    buttons = joystick.get_numbuttons()
    print("Number of buttons: {}".format(buttons))

if joystick_count == 1 :
    while True :
        # Print axes values
        for i in range(axes) :
            axis = joystick.get_axis(i)
            print("Axis {} value: {:f}".format(i,axis))

        for i in range(buttons) :
            button = joystick.get_button(i)
            print("Button {} value {:f}".format(i,button))

        # Wait
        time.sleep(1)
<commit_msg>More bluetooth changes<commit_after># Test of receiving Bluetooth controller commands on Raspberry Pi 3
# Murray Ireland
# 22/12/2016
__author__ = 'Murray Ireland'

# Load library functions
import time
import pygame
#import RPi.GPIO

# Initialise
pygame.init()
pygame.joystick.init()

# Number of joysticks available
joystick_count = pygame.joystick.get_count()

if joystick_count == 0:
    print "No controllers available"
else :
    joystick = pygame.joystick.Joystick(0)
    joystick.init()

    # Get the joystick name
    name = joystick.get_name()
    print "Joystick name: {}".format(name)

    # Get joystick axes
    axes = joystick.get_numaxes()
    print "Number of axes: {}".format(axes)

    # Get joystick buttons
    buttons = joystick.get_numbuttons()
    print "Number of buttons: {}".format(buttons)

if joystick_count == 1:
    while False:
        # Print axes values
        for i in range(axes):
            axis = joystick.get_axis(i)
            print "Axis {} value: {:f}".format(i, axis)

        for i in range(buttons):
            button = joystick.get_button(i)
            print "Button {} value {:f}".format(i, button)

        # Wait
        time.sleep(1)
<|endoftext|>


<commit_before>""" A stylish alternative for caching your map tiles.

TileStache is a Python-based server application that can serve up map tiles
based on rendered geographic data. You might be familiar with TileCache
(http://tilecache.org), the venerable open source WMS server from MetaCarta.
TileStache is similar, but we hope simpler and better-suited to the needs of
designers and cartographers.

Documentation available at http://tilestache.org/doc/
"""
from __future__ import print_function
import os.path

__version__ = open(os.path.join(os.path.dirname(__file__), 'VERSION')).read().strip()

import re

from sys import stdout
try:
    from urlparse import parse_qs
except ImportError:
    from cgi import parse_qs
try:
    from io import StringIO
except ImportError:
    # Python 2
    from StringIO import StringIO
from os.path import dirname, join as pathjoin, realpath
from datetime import datetime, timedelta
try:
    from urllib.parse import urljoin, urlparse
except ImportError:
    # Python 2
    from urlparse import urljoin, urlparse
from wsgiref.headers import Headers
try:
    from urllib.request import urlopen
except ImportError:
    # Python 2
    from urllib import urlopen
from os import getcwd
from time import time

try:
    import http.client as httplib
except ImportError:
    # Python 2
    import httplib
import logging

try:
    from json import load as json_load
    from json import loads as json_loads
except ImportError:
    from simplejson import load as json_load
    from simplejson import loads as json_loads

from ModestMaps.Core import Coordinate

# dictionary of configuration objects for requestLayer().
_previous_configs = {}

from . import Core
from . import Config

# regular expression for PATH_INFO
_pathinfo_pat = re.compile(r'^/?(?P<l>\w.+)/(?P<z>\d+)/(?P<x>-?\d+)/(?P<y>-?\d+)\.(?P<e>\w+)$')
_preview_pat = re.compile(r'^/?(?P<l>\w.+)/(preview\.html)?$')

def getTile(layer, coord, extension, ignore_cached=False):
    ''' Get a type string and tile binary for a given request layer tile.

        This function is documented as part of TileStache's public API:
            http://tilestache.org/doc/#tilestache-gettile

        Arguments:
        - layer: instance of Core.Layer to render.
        - coord: one ModestMaps.Core.Coordinate corresponding to a single tile.
        - extension: filename extension to choose response type, e.g. "png" or "jpg".
        - ignore_cached: always re-render the tile, whether it's in the cache or not.

        This is the main entry point, after site configuration has been loaded
        and individual tiles need to be rendered.
    '''
    status_code, headers, body = layer.getTileResponse(coord, extension, ignore_cached)
    mime = headers.get('Content-Type')

    return mime, body

def getPreview(layer):
    """ Get a type string and dynamic map viewer HTML for a given layer.
    """
    return 200, Headers([('Content-Type', 'text/html')]), Core._preview(layer)

def parseConfig(configHandle):
    """ Parse a configuration file and return a Configuration object.

        Configuration could be a Python dictionary or a file formatted as JSON. In both cases
        it needs to be formatted with two sections, "cache" and "layers":

          {
            "cache": { ... },
            "layers": {
              "layer-1": { ... },
              "layer-2": { ... },
              ...
            }
          }

        The full path to the file is significant, used to
        resolve any relative paths found in the configuration.

        See the Caches module for more information on the "caches" section,
        and the Core and Providers modules for more information on the
        "layers" section.
    """
    if isinstance(configHandle, dict):
        config_dict = configHandle
        dirpath = '.'
    else:
        config_dict = json_load(urlopen(configHandle))
        scheme, host, path, p, q, f = urlparse(configHandle)

        if scheme == '':
            scheme = 'file'
            path = realpath(path)

        dirpath = '%s://%s%s' % (scheme, host, dirname(path).rstrip('/') + '/')

    return Config.buildConfiguration(config_dict, dirpath)

parseConfigfile = parseConfig  # Deprecated function

def splitPathInfo(pathinfo):
    """ Converts a PATH_INFO string to layer name, coordinate, and extension parts.

        Example: "/layer/0/0/0.png", leading "/" optional.
    """
    if pathinfo == '/':
        return None, None, None

    if _pathinfo_pat.match(pathinfo or ''):
        path = _pathinfo_pat.match(pathinfo)
        layer, row, column, zoom, extension = [path.group(p) for p in 'lyxze']
        coord = Coordinate(int(row), int(column), int(zoom))

    elif _preview_pat.match(pathinfo or ''):
        path = _preview_pat.match(pathinfo)
        layer, extension = path.group('l'), 'html'
        coord = None

    else:
        raise Core.KnownUnknown('Bad path: "%s". I was expecting something more like "/example/0/0/0.png"' % pathinfo)

    return layer, coord, extension

def mergePathInfo(layer, coord, extension):
    """ Converts layer name, coordinate and extension back to a PATH_INFO string.

        See also splitPathInfo().
    """
    z = coord.zoom
    x = coord.column
    y = coord.row

    return '/%(layer)s/%(z)d/%(x)d/%(y)d.%(extension)s' % locals()

def requestLayer(config, path_info):
    """ Return a Layer.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.
    """
    if type(config) in (str, unicode):
        #
        # Should be a path to a configuration file we can load;
        # build a tuple key into previously-seen config objects.
        #
        key = hasattr(config, '__hash__') and (config, getcwd())

        if key in _previous_configs:
            config = _previous_configs[key]

        else:
            config = parseConfig(config)

            if key:
                _previous_configs[key] = config

    else:
        assert hasattr(config, 'cache'), 'Configuration object must have a cache.'
        assert hasattr(config, 'layers'), 'Configuration object must have layers.'
        assert hasattr(config, 'dirpath'), 'Configuration object must have a dirpath.'

    # ensure that path_info is at least a single "/"
    path_info = '/' + (path_info or '').lstrip('/')

    if path_info == '/':
        return Core.Layer(config, None, None)

    layername = splitPathInfo(path_info)[0]

    if layername not in config.layers:
        raise Core.KnownUnknown('"%s" is not a layer I know about. Here are some that I do know about: %s.' % (layername, ', '.join(sorted(config.layers.keys()))))

    return config.layers[layername]

def requestHandler(config_hint, path_info, query_string=None):
    """ Generate a mime-type and response body for a given request.

        This function is documented as part of TileStache's public API:
            http://tilestache.org/doc/#tilestache-requesthandler

        TODO: replace this with requestHandler2() in TileStache 2.0.0.

        Calls requestHandler2().
    """
    status_code, headers, content = requestHandler2(config_hint, path_info, query_string)
    mimetype = headers.get('Content-Type')

    return mimetype, content

def requestHandler2(config_hint, path_info, query_string=None, script_name=''):
    """ Generate a set of headers and response body for a given request.

        TODO: Replace requestHandler() with this function in TileStache 2.0.0.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config_hint parameter can be a path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.

        Query string is optional, currently used for JSON callbacks.

        Calls Layer.getTileResponse() to render actual tiles, and getPreview() to render preview.html.
    """
    headers = Headers([])

    try:
        # ensure that path_info is at least a single "/"
        path_info = '/' + (path_info or '').lstrip('/')

        layer = requestLayer(config_hint, path_info)
        query = parse_qs(query_string or '')
        try:
            callback = query['callback'][0]
        except KeyError:
            callback = None

        #
        # Special case for index page.
        #
        if path_info == '/':
            mimetype, content = getattr(layer.config, 'index', ('text/plain', 'TileStache says hello.'))
            return 200, Headers([('Content-Type', mimetype)]), content

        coord, extension = splitPathInfo(path_info)[1:]

        if extension == 'html' and coord is None:
            status_code, headers, content = getPreview(layer)

        elif extension.lower() in layer.redirects:
            other_extension = layer.redirects[extension.lower()]

            redirect_uri = script_name
            redirect_uri += mergePathInfo(layer.name(), coord, other_extension)

            if query_string:
                redirect_uri += '?' + query_string

            headers['Location'] = redirect_uri
            headers['Content-Type'] = 'text/plain'

            return 302, headers, 'You are being redirected to %s\n' % redirect_uri

        else:
            status_code, headers, content = layer.getTileResponse(coord, extension)

        if layer.allowed_origin:
            headers.setdefault('Access-Control-Allow-Origin', layer.allowed_origin)

        if callback and 'json' in headers['Content-Type']:
            headers['Content-Type'] = 'application/javascript; charset=utf-8'
            content = '%s(%s)' % (callback, content)

        if layer.max_cache_age is not None:
            expires = datetime.utcnow() + timedelta(seconds=layer.max_cache_age)
            headers.setdefault('Expires', expires.strftime('%a, %d %b %Y %H:%M:%S GMT'))
            headers.setdefault('Cache-Control', 'public, max-age=%d' % layer.max_cache_age)

    except Core.KnownUnknown as e:
        out = StringIO()

        print >> out, 'Known unknown!'
        print >> out, e
        print >> out, ''
        print >> out, '\n'.join(Core._rummy())

        headers['Content-Type'] = 'text/plain'
        status_code, content = 500, out.getvalue()

    return status_code, headers, content

def cgiHandler(environ, config='./tilestache.cfg', debug=False):
    """ Read environment PATH_INFO, load up configuration, talk to stdout by CGI.

        This function is documented as part of TileStache's public API:
            http://tilestache.org/doc/#cgi

        Calls requestHandler().

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.
    """
    if debug:
        import cgitb
        cgitb.enable()

    path_info = environ.get('PATH_INFO', None)
    query_string = environ.get('QUERY_STRING', None)
    script_name = environ.get('SCRIPT_NAME', None)

    status_code, headers, content = requestHandler2(config, path_info, query_string, script_name)

    headers.setdefault('Content-Length', str(len(content)))

    # output the status code as a header
    stdout.write('Status: %d\n' % status_code)

    # output gathered headers
    for k, v in headers.items():
        stdout.write('%s: %s\n' % (k, v))

    stdout.write('\n')
    stdout.write(content)

class WSGITileServer:
    """ Create a WSGI application that can handle requests from any server that talks WSGI.

        This class is documented as part of TileStache's public API:
            http://tilestache.org/doc/#wsgi

        The WSGI application is an instance of this class. Example:

          app = WSGITileServer('/path/to/tilestache.cfg')
          werkzeug.serving.run_simple('localhost', 8080, app)
    """

    def __init__(self, config, autoreload=False):
        """ Initialize a callable WSGI instance.

            Config parameter can be a file path string for a JSON configuration
            file or a configuration object with 'cache', 'layers', and
            'dirpath' properties.

            Optional autoreload boolean parameter causes config to be re-read
            on each request, applicable only when config is a JSON file.
        """

        if type(config) in (str, unicode, dict):
            self.autoreload = autoreload
            self.config_path = config

            try:
                self.config = parseConfig(config)
            except:
                print("Error loading Tilestache config:")
                raise

        else:
            assert hasattr(config, 'cache'), 'Configuration object must have a cache.'
            assert hasattr(config, 'layers'), 'Configuration object must have layers.'
            assert hasattr(config, 'dirpath'), 'Configuration object must have a dirpath.'

            self.autoreload = False
            self.config_path = None
            self.config = config

    def __call__(self, environ, start_response):
        """
        """
        if self.autoreload: # re-parse the config file on every request
            try:
                self.config = parseConfig(self.config_path)
            except Exception as e:
                raise Core.KnownUnknown("Error loading Tilestache config file:\n%s" % str(e))

        try:
            layer, coord, ext = splitPathInfo(environ['PATH_INFO'])
        except Core.KnownUnknown as e:
            return self._response(start_response, 400, str(e))

        #
        # WSGI behavior is different from CGI behavior, because we may not want
        # to return a chatty rummy for likely-deployed WSGI vs. testing CGI.
        #
        if layer and layer not in self.config.layers:
            return self._response(start_response, 404)

        path_info = environ.get('PATH_INFO', None)
        query_string = environ.get('QUERY_STRING', None)
        script_name = environ.get('SCRIPT_NAME', None)

        status_code, headers, content = requestHandler2(self.config, path_info, query_string, script_name)

        return self._response(start_response, status_code, str(content), headers)

    def _response(self, start_response, code, content='', headers=None):
        """
        """
        headers = headers or Headers([])

        if content:
            headers.setdefault('Content-Length', str(len(content)))

        start_response('%d %s' % (code, httplib.responses[code]), headers.items())
        return [content]

def modpythonHandler(request):
    """ Handle a mod_python request.

        TODO: Upgrade to new requestHandler() so this can return non-200 HTTP.

        Calls requestHandler().

        Example Apache configuration for TileStache:

        <Directory /home/migurski/public_html/TileStache>
            AddHandler mod_python .py
            PythonHandler TileStache::modpythonHandler
            PythonOption config /etc/tilestache.cfg
        </Directory>

        Configuration options, using PythonOption directive:
        - config: path to configuration file, defaults to "tilestache.cfg",
            using request.filename as the current working directory.
    """
    from mod_python import apache

    config_path = request.get_options().get('config', 'tilestache.cfg')
    config_path = realpath(pathjoin(dirname(request.filename), config_path))

    path_info = request.path_info
    query_string = request.args

    mimetype, content = requestHandler(config_path, path_info, query_string)

    request.status = apache.HTTP_OK
    request.content_type = mimetype
    request.set_content_length(len(content))
    request.send_http_header()

    request.write(content)

    return apache.OK
<commit_msg>Fixed schema-less urlopen() usage<commit_after>""" A stylish alternative for caching your map tiles.

TileStache is a Python-based server application that can serve up map tiles
based on rendered geographic data. You might be familiar with TileCache
(http://tilecache.org), the venerable open source WMS server from MetaCarta.
TileStache is similar, but we hope simpler and better-suited to the needs of
designers and cartographers.

Documentation available at http://tilestache.org/doc/
"""
from __future__ import print_function
import os.path

__version__ = open(os.path.join(os.path.dirname(__file__), 'VERSION')).read().strip()

import re

from sys import stdout
try:
    from urlparse import parse_qs
except ImportError:
    from cgi import parse_qs
try:
    from io import StringIO
except ImportError:
    # Python 2
    from StringIO import StringIO
from os.path import dirname, join as pathjoin, realpath
from datetime import datetime, timedelta
try:
    from urllib.parse import urljoin, urlparse
except ImportError:
    # Python 2
    from urlparse import urljoin, urlparse
from wsgiref.headers import Headers
try:
    from urllib.request import urlopen
except ImportError:
    # Python 2
    from urllib import urlopen
from os import getcwd
from time import time

try:
    import http.client as httplib
except ImportError:
    # Python 2
    import httplib
import logging

try:
    from json import load as json_load
    from json import loads as json_loads
except ImportError:
    from simplejson import load as json_load
    from simplejson import loads as json_loads

from ModestMaps.Core import Coordinate

# dictionary of configuration objects for requestLayer().
_previous_configs = {}

from . import Core
from . import Config

# regular expression for PATH_INFO
_pathinfo_pat = re.compile(r'^/?(?P<l>\w.+)/(?P<z>\d+)/(?P<x>-?\d+)/(?P<y>-?\d+)\.(?P<e>\w+)$')
_preview_pat = re.compile(r'^/?(?P<l>\w.+)/(preview\.html)?$')

def getTile(layer, coord, extension, ignore_cached=False):
    ''' Get a type string and tile binary for a given request layer tile.

        This function is documented as part of TileStache's public API:
            http://tilestache.org/doc/#tilestache-gettile

        Arguments:
        - layer: instance of Core.Layer to render.
        - coord: one ModestMaps.Core.Coordinate corresponding to a single tile.
        - extension: filename extension to choose response type, e.g. "png" or "jpg".
        - ignore_cached: always re-render the tile, whether it's in the cache or not.

        This is the main entry point, after site configuration has been loaded
        and individual tiles need to be rendered.
    '''
    status_code, headers, body = layer.getTileResponse(coord, extension, ignore_cached)
    mime = headers.get('Content-Type')

    return mime, body

def getPreview(layer):
    """ Get a type string and dynamic map viewer HTML for a given layer.
    """
    return 200, Headers([('Content-Type', 'text/html')]), Core._preview(layer)

def parseConfig(configHandle):
    """ Parse a configuration file and return a Configuration object.

        Configuration could be a Python dictionary or a file formatted as JSON. In both cases
        it needs to be formatted with two sections, "cache" and "layers":

          {
            "cache": { ... },
            "layers": {
              "layer-1": { ... },
              "layer-2": { ... },
              ...
            }
          }

        The full path to the file is significant, used to
        resolve any relative paths found in the configuration.

        See the Caches module for more information on the "caches" section,
        and the Core and Providers modules for more information on the
        "layers" section.
    """
    if isinstance(configHandle, dict):
        config_dict = configHandle
        dirpath = '.'
    else:
        scheme, host, path, p, q, f = urlparse(configHandle)

        if scheme == '':
            scheme = 'file'
            path = realpath(path)

        if scheme == 'file':
            with open(path) as file:
                config_dict = json_load(file)
        else:
            config_dict = json_load(urlopen(configHandle))

        dirpath = '%s://%s%s' % (scheme, host, dirname(path).rstrip('/') + '/')

    return Config.buildConfiguration(config_dict, dirpath)

parseConfigfile = parseConfig  # Deprecated function

def splitPathInfo(pathinfo):
    """ Converts a PATH_INFO string to layer name, coordinate, and extension parts.

        Example: "/layer/0/0/0.png", leading "/" optional.
    """
    if pathinfo == '/':
        return None, None, None

    if _pathinfo_pat.match(pathinfo or ''):
        path = _pathinfo_pat.match(pathinfo)
        layer, row, column, zoom, extension = [path.group(p) for p in 'lyxze']
        coord = Coordinate(int(row), int(column), int(zoom))

    elif _preview_pat.match(pathinfo or ''):
        path = _preview_pat.match(pathinfo)
        layer, extension = path.group('l'), 'html'
        coord = None

    else:
        raise Core.KnownUnknown('Bad path: "%s". I was expecting something more like "/example/0/0/0.png"' % pathinfo)

    return layer, coord, extension

def mergePathInfo(layer, coord, extension):
    """ Converts layer name, coordinate and extension back to a PATH_INFO string.

        See also splitPathInfo().
    """
    z = coord.zoom
    x = coord.column
    y = coord.row

    return '/%(layer)s/%(z)d/%(x)d/%(y)d.%(extension)s' % locals()

def requestLayer(config, path_info):
    """ Return a Layer.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.
    """
    if type(config) in (str, unicode):
        #
        # Should be a path to a configuration file we can load;
        # build a tuple key into previously-seen config objects.
        #
        key = hasattr(config, '__hash__') and (config, getcwd())

        if key in _previous_configs:
            config = _previous_configs[key]

        else:
            config = parseConfig(config)

            if key:
                _previous_configs[key] = config

    else:
        assert hasattr(config, 'cache'), 'Configuration object must have a cache.'
        assert hasattr(config, 'layers'), 'Configuration object must have layers.'
        assert hasattr(config, 'dirpath'), 'Configuration object must have a dirpath.'

    # ensure that path_info is at least a single "/"
    path_info = '/' + (path_info or '').lstrip('/')

    if path_info == '/':
        return Core.Layer(config, None, None)

    layername = splitPathInfo(path_info)[0]

    if layername not in config.layers:
        raise Core.KnownUnknown('"%s" is not a layer I know about. Here are some that I do know about: %s.' % (layername, ', '.join(sorted(config.layers.keys()))))

    return config.layers[layername]

def requestHandler(config_hint, path_info, query_string=None):
    """ Generate a mime-type and response body for a given request.

        This function is documented as part of TileStache's public API:
            http://tilestache.org/doc/#tilestache-requesthandler

        TODO: replace this with requestHandler2() in TileStache 2.0.0.

        Calls requestHandler2().
    """
    status_code, headers, content = requestHandler2(config_hint, path_info, query_string)
    mimetype = headers.get('Content-Type')

    return mimetype, content

def requestHandler2(config_hint, path_info, query_string=None, script_name=''):
    """ Generate a set of headers and response body for a given request.

        TODO: Replace requestHandler() with this function in TileStache 2.0.0.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config_hint parameter can be a path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.

        Query string is optional, currently used for JSON callbacks.

        Calls Layer.getTileResponse() to render actual tiles, and getPreview() to render preview.html.
    """
    headers = Headers([])

    try:
        # ensure that path_info is at least a single "/"
        path_info = '/' + (path_info or '').lstrip('/')

        layer = requestLayer(config_hint, path_info)
        query = parse_qs(query_string or '')
        try:
            callback = query['callback'][0]
        except KeyError:
            callback = None

        #
        # Special case for index page.
        #
        if path_info == '/':
            mimetype, content = getattr(layer.config, 'index', ('text/plain', 'TileStache says hello.'))
            return 200, Headers([('Content-Type', mimetype)]), content

        coord, extension = splitPathInfo(path_info)[1:]

        if extension == 'html' and coord is None:
            status_code, headers, content = getPreview(layer)

        elif extension.lower() in layer.redirects:
            other_extension = layer.redirects[extension.lower()]

            redirect_uri = script_name
            redirect_uri += mergePathInfo(layer.name(), coord, other_extension)

            if query_string:
                redirect_uri += '?' + query_string

            headers['Location'] = redirect_uri
            headers['Content-Type'] = 'text/plain'

            return 302, headers, 'You are being redirected to %s\n' % redirect_uri

        else:
            status_code, headers, content = layer.getTileResponse(coord, extension)

        if layer.allowed_origin:
            headers.setdefault('Access-Control-Allow-Origin', layer.allowed_origin)

        if callback and 'json' in headers['Content-Type']:
            headers['Content-Type'] = 'application/javascript; charset=utf-8'
            content = '%s(%s)' % (callback, content)

        if layer.max_cache_age is not None:
            expires = datetime.utcnow() + timedelta(seconds=layer.max_cache_age)
            headers.setdefault('Expires', expires.strftime('%a, %d %b %Y %H:%M:%S GMT'))
            headers.setdefault('Cache-Control', 'public, max-age=%d' % layer.max_cache_age)

    except Core.KnownUnknown as e:
        out = StringIO()

        print >> out, 'Known unknown!'
        print >> out, e
        print >> out, ''
        print >> out, '\n'.join(Core._rummy())

        headers['Content-Type'] = 'text/plain'
        status_code, content = 500, out.getvalue()

    return status_code, headers, content

def cgiHandler(environ, config='./tilestache.cfg', debug=False):
    """ Read environment PATH_INFO, load up configuration, talk to stdout by CGI.

        This function is documented as part of TileStache's public API:
            http://tilestache.org/doc/#cgi

        Calls requestHandler().

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.
    """
    if debug:
        import cgitb
        cgitb.enable()

    path_info = environ.get('PATH_INFO', None)
    query_string = environ.get('QUERY_STRING', None)
    script_name = environ.get('SCRIPT_NAME', None)

    status_code, headers, content = requestHandler2(config, path_info, query_string, script_name)

    headers.setdefault('Content-Length', str(len(content)))

    # output the status code as a header
    stdout.write('Status: %d\n' % status_code)

    # output gathered headers
    for k, v in headers.items():
        stdout.write('%s: %s\n' % (k, v))

    stdout.write('\n')
    stdout.write(content)

class WSGITileServer:
    """ Create a WSGI application that can handle requests from any server that talks WSGI.

        This class is documented as part of TileStache's public API:
            http://tilestache.org/doc/#wsgi

        The WSGI application is an instance of this class. Example:

          app = WSGITileServer('/path/to/tilestache.cfg')
          werkzeug.serving.run_simple('localhost', 8080, app)
    """

    def __init__(self, config, autoreload=False):
        """ Initialize a callable WSGI instance.

            Config parameter can be a file path string for a JSON configuration
            file or a configuration object with 'cache', 'layers', and
            'dirpath' properties.

            Optional autoreload boolean parameter causes config to be re-read
            on each request, applicable only when config is a JSON file.
        """

        if type(config) in (str, unicode, dict):
            self.autoreload = autoreload
            self.config_path = config

            try:
                self.config = parseConfig(config)
            except:
                print("Error loading Tilestache config:")
                raise

        else:
            assert hasattr(config, 'cache'), 'Configuration object must have a cache.'
            assert hasattr(config, 'layers'), 'Configuration object must have layers.'
            assert hasattr(config, 'dirpath'), 'Configuration object must have a dirpath.'

            self.autoreload = False
            self.config_path = None
            self.config = config

    def __call__(self, environ, start_response):
        """
        """
        if self.autoreload: # re-parse the config file on every request
            try:
                self.config = parseConfig(self.config_path)
            except Exception as e:
                raise Core.KnownUnknown("Error loading Tilestache config file:\n%s" % str(e))

        try:
            layer, coord, ext = splitPathInfo(environ['PATH_INFO'])
        except Core.KnownUnknown as e:
            return self._response(start_response, 400, str(e))

        #
        # WSGI behavior is different from CGI behavior, because we may not want
        # to return a chatty rummy for likely-deployed WSGI vs. testing CGI.
        #
        if layer and layer not in self.config.layers:
            return self._response(start_response, 404)

        path_info = environ.get('PATH_INFO', None)
        query_string = environ.get('QUERY_STRING', None)
        script_name = environ.get('SCRIPT_NAME', None)

        status_code, headers, content = requestHandler2(self.config, path_info, query_string, script_name)

        return self._response(start_response, status_code, str(content), headers)

    def _response(self, start_response, code, content='', headers=None):
        """
        """
        headers = headers or Headers([])

        if content:
            headers.setdefault('Content-Length', str(len(content)))

        start_response('%d %s' % (code, httplib.responses[code]), headers.items())
        return [content]

def modpythonHandler(request):
    """ Handle a mod_python request.

        TODO: Upgrade to new requestHandler() so this can return non-200 HTTP.

        Calls requestHandler().

        Example Apache configuration for TileStache:

        <Directory /home/migurski/public_html/TileStache>
            AddHandler mod_python .py
            PythonHandler TileStache::modpythonHandler
            PythonOption config /etc/tilestache.cfg
        </Directory>

        Configuration options, using PythonOption directive:
        - config: path to configuration file, defaults to "tilestache.cfg",
            using request.filename as the current working directory.
    """
    from mod_python import apache

    config_path = request.get_options().get('config', 'tilestache.cfg')
    config_path = realpath(pathjoin(dirname(request.filename), config_path))

    path_info = request.path_info
    query_string = request.args

    mimetype, content = requestHandler(config_path, path_info, query_string)

    request.status = apache.HTTP_OK
    request.content_type = mimetype
    request.set_content_length(len(content))
    request.send_http_header()

    request.write(content)

    return apache.OK
<|endoftext|>


<commit_before>#!/usr/bin/env python

################################################################################
# Copyright 2016 William Chan <williamchan@cmu.edu>.
################################################################################

import argparse
import google
import kaldi_io
import numpy as np
import os
import re
import os
import sys
import tensorflow as tf
import tensorflow.core.framework.token_model_pb2 as token_model_pb2

def main():
  parser = argparse.ArgumentParser(description='SPEECH3 (C) 2015 William Chan <williamchan@cmu.edu>')
  parser.add_argument('--kaldi_scp', type=str)
  parser.add_argument('--kaldi_txt', type=str)
  parser.add_argument('--phones', type=str)
  parser.add_argument('--remap', type=str)
  parser.add_argument('--tf_records', type=str)
  args   = vars(parser.parse_args())

  convert(
      args['kaldi_scp'], args['kaldi_txt'], args['phones'], args['remap'],
      args['tf_records'])

def load_phones(phones_txt):
  lines = [line.strip() for line in open(phones_txt, 'r')]
  phones = []
  for line in lines:
    cols = line.split(" ")
    assert len(cols) == 2

    phone = cols[0]
    phones.append(phone)
  return phones

def token_model_add_token(token_model_proto, token_id, token_string):
  token = token_model_proto.tokens.add()
  token.token_id = int(token_id)
  token.token_string = str(token_string)

def create_phone_token_model(phones_txt):
  token_model_proto = token_model_pb2.TokenModelProto()

  token_model_proto.token_sos = 0
  token_model_proto.token_string_sos = "<S>"
  token_model_add_token(
      token_model_proto, token_model_proto.token_sos,
      token_model_proto.token_string_sos)

  token_model_proto.token_eos = 1
  token_model_proto.token_string_eos = "</S>"
  token_model_add_token(
      token_model_proto, token_model_proto.token_eos,
      token_model_proto.token_string_eos)

  token_model_proto.token_eow = 2
  token_model_proto.token_string_eow = " "
  token_model_add_token(
      token_model_proto, token_model_proto.token_eow,
      token_model_proto.token_string_eow)

  token_model_proto.token_unk = 3
  token_model_proto.token_string_unk = "<UNK>"
  token_model_add_token(
      token_model_proto, token_model_proto.token_unk,
      token_model_proto.token_string_unk)

  token_model_proto.token_blank = 4
  token_model_proto.token_string_blank = "<BLANK>"
  token_model_add_token(
      token_model_proto, token_model_proto.token_blank,
      token_model_proto.token_string_blank)

  token_id = 5
  token_map = {}

  phones = load_phones(phones_txt)
  for phone in phones:
    token_model_add_token(token_model_proto, token_id, phone)
    token_map[phone] = token_id

    token_id = token_id + 1

  with open("speech4/conf/timit/token_model.pbtxt", "w") as proto_file:
    proto_file.write(str(token_model_proto))

  return token_model_proto, token_map

def load_text_map(kaldi_txt, phone_map):
  lines = [line.strip() for line in open(kaldi_txt, 'r')]
  text_map = {}
  for line in lines:
    cols = line.split(" ", 1)
    uttid = cols[0]
    text = cols[1]
    text_map[uttid] = text
  return text_map

def extract_text_line(line):
  cols = line.split(" ")
  uttid = cols[0]

def convert(
    kaldi_scp, kaldi_txt, phones_txt, remap_txt, tf_records):
  token_model_proto, phone_map = create_phone_token_model(phones_txt)
  text_map = load_text_map(kaldi_txt, phone_map)

  tf_record_writer = tf.python_io.TFRecordWriter(tf_records)
  kaldi_feat_reader = kaldi_io.SequentialBaseFloatMatrixReader(kaldi_scp)

  for uttid, feats in kaldi_feat_reader:
    text = text_map[uttid]
    tokens = [phone_map[phone] for phone in text.split(" ")]
    tokens = [token_model_proto.token_sos] * 2 + tokens + [token_model_proto.token_eos]
    tokens = [int(token) for token in tokens]
    print tokens

    example = tf.train.Example(features=tf.train.Features(feature={
        'features_len': tf.train.Feature(int64_list=tf.train.Int64List(value=[feats.shape[0]])),
        'features': tf.train.Feature(float_list=tf.train.FloatList(value=feats.flatten('C').tolist())),
        'tokens': tf.train.Feature(int64_list=tf.train.Int64List(value=tokens)),
        'uttid': tf.train.Feature(bytes_list=tf.train.BytesList(value=[str(uttid)])),
        'text': tf.train.Feature(bytes_list=tf.train.BytesList(value=[text]))}))
    tf_record_writer.write(example.SerializeToString())

if __name__ == '__main__':
  main()
<commit_msg>timit.py<commit_after>#!/usr/bin/env python

################################################################################
# Copyright 2016 William Chan <williamchan@cmu.edu>.
################################################################################

import argparse
import google
import kaldi_io
import numpy as np
import os
import re
import os
import sys
import tensorflow as tf
import tensorflow.core.framework.token_model_pb2 as token_model_pb2

def main():
  parser = argparse.ArgumentParser(description='SPEECH3 (C) 2015 William Chan <williamchan@cmu.edu>')
  parser.add_argument('--kaldi_scp', type=str)
  parser.add_argument('--kaldi_txt', type=str)
  parser.add_argument('--phones', type=str)
  parser.add_argument('--remap', type=str)
  parser.add_argument('--tf_records', type=str)
  args   = vars(parser.parse_args())

  convert(
      args['kaldi_scp'], args['kaldi_txt'], args['phones'], args['remap'],
      args['tf_records'])

def load_phones(phones_txt):
  lines = [line.strip() for line in open(phones_txt, 'r')]
  phones = []
  for line in lines:
    cols = line.split(" ")
    assert len(cols) == 2

    phone = cols[0]
    phones.append(phone)
  return phones

def token_model_add_token(token_model_proto, token_id, token_string):
  token = token_model_proto.tokens.add()
  token.token_id = int(token_id)
  token.token_string = str(token_string)

def create_phone_token_model(phones_txt):
  token_model_proto = token_model_pb2.TokenModelProto()

  token_model_proto.token_sos = 0
  token_model_proto.token_string_sos = "<S>"
  token_model_add_token(
      token_model_proto, token_model_proto.token_sos,
      token_model_proto.token_string_sos)

  token_model_proto.token_eos = 1
  token_model_proto.token_string_eos = "</S>"
  token_model_add_token(
      token_model_proto, token_model_proto.token_eos,
      token_model_proto.token_string_eos)

  token_model_proto.token_eow = 2
  token_model_proto.token_string_eow = " "
  token_model_add_token(
      token_model_proto, token_model_proto.token_eow,
      token_model_proto.token_string_eow)

  token_model_proto.token_unk = 3
  token_model_proto.token_string_unk = "<UNK>"
  token_model_add_token(
      token_model_proto, token_model_proto.token_unk,
      token_model_proto.token_string_unk)

  token_model_proto.token_blank = 4
  token_model_proto.token_string_blank = "<BLANK>"
  token_model_add_token(
      token_model_proto, token_model_proto.token_blank,
      token_model_proto.token_string_blank)

  token_id = 5
  token_map = {}

  phones = load_phones(phones_txt)
  for phone in phones:
    token_model_add_token(token_model_proto, token_id, phone)
    token_map[phone] = token_id

    token_id = token_id + 1

  with open("speech4/conf/timit/token_model.pbtxt", "w") as proto_file:
    proto_file.write(str(token_model_proto))

  return token_model_proto, token_map

def load_text_map(kaldi_txt, phone_map):
  lines = [line.strip() for line in open(kaldi_txt, 'r')]
  text_map = {}
  for line in lines:
    cols = line.split(" ", 1)
    uttid = cols[0]
    text = cols[1]
    text_map[uttid] = text
  return text_map

def extract_text_line(line):
  cols = line.split(" ")
  uttid = cols[0]

def convert(
    kaldi_scp, kaldi_txt, phones_txt, remap_txt, tf_records):
  token_model_proto, phone_map = create_phone_token_model(phones_txt)
  text_map = load_text_map(kaldi_txt, phone_map)

  tf_record_writer = tf.python_io.TFRecordWriter(tf_records)
  kaldi_feat_reader = kaldi_io.SequentialBaseFloatMatrixReader(kaldi_scp)

  features_len = 0
  features_len_max = 0
  tokens_len = 0
  tokens_len_max = 0
  feature_token_ratio_min = 10
  pad_min = 1e8
  for uttid, feats in kaldi_feat_reader:
    text = text_map[uttid]
    tokens = [phone_map[phone] for phone in text.split(" ")]
    # tokens = [token_model_proto.token_sos] * 2 + tokens + [token_model_proto.token_eos]
    tokens = [int(token) for token in tokens]
    features_len_max = max(features_len_max, feats.shape[0])
    features_len += feats.shape[0]
    tokens_len_max = max(tokens_len_max, len(tokens))
    tokens_len += len(tokens)
    feature_token_ratio_min = min(feature_token_ratio_min, feats.shape[0] / len(tokens))
    pad_min = min(pad_min, features_len / 4 - tokens_len)

    example = tf.train.Example(features=tf.train.Features(feature={
        'features_len': tf.train.Feature(int64_list=tf.train.Int64List(value=[feats.shape[0]])),
        'features': tf.train.Feature(float_list=tf.train.FloatList(value=feats.flatten('C').tolist())),
        'tokens': tf.train.Feature(int64_list=tf.train.Int64List(value=tokens)),
        'uttid': tf.train.Feature(bytes_list=tf.train.BytesList(value=[str(uttid)])),
        'text': tf.train.Feature(bytes_list=tf.train.BytesList(value=[text]))}))
    tf_record_writer.write(example.SerializeToString())
  print("features_len_max: %d" % features_len_max)
  print("tokens_len_max: %d" % tokens_len_max)
  print("feature_token_ratio_min: %f" % feature_token_ratio_min)
  print("features_len: %d" % features_len)
  print("tokens_len: %d" % tokens_len)
  print("pad_min: %d" % pad_min)

if __name__ == '__main__':
  main()
<|endoftext|>