<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# [The "New BSD" license]
# Copyright (c) 2014 The Board of Trustees of The University of Alabama
# All rights reserved.
#
# See LICENSE for details.

import csv
import sys
import os.path
import random
from collections import namedtuple

import click
import dulwich
import dulwich.repo
from gensim.corpora import MalletCorpus, Dictionary
from gensim.models import LdaModel

import utils
from corpora import MultiTextCorpus, ChangesetCorpus

import logging

logger = logging.getLogger('mct')

class Config:
    def __init__(self):
        self.path = './'
        self.project = None
        self.repo = None
        self.file_corpus_fname = ''
        self.changeset_corpus_fname = ''
        self.file_model_fname = ''
        self.changeset_model_fname = ''
        self.passes = 10
        self.num_topics = 100
        self.alpha = 'symmetric' # or can set a float
        # set all possible config options here

def error(msg, errorno=1):
    logger.error(msg)
    sys.exit(errorno)

pass_config = click.make_pass_decorator(Config, ensure=True)

@click.group()
@click.option('--num-topics', default=100)
@click.option('--verbose', is_flag=True)
@click.option('--path', default='data/',
        help="Set the directory to work within")
@click.argument('project')
@pass_config
def main(config, verbose, path, project, num_topics):
    """
    Modeling Changeset Topics
    """

    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(name)s : %(funcName)s : %(message)s')

    if verbose:
        logging.root.setLevel(level=logging.DEBUG)
    else:
        logging.root.setLevel(level=logging.INFO)

    # Only set config items here, this function is unused otherwise.
    config.path = path
    if not config.path.endswith('/'):
        config.path += '/'

    utils.mkdir(config.path)

    with open("projects.csv", 'r') as f:
        reader = csv.reader(f)
        header = next(reader)
        Project = namedtuple('Project',  ' '.join(header))

        # figure out which column index contains the project name
        name_idx = header.index("name")

        # find the project in the csv, adding it's info to config
        for row in reader:
            if project == row[name_idx]:
                # 🎶  do you believe in magicccccc
                # in a young girl's heart? 🎶
                config.project = Project(*row)
                break

        # we can access project info by:
        #    config.project.url => "http://..."
        #    config.project.name => "Blah Name"

        if config.project is None:
            error("Could not find the project '%s' in 'projects.csv'!" % project)

    corpus_fname = (config.path +
                    config.project.name + '-' +
                    config.project.commit[:8] + '-' +
                    '%s')

    model_fname = (config.path +
                    config.project.name + '-' +
                    config.project.commit[:8] + '-' +
                    str(config.passes) + 'passes-' +
                    str(config.alpha) + 'alpha-' +
                    str(config.num_topics) + 'topics-' +
                    '%s')

    config.file_corpus_fname = corpus_fname % 'file.mallet'
    config.changeset_corpus_fname = corpus_fname % 'changeset.mallet'

    config.file_model_fname = model_fname % 'file.lda'
    config.changeset_model_fname = model_fname % 'changeset.lda'

    config.num_topics = num_topics

@main.command()
@pass_config
@click.pass_context
def corpora(context, config):
    """
    Builds the basic corpora for a project
    """

    git_path = config.path + config.project.name
    # open the repo
    try:
        config.repo = dulwich.repo.Repo(git_path)
    except dulwich.errors.NotGitRepository:
        error('Repository not cloned yet! Clone command: '
                'git clone %s %s' % (config.project.url, git_path))

    logger.info('Creating corpora for: %s' % config.project.name)

    if not os.path.exists(config.file_corpus_fname):
        logger.info('Creating file-based corpus out of source files for '
            'release %s at commit %s' % (
                config.project.release, config.project.commit))

        file_corpus = MultiTextCorpus(config.repo, config.project.commit, lazy_dict=True)
        file_corpus.metadata = True
        MalletCorpus.serialize(config.file_corpus_fname, file_corpus,
                id2word=file_corpus.id2word, metadata=True)
        file_corpus.metadata = False
        file_corpus.id2word.save_as_text(config.file_corpus_fname + '.dict')

    if not os.path.exists(config.changeset_corpus_fname):
        logger.info('Creating changeset-based corpus out of source files for '
            'release %s for all commits reachable from %s' % (
                config.project.release, config.project.commit))

        changeset_corpus = ChangesetCorpus(config.repo, config.project.commit, lazy_dict=True)
        changeset_corpus.metadata = True
        MalletCorpus.serialize(config.changeset_corpus_fname, changeset_corpus,
                id2word=changeset_corpus.id2word, metadata=True)
        changeset_corpus.metadata = False
        changeset_corpus.id2word.save_as_text(config.changeset_corpus_fname + '.dict')

@main.command()
@pass_config
@click.pass_context
def model(context, config):
    """
    Builds a model for the corpora
    """
    logger.info('Building topic models for: %s' % config.project.name)

    if not os.path.exists(config.file_model_fname):
        try:
            file_dict = Dictionary.load_from_text(config.file_corpus_fname + '.dict')
            file_corpus = MalletCorpus(config.file_corpus_fname, id2word=file_dict)
            logger.info('Opened previously created corpus at file %s' % config.file_corpus_fname)
        # build one if it doesnt exist
        except:
            error('Corpora for building file models not found!')

        # TODO
        # Maybe look into various settings for num_topics?
        file_model = LdaModel(file_corpus,
                id2word=file_corpus.id2word,
                alpha=config.alpha,
                passes=config.passes,
                num_topics=config.num_topics)

        file_model.save(config.file_model_fname)

    if not os.path.exists(config.changeset_model_fname):
        try:
            changeset_dict = Dictionary.load_from_text(config.changeset_corpus_fname + '.dict')
            changeset_corpus = MalletCorpus(config.changeset_corpus_fname, id2word=changeset_dict)
            logger.info('Opened previously created corpus at file %s' % config.changeset_corpus_fname)
        # build one if it doesnt exist
        except:
            error('Corpora for building changeset models not found!')

        changeset_model = LdaModel(changeset_corpus,
                id2word=changeset_corpus.id2word,
                alpha=config.alpha,
                passes=config.passes,
                num_topics=config.num_topics)

        changeset_model.save(config.changeset_model_fname)

@main.command()
@pass_config
@click.pass_context
def evaluate(context, config):
    """
    Evalutates the models
    """
    try:
        file_model = LdaModel.load(config.file_model_fname)
        logger.info('Opened previously created model at file %s' % config.file_model_fname)
    except:
        error('Cannot evalutate LDA models not built yet!')

    try:
        changeset_model = LdaModel.load(config.changeset_model_fname)
        logger.info('Opened previously created model at changeset %s' % config.changeset_model_fname)
    except:
        error('Cannot evalutate LDA models not built yet!')

    logger.info('Evalutating models for: %s' % config.project.name)

    file_scores = utils.score(file_model, utils.kullback_leibler_divergence)
    changeset_scores = utils.score(changeset_model, utils.kullback_leibler_divergence)

    file_total = sum([x[1] for x in file_scores])
    changeset_total = sum([x[1] for x in changeset_scores])

    logger.info("File model KL: %f" % file_total)
    logger.info("Changeset model KL: %f" % changeset_total)
    logger.info("File model KL mean: %f" % (file_total / len(file_scores)))
    logger.info("Changeset model KL mean: %f" % (changeset_total / len(changeset_scores)))
    with open(config.path + 'evaluate-results.csv', 'a') as f:
        f.write('%s,"%s",%f,%f' % (config.project.name, str(file_model), file_total, changeset_total))

@main.command()
@pass_config
@click.pass_context
def evaluate_corpora(context, config):
    try:
        file_dict = Dictionary.load_from_text(config.file_corpus_fname + '.dict')
        file_corpus = MalletCorpus(config.file_corpus_fname, id2word=file_dict)
    except:
        error('Corpora not built yet -- cannot evaluate')

    file_word_freq = list(reversed(sorted(count_words(file_corpus))))
    print("Top 10 words in files:", file_word_freq[:10])
    print("Bottom 10 words in files:", file_word_freq[-10:])

    try:
        changeset_dict = Dictionary.load_from_text(config.changeset_corpus_fname + '.dict')
        changeset_corpus = MalletCorpus(config.changeset_corpus_fname, id2word=changeset_dict)
    except:
        error('Corpora not built yet -- cannot evaluate')
    changeset_word_freq = list(reversed(sorted(count_words(changeset_corpus))))
    print("Top 10 words in changesets:", changeset_word_freq[:10])
    print("Bottom 10 words in changesets:", changeset_word_freq[-10:])

def count_words(corpus):
    word_freq = dict()
    for doc in corpus:
        for word, count in doc:
            if word not in word_freq:
                word_freq[word] = 0

            word_freq[word] += count

    for word_id, freq in word_freq.items():
        yield freq, corpus.id2word[word_id]

def perplexity(corpus, model):
    held_out = list()
    training = list()
    target_len = int(0.1 * len(corpus))
    logger.info('Calculating perplexity with held-out %d of %d documents' % (target_len, len(corpus)))

    ids = set()
    while len(ids) < target_len:
        ids.add(random.randint(0, len(corpus)))

    for doc_id, doc in enumerate(corpus):
        if doc_id in ids:
            held_out.append(doc)
        else:
            training.append(doc)

    model.update(training)
    return model.log_perplexity(held_out)

@main.command()
@pass_config
@click.pass_context
def evaluate_perplexity(context, config):
    try:
        file_dict = Dictionary.load_from_text(config.file_corpus_fname + '.dict')
        file_corpus = MalletCorpus(config.file_corpus_fname, id2word=file_dict)
    except:
        error('Corpora not built yet -- cannot evaluate')
    try:
        changeset_dict = Dictionary.load_from_text(config.changeset_corpus_fname + '.dict')
        changeset_corpus = MalletCorpus(config.changeset_corpus_fname, id2word=changeset_dict)
    except:
        error('Corpora not built yet -- cannot evaluate')

    file_model = LdaModel(
            id2word=file_corpus.id2word,
            alpha=config.alpha,
            passes=config.passes,
            num_topics=config.num_topics)

    changeset_model = LdaModel(
            id2word=changeset_corpus.id2word,
            alpha=config.alpha,
            passes=config.passes,
            num_topics=config.num_topics)

    logger.info('Calculating file corpus perplexity for: %s' % config.project.name)
    file_pwb = 0
    file_pwb = perplexity(file_corpus, file_model)

    logger.info('Calculating changeset corpus perplexity for: %s' % config.project.name)
    changeset_pwb = 0
    changeset_pwb = perplexity(changeset_corpus, changeset_model)

    with open(config.path + 'evaluate-perplexity-results.csv', 'a') as f:
        f.write('%s,"%s",%f,%f' % (config.project.name, str(file_model), file_pwb, changeset_pwb))

@main.command()
@pass_config
@click.pass_context
def run_all(context, config):
    """
    Runs corpora, preprocess, model, and evaluate in one shot.
    """
    logger.info('Doing everything for: %s' % config.project.name)
    context.forward(corpora)
    context.forward(model)
    context.forward(evaluate)
<commit_msg>Better csv<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# [The "New BSD" license]
# Copyright (c) 2014 The Board of Trustees of The University of Alabama
# All rights reserved.
#
# See LICENSE for details.

import csv
import sys
import os.path
import random
from collections import namedtuple

import click
import dulwich
import dulwich.repo
from gensim.corpora import MalletCorpus, Dictionary
from gensim.models import LdaModel

import utils
from corpora import MultiTextCorpus, ChangesetCorpus

import logging

logger = logging.getLogger('mct')

class Config:
    def __init__(self):
        self.path = './'
        self.project = None
        self.repo = None
        self.file_corpus_fname = ''
        self.changeset_corpus_fname = ''
        self.file_model_fname = ''
        self.changeset_model_fname = ''
        self.passes = 10
        self.num_topics = 100
        self.alpha = 'symmetric' # or can set a float
        # set all possible config options here

def error(msg, errorno=1):
    logger.error(msg)
    sys.exit(errorno)

pass_config = click.make_pass_decorator(Config, ensure=True)

@click.group()
@click.option('--num-topics', default=100)
@click.option('--verbose', is_flag=True)
@click.option('--path', default='data/',
        help="Set the directory to work within")
@click.argument('project')
@pass_config
def main(config, verbose, path, project, num_topics):
    """
    Modeling Changeset Topics
    """

    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(name)s : %(funcName)s : %(message)s')

    if verbose:
        logging.root.setLevel(level=logging.DEBUG)
    else:
        logging.root.setLevel(level=logging.INFO)

    # Only set config items here, this function is unused otherwise.
    config.path = path
    if not config.path.endswith('/'):
        config.path += '/'

    utils.mkdir(config.path)

    with open("projects.csv", 'r') as f:
        reader = csv.reader(f)
        header = next(reader)
        Project = namedtuple('Project',  ' '.join(header))

        # figure out which column index contains the project name
        name_idx = header.index("name")

        # find the project in the csv, adding it's info to config
        for row in reader:
            if project == row[name_idx]:
                # 🎶  do you believe in magicccccc
                # in a young girl's heart? 🎶
                config.project = Project(*row)
                break

        # we can access project info by:
        #    config.project.url => "http://..."
        #    config.project.name => "Blah Name"

        if config.project is None:
            error("Could not find the project '%s' in 'projects.csv'!" % project)

    corpus_fname = (config.path +
                    config.project.name + '-' +
                    config.project.commit[:8] + '-' +
                    '%s')

    model_fname = (config.path +
                    config.project.name + '-' +
                    config.project.commit[:8] + '-' +
                    str(config.passes) + 'passes-' +
                    str(config.alpha) + 'alpha-' +
                    str(config.num_topics) + 'topics-' +
                    '%s')

    config.file_corpus_fname = corpus_fname % 'file.mallet'
    config.changeset_corpus_fname = corpus_fname % 'changeset.mallet'

    config.file_model_fname = model_fname % 'file.lda'
    config.changeset_model_fname = model_fname % 'changeset.lda'

    config.num_topics = num_topics

@main.command()
@pass_config
@click.pass_context
def corpora(context, config):
    """
    Builds the basic corpora for a project
    """

    git_path = config.path + config.project.name
    # open the repo
    try:
        config.repo = dulwich.repo.Repo(git_path)
    except dulwich.errors.NotGitRepository:
        error('Repository not cloned yet! Clone command: '
                'git clone %s %s' % (config.project.url, git_path))

    logger.info('Creating corpora for: %s' % config.project.name)

    if not os.path.exists(config.file_corpus_fname):
        logger.info('Creating file-based corpus out of source files for '
            'release %s at commit %s' % (
                config.project.release, config.project.commit))

        file_corpus = MultiTextCorpus(config.repo, config.project.commit, lazy_dict=True)
        file_corpus.metadata = True
        MalletCorpus.serialize(config.file_corpus_fname, file_corpus,
                id2word=file_corpus.id2word, metadata=True)
        file_corpus.metadata = False
        file_corpus.id2word.save_as_text(config.file_corpus_fname + '.dict')

    if not os.path.exists(config.changeset_corpus_fname):
        logger.info('Creating changeset-based corpus out of source files for '
            'release %s for all commits reachable from %s' % (
                config.project.release, config.project.commit))

        changeset_corpus = ChangesetCorpus(config.repo, config.project.commit, lazy_dict=True)
        changeset_corpus.metadata = True
        MalletCorpus.serialize(config.changeset_corpus_fname, changeset_corpus,
                id2word=changeset_corpus.id2word, metadata=True)
        changeset_corpus.metadata = False
        changeset_corpus.id2word.save_as_text(config.changeset_corpus_fname + '.dict')

@main.command()
@pass_config
@click.pass_context
def model(context, config):
    """
    Builds a model for the corpora
    """
    logger.info('Building topic models for: %s' % config.project.name)

    if not os.path.exists(config.file_model_fname):
        try:
            file_dict = Dictionary.load_from_text(config.file_corpus_fname + '.dict')
            file_corpus = MalletCorpus(config.file_corpus_fname, id2word=file_dict)
            logger.info('Opened previously created corpus at file %s' % config.file_corpus_fname)
        # build one if it doesnt exist
        except:
            error('Corpora for building file models not found!')

        # TODO
        # Maybe look into various settings for num_topics?
        file_model = LdaModel(file_corpus,
                id2word=file_corpus.id2word,
                alpha=config.alpha,
                passes=config.passes,
                num_topics=config.num_topics)

        file_model.save(config.file_model_fname)

    if not os.path.exists(config.changeset_model_fname):
        try:
            changeset_dict = Dictionary.load_from_text(config.changeset_corpus_fname + '.dict')
            changeset_corpus = MalletCorpus(config.changeset_corpus_fname, id2word=changeset_dict)
            logger.info('Opened previously created corpus at file %s' % config.changeset_corpus_fname)
        # build one if it doesnt exist
        except:
            error('Corpora for building changeset models not found!')

        changeset_model = LdaModel(changeset_corpus,
                id2word=changeset_corpus.id2word,
                alpha=config.alpha,
                passes=config.passes,
                num_topics=config.num_topics)

        changeset_model.save(config.changeset_model_fname)

@main.command()
@pass_config
@click.pass_context
def evaluate(context, config):
    """
    Evalutates the models
    """
    try:
        file_model = LdaModel.load(config.file_model_fname)
        logger.info('Opened previously created model at file %s' % config.file_model_fname)
    except:
        error('Cannot evalutate LDA models not built yet!')

    try:
        changeset_model = LdaModel.load(config.changeset_model_fname)
        logger.info('Opened previously created model at changeset %s' % config.changeset_model_fname)
    except:
        error('Cannot evalutate LDA models not built yet!')

    logger.info('Evalutating models for: %s' % config.project.name)

    file_scores = utils.score(file_model, utils.kullback_leibler_divergence)
    changeset_scores = utils.score(changeset_model, utils.kullback_leibler_divergence)

    file_total = sum([x[1] for x in file_scores])
    changeset_total = sum([x[1] for x in changeset_scores])

    logger.info("File model KL: %f" % file_total)
    logger.info("Changeset model KL: %f" % changeset_total)
    logger.info("File model KL mean: %f" % (file_total / len(file_scores)))
    logger.info("Changeset model KL mean: %f" % (changeset_total / len(changeset_scores)))
    with open(config.path + 'evaluate-results.csv', 'a') as f:
        f.write('%s,%f,%f' % (config.file_model_fname, file_total, changeset_total))

@main.command()
@pass_config
@click.pass_context
def evaluate_corpora(context, config):
    try:
        file_dict = Dictionary.load_from_text(config.file_corpus_fname + '.dict')
        file_corpus = MalletCorpus(config.file_corpus_fname, id2word=file_dict)
    except:
        error('Corpora not built yet -- cannot evaluate')

    file_word_freq = list(reversed(sorted(count_words(file_corpus))))
    print("Top 10 words in files:", file_word_freq[:10])
    print("Bottom 10 words in files:", file_word_freq[-10:])

    try:
        changeset_dict = Dictionary.load_from_text(config.changeset_corpus_fname + '.dict')
        changeset_corpus = MalletCorpus(config.changeset_corpus_fname, id2word=changeset_dict)
    except:
        error('Corpora not built yet -- cannot evaluate')
    changeset_word_freq = list(reversed(sorted(count_words(changeset_corpus))))
    print("Top 10 words in changesets:", changeset_word_freq[:10])
    print("Bottom 10 words in changesets:", changeset_word_freq[-10:])

def count_words(corpus):
    word_freq = dict()
    for doc in corpus:
        for word, count in doc:
            if word not in word_freq:
                word_freq[word] = 0

            word_freq[word] += count

    for word_id, freq in word_freq.items():
        yield freq, corpus.id2word[word_id]

def perplexity(corpus, model):
    held_out = list()
    training = list()
    target_len = int(0.1 * len(corpus))
    logger.info('Calculating perplexity with held-out %d of %d documents' % (target_len, len(corpus)))

    ids = set()
    while len(ids) < target_len:
        ids.add(random.randint(0, len(corpus)))

    for doc_id, doc in enumerate(corpus):
        if doc_id in ids:
            held_out.append(doc)
        else:
            training.append(doc)

    model.update(training)
    return model.log_perplexity(held_out)

@main.command()
@pass_config
@click.pass_context
def evaluate_perplexity(context, config):
    try:
        file_dict = Dictionary.load_from_text(config.file_corpus_fname + '.dict')
        file_corpus = MalletCorpus(config.file_corpus_fname, id2word=file_dict)
    except:
        error('Corpora not built yet -- cannot evaluate')
    try:
        changeset_dict = Dictionary.load_from_text(config.changeset_corpus_fname + '.dict')
        changeset_corpus = MalletCorpus(config.changeset_corpus_fname, id2word=changeset_dict)
    except:
        error('Corpora not built yet -- cannot evaluate')

    file_model = LdaModel(
            id2word=file_corpus.id2word,
            alpha=config.alpha,
            passes=config.passes,
            num_topics=config.num_topics)

    changeset_model = LdaModel(
            id2word=changeset_corpus.id2word,
            alpha=config.alpha,
            passes=config.passes,
            num_topics=config.num_topics)

    logger.info('Calculating file corpus perplexity for: %s' % config.project.name)
    file_pwb = 0
    file_pwb = perplexity(file_corpus, file_model)

    logger.info('Calculating changeset corpus perplexity for: %s' % config.project.name)
    changeset_pwb = 0
    changeset_pwb = perplexity(changeset_corpus, changeset_model)

    with open(config.path + 'evaluate-perplexity-results.csv', 'a') as f:
        f.write('%s,%f,%f' % (config.file_model_fname, file_pwb, changeset_pwb))

@main.command()
@pass_config
@click.pass_context
def run_all(context, config):
    """
    Runs corpora, preprocess, model, and evaluate in one shot.
    """
    logger.info('Doing everything for: %s' % config.project.name)
    context.forward(corpora)
    context.forward(model)
    context.forward(evaluate)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Main recognizer loop: wait for a trigger then perform and handle
recognition."""

import logging
import os
import sys
import threading
import time

import configargparse
from googlesamples.assistant import auth_helpers

import audio
import action
import i18n
import speech
import tts

# =============================================================================
#
# Hey, Makers!
#
# Are you looking for actor.add_keyword? Do you want to add a new command?
# You need to edit src/action.py. Check out the instructions at:
# https://aiyprojects.withgoogle.com/voice/#makers-guide-3-3--create-a-new-voice-command-or-action
#
# =============================================================================

logging.basicConfig(
    level=logging.INFO,
    format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s"
)
logger = logging.getLogger('main')

CACHE_DIR = os.getenv('XDG_CACHE_HOME') or os.path.expanduser('~/.cache')
VR_CACHE_DIR = os.path.join(CACHE_DIR, 'voice-recognizer')

CONFIG_DIR = os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config')
CONFIG_FILES = [
    '/etc/voice-recognizer.ini',
    os.path.join(CONFIG_DIR, 'voice-recognizer.ini')
]

# Legacy fallback: old locations of secrets/credentials.
OLD_CLIENT_SECRETS = os.path.expanduser('~/client_secrets.json')
OLD_SERVICE_CREDENTIALS = os.path.expanduser('~/credentials.json')

ASSISTANT_CREDENTIALS = os.path.join(VR_CACHE_DIR, 'assistant_credentials.json')
ASSISTANT_OAUTH_SCOPE = 'https://www.googleapis.com/auth/assistant-sdk-prototype'

PID_FILE = '/run/user/%d/voice-recognizer.pid' % os.getuid()

def try_to_get_credentials(client_secrets):
    """Try to get credentials, or print an error and quit on failure."""

    if os.path.exists(ASSISTANT_CREDENTIALS):
        return auth_helpers.load_credentials(
            ASSISTANT_CREDENTIALS, scopes=[ASSISTANT_OAUTH_SCOPE])

    if not os.path.exists(VR_CACHE_DIR):
        os.mkdir(VR_CACHE_DIR)

    if not os.path.exists(client_secrets) and os.path.exists(OLD_CLIENT_SECRETS):
        client_secrets = OLD_CLIENT_SECRETS

    if not os.path.exists(client_secrets):
        print('You need client secrets to use the Assistant API.')
        print('Follow these instructions:')
        print('    https://developers.google.com/api-client-library/python/auth/installed-app'
              '#creatingcred')
        print('and put the file at', client_secrets)
        sys.exit(1)

    if not os.getenv('DISPLAY') and not sys.stdout.isatty():
        print("""
To use the Assistant API, manually start the application from the dev terminal.
See the "Turn on the Assistant API" section of the Voice Recognizer
User's Guide for more info.""")
        sys.exit(1)

    credentials = auth_helpers.credentials_flow_interactive(
        client_secrets, scopes=[ASSISTANT_OAUTH_SCOPE])
    auth_helpers.save_credentials(ASSISTANT_CREDENTIALS, credentials)
    logging.info('OAuth credentials initialized: %s', ASSISTANT_CREDENTIALS)
    return credentials

def create_pid_file(file_name):
    with open(file_name, 'w') as pid_file:
        pid_file.write("%d" % os.getpid())

def main():
    parser = configargparse.ArgParser(
        default_config_files=CONFIG_FILES,
        description="Act on voice commands using Google's speech recognition")
    parser.add_argument('-I', '--input-device', default='default',
                        help='Name of the audio input device')
    parser.add_argument('-O', '--output-device', default='default',
                        help='Name of the audio output device')
    parser.add_argument('-T', '--trigger', default='gpio',
                        help='Trigger to use {\'clap\', \'gpio\'}')
    parser.add_argument('--cloud-speech', action='store_true',
                        help='Use the Cloud Speech API instead of the Assistant API')
    parser.add_argument('-L', '--language', default='en-US',
                        help='Language code to use for speech (default: en-US)')
    parser.add_argument('-l', '--led-fifo', default='/tmp/status-led',
                        help='Status led control fifo')
    parser.add_argument('-p', '--pid-file', default=PID_FILE,
                        help='File containing our process id for monitoring')
    parser.add_argument('--audio-logging', action='store_true',
                        help='Log all requests and responses to WAV files in /tmp')
    parser.add_argument('--assistant-secrets',
                        default=os.path.expanduser('~/assistant.json'),
                        help='Path to client secrets for the Assistant API')
    parser.add_argument('--cloud-speech-secrets',
                        default=os.path.expanduser('~/cloud_speech.json'),
                        help='Path to service account credentials for the '
                        'Cloud Speech API')

    args = parser.parse_args()

    create_pid_file(args.pid_file)
    i18n.set_language_code(args.language, gettext_install=True)

    player = audio.Player(args.output_device)

    if args.cloud_speech:
        credentials_file = os.path.expanduser(args.cloud_speech_secrets)
        if not os.path.exists(credentials_file) and os.path.exists(OLD_SERVICE_CREDENTIALS):
            credentials_file = OLD_SERVICE_CREDENTIALS
        recognizer = speech.CloudSpeechRequest(credentials_file)
    else:
        credentials = try_to_get_credentials(
            os.path.expanduser(args.assistant_secrets))
        recognizer = speech.AssistantSpeechRequest(credentials)

    recorder = audio.Recorder(
        input_device=args.input_device, channels=1,
        bytes_per_sample=speech.AUDIO_SAMPLE_SIZE,
        sample_rate_hz=speech.AUDIO_SAMPLE_RATE_HZ)
    with recorder:
        do_recognition(args, recorder, recognizer, player)

def do_recognition(args, recorder, recognizer, player):
    """Configure and run the recognizer."""
    say = tts.create_say(player)

    actor = action.make_actor(say)

    if args.cloud_speech:
        action.add_commands_just_for_cloud_speech_api(actor, say)

    recognizer.add_phrases(actor)
    recognizer.set_audio_logging_enabled(args.audio_logging)

    if args.trigger == 'gpio':
        import triggers.gpio
        triggerer = triggers.gpio.GpioTrigger(channel=23)
        msg = 'Press the button on GPIO 23'
    elif args.trigger == 'clap':
        import triggers.clap
        triggerer = triggers.clap.ClapTrigger(recorder)
        msg = 'Clap your hands'
    else:
        logger.error("Unknown trigger '%s'", args.trigger)
        return

    mic_recognizer = SyncMicRecognizer(
        actor, recognizer, recorder, player, say, triggerer, led_fifo=args.led_fifo)

    with mic_recognizer:
        if sys.stdout.isatty():
            print(msg + ' then speak, or press Ctrl+C to quit...')

        # wait for KeyboardInterrupt
        while True:
            time.sleep(1)

class SyncMicRecognizer(object):

    """Detects triggers and runs recognition in a background thread.

    This is a context manager, so it will clean up the background thread if the
    main program is interrupted.
    """

    # pylint: disable=too-many-instance-attributes

    def __init__(self, actor, recognizer, recorder, player, say, triggerer, led_fifo):
        self.actor = actor
        self.player = player
        self.recognizer = recognizer
        self.recognizer.set_endpointer_cb(self.endpointer_cb)
        self.recorder = recorder
        self.say = say
        self.triggerer = triggerer
        self.triggerer.set_callback(self.recognize)

        self.running = False

        if led_fifo and os.path.exists(led_fifo):
            self.led_fifo = led_fifo
        else:
            if led_fifo:
                logger.warning(
                    'File %s specified for --led-fifo does not exist.',
                    led_fifo)
            self.led_fifo = None
        self.recognizer_event = threading.Event()

    def __enter__(self):
        self.running = True
        threading.Thread(target=self._recognize).start()
        self.triggerer.start()
        self._status('ready')

    def __exit__(self, *args):
        self.running = False
        self.recognizer_event.set()

        self.recognizer.end_audio()

    def _status(self, status):
        if self.led_fifo:
            with open(self.led_fifo, 'w') as led:
                led.write(status + '\n')
        logger.info('%s...', status)

    def recognize(self):
        if self.recognizer_event.is_set():
            # Duplicate trigger (eg multiple button presses)
            return

        self.recognizer.reset()
        self.recorder.add_processor(self.recognizer)
        self._status('listening')
        # Tell recognizer to run
        self.recognizer_event.set()

    def endpointer_cb(self):
        self.recorder.del_processor(self.recognizer)
        self._status('thinking')

    def _recognize(self):
        while self.running:
            self.recognizer_event.wait()
            if not self.running:
                break

            logger.info('recognizing...')
            try:
                self._handle_result(self.recognizer.do_request())
            except speech.Error:
                logger.exception('Unexpected error')
                self.say(_('Unexpected error. Try again or check the logs.'))

            self.recognizer_event.clear()
            self.triggerer.start()
            self._status('ready')

    def _handle_result(self, result):
        if result.transcript and self.actor.handle(result.transcript):
            logger.info('handled local command: %s', result.transcript)
        elif result.response_audio:
            self._play_assistant_response(result.response_audio)
        elif result.transcript:
            logger.warning('%r was not handled', result.transcript)
            self.say(_("I don’t know how to answer that."))
        else:
            logger.warning('no command recognized')
            self.say(_("Could you try that again?"))

    def _play_assistant_response(self, audio_bytes):
        bytes_per_sample = speech.AUDIO_SAMPLE_SIZE
        sample_rate_hz = speech.AUDIO_SAMPLE_RATE_HZ
        logger.info('Playing %.4f seconds of audio...',
                    len(audio_bytes) / (bytes_per_sample * sample_rate_hz))
        self.player.play_bytes(audio_bytes, sample_width=bytes_per_sample,
                               sample_rate=sample_rate_hz)

if __name__ == '__main__':
    main()
<commit_msg>Stay silent when we don't recognize a command<commit_after>#!/usr/bin/env python3
# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Main recognizer loop: wait for a trigger then perform and handle
recognition."""

import logging
import os
import sys
import threading
import time

import configargparse
from googlesamples.assistant import auth_helpers

import audio
import action
import i18n
import speech
import tts

# =============================================================================
#
# Hey, Makers!
#
# Are you looking for actor.add_keyword? Do you want to add a new command?
# You need to edit src/action.py. Check out the instructions at:
# https://aiyprojects.withgoogle.com/voice/#makers-guide-3-3--create-a-new-voice-command-or-action
#
# =============================================================================

logging.basicConfig(
    level=logging.INFO,
    format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s"
)
logger = logging.getLogger('main')

CACHE_DIR = os.getenv('XDG_CACHE_HOME') or os.path.expanduser('~/.cache')
VR_CACHE_DIR = os.path.join(CACHE_DIR, 'voice-recognizer')

CONFIG_DIR = os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config')
CONFIG_FILES = [
    '/etc/voice-recognizer.ini',
    os.path.join(CONFIG_DIR, 'voice-recognizer.ini')
]

# Legacy fallback: old locations of secrets/credentials.
OLD_CLIENT_SECRETS = os.path.expanduser('~/client_secrets.json')
OLD_SERVICE_CREDENTIALS = os.path.expanduser('~/credentials.json')

ASSISTANT_CREDENTIALS = os.path.join(VR_CACHE_DIR, 'assistant_credentials.json')
ASSISTANT_OAUTH_SCOPE = 'https://www.googleapis.com/auth/assistant-sdk-prototype'

PID_FILE = '/run/user/%d/voice-recognizer.pid' % os.getuid()

def try_to_get_credentials(client_secrets):
    """Try to get credentials, or print an error and quit on failure."""

    if os.path.exists(ASSISTANT_CREDENTIALS):
        return auth_helpers.load_credentials(
            ASSISTANT_CREDENTIALS, scopes=[ASSISTANT_OAUTH_SCOPE])

    if not os.path.exists(VR_CACHE_DIR):
        os.mkdir(VR_CACHE_DIR)

    if not os.path.exists(client_secrets) and os.path.exists(OLD_CLIENT_SECRETS):
        client_secrets = OLD_CLIENT_SECRETS

    if not os.path.exists(client_secrets):
        print('You need client secrets to use the Assistant API.')
        print('Follow these instructions:')
        print('    https://developers.google.com/api-client-library/python/auth/installed-app'
              '#creatingcred')
        print('and put the file at', client_secrets)
        sys.exit(1)

    if not os.getenv('DISPLAY') and not sys.stdout.isatty():
        print("""
To use the Assistant API, manually start the application from the dev terminal.
See the "Turn on the Assistant API" section of the Voice Recognizer
User's Guide for more info.""")
        sys.exit(1)

    credentials = auth_helpers.credentials_flow_interactive(
        client_secrets, scopes=[ASSISTANT_OAUTH_SCOPE])
    auth_helpers.save_credentials(ASSISTANT_CREDENTIALS, credentials)
    logging.info('OAuth credentials initialized: %s', ASSISTANT_CREDENTIALS)
    return credentials

def create_pid_file(file_name):
    with open(file_name, 'w') as pid_file:
        pid_file.write("%d" % os.getpid())

def main():
    parser = configargparse.ArgParser(
        default_config_files=CONFIG_FILES,
        description="Act on voice commands using Google's speech recognition")
    parser.add_argument('-I', '--input-device', default='default',
                        help='Name of the audio input device')
    parser.add_argument('-O', '--output-device', default='default',
                        help='Name of the audio output device')
    parser.add_argument('-T', '--trigger', default='gpio',
                        help='Trigger to use {\'clap\', \'gpio\'}')
    parser.add_argument('--cloud-speech', action='store_true',
                        help='Use the Cloud Speech API instead of the Assistant API')
    parser.add_argument('-L', '--language', default='en-US',
                        help='Language code to use for speech (default: en-US)')
    parser.add_argument('-l', '--led-fifo', default='/tmp/status-led',
                        help='Status led control fifo')
    parser.add_argument('-p', '--pid-file', default=PID_FILE,
                        help='File containing our process id for monitoring')
    parser.add_argument('--audio-logging', action='store_true',
                        help='Log all requests and responses to WAV files in /tmp')
    parser.add_argument('--assistant-secrets',
                        default=os.path.expanduser('~/assistant.json'),
                        help='Path to client secrets for the Assistant API')
    parser.add_argument('--cloud-speech-secrets',
                        default=os.path.expanduser('~/cloud_speech.json'),
                        help='Path to service account credentials for the '
                        'Cloud Speech API')

    args = parser.parse_args()

    create_pid_file(args.pid_file)
    i18n.set_language_code(args.language, gettext_install=True)

    player = audio.Player(args.output_device)

    if args.cloud_speech:
        credentials_file = os.path.expanduser(args.cloud_speech_secrets)
        if not os.path.exists(credentials_file) and os.path.exists(OLD_SERVICE_CREDENTIALS):
            credentials_file = OLD_SERVICE_CREDENTIALS
        recognizer = speech.CloudSpeechRequest(credentials_file)
    else:
        credentials = try_to_get_credentials(
            os.path.expanduser(args.assistant_secrets))
        recognizer = speech.AssistantSpeechRequest(credentials)

    recorder = audio.Recorder(
        input_device=args.input_device, channels=1,
        bytes_per_sample=speech.AUDIO_SAMPLE_SIZE,
        sample_rate_hz=speech.AUDIO_SAMPLE_RATE_HZ)
    with recorder:
        do_recognition(args, recorder, recognizer, player)

def do_recognition(args, recorder, recognizer, player):
    """Configure and run the recognizer."""
    say = tts.create_say(player)

    actor = action.make_actor(say)

    if args.cloud_speech:
        action.add_commands_just_for_cloud_speech_api(actor, say)

    recognizer.add_phrases(actor)
    recognizer.set_audio_logging_enabled(args.audio_logging)

    if args.trigger == 'gpio':
        import triggers.gpio
        triggerer = triggers.gpio.GpioTrigger(channel=23)
        msg = 'Press the button on GPIO 23'
    elif args.trigger == 'clap':
        import triggers.clap
        triggerer = triggers.clap.ClapTrigger(recorder)
        msg = 'Clap your hands'
    else:
        logger.error("Unknown trigger '%s'", args.trigger)
        return

    mic_recognizer = SyncMicRecognizer(
        actor, recognizer, recorder, player, say, triggerer, led_fifo=args.led_fifo)

    with mic_recognizer:
        if sys.stdout.isatty():
            print(msg + ' then speak, or press Ctrl+C to quit...')

        # wait for KeyboardInterrupt
        while True:
            time.sleep(1)

class SyncMicRecognizer(object):

    """Detects triggers and runs recognition in a background thread.

    This is a context manager, so it will clean up the background thread if the
    main program is interrupted.
    """

    # pylint: disable=too-many-instance-attributes

    def __init__(self, actor, recognizer, recorder, player, say, triggerer, led_fifo):
        self.actor = actor
        self.player = player
        self.recognizer = recognizer
        self.recognizer.set_endpointer_cb(self.endpointer_cb)
        self.recorder = recorder
        self.say = say
        self.triggerer = triggerer
        self.triggerer.set_callback(self.recognize)

        self.running = False

        if led_fifo and os.path.exists(led_fifo):
            self.led_fifo = led_fifo
        else:
            if led_fifo:
                logger.warning(
                    'File %s specified for --led-fifo does not exist.',
                    led_fifo)
            self.led_fifo = None
        self.recognizer_event = threading.Event()

    def __enter__(self):
        self.running = True
        threading.Thread(target=self._recognize).start()
        self.triggerer.start()
        self._status('ready')

    def __exit__(self, *args):
        self.running = False
        self.recognizer_event.set()

        self.recognizer.end_audio()

    def _status(self, status):
        if self.led_fifo:
            with open(self.led_fifo, 'w') as led:
                led.write(status + '\n')
        logger.info('%s...', status)

    def recognize(self):
        if self.recognizer_event.is_set():
            # Duplicate trigger (eg multiple button presses)
            return

        self.recognizer.reset()
        self.recorder.add_processor(self.recognizer)
        self._status('listening')
        # Tell recognizer to run
        self.recognizer_event.set()

    def endpointer_cb(self):
        self.recorder.del_processor(self.recognizer)
        self._status('thinking')

    def _recognize(self):
        while self.running:
            self.recognizer_event.wait()
            if not self.running:
                break

            logger.info('recognizing...')
            try:
                self._handle_result(self.recognizer.do_request())
            except speech.Error:
                logger.exception('Unexpected error')
                self.say(_('Unexpected error. Try again or check the logs.'))

            self.recognizer_event.clear()
            self.triggerer.start()
            self._status('ready')

    def _handle_result(self, result):
        if result.transcript and self.actor.handle(result.transcript):
            logger.info('handled local command: %s', result.transcript)
        elif result.response_audio:
            self._play_assistant_response(result.response_audio)
        elif result.transcript:
            logger.warning('%r was not handled', result.transcript)
        else:
            logger.warning('no command recognized')

    def _play_assistant_response(self, audio_bytes):
        bytes_per_sample = speech.AUDIO_SAMPLE_SIZE
        sample_rate_hz = speech.AUDIO_SAMPLE_RATE_HZ
        logger.info('Playing %.4f seconds of audio...',
                    len(audio_bytes) / (bytes_per_sample * sample_rate_hz))
        self.player.play_bytes(audio_bytes, sample_width=bytes_per_sample,
                               sample_rate=sample_rate_hz)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from mmhandler import MmHandler
# from sql import SQL
import config
import telebot
import re

"""
TODO:
Оповещения (Ты забыл обо мне?)
Разобрать выражение на естественном языке
11) start
12) help
13) main
14) формирование ответного сообщения
прикрутить передачу trace ошибок из SQL
"""

bot = telebot.TeleBot(config.token)
handler = MmHandler(0)  # по умолчанию user_id = 0

# прикрутить вебхуки
# прикрутить разбор сообщения на естественном языке

# для тестирования
@bot.message_handler(commands=['start'])
def start(message):
    MmHandler.user_id = message.chat.id
    if MmHandler.start:
        bot.send_message(message.chat.id, 'Hello. Let\'s start from help command')
    else:
        bot.send_message(message.chat.id, 'Undefined error')

@bot.message_handler(commands=['help'])
def help(message):
    help_str = "добавить доход:\n+ {сумма} {название} {коммент}\nдобавить фиксированный доход:\n+ {сумма} {название} {дата} {коммент}\nдобавить расход:\n- {сумма} {категория} {коммент}\nпоказать категории:\nпокажи категории\nдобавить категорию:\nдобавь категорию {название}\nудалить категорию:\nудали категорию {название}\nпосмотреть отчет за месяц:\nотчет за {месяц}\nпосмотреть отчет за определенный период:\nотчет с {начальная дата} по {конечная дата}\nформат даты: xx.xx.xxxx"
    bot.send_message(message.chat.id, 'Hello. This you can do:')
    bot.send_message(message.chat.id, help_str)

@bot.message_handler(content_types=["text"])
def parse(message):
    handler.user_id = message.chat.id
    str = message.text.split()
    length = len(str)
    if length == 0:
        pass  # вставить вызов help
    elif str[0] == '+':
        if length == 3:
            if re.match('\d+', str[1]) and re.match('[а-яА-я]+', str[2]):
                handler.add_operation(str[1], str[2])
                # вызов доход amount category
        elif length >= 4:
            if length >= 4 and re.match('\d+', str[1]) and re.match('[а-яА-я]+', str[2]):
                buf = ' '.join(str[3:length])
                handler.add_operation(str[1], str[2], buf)
                # вызов доход amount category description
    elif str[0] == '-':
        if length == 3 and re.match('\d+', str[1]) and re.match('[а-яА-я]+', str[2]):
            handler.add_operation(str[1], str[2])
            # расход amount category
        elif length > 3 and re.match('\d+', str[1]) and re.match('[а-яА-я]+', str[2]):
            buf = ' '.join(str[3:length])
            handler.add_operation(str[1], str[2], buf)
            # расход amount category description
    elif str[0] == 'покажи':
        if str[1] == 'категории':
            handler.show_categories()
            # категории
    elif str[0] == 'добавь' and str[1] == 'категорию' and re.match('[а-яА-я]+', str[2]):
        handler.add_category(str[2])
        # добавь категорию
    elif str[0] == 'удали' and str[1] == 'категорию' and re.match('[а-яА-я]+', str[2]):
        handler.del_category(str[2])
        # удали категорию
    elif str[0] == 'отчет':
        if str[1] == 'за' and re.match('[а-яА-я]+', str[2]):
            handler.view_report(str[2])
            # отчет за период
        if str[1] == 'с' and re.match('\d{1,2}.\d{1,2}.\d{4}', str[2]) and str[3] == 'по' and \
                re.match('\d{1,2}.\d{1,2}.\d{4}', str[4]):
            handler.view_custom_report(str[2], str[4])
            # отчет с date по date
    else:
        pass  # вызов help

# бесконечная петля опроса
if __name__ == '__main__':
    bot.polling(none_stop=True)
<commit_msg>by Roman with improved date xx-xx-xxxx<commit_after>from mmhandler import MmHandler
# from sql import SQL
import config
import telebot
import re

"""
TODO:
Оповещения (Ты забыл обо мне?)
Разобрать выражение на естественном языке
11) start
12) help
13) main
14) формирование ответного сообщения
прикрутить передачу trace ошибок из SQL
"""

bot = telebot.TeleBot(config.token)
handler = MmHandler(0)  # по умолчанию user_id = 0

# прикрутить вебхуки
# прикрутить разбор сообщения на естественном языке

# для тестирования
@bot.message_handler(commands=['start'])
def start(message):
    MmHandler.user_id = message.chat.id
    if MmHandler.start:
        bot.send_message(message.chat.id, 'Hello. Let\'s start from help command')
    else:
        bot.send_message(message.chat.id, 'Undefined error')

@bot.message_handler(commands=['help'])
def help(message):
    help_str = "добавить доход:\n+ {сумма} {название} {коммент}\nдобавить фиксированный доход:\n+ {сумма} {название} {дата} {коммент}\nдобавить расход:\n- {сумма} {категория} {коммент}\nпоказать категории:\nпокажи категории\nдобавить категорию:\nдобавь категорию {название}\nудалить категорию:\nудали категорию {название}\nпосмотреть отчет за месяц:\nотчет за {месяц}\nпосмотреть отчет за определенный период:\nотчет с {начальная дата} по {конечная дата}\nформат даты: xx.xx.xxxx"
    bot.send_message(message.chat.id, 'Hello. This you can do:')
    bot.send_message(message.chat.id, help_str)

@bot.message_handler(content_types=["text"])
def parse(message):
    handler.user_id = message.chat.id
    str = message.text.split()
    length = len(str)
    if length == 0:
        pass  # вставить вызов help
    elif str[0] == '+':
        if length == 3:
            if re.match('\d+', str[1]) and re.match('[а-яА-я]+', str[2]):
                handler.add_operation(str[1], str[2])
                # вызов доход amount category
        elif length >= 4:
            if length >= 4 and re.match('\d+', str[1]) and re.match('[а-яА-я]+', str[2]):
                buf = ' '.join(str[3:length])
                handler.add_operation(str[1], str[2], buf)
                # вызов доход amount category description
    elif str[0] == '-':
        if length == 3 and re.match('\d+', str[1]) and re.match('[а-яА-я]+', str[2]):
            handler.add_operation(str[1], str[2])
            # расход amount category
        elif length > 3 and re.match('\d+', str[1]) and re.match('[а-яА-я]+', str[2]):
            buf = ' '.join(str[3:length])
            handler.add_operation(str[1], str[2], buf)
            # расход amount category description
    elif str[0] == 'покажи':
        if str[1] == 'категории':
            handler.show_categories()
            # категории
    elif str[0] == 'добавь' and str[1] == 'категорию' and re.match('[а-яА-я]+', str[2]):
        handler.add_category(str[2])
        # добавь категорию
    elif str[0] == 'удали' and str[1] == 'категорию' and re.match('[а-яА-я]+', str[2]):
        handler.del_category(str[2])
        # удали категорию
    elif str[0] == 'отчет':
        if str[1] == 'за' and re.match('[а-яА-я]+', str[2]):
            handler.view_report(str[2])
            # отчет за период
        if str[1] == 'с' and re.match('\d{1,2}-\d{1,2}-\d{4}', str[2]) and str[3] == 'по' and \
                re.match('\d{1,2}-\d{1,2}-\d{4}', str[4]):
            handler.view_custom_report(str[2], str[4])
            # отчет с date по date
    else:
        pass  # вызов help

# бесконечная петля опроса
if __name__ == '__main__':
    bot.polling(none_stop=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# morse-code-twitter

# Standard Library Imports
import time
from threading import Timer

# Pip Imports
import RPi.GPIO as GPIO
import twitter

# Local Imports
import morse
import tweeting

from Adafruit_CharLCD.Adafruit_CharLCD import Adafruit_CharLCD

# Constants
MORSE_PIN = 18
BUZZER_PIN = 14
DOT = 150  # ms
DASH = 3 * DOT

def millis():
    """ Function which mimics Arduino millis() """
    return int(round(time.time() * 1000))

class MorseButton:
    """ Class to keep track of state variables for a morse code button. """
    def __init__(self, pin, buzzer_pin=BUZZER_PIN, tweet_timeout=5.0):
        """ Store pin, set up IO. """
        self.pin = pin
        self.buzzer_pin = buzzer_pin
        self.tweet_timeout = tweet_timeout

        # Set up screen
        self.lcd = Adafruit_CharLCD()
        self.lcd.begin(16, 1)

        # Set up other GPIO pins
        # GPIO.setmode(GPIO.BCM)  # LCD Library uses BCM
        GPIO.setup(self.pin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
        GPIO.setup(self.buzzer_pin, GPIO.OUT)
        GPIO.output(self.buzzer_pin, GPIO.LOW)  # turn buzzer OFF
        GPIO.add_event_detect(self.pin, GPIO.BOTH,
                              callback=self.callback, bouncetime=25)

        # Set up state variables
        self.time_var = 0
        self.curr_tweet = ""
        self.tweet_started = False

        # Indicate everything is ready
        print("Start Tweeting!")

    def handle_tweet(self):
        """ Function which handles out the current tweet. """
        print(self.curr_tweet)

        try:
            decoded = morse.decode_message(self.curr_tweet)
            tweeting.send_tweet(decoded)

        except morse.MorseCodeDecodeError as e:
            print(e.message, ': ',  e.invalid_str)

        except twitter.api.TwitterHTTPError as t:
            pass  # It's fine, probably

        self.curr_tweet = ""
        self.tweet_started = False

    def callback(self, channel):
        """ Callback to handle button rising and falling edges. """
        # Current state of the button is end edge of the transition.
        button_pressed = not GPIO.input(self.pin)

        if button_pressed:  # Button was just pressed.
            duration = millis() - self.time_var

            GPIO.output(self.buzzer_pin, GPIO.HIGH)

            # If the button was already pressed, record spaces, and restart the
            # tweet printing timer.
            if self.tweet_started:
                if DASH < duration <= 2.5 * DASH:
                    self.curr_tweet += ' '
                elif 2.5 * DASH < duration:
                    self.curr_tweet += ' / '

                self.tweet_printer.cancel()
                self.tweet_printer = Timer(self.tweet_timeout,
                                           self.handle_tweet)
                self.tweet_printer.start()

            # If the tweet was not started, start it, and start the timeout.
            else:
                self.tweet_started = True
                self.tweet_printer = Timer(self.tweet_timeout,
                                           self.handle_tweet)
                self.tweet_printer.start()

            self.time_var = millis()  # Keep track of press time.

        else:  # Button was just released.
            duration = millis() - self.time_var

            GPIO.output(self.buzzer_pin, GPIO.LOW)

            # Determine if the duration held was a dot or a dash.
            if duration <= 1.5 * DOT:
                self.curr_tweet += '.'
            else:
                self.curr_tweet += '-'

            self.time_var = millis()  # Keep track of release time.

def main():
    button = MorseButton(MORSE_PIN)
    while(True):
        try:
            button.lcd.clear()
            button.lcd.message(button.curr_tweet)
            time.sleep(0.1)

        except KeyboardInterrupt:
            GPIO.cleanup()
            return

if __name__ == '__main__':
    main()
<commit_msg>moving logic back inside class in new loop() function<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# morse-code-twitter

# Standard Library Imports
import time
from threading import Timer

# Pip Imports
import RPi.GPIO as GPIO
import twitter

# Local Imports
import morse
import tweeting

from Adafruit_CharLCD.Adafruit_CharLCD import Adafruit_CharLCD

# Constants
MORSE_PIN = 18
BUZZER_PIN = 14
DOT = 150  # ms
DASH = 3 * DOT

def millis():
    """ Function which mimics Arduino millis() """
    return int(round(time.time() * 1000))

class MorseButton:
    """ Class to keep track of state variables for a morse code button. """
    def __init__(self, pin, buzzer_pin=BUZZER_PIN, tweet_timeout=5.0):
        """ Store pin, set up IO. """
        self.pin = pin
        self.buzzer_pin = buzzer_pin
        self.tweet_timeout = tweet_timeout

        # Set up screen
        self.lcd = Adafruit_CharLCD()
        self.lcd.begin(16, 1)

        # Set up other GPIO pins
        # GPIO.setmode(GPIO.BCM)  # LCD Library uses BCM
        GPIO.setup(self.pin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
        GPIO.setup(self.buzzer_pin, GPIO.OUT)
        GPIO.output(self.buzzer_pin, GPIO.LOW)  # turn buzzer OFF
        GPIO.add_event_detect(self.pin, GPIO.BOTH,
                              callback=self.callback, bouncetime=25)

        # Set up state variables
        self.time_var = 0
        self.curr_tweet = ""
        self.tweet_started = False

        # Indicate everything is ready
        print("Start Tweeting!")

    def handle_tweet(self):
        """ Function which handles out the current tweet. """
        print(self.curr_tweet)

        try:
            decoded = morse.decode_message(self.curr_tweet)
            tweeting.send_tweet(decoded)

        except morse.MorseCodeDecodeError as e:
            print(e.message, ': ',  e.invalid_str, "")

        except twitter.api.TwitterHTTPError:
            print("Twitter Error, Try Again!")

        self.curr_tweet = ""
        self.tweet_started = False

    def loop(self):
        """ Handle any business which needs to be handled in a while True loop.
        """
        self.lcd.clear()
        self.lcd.message(self.curr_tweet)
        time.sleep(0.1)

    def callback(self, channel):
        """ Callback to handle button rising and falling edges. """
        # Current state of the button is end edge of the transition.
        button_pressed = not GPIO.input(self.pin)

        if button_pressed:  # Button was just pressed.
            duration = millis() - self.time_var

            GPIO.output(self.buzzer_pin, GPIO.HIGH)

            # If the button was already pressed, record spaces, and restart the
            # tweet printing timer.
            if self.tweet_started:
                if DASH < duration <= 2.5 * DASH:
                    self.curr_tweet += ' '
                elif 2.5 * DASH < duration:
                    self.curr_tweet += ' / '

                self.tweet_printer.cancel()
                self.tweet_printer = Timer(self.tweet_timeout,
                                           self.handle_tweet)
                self.tweet_printer.start()

            # If the tweet was not started, start it, and start the timeout.
            else:
                self.tweet_started = True
                self.tweet_printer = Timer(self.tweet_timeout,
                                           self.handle_tweet)
                self.tweet_printer.start()

            self.time_var = millis()  # Keep track of press time.

        else:  # Button was just released.
            duration = millis() - self.time_var

            GPIO.output(self.buzzer_pin, GPIO.LOW)

            # Determine if the duration held was a dot or a dash.
            if duration <= 1.5 * DOT:
                self.curr_tweet += '.'
            else:
                self.curr_tweet += '-'

            self.time_var = millis()  # Keep track of release time.

def main():
    button = MorseButton(MORSE_PIN)
    while(True):
        try:
            button.loop()

        except KeyboardInterrupt:
            GPIO.cleanup()
            return

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import networkx as nx
import random
import sys
import math
import os
import configparse
import datetime
import inspect
import logging
import cProfile
import pstats
import pdb
#local modules
import estrangement
import agglomerate
import optionsadder

def parse_args(reader_functions={}):
    """@brief parse cmd line and conf file options
    @retval (opt, args) as returned by configparse.OptionParser.parse_args"""
    # read in options from cmdline and conffile
    usage="""usage: %prog [options] (--help for help)\n"""

    parser = configparse.OptionParser(description="Estrangement Confinement Algorithm",
         usage=usage)

    optionsadder.add_options(parser, reader_functions=reader_functions)
    (opt, args) = parser.parse_args(files=['./simulation.conf'])
    #check_options(opt, parser)
    return (opt, args)

def summary(g1):
    logging.info(nx.info(g1))
    logging.info("Number of components: %d " % nx.number_connected_components(g1))

def maxQ(g1):
    """ a convenenience function to do plain Q maximization"""
    dictPartition = {}
    dictQ = {}
    # set params to do pure Q maximization
    lambduh = 0.0
    Zgraph = nx.Graph()
    # do 100 runs and pick the best
    for r in range(10*opt.minrepeats):
        # best_partition calls agglomerative lpa
        dictPartition[r] = agglomerate.best_partition(g1, opt, lambduh, Zgraph)
        dictQ[r] = agglomerate.modularity(dictPartition[r], g1)
    logging.info("dictQ = %s", str(dictQ))
    bestr = max(dictQ, key=dictQ.get)
    return dictPartition[bestr]

def read_general(datadir):
    """ generator function to read many datasets including mit and random_with_stable_core"""
    # passed in datadir can be one of these
    repo_datadir = "../../../../datasets/"
    #datadir = "random_with_stable_core"
    datadir = os.path.join(repo_datadir, datadir)
    print "datadir: ", datadir

    if os.path.exists(os.path.join(datadir, "network.merged")):
        g1 = nx.read_edgelist(os.path.join(datadir, "network.merged"), nodetype=int, data=(('weight',float),))
        if not os.path.exists(os.path.join(datadir, "merged_label_dict")):
            merged_label_dict = maxQ(g1)
            with open(os.path.join(datadir, 'merged_label_dict.txt'), 'w') as lf:
                lf.write(repr(merged_label_dict))

    raw_file_list = os.listdir(datadir)
    timestamps = sorted([int(f.rstrip(".ncol")) for f in raw_file_list if f.endswith(".ncol")])

    initial_label_dict_filename = os.path.join(datadir, 'initial_label_dict.txt')

    beginning = True
    for t in timestamps:
        f = str(t) + ".ncol"

        fpath = os.path.join(datadir,f)

        # skip empty files but increment timestamp
        if os.path.getsize(fpath) == 0:
            continue

        g1 = nx.read_edgelist(fpath, nodetype=int, data=(('weight',float),))

        summary(g1)

        if beginning is True:
            # when called for the first time just return initial_label_dict
            if not os.path.exists(initial_label_dict_filename):
                initial_label_dict = maxQ(g1)
                with open(initial_label_dict_filename, 'w') as lf:
                    lf.write(repr(initial_label_dict))

            with open(initial_label_dict_filename, 'r') as lf:
                initial_label_dict = eval(lf.read())
            yield (t, g1, initial_label_dict)
            beginning = False
        else:
            yield (t, g1, None)

if __name__ == '__main__':

    # get names of all functions in this module
    all_functions = inspect.getmembers(sys.modules[__name__], inspect.isfunction)
    reader_functions = dict([ t for t in all_functions if t[0].startswith("read") ])

    (opt, args) = parse_args(reader_functions)
    print(opt)

    # Write all option settings to the log file
    #opt_dict = eval(str(opt))
    with open("options.log", 'w') as optf:
       optf.write(str(opt))

    logging.basicConfig(level=getattr(logging, opt.loglevel.upper(), None))

    if opt.profiler_on is True:
        # run main through the profiler
        cProfile.run('estrangement.ERA(reader_functions[opt.graph_reader_fn], opt)',
            "simulation.prof")
        # print 40 most expensive fuctions sorted by tot_time, after we are done
        st = pstats.Stats("simulation.prof")
        st.sort_stats('time')
        st.print_stats(40)
    else:
        # run without profiler
        estrangement.ERA(reader_functions[opt.graph_reader_fn], opt)

    print "#################### DONE ##############################"
<commit_msg>do not create merged graph<commit_after>#!/usr/bin/env python

import networkx as nx
import random
import sys
import math
import os
import configparse
import datetime
import inspect
import logging
import cProfile
import pstats
import pdb
#local modules
import estrangement
import agglomerate
import optionsadder

def parse_args(reader_functions={}):
    """@brief parse cmd line and conf file options
    @retval (opt, args) as returned by configparse.OptionParser.parse_args"""
    # read in options from cmdline and conffile
    usage="""usage: %prog [options] (--help for help)\n"""

    parser = configparse.OptionParser(description="Estrangement Confinement Algorithm",
         usage=usage)

    optionsadder.add_options(parser, reader_functions=reader_functions)
    (opt, args) = parser.parse_args(files=['./simulation.conf'])
    #check_options(opt, parser)
    return (opt, args)

def summary(g1):
    logging.info(nx.info(g1))
    logging.info("Number of components: %d " % nx.number_connected_components(g1))

def maxQ(g1):
    """ a convenenience function to do plain Q maximization"""
    dictPartition = {}
    dictQ = {}
    # set params to do pure Q maximization
    lambduh = 0.0
    Zgraph = nx.Graph()
    # do 100 runs and pick the best
    for r in range(10*opt.minrepeats):
        # best_partition calls agglomerative lpa
        dictPartition[r] = agglomerate.best_partition(g1, opt, lambduh, Zgraph)
        dictQ[r] = agglomerate.modularity(dictPartition[r], g1)
    logging.info("dictQ = %s", str(dictQ))
    bestr = max(dictQ, key=dictQ.get)
    return dictPartition[bestr]

def read_general(datadir):
    """ generator function to read many datasets including mit and random_with_stable_core"""
    # passed in datadir can be one of these
    repo_datadir = "../../../../datasets/"
    #datadir = "random_with_stable_core"
    datadir = os.path.join(repo_datadir, datadir)
    print "datadir: ", datadir

    raw_file_list = os.listdir(datadir)
    timestamps = sorted([int(f.rstrip(".ncol")) for f in raw_file_list if f.endswith(".ncol")])

    initial_label_dict_filename = os.path.join(datadir, 'initial_label_dict.txt')

    beginning = True
    for t in timestamps:
        f = str(t) + ".ncol"

        fpath = os.path.join(datadir,f)

        # skip empty files but increment timestamp
        if os.path.getsize(fpath) == 0:
            continue

        g1 = nx.read_edgelist(fpath, nodetype=int, data=(('weight',float),))

        summary(g1)

        if beginning is True:
            # when called for the first time just return initial_label_dict
            if not os.path.exists(initial_label_dict_filename):
                initial_label_dict = maxQ(g1)
                with open(initial_label_dict_filename, 'w') as lf:
                    lf.write(repr(initial_label_dict))

            with open(initial_label_dict_filename, 'r') as lf:
                initial_label_dict = eval(lf.read())
            yield (t, g1, initial_label_dict)
            beginning = False
        else:
            yield (t, g1, None)

if __name__ == '__main__':

    # get names of all functions in this module
    all_functions = inspect.getmembers(sys.modules[__name__], inspect.isfunction)
    reader_functions = dict([ t for t in all_functions if t[0].startswith("read") ])

    (opt, args) = parse_args(reader_functions)
    print(opt)

    # Write all option settings to the log file
    #opt_dict = eval(str(opt))
    with open("options.log", 'w') as optf:
       optf.write(str(opt))

    logging.basicConfig(level=getattr(logging, opt.loglevel.upper(), None))

    if opt.profiler_on is True:
        # run main through the profiler
        cProfile.run('estrangement.ERA(reader_functions[opt.graph_reader_fn], opt)',
            "simulation.prof")
        # print 40 most expensive fuctions sorted by tot_time, after we are done
        st = pstats.Stats("simulation.prof")
        st.sort_stats('time')
        st.print_stats(40)
    else:
        # run without profiler
        estrangement.ERA(reader_functions[opt.graph_reader_fn], opt)

    print "#################### DONE ##############################"
<|endoftext|>


<commit_before># basic types
# filenames are always absolute/ready to be read

import os.path
import io
import zipfile
import subprocess

import data,parse

# приложения, которые парсим
class Document:
	def __init__(self,law,data):
		# fix data
		if isinstance(data['forYear'],(str,int)):
			data['forYear']=(data['forYear'],)
		data['forYear']=tuple(str(y) for y in data['forYear'])
		if isinstance(data['total'],(str,int)):
			data['total']=(data['total'],)
		if 'delta' not in data:
			data['delta']=False
		# init
		self.law=law
		self.forYears=data['forYear']
		self.code=self.law.code+'-'+','.join(self.forYears)
		self.pdfFilename=self.law.environment.rootPath+'/pdf/'+self.code+'.pdf'
		self.txtFilename=self.law.environment.rootPath+'/txt/'+self.code+'.txt'
		self.zipContents=data['zipContents']
		self.totals=data['total']
		if len(self.forYears)!=len(self.totals):
			raise Exception('invalid number of columns')
		self.nCols=len(self.totals)
		self.delta=data['delta']
		self.appendixNumber=data['appendixNumber']
		self.title=data['title']
	def hasPdf(self):
		return os.path.isfile(self.pdfFilename)
	def writePdf(self):
		# attempt to read before opening pdfFile - otherwise might get zero-length file on error
		bytes=None
		for s in self.zipContents.split('/'):
			if bytes is None:
				zipFile=zipfile.ZipFile(law.zipFilename)
			else:
				zipFile=zipfile.ZipFile(io.BytesIO(bytes))
			bytes=zipFile.read(s)
			zipFile.close()
		# write pdf
		pdfFile=open(self.pdfFilename,'wb')
		pdfFile.write(bytes)
		pdfFile.close()
	def hasTxt(self):
		return os.path.isfile(self.txtFilename)
	def writeTxt(self):
		status=subprocess.call(['java','-jar',self.law.environment.pdfboxFilename,'ExtractText','-encoding','UTF-8','-sort',self.pdfFilename,self.txtFilename])
		if status!=0:
			raise Exception('external command failure')
	def getFilenameParensPart(self,depth,stairs,sums):
		return '('+str(depth)+(',stairs' if stairs else '')+(',sums' if sums else '')+')'
	def getCsvFilename(self,depth,stairs,sums):
		return self.law.environment.rootPath+'/csv/'+self.code+self.getFilenameParensPart(depth,stairs,sums)+'.csv'
	def getXlsFilename(self,depth,stairs):
		sums=True
		return self.law.environment.rootPath+'/xls/'+self.code+self.getFilenameParensPart(depth,stairs,sums)+'.xls'
	def hasCsvsAndXlss(self):
		for depth in range(1,4):
			for stairs in (False,True):
				for sums in (False,True):
					if not os.path.isfile(self.getCsvFilename(depth,stairs,sums)):
						return False
				if not os.path.isfile(self.getXlsFilename(depth,stairs)):
					return False
		return True
	def writeCsvsAndXlss(self):
		header=[('Изменение суммы' if self.delta else 'Сумма')+' на '+year+' г. (тыс. руб.)' for year in self.forYears]
		for depth in range(1,4):
			spreadsheet=parse.Spreadsheet(self.nCols,depth)
			spreadsheet.read(self.txtFilename)
			spreadsheet.check(self.totals)
			spreadsheet.setDocumentTitle('Приложение '+str(self.appendixNumber)+' к Закону Санкт-Петербурга «'+self.law.title+'»')
			spreadsheet.setTableTitle(self.title)
			spreadsheet.setAmountHeader(header)
			for stairs in (False,True):
				for sums in (False,True):
					csvFilename=self.getCsvFilename(depth,stairs,sums)
					spreadsheet.writeCsv(csvFilename,stairs,sums)
				xlsFilename=self.getXlsFilename(depth,stairs)
				spreadsheet.writeXls(xlsFilename,self.delta,stairs)

# закон или законопроект, в к-рый входит несколько приложений - с ними отдельно разбираться
class Law:
	def __init__(self,environment,data):
		# year = на какой год (первый из (трёх) годов) (FIXME update)
		# date = дата опубликования
		# viewUrl = url для просмотра
		# downloadUrl = url для скачивания
		#	из него будет делаться путь к скачанным файлам в /zip
		# version = (FIXME update)
		#	0 - исходный закон/проект
		#	1,2,3,... - n-я корректировка (пока была только одна корректировка, даты есть только для 2013)
		#		бывает и больше корректировок
		#	-1 - закон об исполнении (его проекта нет, в зипе только приложения)
		if 'documents' not in data:
			data['documents']=[]
		self.environment=environment
		self.code=data['code']
		if not data['downloadUrl'].startswith(self.environment.rootUrl):
			raise Exception('invalid download url')
		self.zipFilename=self.environment.rootPath+'/zip/'+data['downloadUrl'][len(self.environment.rootUrl):]
		self.title=data['title']
		self.documents=(Document(self,documentData) for documentData in data['documents'])
	def hasZip(self):
		return os.path.isfile(self.zipFilename)

class Environment:
	def __init__(self,data):
		self.rootPath=os.path.realpath(
			os.path.dirname(os.path.realpath(__file__))+'/..'
		)
		self.rootUrl=data['rootUrl']
		self.pdfboxFilename=self.rootPath+'/bin/'+data['pdfboxJar']
		self.laws=(Law(self,lawData) for lawData in data['laws'])

e=Environment(data.data)
for law in e.laws:
	if not law.hasZip():
		raise Exception(law.zipFilename+' has to be downloaded')
	for document in law.documents:
		if not document.hasPdf():
			document.writePdf()
		if not document.hasTxt():
			document.writeTxt()
		if not document.hasCsvsAndXlss():
			document.writeCsvsAndXlss()
<commit_msg>document code added to exception message<commit_after># basic types
# filenames are always absolute/ready to be read

import sys
import os.path
import io
import zipfile
import subprocess

import data,parse

# приложения, которые парсим
class Document:
	def __init__(self,law,data):
		# fix data
		if isinstance(data['forYear'],(str,int)):
			data['forYear']=(data['forYear'],)
		data['forYear']=tuple(str(y) for y in data['forYear'])
		if isinstance(data['total'],(str,int)):
			data['total']=(data['total'],)
		if 'delta' not in data:
			data['delta']=False
		# init
		self.law=law
		self.forYears=data['forYear']
		self.code=self.law.code+'-'+','.join(self.forYears)
		self.pdfFilename=self.law.environment.rootPath+'/pdf/'+self.code+'.pdf'
		self.txtFilename=self.law.environment.rootPath+'/txt/'+self.code+'.txt'
		self.zipContents=data['zipContents']
		self.totals=data['total']
		if len(self.forYears)!=len(self.totals):
			raise Exception('invalid number of columns')
		self.nCols=len(self.totals)
		self.delta=data['delta']
		self.appendixNumber=data['appendixNumber']
		self.title=data['title']
	def hasPdf(self):
		return os.path.isfile(self.pdfFilename)
	def writePdf(self):
		# attempt to read before opening pdfFile - otherwise might get zero-length file on error
		bytes=None
		for s in self.zipContents.split('/'):
			if bytes is None:
				zipFile=zipfile.ZipFile(law.zipFilename)
			else:
				zipFile=zipfile.ZipFile(io.BytesIO(bytes))
			bytes=zipFile.read(s)
			zipFile.close()
		# write pdf
		pdfFile=open(self.pdfFilename,'wb')
		pdfFile.write(bytes)
		pdfFile.close()
	def hasTxt(self):
		return os.path.isfile(self.txtFilename)
	def writeTxt(self):
		status=subprocess.call(['java','-jar',self.law.environment.pdfboxFilename,'ExtractText','-encoding','UTF-8','-sort',self.pdfFilename,self.txtFilename])
		if status!=0:
			raise Exception('external command failure')
	def getFilenameParensPart(self,depth,stairs,sums):
		return '('+str(depth)+(',stairs' if stairs else '')+(',sums' if sums else '')+')'
	def getCsvFilename(self,depth,stairs,sums):
		return self.law.environment.rootPath+'/csv/'+self.code+self.getFilenameParensPart(depth,stairs,sums)+'.csv'
	def getXlsFilename(self,depth,stairs):
		sums=True
		return self.law.environment.rootPath+'/xls/'+self.code+self.getFilenameParensPart(depth,stairs,sums)+'.xls'
	def hasCsvsAndXlss(self):
		for depth in range(1,4):
			for stairs in (False,True):
				for sums in (False,True):
					if not os.path.isfile(self.getCsvFilename(depth,stairs,sums)):
						return False
				if not os.path.isfile(self.getXlsFilename(depth,stairs)):
					return False
		return True
	def writeCsvsAndXlss(self):
		header=[('Изменение суммы' if self.delta else 'Сумма')+' на '+year+' г. (тыс. руб.)' for year in self.forYears]
		for depth in range(1,4):
			spreadsheet=parse.Spreadsheet(self.nCols,depth)
			spreadsheet.read(self.txtFilename)
			spreadsheet.check(self.totals)
			spreadsheet.setDocumentTitle('Приложение '+str(self.appendixNumber)+' к Закону Санкт-Петербурга «'+self.law.title+'»')
			spreadsheet.setTableTitle(self.title)
			spreadsheet.setAmountHeader(header)
			for stairs in (False,True):
				for sums in (False,True):
					csvFilename=self.getCsvFilename(depth,stairs,sums)
					spreadsheet.writeCsv(csvFilename,stairs,sums)
				xlsFilename=self.getXlsFilename(depth,stairs)
				spreadsheet.writeXls(xlsFilename,self.delta,stairs)

# закон или законопроект, в к-рый входит несколько приложений - с ними отдельно разбираться
class Law:
	def __init__(self,environment,data):
		# year = на какой год (первый из (трёх) годов) (FIXME update)
		# date = дата опубликования
		# viewUrl = url для просмотра
		# downloadUrl = url для скачивания
		#	из него будет делаться путь к скачанным файлам в /zip
		# version = (FIXME update)
		#	0 - исходный закон/проект
		#	1,2,3,... - n-я корректировка (пока была только одна корректировка, даты есть только для 2013)
		#		бывает и больше корректировок
		#	-1 - закон об исполнении (его проекта нет, в зипе только приложения)
		if 'documents' not in data:
			data['documents']=[]
		self.environment=environment
		self.code=data['code']
		if not data['downloadUrl'].startswith(self.environment.rootUrl):
			raise Exception('invalid download url')
		self.zipFilename=self.environment.rootPath+'/zip/'+data['downloadUrl'][len(self.environment.rootUrl):]
		self.title=data['title']
		self.documents=(Document(self,documentData) for documentData in data['documents'])
	def hasZip(self):
		return os.path.isfile(self.zipFilename)

class Environment:
	def __init__(self,data):
		self.rootPath=os.path.realpath(
			os.path.dirname(os.path.realpath(__file__))+'/..'
		)
		self.rootUrl=data['rootUrl']
		self.pdfboxFilename=self.rootPath+'/bin/'+data['pdfboxJar']
		self.laws=(Law(self,lawData) for lawData in data['laws'])

env=Environment(data.data)
for law in env.laws:
	if not law.hasZip():
		raise Exception(law.zipFilename+' has to be downloaded')
	for document in law.documents:
		try:
			if not document.hasPdf():
				document.writePdf()
			if not document.hasTxt():
				document.writeTxt()
			if not document.hasCsvsAndXlss():
				document.writeCsvsAndXlss()
		except Exception as e:
			tb=sys.exc_info()[2]
			raise Exception('error in document '+document.code+': '+str(e)).with_traceback(tb)
<|endoftext|>