<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import configparser
import re
import tempfile
import textwrap
import time
import tweepy

import chem_bot
from chem_bot import SmilesEncoder

__CONFIG_FILE__ = 'twitter.config'

def oauth(auth, config):
    if auth.access_token == '':
        try:
            redirect_url = auth.get_authorization_url()
            print('Get authorized PIN code from url.')
            print(redirect_url)
            pin = input('PIN: ')
            auth.get_access_token(pin)
            config.set('tokens', 'access_token', auth.access_token)
            config.set(
                'tokens', 'access_token_secret', auth.access_token_secret)
            with open(__CONFIG_FILE__, 'w') as f:
                config.write(f)
            return auth
        except tweepy.TweepError:
            raise Exception(textwrap.dedent("""\
                Error: Failed to get request token.
                Please check [twitter.config].\
                """))
    else:
        return auth

def _tweet_test(api, smiles, screen_name):
    return api.update_status('tes')

def get_config():
    config = configparser.ConfigParser()
    config.read(__CONFIG_FILE__)
    auth = tweepy.OAuthHandler(
        config.get('tokens', 'consumer_key'),
        config.get('tokens', 'consumer_secret'))
    auth.set_access_token(
        config.get('tokens', 'access_token'),
        config.get('tokens', 'access_token_secret'))
    return tweepy.API(auth), config

class Listner(tweepy.StreamListener):
    """Streaming time-line."""
    def __init__(self, api=None):
        super().__init__(api)
        self.iupac_prefix = config.get('general', 'iupac_prefix')
        self.smiles_prefix = config.get('general', 'smiles_prefix')
        self.opsin = config.get('general', 'opsin')

    def on_status(self, status):
        if str(status.text).startswith(self.iupac_prefix):
            print(
                '[CATCH] @{0} >>> {1}'
                .format(status.author.screen_name, status.text))
            iupac = status.text.split(self.iupac_prefix)[1].lstrip(' ')
            if self.check_ascii(iupac):
                smiles, error = chem_bot.util.converter.iupac_to_smiles(
                    iupac, self.opsin)
            else:
                smiles = ''
                error = iupac

            if smiles == '':
                self.reply_iupac_convert_error(
                    error, status.id, status.author.screen_name)
                print('[BOT] Error: {0}'.format(error))
            else:
                self.reply_with_png(
                    api,
                    smiles,
                    status.id,
                    status.author.screen_name,
                    descriptor_type='IUPAC名')
            print('[BOT] continue streaming...')

        if str(status.text).startswith(self.smiles_prefix):
            print(
                '[CATCH] @{0} >>> {1}'
                .format(status.author.screen_name, status.text))
            command = status.text.split(self.smiles_prefix)[1].lstrip(' ')
            smiles, option_d = self.parse_tweet_command(command)
            self.reply_with_png(
                api,
                smiles,
                status.id,
                status.author.screen_name,
                option_d=option_d)
            print('[BOT] continue streaming...')

        return True

    def parse_tweet_command(self, command):
        """Parse command into SMILES/IUPAC and options.

        If command not has any options, return (smile/iupac, None)
        """
        try:
            discriptor, trail_line = re.split(r',[ ]*| +', command, maxsplit=1)
        except ValueError:
            return command, None

        options = re.split(r', *', trail_line)
        option_d = dict([re.split(r': *', x) for x in options])
        return discriptor, option_d

    def reply_iupac_convert_error(self, error, s_id, screen_name):
        """Tweet for reply about iupac convert error."""
        print('[BOT] IUPAC conversion error')
        tweet = '@{0} '.format(screen_name)
        return self.tweet_error_message(
            tweet + 'すまない。私の化学目録に「{0}」という文字は無かった。'
            .format(error), s_id)

    def reply_with_png(self, api, smiles,
                       s_id, screen_name, option_d=None,
                       descriptor_type='SMILES'):
        """Tweet chem graph to user"""
        print('[SMILES]: {0}'.format(smiles))
        tweet = '@{0}'.format(screen_name)

        if smiles == '':
            tweet += (
                '{0}を入力し忘れていないか確認してもらえないだろうか。'
                .format(descriptor_type))
            return self.tweet_error_message(tweet, s_id)

        if self.check_ascii(smiles):
            encoder = SmilesEncoder(smiles)
        else:
            tweet += ' おや、SMILESに使えない文字が入っているようだ。'
            return self.tweet_error_message(tweet, s_id)

        if encoder.mol is None:
            print('Encoding error for [ {0} ]'.format(smiles))
            tweet += (
                ' すまない。この{0}は上手く変換できなかったようだ。'
                .format(descriptor_type))
            tweet += '"{0}"'.format(smiles)
            return self.tweet_error_message(tweet, s_id)

        png_binary = encoder.to_png()
        image = tempfile.TemporaryFile()
        image.write(png_binary)
        image.seek(0)
        try:
            api.update_with_media(
                filename='{0}.png'.format(smiles),
                status=self.string_trimmer(tweet),
                in_reply_to_status_id=s_id,
                file=image)
        except Exception as e:
            print(e)
        return True

    def tweet_error_message(self, tweet, s_id):
            try:
                api.update_status(self.string_trimmer(tweet), s_id)
            except tweepy.error.TweepError as e:
                print('Error: {0}'.format(e))
            return False

    def check_ascii(self, smiles):
        """Except multibyte characters in strings."""
        try:
            smiles.encode('ascii')
            return True
        except UnicodeEncodeError:
            return False

    def string_trimmer(self, line):
        """Wrap end of strings to under 140 characters."""
        if len(line) > 140:
            return line[:137] + '...'
        else:
            return line

def _main():
    global config
    global api
    api, config = get_config()
    oauth(api.auth, config)
    print('[BOT] OAuth is OK')
    print('[BOT] start streaming...')
    stream = tweepy.Stream(api.auth, Listner(), secure=True)
    while True:
        try:
            stream.userstream()
        except Exception:
            time.sleep(60)
            stream = tweepy.Stream(api.auth, Listner(), secure=True)

if __name__ == '__main__':
    _main()
<commit_msg>:penguin: Reply with conveted SMILES<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import configparser
import re
import tempfile
import textwrap
import time
import tweepy

import chem_bot
from chem_bot import SmilesEncoder

__CONFIG_FILE__ = 'twitter.config'

def oauth(auth, config):
    if auth.access_token == '':
        try:
            redirect_url = auth.get_authorization_url()
            print('Get authorized PIN code from url.')
            print(redirect_url)
            pin = input('PIN: ')
            auth.get_access_token(pin)
            config.set('tokens', 'access_token', auth.access_token)
            config.set(
                'tokens', 'access_token_secret', auth.access_token_secret)
            with open(__CONFIG_FILE__, 'w') as f:
                config.write(f)
            return auth
        except tweepy.TweepError:
            raise Exception(textwrap.dedent("""\
                Error: Failed to get request token.
                Please check [twitter.config].\
                """))
    else:
        return auth

def _tweet_test(api, smiles, screen_name):
    return api.update_status('tes')

def get_config():
    config = configparser.ConfigParser()
    config.read(__CONFIG_FILE__)
    auth = tweepy.OAuthHandler(
        config.get('tokens', 'consumer_key'),
        config.get('tokens', 'consumer_secret'))
    auth.set_access_token(
        config.get('tokens', 'access_token'),
        config.get('tokens', 'access_token_secret'))
    return tweepy.API(auth), config

class Listner(tweepy.StreamListener):
    """Streaming time-line."""
    def __init__(self, api=None):
        super().__init__(api)
        self.iupac_prefix = config.get('general', 'iupac_prefix')
        self.smiles_prefix = config.get('general', 'smiles_prefix')
        self.opsin = config.get('general', 'opsin')

    def on_status(self, status):
        if str(status.text).startswith(self.iupac_prefix):
            print(
                '[CATCH] @{0} >>> {1}'
                .format(status.author.screen_name, status.text))
            iupac = status.text.split(self.iupac_prefix)[1].lstrip(' ')
            if self.check_ascii(iupac):
                smiles, error = chem_bot.util.converter.iupac_to_smiles(
                    iupac, self.opsin)
            else:
                smiles = ''
                error = iupac

            if smiles == '':
                self.reply_iupac_convert_error(
                    error, status.id, status.author.screen_name)
                print('[BOT] Error: {0}'.format(error))
            else:
                self.reply_with_png(
                    api,
                    smiles,
                    status.id,
                    status.author.screen_name,
                    descriptor_type='IUPAC名',
                    with_smiles=True)
            print('[BOT] continue streaming...')

        if str(status.text).startswith(self.smiles_prefix):
            print(
                '[CATCH] @{0} >>> {1}'
                .format(status.author.screen_name, status.text))
            command = status.text.split(self.smiles_prefix)[1].lstrip(' ')
            smiles, option_d = self.parse_tweet_command(command)
            self.reply_with_png(
                api,
                smiles,
                status.id,
                status.author.screen_name,
                option_d=option_d)
            print('[BOT] continue streaming...')

        return True

    def parse_tweet_command(self, command):
        """Parse command into SMILES/IUPAC and options.

        If command not has any options, return (smile/iupac, None)
        """
        try:
            discriptor, trail_line = re.split(r',[ ]*| +', command, maxsplit=1)
        except ValueError:
            return command, None

        options = re.split(r', *', trail_line)
        option_d = dict([re.split(r': *', x) for x in options])
        return discriptor, option_d

    def reply_iupac_convert_error(self, error, s_id, screen_name):
        """Tweet for reply about iupac convert error."""
        print('[BOT] IUPAC conversion error')
        tweet = '@{0} '.format(screen_name)
        return self.tweet_error_message(
            tweet + 'すまない。私の化学目録に「{0}」という文字は無かった。'
            .format(error), s_id)

    def reply_with_png(self, api, smiles,
                       s_id, screen_name, option_d=None,
                       descriptor_type='SMILES', with_smiles=False):
        """Tweet chem graph to user"""
        print('[SMILES]: {0}'.format(smiles))
        tweet = '@{0}'.format(screen_name)

        if smiles == '':
            tweet += (
                '{0}を入力し忘れていないか確認してもらえないだろうか。'
                .format(descriptor_type))
            return self.tweet_error_message(tweet, s_id)

        if self.check_ascii(smiles):
            encoder = SmilesEncoder(smiles)
        else:
            tweet += ' おや、SMILESに使えない文字が入っているようだ。'
            return self.tweet_error_message(tweet, s_id)

        if encoder.mol is None:
            print('Encoding error for [ {0} ]'.format(smiles))
            tweet += (
                ' すまない。この{0}は上手く変換できなかったようだ。'
                .format(descriptor_type))
            tweet += '"{0}"'.format(smiles)
            return self.tweet_error_message(tweet, s_id)

        png_binary = encoder.to_png()
        image = tempfile.TemporaryFile()
        image.write(png_binary)
        image.seek(0)

        if with_smiles:
            tweet = '{0} "{1}"'.format(tweet, smiles)

        try:
            api.update_with_media(
                filename='{0}.png'.format(smiles),
                status=self.string_trimmer(tweet),
                in_reply_to_status_id=s_id,
                file=image)
        except Exception as e:
            print(e)
        return True

    def tweet_error_message(self, tweet, s_id):
            try:
                api.update_status(self.string_trimmer(tweet), s_id)
            except tweepy.error.TweepError as e:
                print('Error: {0}'.format(e))
            return False

    def check_ascii(self, smiles):
        """Except multibyte characters in strings."""
        try:
            smiles.encode('ascii')
            return True
        except UnicodeEncodeError:
            return False

    def string_trimmer(self, line):
        """Wrap end of strings to under 140 characters."""
        if len(line) > 140:
            return line[:137] + '...'
        else:
            return line

def _main():
    global config
    global api
    api, config = get_config()
    oauth(api.auth, config)
    print('[BOT] OAuth is OK')
    print('[BOT] start streaming...')
    stream = tweepy.Stream(api.auth, Listner(), secure=True)
    while True:
        try:
            stream.userstream()
        except Exception:
            time.sleep(60)
            stream = tweepy.Stream(api.auth, Listner(), secure=True)

if __name__ == '__main__':
    _main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

from collections import Counter
import heapq
import logging
import optparse
import sys

LOG = logging.getLogger(__name__)

BOS = "<s>"
EOS = "</s>"
UNK = "<unk>"

def replace_tags(tokens,tags,vocab):
  for i,t in enumerate(tokens):
    if not t in vocab:
      if i < len(tags):
        tokens[i] = tags[i]
      else:
	print "Error: missing tags for index i:", i
        print ' '.join(tokens)
        print ' '.join(tags)
	tokens[i] = UNK

def replace_unks(tokens,vocab):
  for i,t in enumerate(tokens):
    if not t in vocab:
      tokens[i] = UNK

def get_ngrams(corpus_stem, align_file, tagged_stem, svocab, tvocab, slang,tlang, m, n, ofh):
    """
      m - source context
      n - target context

      returns set of tags used
    """
    tags = Counter()
    sfh = open(corpus_stem + "." + slang)
    tfh = open(corpus_stem + "." + tlang)
    afh = open(align_file)
    fhs = [sfh,tfh,afh]
    if tagged_stem:
      fhs.append(open(tagged_stem + "." + slang))
      fhs.append(open(tagged_stem + "." + tlang))

    count  = 0
    ngrams = 0
    LOG.info("Extracting ngrams")
    for lines  in zip(*fhs):
      stokens = lines[0][:-1].split()
      ttokens = lines[1][:-1].split()
      stokens.append(EOS)
      ttokens.append(EOS)
      if tagged_stem:
        stags = lines[3][:-1].split()
        ttags = lines[4][:-1].split()
        stags.append(EOS)
        ttags.append(EOS)
        tags.update(stags)
        tags.update(ttags)
        replace_tags(stokens,stags,svocab)
        replace_tags(ttokens,ttags,tvocab)
      else:
        replace_unks(stokens,svocab)
        replace_unks(ttokens,tvocab)
      # list aligns for each target
      # Note: align specifies source -> target
      target_aligns = [[] for t in range(len(ttokens))]
      for atoken in lines[2][:-1].split():
        spos,tpos = atoken.split("-")
        spos,tpos = int(spos), int(tpos)
        target_aligns[tpos].append(spos)
      #EOS alignment
      target_aligns[-1] = [len(stokens)-1]

      for tpos,spos_list in enumerate(target_aligns):
        # Affiliation heuristics - see Devlin t al. p1371
        if not spos_list:
          #tpos has no alignment, look right, then left, then right-right, then left-left etc
          rpos = tpos+1
          lpos = tpos-1
          while rpos < len(ttokens) or lpos >= 0:
            if rpos < len(ttokens) and target_aligns[rpos]:
              spos_list = target_aligns[rpos]
              break
            if lpos >= 0 and target_aligns[lpos]:
              spos_list = target_aligns[lpos]
              break
            rpos += 1
            lpos -= 1

        if not spos_list:
          raise Exception("No alignments in sentence \nSRC: " +  lines[0][:-1] + "\nTGT: " +  lines[1][:-1])
        spos = (max(spos_list) + min(spos_list)) / 2

        # source-context, target-context, predicted word
        for i in range(max(0,m-spos)):
          print>>ofh, BOS,
          #print [spos-m/2,spos+m/2+1], stokens[spos-m/2:spos+m/2+1]
        print>>ofh, " ".join([s for s in stokens[max(0,spos-m):spos+m+1]]),
        for i in range(max(0,spos+m+1-len(stokens))):
          print>>ofh, EOS,
        for i in range(max(0,n-(tpos+1))):
          print>>ofh, BOS,
        print>>ofh, " ".join([t for t in ttokens[max(0,tpos+1-n):tpos+1]]),
        print>>ofh
        ngrams += 1

      count += 1
      if count % 1000 == 0: sys.stderr.write(".")
      if count % 50000 == 0:  sys.stderr.write(" [%d]\n" % count)
    ofh.close()
    sys.stderr.write("\n")
    LOG.info("Extracted %d ngrams" % ngrams)
    return tags

<commit_msg>improve bilingualLM alignment heuristics consistency<commit_after>#!/usr/bin/env python

from collections import Counter
import heapq
import logging
import optparse
import sys

LOG = logging.getLogger(__name__)

BOS = "<s>"
EOS = "</s>"
UNK = "<unk>"

def replace_tags(tokens,tags,vocab):
  for i,t in enumerate(tokens):
    if not t in vocab:
      if i < len(tags):
        tokens[i] = tags[i]
      else:
	print "Error: missing tags for index i:", i
        print ' '.join(tokens)
        print ' '.join(tags)
	tokens[i] = UNK

def replace_unks(tokens,vocab):
  for i,t in enumerate(tokens):
    if not t in vocab:
      tokens[i] = UNK

def get_ngrams(corpus_stem, align_file, tagged_stem, svocab, tvocab, slang,tlang, m, n, ofh):
    """
      m - source context
      n - target context

      returns set of tags used
    """
    tags = Counter()
    sfh = open(corpus_stem + "." + slang)
    tfh = open(corpus_stem + "." + tlang)
    afh = open(align_file)
    fhs = [sfh,tfh,afh]
    if tagged_stem:
      fhs.append(open(tagged_stem + "." + slang))
      fhs.append(open(tagged_stem + "." + tlang))

    count  = 0
    ngrams = 0
    LOG.info("Extracting ngrams")
    for lines  in zip(*fhs):
      stokens = lines[0][:-1].split()
      ttokens = lines[1][:-1].split()
      stokens.append(EOS)
      ttokens.append(EOS)
      if tagged_stem:
        stags = lines[3][:-1].split()
        ttags = lines[4][:-1].split()
        stags.append(EOS)
        ttags.append(EOS)
        tags.update(stags)
        tags.update(ttags)
        replace_tags(stokens,stags,svocab)
        replace_tags(ttokens,ttags,tvocab)
      else:
        replace_unks(stokens,svocab)
        replace_unks(ttokens,tvocab)
      # list aligns for each target
      # Note: align specifies source -> target
      target_aligns = [[] for t in range(len(ttokens))]
      for atoken in lines[2][:-1].split():
        spos,tpos = atoken.split("-")
        spos,tpos = int(spos), int(tpos)
        target_aligns[tpos].append(spos)
      #EOS alignment
      target_aligns[-1] = [len(stokens)-1]

      for tpos,spos_list in enumerate(target_aligns):
        # Affiliation heuristics - see Devlin t al. p1371
        if not spos_list:
          #tpos has no alignment, look right, then left, then right-right, then left-left etc
          rpos = tpos+1
          lpos = tpos-1
          while rpos < len(ttokens) or lpos >= 0:
            if rpos < len(ttokens) and target_aligns[rpos]:
              spos_list = target_aligns[rpos]
              break
            if lpos >= 0 and target_aligns[lpos]:
              spos_list = target_aligns[lpos]
              break
            rpos += 1
            lpos -= 1

        if not spos_list:
          raise Exception("No alignments in sentence \nSRC: " +  lines[0][:-1] + "\nTGT: " +  lines[1][:-1])
        midpos = (len(spos_list)-1) / 2
        spos = sorted(spos_list)[midpos]

        # source-context, target-context, predicted word
        for i in range(max(0,m-spos)):
          print>>ofh, BOS,
          #print [spos-m/2,spos+m/2+1], stokens[spos-m/2:spos+m/2+1]
        print>>ofh, " ".join([s for s in stokens[max(0,spos-m):spos+m+1]]),
        for i in range(max(0,spos+m+1-len(stokens))):
          print>>ofh, EOS,
        for i in range(max(0,n-(tpos+1))):
          print>>ofh, BOS,
        print>>ofh, " ".join([t for t in ttokens[max(0,tpos+1-n):tpos+1]]),
        print>>ofh
        ngrams += 1

      count += 1
      if count % 1000 == 0: sys.stderr.write(".")
      if count % 50000 == 0:  sys.stderr.write(" [%d]\n" % count)
    ofh.close()
    sys.stderr.write("\n")
    LOG.info("Extracted %d ngrams" % ngrams)
    return tags

<|endoftext|>


<commit_before>"""
 Copyright 2020 Google Inc.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""
"""This module provides a service for adding credentials to the agency
dashboard.

The module is used in conjunction with the rest of the agency dashboard solution
to provide the credentials required to acccess the Ads accounts that will be the
basis for reporting.

This must be deployed as part of the agency dashboard Google Cloud AppEngine
app. Optionally, this component can be left out of the deployment and the
required credentials can be inserted manually into the project firestore store.

The required credentials must be located in a document stored at
/agency_ads/credentials and have the following fields:
- mcc_id: the account id of the management account to be used with the app
- client_id: the client ID created in the Google API console
- client_secret: the client secret generated with the above client id
- developer_token: the developer token for the account to be used with the app
- refresh_token: an oauth2 refresh token generated using the client id and
secret above
"""

import logging
import os

from bottle import Bottle
from bottle import HTTPError
from bottle import redirect
from bottle import request
from bottle import template
from bottle import view
from google_auth_oauthlib.flow import Flow
from oauthlib.oauth2.rfc6749.errors import InvalidGrantError

import google.cloud.exceptions
import google.cloud.firestore
import google.cloud.logging

app = Bottle()
logging_client = google.cloud.logging.Client()
logging_handler = logging_client.get_default_handler()
logger = logging.getLogger('Config-Service')
logger.addHandler(logging_handler)

project_name = os.environ['GOOGLE_CLOUD_PROJECT']
redirect_uri = f'https://config-service.{project_name}.appspot.com/config_end'

def client_config_exists():
  """Retrives the Adwords client configuation from firestore.

  Retrieves the Adwords client configuration from firestore if it is available.
  If the client_config doc is not found in the data store, a NotFound exception
  is raised.

  Returns:
    A dict containing the web client configuration for Google Adwords

  Raises:
    google.cloud.exceptions.NotFound: the client_config document was not in the
    agency_ads collection
  """
  storage_client = google.cloud.firestore.Client()
  try:
    client_config = (
        storage_client.collection('agency_ads').document('credentials').get())
    try:
      client_config.get('client_id')
    except KeyError:
      return False
  except google.cloud.exceptions.NotFound:
    return False

  return True

@app.route('/config')
@view('start_config')
def start_ads_config():
  """This route starts the oauth authentication flow.

  The route displays a page that allows the user to input the information
  required

  The route first checks for the existance of the config. If there is an
  existing config, a message is show to the user to help them avoid writing over
  it.

  Returns:
    An html page with fields for entering credentials.

  """
  return {'client_config_exists': client_config_exists()}

@app.route('/config_end')
def end_ads_config():
  """This route completes the oauth flow and saves the returned refresh token.

  The oauth state saved in the start of the flow is also removed from the
  credentials doc.

  Returns:
    On success, an HTML page is returned letting the user know the authorization
    was successful. On failure, the user is returned the page to enter their
    credentials with an error message.
  """
  storage_client = google.cloud.firestore.Client()
  try:
    credentials_doc = (
        storage_client.collection('agency_ads').document('credentials').get())
    client_id = credentials_doc.get('client_id')
    client_secret = credentials_doc.get('client_secret')
    oauth_state = credentials_doc.get('oauth_state')
  except (google.cloud.exceptions.NotFound, KeyError):
    logger.exception('Unable to load ads credentials.')
    return template(
        'start_config', error='Error loading credentials after oauth')

  auth_code = request.query.get('code')
  if not auth_code:
    return template('start_config', error='No authorization code in request.')

  client_config = {
      'web': {
          'client_id': client_id,
          'client_secret': client_secret,
          'auth_uri': 'https://accounts.google.com/o/oauth2/auth',
          'token_uri': 'https://accounts.google.com/o/oauth2/token',
      }
  }
  flow = Flow.from_client_config(
      client_config,
      scopes=['https://www.googleapis.com/auth/adwords'],
      state=oauth_state)
  flow.redirect_uri = redirect_uri
  try:
    flow.fetch_token(code=auth_code)
  except InvalidGrantError:
    logger.exception('Error fetching refresh token after oauth')
    return template('start_config', error='Error retreiving refresh token.')

  storage_client = google.cloud.firestore.Client()
  try:
    credentials_doc = storage_client.collection('agency_ads').document(
        'credentials')
    credentials_doc.update({
        'refresh_token': flow.credentials.refresh_token,
        'oauth_state': google.cloud.firestore.DELETE_FIELD
    })
  except google.cloud.exceptions.NotFound:
    logger.exception('Error finding or updating credentials in firestore.')
    return template(
        'start_config', error='Error updating credentials doc in firestore')

  return template('config_complete')

@app.route('/config_upload_client', method='POST')
def save_client_config():
  """Route used to save the client configuration JSON file to firestore."""
  mcc_id = request.forms.get('mcc_id')
  client_id = request.forms.get('client_id')
  client_secret = request.forms.get('client_secret')
  developer_token = request.forms.get('developer_token')
  psi_api_token = request.forms.get('psi_api_token')

  client_config = {
      'web': {
          'client_id': client_id,
          'client_secret': client_secret,
          'auth_uri': 'https://accounts.google.com/o/oauth2/auth',
          'token_uri': 'https://accounts.google.com/o/oauth2/token',
      }
  }
  flow = Flow.from_client_config(
      client_config, scopes=['https://www.googleapis.com/auth/adwords'])
  flow.redirect_uri = redirect_uri
  auth_url, oauth_state = flow.authorization_url(prompt='consent')

  storage_client = google.cloud.firestore.Client()
  try:
    credentials_doc = storage_client.collection('agency_ads').document(
        'credentials')
    credentials_content = {
        'mcc_id': mcc_id,
        'client_id': client_id,
        'client_secret': client_secret,
        'developer_token': developer_token,
        'psi_api_token': psi_api_token,
        'oauth_state': oauth_state
    }
    credentials_doc.set(credentials_content)
  except google.cloud.exceptions.NotFound:
    logger.exception('Unable to find ads credentials.')
    raise HTTPError(500, 'Unable to find ads credentials.')

  redirect(auth_url)
<commit_msg>Updated getting app url to allow for region in url.<commit_after>"""
 Copyright 2020 Google Inc.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""
"""This module provides a service for adding credentials to the agency
dashboard.

The module is used in conjunction with the rest of the agency dashboard solution
to provide the credentials required to acccess the Ads accounts that will be the
basis for reporting.

This must be deployed as part of the agency dashboard Google Cloud AppEngine
app. Optionally, this component can be left out of the deployment and the
required credentials can be inserted manually into the project firestore store.

The required credentials must be located in a document stored at
/agency_ads/credentials and have the following fields:
- mcc_id: the account id of the management account to be used with the app
- client_id: the client ID created in the Google API console
- client_secret: the client secret generated with the above client id
- developer_token: the developer token for the account to be used with the app
- refresh_token: an oauth2 refresh token generated using the client id and
secret above
"""

import logging
import os

from bottle import Bottle
from bottle import HTTPError
from bottle import redirect
from bottle import request
from bottle import template
from bottle import view
from google_auth_oauthlib.flow import Flow
from oauthlib.oauth2.rfc6749.errors import InvalidGrantError

import google.cloud.exceptions
import google.cloud.firestore
import google.cloud.logging

app = Bottle()
logging_client = google.cloud.logging.Client()
logging_handler = logging_client.get_default_handler()
logger = logging.getLogger('Config-Service')
logger.addHandler(logging_handler)

def client_config_exists():
  """Retrives the Adwords client configuation from firestore.

  Retrieves the Adwords client configuration from firestore if it is available.
  If the client_config doc is not found in the data store, a NotFound exception
  is raised.

  Returns:
    A dict containing the web client configuration for Google Adwords

  Raises:
    google.cloud.exceptions.NotFound: the client_config document was not in the
    agency_ads collection
  """
  storage_client = google.cloud.firestore.Client()
  try:
    client_config = (
        storage_client.collection('agency_ads').document('credentials').get())
    try:
      client_config.get('client_id')
    except KeyError:
      return False
  except google.cloud.exceptions.NotFound:
    return False

  return True

@app.route('/config')
@view('start_config')
def start_ads_config():
  """This route starts the oauth authentication flow.

  The route displays a page that allows the user to input the information
  required

  The route first checks for the existance of the config. If there is an
  existing config, a message is show to the user to help them avoid writing over
  it.

  Returns:
    An html page with fields for entering credentials.

  """
  return {'client_config_exists': client_config_exists()}

@app.route('/config_end')
def end_ads_config():
  """This route completes the oauth flow and saves the returned refresh token.

  The oauth state saved in the start of the flow is also removed from the
  credentials doc.

  Returns:
    On success, an HTML page is returned letting the user know the authorization
    was successful. On failure, the user is returned the page to enter their
    credentials with an error message.
  """
  storage_client = google.cloud.firestore.Client()
  try:
    credentials_doc = (
        storage_client.collection('agency_ads').document('credentials').get())
    client_id = credentials_doc.get('client_id')
    client_secret = credentials_doc.get('client_secret')
    oauth_state = credentials_doc.get('oauth_state')
  except (google.cloud.exceptions.NotFound, KeyError):
    logger.exception('Unable to load ads credentials.')
    return template(
        'start_config', error='Error loading credentials after oauth')

  auth_code = request.query.get('code')
  if not auth_code:
    return template('start_config', error='No authorization code in request.')

  client_config = {
      'web': {
          'client_id': client_id,
          'client_secret': client_secret,
          'auth_uri': 'https://accounts.google.com/o/oauth2/auth',
          'token_uri': 'https://accounts.google.com/o/oauth2/token',
      }
  }
  flow = Flow.from_client_config(
      client_config,
      scopes=['https://www.googleapis.com/auth/adwords'],
      state=oauth_state)

  req = urllib.parse.urlparse(self.request.url)
  redirect_uri = f'{req.schema}://{req.hostname}/config_end'
  flow.redirect_uri = redirect_uri
  try:
    flow.fetch_token(code=auth_code)
  except InvalidGrantError:
    logger.exception('Error fetching refresh token after oauth')
    return template('start_config', error='Error retreiving refresh token.')

  storage_client = google.cloud.firestore.Client()
  try:
    credentials_doc = storage_client.collection('agency_ads').document(
        'credentials')
    credentials_doc.update({
        'refresh_token': flow.credentials.refresh_token,
        'oauth_state': google.cloud.firestore.DELETE_FIELD
    })
  except google.cloud.exceptions.NotFound:
    logger.exception('Error finding or updating credentials in firestore.')
    return template(
        'start_config', error='Error updating credentials doc in firestore')

  return template('config_complete')

@app.route('/config_upload_client', method='POST')
def save_client_config():
  """Route used to save the client configuration JSON file to firestore."""
  mcc_id = request.forms.get('mcc_id')
  client_id = request.forms.get('client_id')
  client_secret = request.forms.get('client_secret')
  developer_token = request.forms.get('developer_token')
  psi_api_token = request.forms.get('psi_api_token')

  client_config = {
      'web': {
          'client_id': client_id,
          'client_secret': client_secret,
          'auth_uri': 'https://accounts.google.com/o/oauth2/auth',
          'token_uri': 'https://accounts.google.com/o/oauth2/token',
      }
  }
  flow = Flow.from_client_config(
      client_config, scopes=['https://www.googleapis.com/auth/adwords'])
  flow.redirect_uri = redirect_uri
  auth_url, oauth_state = flow.authorization_url(prompt='consent')

  storage_client = google.cloud.firestore.Client()
  try:
    credentials_doc = storage_client.collection('agency_ads').document(
        'credentials')
    credentials_content = {
        'mcc_id': mcc_id,
        'client_id': client_id,
        'client_secret': client_secret,
        'developer_token': developer_token,
        'psi_api_token': psi_api_token,
        'oauth_state': oauth_state
    }
    credentials_doc.set(credentials_content)
  except google.cloud.exceptions.NotFound:
    logger.exception('Unable to find ads credentials.')
    raise HTTPError(500, 'Unable to find ads credentials.')

  redirect(auth_url)
<|endoftext|>


<commit_before>import param
import numpy as np
from cartopy import crs as ccrs
from cartopy.feature import Feature as cFeature
from cartopy.io.img_tiles import GoogleTiles as cGoogleTiles
from cartopy.io.shapereader import Reader
from holoviews.core import Element2D, Dimension, Dataset as HvDataset, NdOverlay
from holoviews.core.util import basestring, pd
from holoviews.element import (Text as HVText, Path as HVPath,
                               Polygons as HVPolygons, GridImage)

from shapely.geometry.base import BaseGeometry
from shapely.geometry import (MultiLineString, LineString,
                              MultiPolygon, Polygon)

try:
    from iris.cube import Cube
except ImportError:
    Cube = None

try:
    from bokeh.models import WMTSTileSource
except:
    WMTSTileSource = None

geographic_types = (cGoogleTiles, cFeature)

def is_geographic(element, kdims=None):
    """
    Utility to determine whether the supplied element optionally
    a subset of its key dimensions represent a geographic coordinate
    system.
    """
    if isinstance(element, NdOverlay):
        element = element.last

    if kdims:
        kdims = [element.get_dimension(d) for d in kdims]
    else:
        kdims = element.kdims

    if len(kdims) != 2:
        return False
    if isinstance(element.data, geographic_types) or isinstance(element, WMTS):
        return True
    elif isinstance(element, _Element):
        return kdims == element.kdims and element.crs
    else:
        return False

class _Element(Element2D):
    """
    Baseclass for Element2D types with associated cartopy
    coordinate reference system.
    """

    _abstract = True

    crs = param.ClassSelector(class_=ccrs.CRS, doc="""
        Cartopy coordinate-reference-system specifying the
        coordinate system of the data. Inferred automatically
        when _Element wraps cartopy Feature object.""")

    kdims = param.List(default=[Dimension('Longitude'), Dimension('Latitude')])

    def __init__(self, data, **kwargs):
        crs = None
        crs_data = data.data if isinstance(data, HvDataset) else data
        if Cube and isinstance(crs_data, Cube):
            coord_sys = crs_data.coord_system()
            if hasattr(coord_sys, 'as_cartopy_projection'):
                crs = coord_sys.as_cartopy_projection()
        elif isinstance(crs_data, (cFeature, cGoogleTiles)):
            crs = crs_data.crs

        supplied_crs = kwargs.get('crs', None)
        if supplied_crs and crs and crs != supplied_crs:
            raise ValueError('Supplied coordinate reference '
                             'system must match crs of the data.')
        elif crs:
            kwargs['crs'] = crs
        super(_Element, self).__init__(data, **kwargs)

    def clone(self, data=None, shared_data=True, new_type=None,
              *args, **overrides):
        if 'crs' not in overrides:
            overrides['crs'] = self.crs
        return super(_Element, self).clone(data, shared_data, new_type,
                                           *args, **overrides)

class _GeoFeature(_Element):
    """
    Baseclass for geographic types without their own data.
    """

    _auxiliary_component = True

    def dimension_values(self, dim):
        """
        _GeoFeature types do not contain actual data.
        """
        return []

class Feature(_GeoFeature):
    """
    A Feature represents a geographical feature
    specified as a cartopy Feature type.
    """

    group = param.String(default='Feature')

    def __init__(self, data, **params):
        if not isinstance(data, cFeature):
            raise TypeError('%s data has to be an cartopy Feature type'
                            % type(data).__name__)
        super(Feature, self).__init__(data, **params)

class WMTS(_GeoFeature):
    """
    The WMTS Element represents a Web Map Tile Service
    specified as a tuple of the API URL and
    """

    group = param.String(default='WMTS')

    layer = param.String(doc="The layer on the tile service")

    def __init__(self, data, **params):
        if isinstance(data, tuple):
            data = data
        else:
            data = (data,)

        for d in data:
            if WMTSTileSource and isinstance(d, WMTSTileSource):
                if 'crs' not in params:
                    params['crs'] = ccrs.GOOGLE_MERCATOR
            elif not isinstance(data, basestring):
                if 'crs' not in params and not self.crs:
                    raise Exception('Must supply coordinate reference '
                                    'system with cartopy WMTS URL.')
            else:
                raise TypeError('%s data has to be a tile service URL'
                                % type(data).__name__)
        super(WMTS, self).__init__(data, **params)

class Tiles(_GeoFeature):
    """
    Tiles represents an image tile source to dynamically
    load data from depending on the zoom level.
    """

    group = param.String(default='Tiles')

    def __init__(self, data, **params):
        if not isinstance(data, cGoogleTiles):
            raise TypeError('%s data has to be a cartopy GoogleTiles type'
                            % type(data).__name__)
        super(Tiles, self).__init__(data, **params)

class Dataset(_Element, HvDataset):
    """
    Coordinate system aware version of a HoloViews dataset.
    """

class Points(Dataset):
    """
    Points represent a collection of points with
    an associated cartopy coordinate-reference system.
    """

    group = param.String(default='Points')

class LineContours(_Element, GridImage):
    """
    Contours represents a 2D array of some quantity with
    some associated coordinates, which may be discretized
    into one or more line contours.
    """

    vdims = param.List(default=[Dimension('z')], bounds=(1, 1))

    group = param.String(default='LineContours')

class FilledContours(LineContours):
    """
    Contours represents a 2D array of some quantity with
    some associated coordinates, which may be discretized
    into one or more filled contours.
    """

    group = param.String(default='FilledContours')

class Image(_Element, GridImage):
    """
    Image represents a 2D array of some quantity with
    some associated coordinates.
    """

    vdims = param.List(default=[Dimension('z')], bounds=(1, 1))

    group = param.String(default='Image')

class Text(HVText, _Element):
    """
    An annotation containing some text at an x, y coordinate
    along with a coordinate reference system.
    """

class Path(_Element, HVPath):
    """
    The Path Element contains a list of Paths stored as Nx2 numpy
    arrays along with a coordinate reference system.
    """

    def geom(self):
        """
        Returns Path as a shapely geometry.
        """
        lines = []
        for path in self.data:
            lines.append(LineString(path))
        return MultiLineString(lines)

class Polygons(_Element, HVPolygons):
    """
    Polygons is a Path Element type that may contain any number of
    closed paths with an associated value and a coordinate reference
    system.
    """

    def geom(self):
        """
        Returns Polygons as a shapely geometry.
        """
        polys = []
        for poly in self.data:
            polys.append(Polygon(poly))
        return MultiPolygon(polys)

class Shape(_Element):
    """
    Shape wraps any shapely geometry type.
    """

    group = param.String(default='Shape')

    level = param.Number(default=None, doc="""
        Optional level associated with the set of Contours.""")

    vdims = param.List(default=[Dimension('Level')], doc="""
        Contours optionally accept a value dimension, corresponding
        to the supplied values.""", bounds=(1,1))

    def __init__(self, data, **params):
        if not isinstance(data, BaseGeometry):
            raise TypeError('%s data has to be a shapely geometry type.'
                            % type(data).__name__)
        super(Shape, self).__init__(data, **params)

    @classmethod
    def from_shapefile(cls, shapefile, *args, **kwargs):
        """
        Loads a shapefile from disk and optionally merges
        it with a dataset. See ``from_records`` for full
        signature.
        """
        reader = Reader(shapefile)
        return cls.from_records(reader.records(), *args, **kwargs)

    @classmethod
    def from_records(cls, records, dataset=None, on=None,
                     value=None, index=[], **kwargs):
        """
        Load data from a collection of
        ``cartopy.io.shapereader.Record`` objects and optionally merge
        it with a dataset to assign values to each polygon and form a
        chloropleth. Supplying just records will return an NdOverlay
        of Shape Elements with a numeric index. If a dataset is
        supplied, a mapping between the attribute names in the records
        and the dimension names in the dataset must be supplied. The
        values assigned to each shape file can then be drawn from the
        dataset by supplying a ``value`` and keys the Shapes are
        indexed by specifying one or index dimensions.

        * records - An iterator of cartopy.io.shapereader.Record
                    objects.
        * dataset - Any HoloViews Dataset type.
        * on      - A mapping between the attribute names in
                    the records and the dimensions in the dataset.
        * value   - The value dimension in the dataset the
                    values will be drawn from.
        * index   - One or more dimensions in the dataset
                    the Shapes will be indexed by.

        Returns an NdOverlay of Shapes.
        """
        if dataset is not None and not on:
            raise ValueError('To merge dataset with shapes mapping '
                             'must define attribute(s) to merge on.')

        if pd and isinstance(dataset, pd.DataFrame):
            dataset = Dataset(dataset)

        if not isinstance(on, (dict, list)):
            on = [on]
        if on and not isinstance(on, dict):
            on = {o: o for o in on}
        if not isinstance(index, list):
            index = [index]

        kdims = []
        for ind in (index if index else ['Index']):
            if dataset and dataset.get_dimension(ind):
                dim = dataset.get_dimension(ind)
            else:
                dim = Dimension(ind)
            kdims.append(dim)

        ddims = []
        if dataset:
            vdim = dataset.get_dimension(value)
            kwargs['vdims'] = [vdim]
            if not vdim:
                raise ValueError('Value dimension not found '
                                 'in dataset: {}'.format(vdim))
            ddims = dataset.dimensions()

        chloropleth = NdOverlay(kdims=kdims)
        for i, rec in enumerate(records):
            if dataset:
                selection = {dim: rec.attributes.get(attr, None)
                             for attr, dim in on.items()}
                row = dataset.select(**selection)
                if not len(row):
                    continue
                if value:
                    value = row[vdim.name][0]
                    kwargs['level'] = value
            if index:
                key = []
                for kdim in kdims:
                    if kdim in ddims:
                        k = row[kdim.name][0]
                    elif kdim.name in rec.attributes:
                        k = rec.attributes[kdim.name]
                    else:
                        raise ValueError('%s could not be found' % kdim)
                    key.append(k)
                key = tuple(key)
            else:
                key = i
            chloropleth[key] = Shape(rec.geometry, **kwargs)
        return chloropleth

    def dimension_values(self, dimension):
        """
        Shapes do not support convert to array values.
        """
        dim = self.get_dimension(dimension)
        if dim in self.vdims:
            return [self.level]
        else:
            return []

    def range(self, dimension):
        dim = self.get_dimension(dimension)
        if dim.range != (None, None):
            return dim.range

        idx = self.get_dimension_index(dimension)
        if idx == 2:
            return self.level, self.level
        if idx in [0, 1]:
            l, b, r, t = self.data.bounds
            if idx == 0:
                return l, r
            elif idx == 1:
                return b, t
        else:
            return (np.NaN, np.NaN)

    def geom(self):
        """
        Returns Shape as a shapely geometry
        """
        return self.data

    def __len__(self):
        return len(self.data)
<commit_msg>gv.Image now inherits from hv.Image<commit_after>import param
import numpy as np
from cartopy import crs as ccrs
from cartopy.feature import Feature as cFeature
from cartopy.io.img_tiles import GoogleTiles as cGoogleTiles
from cartopy.io.shapereader import Reader
from holoviews.core import Element2D, Dimension, Dataset as HvDataset, NdOverlay
from holoviews.core.util import basestring, pd
from holoviews.element import (Text as HvText, Path as HvPath,
                               Polygons as HvPolygons, Image as HvImage)

from shapely.geometry.base import BaseGeometry
from shapely.geometry import (MultiLineString, LineString,
                              MultiPolygon, Polygon)

try:
    from iris.cube import Cube
except ImportError:
    Cube = None

try:
    from bokeh.models import WMTSTileSource
except:
    WMTSTileSource = None

geographic_types = (cGoogleTiles, cFeature)

def is_geographic(element, kdims=None):
    """
    Utility to determine whether the supplied element optionally
    a subset of its key dimensions represent a geographic coordinate
    system.
    """
    if isinstance(element, NdOverlay):
        element = element.last

    if kdims:
        kdims = [element.get_dimension(d) for d in kdims]
    else:
        kdims = element.kdims

    if len(kdims) != 2:
        return False
    if isinstance(element.data, geographic_types) or isinstance(element, WMTS):
        return True
    elif isinstance(element, _Element):
        return kdims == element.kdims and element.crs
    else:
        return False

class _Element(Element2D):
    """
    Baseclass for Element2D types with associated cartopy
    coordinate reference system.
    """

    _abstract = True

    crs = param.ClassSelector(class_=ccrs.CRS, doc="""
        Cartopy coordinate-reference-system specifying the
        coordinate system of the data. Inferred automatically
        when _Element wraps cartopy Feature object.""")

    kdims = param.List(default=[Dimension('Longitude'), Dimension('Latitude')])

    def __init__(self, data, **kwargs):
        crs = None
        crs_data = data.data if isinstance(data, HvDataset) else data
        if Cube and isinstance(crs_data, Cube):
            coord_sys = crs_data.coord_system()
            if hasattr(coord_sys, 'as_cartopy_projection'):
                crs = coord_sys.as_cartopy_projection()
        elif isinstance(crs_data, (cFeature, cGoogleTiles)):
            crs = crs_data.crs

        supplied_crs = kwargs.get('crs', None)
        if supplied_crs and crs and crs != supplied_crs:
            raise ValueError('Supplied coordinate reference '
                             'system must match crs of the data.')
        elif crs:
            kwargs['crs'] = crs
        super(_Element, self).__init__(data, **kwargs)

    def clone(self, data=None, shared_data=True, new_type=None,
              *args, **overrides):
        if 'crs' not in overrides and isinstance(new_type, _Element):
            overrides['crs'] = self.crs
        return super(_Element, self).clone(data, shared_data, new_type,
                                           *args, **overrides)

class _GeoFeature(_Element):
    """
    Baseclass for geographic types without their own data.
    """

    _auxiliary_component = True

    def dimension_values(self, dim):
        """
        _GeoFeature types do not contain actual data.
        """
        return []

class Feature(_GeoFeature):
    """
    A Feature represents a geographical feature
    specified as a cartopy Feature type.
    """

    group = param.String(default='Feature')

    def __init__(self, data, **params):
        if not isinstance(data, cFeature):
            raise TypeError('%s data has to be an cartopy Feature type'
                            % type(data).__name__)
        super(Feature, self).__init__(data, **params)

class WMTS(_GeoFeature):
    """
    The WMTS Element represents a Web Map Tile Service
    specified as a tuple of the API URL and
    """

    group = param.String(default='WMTS')

    layer = param.String(doc="The layer on the tile service")

    def __init__(self, data, **params):
        if isinstance(data, tuple):
            data = data
        else:
            data = (data,)

        for d in data:
            if WMTSTileSource and isinstance(d, WMTSTileSource):
                if 'crs' not in params:
                    params['crs'] = ccrs.GOOGLE_MERCATOR
            elif not isinstance(data, basestring):
                if 'crs' not in params and not self.crs:
                    raise Exception('Must supply coordinate reference '
                                    'system with cartopy WMTS URL.')
            else:
                raise TypeError('%s data has to be a tile service URL'
                                % type(data).__name__)
        super(WMTS, self).__init__(data, **params)

class Tiles(_GeoFeature):
    """
    Tiles represents an image tile source to dynamically
    load data from depending on the zoom level.
    """

    group = param.String(default='Tiles')

    def __init__(self, data, **params):
        if not isinstance(data, cGoogleTiles):
            raise TypeError('%s data has to be a cartopy GoogleTiles type'
                            % type(data).__name__)
        super(Tiles, self).__init__(data, **params)

class Dataset(_Element, HvDataset):
    """
    Coordinate system aware version of a HoloViews dataset.
    """

class Points(Dataset):
    """
    Points represent a collection of points with
    an associated cartopy coordinate-reference system.
    """

    group = param.String(default='Points')

class LineContours(_Element, HvImage):
    """
    Contours represents a 2D array of some quantity with
    some associated coordinates, which may be discretized
    into one or more line contours.
    """

    vdims = param.List(default=[Dimension('z')], bounds=(1, 1))

    group = param.String(default='LineContours')

class FilledContours(LineContours):
    """
    Contours represents a 2D array of some quantity with
    some associated coordinates, which may be discretized
    into one or more filled contours.
    """

    group = param.String(default='FilledContours')

class Image(_Element, HvImage):
    """
    Image represents a 2D array of some quantity with
    some associated coordinates.
    """

    vdims = param.List(default=[Dimension('z')], bounds=(1, 1))

    group = param.String(default='Image')

class Text(HvText, _Element):
    """
    An annotation containing some text at an x, y coordinate
    along with a coordinate reference system.
    """

class Path(_Element, HvPath):
    """
    The Path Element contains a list of Paths stored as Nx2 numpy
    arrays along with a coordinate reference system.
    """

    def geom(self):
        """
        Returns Path as a shapely geometry.
        """
        lines = []
        for path in self.data:
            lines.append(LineString(path))
        return MultiLineString(lines)

class Polygons(_Element, HvPolygons):
    """
    Polygons is a Path Element type that may contain any number of
    closed paths with an associated value and a coordinate reference
    system.
    """

    def geom(self):
        """
        Returns Polygons as a shapely geometry.
        """
        polys = []
        for poly in self.data:
            polys.append(Polygon(poly))
        return MultiPolygon(polys)

class Shape(_Element):
    """
    Shape wraps any shapely geometry type.
    """

    group = param.String(default='Shape')

    level = param.Number(default=None, doc="""
        Optional level associated with the set of Contours.""")

    vdims = param.List(default=[Dimension('Level')], doc="""
        Contours optionally accept a value dimension, corresponding
        to the supplied values.""", bounds=(1,1))

    def __init__(self, data, **params):
        if not isinstance(data, BaseGeometry):
            raise TypeError('%s data has to be a shapely geometry type.'
                            % type(data).__name__)
        super(Shape, self).__init__(data, **params)

    @classmethod
    def from_shapefile(cls, shapefile, *args, **kwargs):
        """
        Loads a shapefile from disk and optionally merges
        it with a dataset. See ``from_records`` for full
        signature.
        """
        reader = Reader(shapefile)
        return cls.from_records(reader.records(), *args, **kwargs)

    @classmethod
    def from_records(cls, records, dataset=None, on=None,
                     value=None, index=[], **kwargs):
        """
        Load data from a collection of
        ``cartopy.io.shapereader.Record`` objects and optionally merge
        it with a dataset to assign values to each polygon and form a
        chloropleth. Supplying just records will return an NdOverlay
        of Shape Elements with a numeric index. If a dataset is
        supplied, a mapping between the attribute names in the records
        and the dimension names in the dataset must be supplied. The
        values assigned to each shape file can then be drawn from the
        dataset by supplying a ``value`` and keys the Shapes are
        indexed by specifying one or index dimensions.

        * records - An iterator of cartopy.io.shapereader.Record
                    objects.
        * dataset - Any HoloViews Dataset type.
        * on      - A mapping between the attribute names in
                    the records and the dimensions in the dataset.
        * value   - The value dimension in the dataset the
                    values will be drawn from.
        * index   - One or more dimensions in the dataset
                    the Shapes will be indexed by.

        Returns an NdOverlay of Shapes.
        """
        if dataset is not None and not on:
            raise ValueError('To merge dataset with shapes mapping '
                             'must define attribute(s) to merge on.')

        if pd and isinstance(dataset, pd.DataFrame):
            dataset = Dataset(dataset)

        if not isinstance(on, (dict, list)):
            on = [on]
        if on and not isinstance(on, dict):
            on = {o: o for o in on}
        if not isinstance(index, list):
            index = [index]

        kdims = []
        for ind in (index if index else ['Index']):
            if dataset and dataset.get_dimension(ind):
                dim = dataset.get_dimension(ind)
            else:
                dim = Dimension(ind)
            kdims.append(dim)

        ddims = []
        if dataset:
            vdim = dataset.get_dimension(value)
            kwargs['vdims'] = [vdim]
            if not vdim:
                raise ValueError('Value dimension not found '
                                 'in dataset: {}'.format(vdim))
            ddims = dataset.dimensions()

        chloropleth = NdOverlay(kdims=kdims)
        for i, rec in enumerate(records):
            if dataset:
                selection = {dim: rec.attributes.get(attr, None)
                             for attr, dim in on.items()}
                row = dataset.select(**selection)
                if not len(row):
                    continue
                if value:
                    value = row[vdim.name][0]
                    kwargs['level'] = value
            if index:
                key = []
                for kdim in kdims:
                    if kdim in ddims:
                        k = row[kdim.name][0]
                    elif kdim.name in rec.attributes:
                        k = rec.attributes[kdim.name]
                    else:
                        raise ValueError('%s could not be found' % kdim)
                    key.append(k)
                key = tuple(key)
            else:
                key = i
            chloropleth[key] = Shape(rec.geometry, **kwargs)
        return chloropleth

    def dimension_values(self, dimension):
        """
        Shapes do not support convert to array values.
        """
        dim = self.get_dimension(dimension)
        if dim in self.vdims:
            return [self.level]
        else:
            return []

    def range(self, dimension):
        dim = self.get_dimension(dimension)
        if dim.range != (None, None):
            return dim.range

        idx = self.get_dimension_index(dimension)
        if idx == 2:
            return self.level, self.level
        if idx in [0, 1]:
            l, b, r, t = self.data.bounds
            if idx == 0:
                return l, r
            elif idx == 1:
                return b, t
        else:
            return (np.NaN, np.NaN)

    def geom(self):
        """
        Returns Shape as a shapely geometry
        """
        return self.data

    def __len__(self):
        return len(self.data)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Licensed under the BSD license.  See full license in LICENSE file.
# http://www.lightshowpi.com/
#
# Author: Todd Giles (todd@lightshowpi.com)
#
# TODO(todd): Refactor the configuration manager into a configuration manager class (to remove
#                  the extensive use of globals currently used).
# TODO(todd): Add a main and allow running configuration manager alone to view the current
#                  configuration, and potentially edit it.
"""Configuration management for the lightshow.

Configuration files are all located in the <homedir>/config directory. This file contains tools to
manage these configuration files.
"""
# The home directory and configuration directory for the application.

import ConfigParser
import ast
import datetime
import fcntl
import logging
import os
import os.path
import sys
import warnings
import json

HOME_DIR = os.getenv("SYNCHRONIZED_LIGHTS_HOME")
if not HOME_DIR:
    print("Need to setup SYNCHRONIZED_LIGHTS_HOME environment variable, "
          "see readme")
    sys.exit()
CONFIG_DIR = HOME_DIR + '/config'
LOG_DIR = HOME_DIR + '/logs'

# Load configuration file, loads defaults from config directory, and then
# overrides from the same directory cfg file, then from /home/pi/.lights.cfg
# and then from ~/.lights.cfg (which will be the root's home).
CONFIG = ConfigParser.RawConfigParser(allow_no_value=True)
CONFIG.readfp(open(CONFIG_DIR + '/defaults.cfg'))
CONFIG.read([CONFIG_DIR + '/overrides.cfg', '/home/pi/.lights.cfg',
             os.path.expanduser('~/.lights.cfg')])

def _as_list(list_str, delimiter=','):
    """Return a list of items from a delimited string (after stripping whitespace)."""
    return [str.strip(item) for item in list_str.split(delimiter)]

# Retrieve hardware configuration
_HARDWARE_CONFIG = {}
def hardware():
    """
    Retrieves the hardware configuration

    loading and parsing it from a file if necessary.
    """

    global _HARDWARE_CONFIG
    if len(_HARDWARE_CONFIG) == 0:
        _HARDWARE_CONFIG = dict(CONFIG.items('hardware'))

        # Devices
        devices = dict()

        try:
            devices = json.loads(_HARDWARE_CONFIG['devices'])
        except Exception as error:
            logging.error("devices not defined or not in JSON format." \
                + str(error))

        _HARDWARE_CONFIG["devices"] = devices

    return _HARDWARE_CONFIG

# Retrieve light show configuration
_LIGHTSHOW_CONFIG = {}
def lightshow():
    """
    Retrieve the lightshow configuration

    loading and parsing it from a file as necessary.
    """
    global _LIGHTSHOW_CONFIG
    if len(_LIGHTSHOW_CONFIG) == 0:
        _LIGHTSHOW_CONFIG = dict(CONFIG.items('lightshow'))

        _LIGHTSHOW_CONFIG['audio_in_channels'] = \
            CONFIG.getint('lightshow', 'audio_in_channels')
        _LIGHTSHOW_CONFIG['audio_in_sample_rate'] = \
            CONFIG.getint('lightshow', 'audio_in_sample_rate')

        # setup up preshow
        preshow = None
        if (CONFIG.get('lightshow', 'preshow_configuration')
            and not CONFIG.get('lightshow', 'preshow_script')):
            try:
                preshow = \
                    json.loads(_LIGHTSHOW_CONFIG['preshow_configuration'])
            except (ValueError, TypeError) as error:
                logging.error("Preshow_configuration not defined or not in "
                              "JSON format." + str(error))
        else:
            if os.path.isfile(_LIGHTSHOW_CONFIG['preshow_script']):
                preshow = _LIGHTSHOW_CONFIG['preshow_script']

        _LIGHTSHOW_CONFIG['preshow'] = preshow

        # setup postshow
        postshow = None
        if (CONFIG.get('lightshow', 'postshow_configuration')
            and not CONFIG.get('lightshow', 'postshow_script')):
            try:
                postshow = \
                    json.loads(_LIGHTSHOW_CONFIG['postshow_configuration'])
            except (ValueError, TypeError) as error:
                logging.error("Postshow_configuration not "
                              "defined or not in JSON format." + str(error))
        else:
            if os.path.isfile(_LIGHTSHOW_CONFIG['postshow_script']):
                postshow = _LIGHTSHOW_CONFIG['postshow_script']
        _LIGHTSHOW_CONFIG['postshow'] = postshow

    return _LIGHTSHOW_CONFIG

_SMS_CONFIG = {}
_WHO_CAN = {}
def sms():
    """Retrieves and validates sms configuration"""
    global _SMS_CONFIG, _WHO_CAN
    if len(_SMS_CONFIG) == 0:
        _SMS_CONFIG = dict(CONFIG.items('sms'))
        _WHO_CAN = dict()
        _WHO_CAN['all'] = set()

        # Commands
        _SMS_CONFIG['commands'] = _as_list(_SMS_CONFIG['commands'])
        for cmd in _SMS_CONFIG['commands']:
            try:
                _SMS_CONFIG[cmd + '_aliases'] = _as_list(_SMS_CONFIG[cmd \
                    + '_aliases'])
            except:
                _SMS_CONFIG[cmd + '_aliases'] = []
            _WHO_CAN[cmd] = set()

        # Groups / Permissions
        _SMS_CONFIG['groups'] = _as_list(_SMS_CONFIG['groups'])
        _SMS_CONFIG['throttled_groups'] = dict()
        for group in _SMS_CONFIG['groups']:
            try:
                _SMS_CONFIG[group + '_users'] = _as_list(_SMS_CONFIG[group \
                    + '_users'])
            except:
                _SMS_CONFIG[group + '_users'] = []
            try:
                _SMS_CONFIG[group + '_commands'] = _as_list(_SMS_CONFIG[group \
                    + '_commands'])
            except:
                _SMS_CONFIG[group + '_commands'] = []
            for cmd in _SMS_CONFIG[group + '_commands']:
                for user in _SMS_CONFIG[group + '_users']:
                    _WHO_CAN[cmd].add(user)

            # Throttle
            try:
                throttled_group_definitions = _as_list(_SMS_CONFIG[group \
                    + '_throttle'])
                throttled_group = dict()
                for definition in throttled_group_definitions:
                    definition = definition.split(':')
                    if len(definition) != 2:
                        warnings.warn(group + "_throttle definitions should "
                                      "be in the form " \
                                          + "[command]:<limit> - " \
                                              + definition.join(':'))
                        continue
                    throttle_command = definition[0]
                    throttle_limit = int(definition[1])
                    throttled_group[throttle_command] = throttle_limit
                _SMS_CONFIG['throttled_groups'][group] = throttled_group
            except:
                warnings.warn("Throttle definition either does not exist or "
                              "is configured "
                              "incorrectly for group: " + group)

        # Blacklist
        _SMS_CONFIG['blacklist'] = _as_list(_SMS_CONFIG['blacklist'])

    return _SMS_CONFIG

_SONG_LIST = []
def songs():
    """Retrieve the song list"""
    if len(_SONG_LIST) == 0:
        pass  # TODO(todd): Load playlist if not already loaded, also refactor
              #             the code that loads the playlist in check_sms and
              #             synchronzied_lights such that we don't duplicate it
              #             there.
    return _SONG_LIST

# Sets the list of songs (if loaded elsewhere, as is done by check_sms)
def set_songs(song_list):
    """
    Sets the list of songs

    if loaded elsewhere, as is done by check_sms for example
    """
    global _SONG_LIST
    _SONG_LIST = song_list

##############################
# Application State Utilities
##############################

# Load application state configuration file from CONFIG directory.
STATE = ConfigParser.RawConfigParser()
STATE_SECTION = 'do_not_modify'
STATE_FILENAME = CONFIG_DIR + '/state.cfg'

# Ensure state file has been created
if not os.path.isfile(STATE_FILENAME):
    open(STATE_FILENAME, 'a').close()

def load_state():
    """Force the state to be reloaded form disk."""
    with open(STATE_FILENAME) as state_fp:
        fcntl.lockf(state_fp, fcntl.LOCK_SH)
        STATE.readfp(state_fp, STATE_FILENAME)
        fcntl.lockf(state_fp, fcntl.LOCK_UN)
load_state()  # Do an initial load

def get_state(name, default=''):
    """
    Get application state

    Return the value of a specific application state variable, or the specified
    default if not able to load it from the state file
    """
    try:
        return STATE.get(STATE_SECTION, name)
    except:
        return default

def update_state(name, value):
    """Update the application state (name / value pair)"""
    value = str(value)
    logging.info('Updating application state {%s: %s}', name, value)
    try:
        STATE.add_section(STATE_SECTION)
    except ConfigParser.DuplicateSectionError:
        pass  # Ok, it's already there
    STATE.set(STATE_SECTION, name, value)
    with open(STATE_FILENAME, 'wb') as state_fp:
        fcntl.lockf(state_fp, fcntl.LOCK_EX)
        STATE.write(state_fp)
        fcntl.lockf(state_fp, fcntl.LOCK_UN)

def has_permission(user, cmd):
    """Returns True iff a user has permissio to execute the given command"""
    blacklisted = user in sms()['blacklist']
    return not blacklisted and (user in _WHO_CAN['all']
                                or 'all' in _WHO_CAN[cmd]
                                or user in _WHO_CAN[cmd])

def is_throttle_exceeded(cmd, user):
    """Returns True if the throttle has been exeeded and False otherwise"""
    # Load throttle STATE
    load_state()
    throttlestate = ast.literal_eval(get_state('throttle', '{}'))
    processcommandflag = -1

    # Analyze throttle timing
    currenttimestamp = datetime.datetime.now()
    throttletimelimit = _SMS_CONFIG['throttle_time_limit_seconds']
    throttlestarttime = datetime.datetime.strptime(
        throttlestate['throttle_timestamp_start'], '%Y-%m-%d %H:%M:%S.%f') \
        if "throttle_timestamp_start" in throttlestate else currenttimestamp
    throttlestoptime = \
        throttlestarttime + datetime.timedelta(seconds=int(throttletimelimit))

    # Compare times and see if we need to reset the throttle STATE
    if (currenttimestamp == throttlestarttime) or \
        (throttlestoptime < currenttimestamp):
        # There is no time recorded or the time has
        # expired reset the throttle STATE
        throttlestate = {}
        throttlestate['throttle_timestamp_start'] = str(currenttimestamp)
        update_state('throttle', throttlestate)

    # ANALYZE THE THROTTLE COMMANDS AND LIMITS
    allthrottlelimit = -1
    cmdthrottlelimit = -1

    # Check to see what group belongs to starting with the first group declared
    throttled_group = None
    for group in _SMS_CONFIG['groups']:
        userlist = _SMS_CONFIG[group + "_users"]
        if user in userlist:
            # The user belongs to this group, check if there
            # are any throttle definitions
            if group in _SMS_CONFIG['throttled_groups']:
                # The group has throttle commands defined,
                # now check if the command is defined
                throttledcommands = _SMS_CONFIG['throttled_groups'][group]

                # Check if all command exists
                if "all" in throttledcommands:
                    allthrottlelimit = int(throttledcommands['all'])

                # Check if the command passed is defined
                if cmd in throttledcommands:
                    cmdthrottlelimit = int(throttledcommands[cmd])

                # A throttle definition was found,
                # we no longer need to check anymore groups
                if allthrottlelimit != -1 or cmdthrottlelimit != -1:
                    throttled_group = group
                    break

    # Process the throttle settings that were found for the throttled group
    if not throttled_group:
        # No throttle limits were found for any group
        return False
    else:
        # Throttle limits were found, check them against throttle STATE limits
        groupthrottlestate = \
          throttlestate[throttled_group] if throttled_group in throttlestate else {}
        groupthrottlecmdlimit = \
          int(groupthrottlestate[cmd]) if cmd in groupthrottlestate else 0

    # Check to see if we need to apply "all"
    if allthrottlelimit != -1:
        groupthrottlealllimit = \
          int(groupthrottlestate['all']) if 'all' in groupthrottlestate else 0

        # Check if "all" throttle limit has been reached
        if groupthrottlealllimit < allthrottlelimit:
            # Not Reached, bump throttle and record
            groupthrottlealllimit = groupthrottlealllimit + 1
            groupthrottlestate['all'] = groupthrottlealllimit
            throttlestate[throttled_group] = groupthrottlestate
            processcommandflag = False
        else:
            # "all" throttle has been reached we
            # dont want to process anything else
            return True

    # Check to see if we need to apply "cmd"
    if cmdthrottlelimit != -1:
        if groupthrottlecmdlimit < cmdthrottlelimit:
            # Not reached, bump throttle
            groupthrottlecmdlimit = groupthrottlecmdlimit + 1
            groupthrottlestate[cmd] = groupthrottlecmdlimit
            throttlestate[throttled_group] = groupthrottlestate
            processcommandflag = False

    # Record the updatedthrottle STATE and return
    update_state('throttle', throttlestate)

    return processcommandflag
<commit_msg>Fixed docstrings, and made a few variable names more pythonic.  also added white space for readability<commit_after>#!/usr/bin/env python
#
# Licensed under the BSD license.  See full license in LICENSE file.
# http://www.lightshowpi.com/
#
# Author: Todd Giles (todd@lightshowpi.com)
#
# TODO(todd): Refactor the configuration manager into a configuration manager class (to remove
# the extensive use of globals currently used).
# TODO(todd): Add a main and allow running configuration manager alone to view the current
#                  configuration, and potentially edit it.

"""Configuration management for the lightshow.

Configuration files are all located in the <homedir>/config directory. This file contains tools to
manage these configuration files.
"""

import ConfigParser
import ast
import datetime
import fcntl
import logging
import os
import os.path
import sys
import warnings
import json

# The home directory and configuration directory for the application.
HOME_DIR = os.getenv("SYNCHRONIZED_LIGHTS_HOME")
if not HOME_DIR:
    print("Need to setup SYNCHRONIZED_LIGHTS_HOME environment variable, "
          "see readme")
    sys.exit()
CONFIG_DIR = HOME_DIR + '/config'
LOG_DIR = HOME_DIR + '/logs'

# Load configuration file, loads defaults from config directory, and then
# overrides from the same directory cfg file, then from /home/pi/.lights.cfg
# and then from ~/.lights.cfg (which will be the root's home).
CONFIG = ConfigParser.RawConfigParser(allow_no_value=True)
CONFIG.readfp(open(CONFIG_DIR + '/defaults.cfg'))
CONFIG.read([CONFIG_DIR + '/overrides.cfg', '/home/pi/.lights.cfg',
             os.path.expanduser('~/.lights.cfg')])

def _as_list(list_str, delimiter=','):
    """Return a list of items from a delimited string (after stripping whitespace).

    :param list_str: string to turn into a list
    :type list_str: str

    :param delimiter: split the string on this
    :type delimiter: str

    :return: string converted to a list
    :rtype: list
    """
    return [str.strip(item) for item in list_str.split(delimiter)]

# Retrieve hardware configuration
_HARDWARE_CONFIG = dict()

def hardware():
    """Retrieves the hardware configuration

    loading and parsing it from a file if necessary.

    :return: _HARDWARE_CONFIG
    :rtype: dict
    """

    global _HARDWARE_CONFIG
    if len(_HARDWARE_CONFIG) == 0:
        _HARDWARE_CONFIG = dict(CONFIG.items('hardware'))

        # Devices
        devices = dict()

        try:
            devices = json.loads(_HARDWARE_CONFIG['devices'])
        except Exception as error:
            logging.error("devices not defined or not in JSON format."
                          + str(error))

        _HARDWARE_CONFIG["devices"] = devices

    return _HARDWARE_CONFIG

# Retrieve light show configuration
_LIGHTSHOW_CONFIG = dict()

def lightshow():
    """Retrieve the lightshow configuration

    loading and parsing it from a file as necessary.

    :return: _LIGHTSHOW_CONFIG
    :rtype: dict
   """
    global _LIGHTSHOW_CONFIG
    if len(_LIGHTSHOW_CONFIG) == 0:
        _LIGHTSHOW_CONFIG = dict(CONFIG.items('lightshow'))

        _LIGHTSHOW_CONFIG['audio_in_channels'] = \
            CONFIG.getint('lightshow', 'audio_in_channels')
        _LIGHTSHOW_CONFIG['audio_in_sample_rate'] = \
            CONFIG.getint('lightshow', 'audio_in_sample_rate')

        # setup up preshow
        preshow = None
        if (CONFIG.get('lightshow',
                       'preshow_configuration') and
                not CONFIG.get('lightshow', 'preshow_script')):

            try:
                preshow = \
                    json.loads(_LIGHTSHOW_CONFIG['preshow_configuration'])
            except (ValueError, TypeError) as error:
                logging.error("Preshow_configuration not defined or not in "
                              "JSON format." + str(error))
        else:
            if os.path.isfile(_LIGHTSHOW_CONFIG['preshow_script']):
                preshow = _LIGHTSHOW_CONFIG['preshow_script']

        _LIGHTSHOW_CONFIG['preshow'] = preshow

        # setup postshow
        postshow = None
        if (CONFIG.get('lightshow',
                       'postshow_configuration') and
                not CONFIG.get('lightshow', 'postshow_script')):
            try:
                postshow = \
                    json.loads(_LIGHTSHOW_CONFIG['postshow_configuration'])
            except (ValueError, TypeError) as error:
                logging.error("Postshow_configuration not "
                              "defined or not in JSON format." + str(error))
        else:
            if os.path.isfile(_LIGHTSHOW_CONFIG['postshow_script']):
                postshow = _LIGHTSHOW_CONFIG['postshow_script']
        _LIGHTSHOW_CONFIG['postshow'] = postshow

    return _LIGHTSHOW_CONFIG

_SMS_CONFIG = dict()
_WHO_CAN = dict()

def sms():
    """Retrieves and validates sms configuration

    :return: _SMS_CONFIG
    :rtype: dict
    """
    global _SMS_CONFIG, _WHO_CAN
    if len(_SMS_CONFIG) == 0:
        _SMS_CONFIG = dict(CONFIG.items('sms'))
        _WHO_CAN = dict()
        _WHO_CAN['all'] = set()

        # Commands
        _SMS_CONFIG['commands'] = _as_list(_SMS_CONFIG['commands'])
        # KeyError is the only parsing error I can get on this one
        # but I did find other issues, duplicate entries, empty entries
        # conflicting entries with other command aliases
        for cmd in _SMS_CONFIG['commands']:
            try:
                _SMS_CONFIG[cmd + '_aliases'] = _as_list(_SMS_CONFIG[cmd + '_aliases'])
            except KeyError:
                _SMS_CONFIG[cmd + '_aliases'] = []

            _WHO_CAN[cmd] = set()

        # Groups / Permissions
        _SMS_CONFIG['groups'] = _as_list(_SMS_CONFIG['groups'])
        _SMS_CONFIG['throttled_groups'] = dict()
        for group in _SMS_CONFIG['groups']:
            try:
                _SMS_CONFIG[group + '_users'] = _as_list(_SMS_CONFIG[group
                                                                     + '_users'])
            except KeyError:
                _SMS_CONFIG[group + '_users'] = []

            try:
                _SMS_CONFIG[group + '_commands'] = _as_list(_SMS_CONFIG[group
                                                                        + '_commands'])
            except KeyError:
                _SMS_CONFIG[group + '_commands'] = []

            for cmd in _SMS_CONFIG[group + '_commands']:
                for user in _SMS_CONFIG[group + '_users']:
                    _WHO_CAN[cmd].add(user)

            # Throttle
            try:
                throttled_group_definitions = _as_list(_SMS_CONFIG[group
                                                                   + '_throttle'])
                throttled_group = dict()
                for definition in throttled_group_definitions:
                    definition = definition.split(':')
                    if len(definition) != 2:
                        warnings.warn(group + "_throttle definitions should "
                                              "be in the form "
                                      + "[command]:<limit> - "
                                      + ":".join(definition))
                        continue
                    throttle_command = definition[0]
                    throttle_limit = int(definition[1])
                    throttled_group[throttle_command] = throttle_limit
                _SMS_CONFIG['throttled_groups'][group] = throttled_group
            except KeyError:
                warnings.warn("Throttle definition either does not exist or "
                              "is configured incorrectly for group: " + group)

        # Blacklist
        _SMS_CONFIG['blacklist'] = _as_list(_SMS_CONFIG['blacklist'])

    return _SMS_CONFIG

_SONG_LIST = []

def songs():
    """Retrieve the song list

    :return: a list of songs
    :rtype: list
    """
    if len(_SONG_LIST) == 0:
        pass  # TODO(todd): Load playlist if not already loaded, also refactor
        #             the code that loads the playlist in check_sms and
        #             synchronzied_lights such that we don't duplicate it
        #             there.
        # NOTE: check_sms imports it's own copy of configuration_manager
        #       so does the hardware_controller and synchronized_lights
        #       if the configuration_manager were a class and only loaded
        #       as a single instance, in say synchronized loaded and that
        #       ran check_sms and passed it a reference to it's configuration_manager
        #       then it would work.  There are still other issues like
        #       command.py needing a copy of the configs too, it does not
        #       affect this issue but does use limited resources, But we
        #       could combine check_sms and commands into a single file
        #       They would live happily together and share global resources
        #       The way it is done now is the most appropriate,  The only other
        #       way I can think of is with pipes and streams and that is way
        #       overkill. mmap maybe, but same thing just shifted from a file
        #       to memory, and a lot of unnecessary code.
        #       Just thinking out loud.  Maybe someone will see this and have
        #       a really good idea.
        #
    return _SONG_LIST

def set_songs(song_list):
    """Sets the list of songs

    if loaded elsewhere, as is done by check_sms for example

    :param song_list: a list of songs
    :type song_list: list
    """
    global _SONG_LIST
    _SONG_LIST = song_list

##############################
# Application State Utilities
##############################

# Load application state configuration file from CONFIG directory.
STATE = ConfigParser.RawConfigParser()
STATE_SECTION = 'do_not_modify'
STATE_FILENAME = CONFIG_DIR + '/state.cfg'

# Ensure state file has been created
if not os.path.isfile(STATE_FILENAME):
    open(STATE_FILENAME, 'a').close()

def load_state():
    """Force the state to be reloaded form disk."""
    with open(STATE_FILENAME) as state_fp:
        fcntl.lockf(state_fp, fcntl.LOCK_SH)
        STATE.readfp(state_fp, STATE_FILENAME)
        fcntl.lockf(state_fp, fcntl.LOCK_UN)

load_state()  # Do an initial load

def get_state(name, default=''):
    """
    Get application state

    Return the value of a specific application state variable, or the specified
    default if not able to load it from the state file

    :param name: option to load from state file
    :type name: str

    :param default: return if not able to load option from state file
    :type default: str

    :return: the current state
    :rtype: str
    """
    try:
        return STATE.get(STATE_SECTION, name)
    except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
        return default

def update_state(name, value):
    """Update the application state (name / value pair)

    :param name: option name to update
    :type name: str

    :param value: value to update option name to
    :type value: str
    """
    value = str(value)
    logging.info('Updating application state {%s: %s}', name, value)

    try:
        STATE.add_section(STATE_SECTION)
    except ConfigParser.DuplicateSectionError:
        pass  # Ok, it's already there

    STATE.set(STATE_SECTION, name, value)

    with open(STATE_FILENAME, 'wb') as state_fp:
        fcntl.lockf(state_fp, fcntl.LOCK_EX)
        STATE.write(state_fp)
        fcntl.lockf(state_fp, fcntl.LOCK_UN)

def has_permission(user, cmd):
    """Returns True if a user has permission to execute the given command
    :param user: the user trying to execute the command
    :type user: str

    :param cmd: the command at question
    :type cmd: str

    :return: user has permission
    :rtype: bool
    """
    blacklisted = user in sms()['blacklist']

    return not blacklisted and (user in _WHO_CAN['all']
                                or 'all' in _WHO_CAN[cmd]
                                or user in _WHO_CAN[cmd])

def is_throttle_exceeded(cmd, user):
    """Returns True if the throttle has been exceeded and False otherwise

    :param cmd: the command at question
    :type cmd: str

    :param user: the user trying to execute the command
    :type user: str

    :return: has throttle been exceeded
    :rtype: bool
    """
    # Load throttle STATE
    load_state()
    throttle_state = ast.literal_eval(get_state('throttle', '{}'))
    process_command_flag = -1

    # Analyze throttle timing
    current_timestamp = datetime.datetime.now()
    throttle_timelimit = _SMS_CONFIG['throttle_time_limit_seconds']
    throttle_starttime = datetime.datetime.strptime(
        throttle_state['throttle_timestamp_start'], '%Y-%m-%d %H:%M:%S.%f') \
        if "throttle_timestamp_start" in throttle_state else current_timestamp
    throttle_stoptime = \
        throttle_starttime + datetime.timedelta(seconds=int(throttle_timelimit))

    # Compare times and see if we need to reset the throttle STATE
    if (current_timestamp == throttle_starttime) or \
            (throttle_stoptime < current_timestamp):
        # There is no time recorded or the time has
        # expired reset the throttle STATE
        throttle_state = dict()
        throttle_state['throttle_timestamp_start'] = str(current_timestamp)
        update_state('throttle', throttle_state)

    # ANALYZE THE THROTTLE COMMANDS AND LIMITS
    all_throttle_limit = -1
    cmd_throttle_limit = -1

    # Check to see what group belongs to starting with the first group declared
    throttled_group = None
    for group in _SMS_CONFIG['groups']:
        userlist = _SMS_CONFIG[group + "_users"]

        if user in userlist:
            # The user belongs to this group, check if there
            # are any throttle definitions
            if group in _SMS_CONFIG['throttled_groups']:
                # The group has throttle commands defined,
                # now check if the command is defined
                throttled_commands = _SMS_CONFIG['throttled_groups'][group]

                # Check if all command exists
                if "all" in throttled_commands:
                    all_throttle_limit = int(throttled_commands['all'])

                # Check if the command passed is defined
                if cmd in throttled_commands:
                    cmd_throttle_limit = int(throttled_commands[cmd])

                # A throttle definition was found,
                # we no longer need to check anymore groups
                if all_throttle_limit != -1 or cmd_throttle_limit != -1:
                    throttled_group = group
                    break

    # Process the throttle settings that were found for the throttled group
    if not throttled_group:
        # No throttle limits were found for any group
        return False
    else:
        # Throttle limits were found, check them against throttle STATE limits
        if throttled_group in throttle_state:
            group_throttle_state = throttle_state[throttled_group]
        else:
            group_throttle_state = dict()

        if cmd in group_throttle_state:
            group_throttle_cmd_limit = int(group_throttle_state[cmd])
        else:
            group_throttle_cmd_limit = 0

    # Check to see if we need to apply "all"
    if all_throttle_limit != -1:
        groupthrottlealllimit = \
            int(group_throttle_state['all']) if 'all' in group_throttle_state else 0

        # Check if "all" throttle limit has been reached
        if groupthrottlealllimit < all_throttle_limit:
            # Not Reached, bump throttle and record
            groupthrottlealllimit += 1
            group_throttle_state['all'] = groupthrottlealllimit
            throttle_state[throttled_group] = group_throttle_state
            process_command_flag = False
        else:
            # "all" throttle has been reached we
            # dont want to process anything else
            return True

    # Check to see if we need to apply "cmd"
    if cmd_throttle_limit != -1:
        if group_throttle_cmd_limit < cmd_throttle_limit:
            # Not reached, bump throttle
            group_throttle_cmd_limit += 1
            group_throttle_state[cmd] = group_throttle_cmd_limit
            throttle_state[throttled_group] = group_throttle_state
            process_command_flag = False

    # Record the updatedthrottle STATE and return
    update_state('throttle', throttle_state)

    return process_command_flag
<|endoftext|>


<commit_before>#!/usr/bin/python

import time
import datetime

from Adafruit_LED_Backpack import SevenSegment

# ===========================================================================
# Clock Example
# ===========================================================================
segment = SevenSegment.SevenSegment(address=0x70)

# Initialize the display. Must be called once before using the display.
segment.begin()

print "Press CTRL+Z to exit"

# Continually update the time on a 4 char, 7-segment display
while(True):
  now = datetime.datetime.now()
  hour = now.hour
  minute = now.minute
  second = now.second

  if hour >= 18 or minute > 32 or hour < 7:
    segment.set_brightness(0)
  else:
    segment.set_brightness(5)

  if hour == 24:
    hour = 12
  else:
    hour = hour % 12

  A = int(hour / 10)

  if A == 0:
    A = ' '

  segment.clear()
  # Set hours
  segment.set_digit(0, A)     # Tens
  segment.set_digit(1, hour % 10)          # Ones
  # Set minutes
  segment.set_digit(2, int(minute / 10))   # Tens
  segment.set_digit(3, minute % 10)        # Ones
  # Toggle colon
  segment.set_colon(second % 2)              # Toggle colon at 1Hz

  # Write the display buffer to the hardware.  This must be called to
  # update the actual display LEDs.
  segment.write_display()

  # Wait a quarter second (less than 1 second to prevent colon blinking getting$
  time.sleep(0.25)
<commit_msg>Update clock.py<commit_after>#!/usr/bin/python

import time
import datetime

from Adafruit_LED_Backpack import SevenSegment

# ===========================================================================
# Clock Example
# ===========================================================================
segment = SevenSegment.SevenSegment(address=0x70)

# Initialize the display. Must be called once before using the display.
segment.begin()

print "Press CTRL+Z to exit"

# Continually update the time on a 4 char, 7-segment display
while(True):
  now = datetime.datetime.now()
  hour = now.hour
  minute = now.minute
  second = now.second

  if hour >= 18 or hour < 7:
    segment.set_brightness(0)
  else:
    segment.set_brightness(5)

  if hour == 24:
    hour = 12
  else:
    hour = hour % 12

  A = int(hour / 10)

  if A == 0:
    A = ' '

  segment.clear()
  # Set hours
  segment.set_digit(0, A)     # Tens
  segment.set_digit(1, hour % 10)          # Ones
  # Set minutes
  segment.set_digit(2, int(minute / 10))   # Tens
  segment.set_digit(3, minute % 10)        # Ones
  # Toggle colon
  segment.set_colon(second % 2)              # Toggle colon at 1Hz

  # Write the display buffer to the hardware.  This must be called to
  # update the actual display LEDs.
  segment.write_display()

  # Wait a quarter second (less than 1 second to prevent colon blinking getting$
  time.sleep(0.25)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys,ast
import openvgdb

string='{"game":{"rom":{"serial":"romSerial","image":"romFileName"},"name":"romExtensionlessFileName","description":"romExtensionlessFileName"}}'
par=ast.literal_eval(string)
games=[]
game=[]
val='game'

def parse(a,level):
    ret=[]
    last=[]
    check=['crc','md5','sha1','serial']
    for entry in a:
        if type(a[entry])==type({}) and entry=="game":
            last.append("\t"*level+str(entry)+" ( ")
            last.append(parse(a[entry],level+1))
            last.append("\t"*level+")\n")
        elif type(a[entry])==type({}) and entry=="rom":
            last.append("\t"*level+str(entry)+" ( ")
            last.append(parse(a[entry],level)[:-1])
#            print parse(a[entry],level+1)[:-1]
            last.append("\t"*level+")\n")
        else:
            value=game[a[entry]]
            if type(value)==type(42) or entry in check:
                ret.append("\t"*level+str(entry)+" "+str(value)+" ")
            else:
                try:
                    ret.append("\t"*level+str(entry)+' "'+str(value).encode('utf-8')+"\" ")
                except:
                    print 'Couldn\'t parse'
                    print value
        ret.append("\n")
    return "".join(ret)+"".join(last)

if len(sys.argv)%2!=0:
    print 'Something is wrong.Check your parameters again (the count is off)'
    quit()

O=openvgdb.vgdb('openvgdb.sqlite')
print 'Getting the game list.This can take a while...'
header="clrmamepro ( \n\
name \""+O.get_systems()[int(sys.argv[1])]+"\"\n\
)\n"
games=O.get_console(sys.argv[1])
print 'Done.Starting to convert'
#print game
#print parse(par,0)
#f.write(parse(par,0))
#quit()

for i in range(0,(len(sys.argv)-2)/2):
    string=sys.argv[2+i*2]
    par=ast.literal_eval(string)
    f=open(sys.argv[3+i*2],"w")
    print 'writing '+sys.argv[3+i*2]
    f.write(header)
    for game in games:
        f.write(parse(par,0))
    f.close()
print 'Done everything went according to plan'
<commit_msg>replaced tab with space for rom<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys,ast
import openvgdb

string='{"game":{"rom":{"serial":"romSerial","image":"romFileName"},"name":"romExtensionlessFileName","description":"romExtensionlessFileName"}}'
par=ast.literal_eval(string)
games=[]
game=[]
val='game'

def parse(a,level):
    ret=[]
    last=[]
    check=['crc','md5','sha1','serial']
    for entry in a:
        if type(a[entry])==type({}) and entry=="game":
            last.append("\t"*level+str(entry)+" ( ")
            last.append(parse(a[entry],level+1))
            last.append("\t"*level+")\n")
        elif type(a[entry])==type({}) and entry=="rom":
            last.append(" "*level+str(entry)+" ( ")
            last.append(parse(a[entry],level)[:-1])
#            print parse(a[entry],level+1)[:-1]
            last.append(" "*level+")\n")
        else:
            value=game[a[entry]]
            if type(value)==type(42) or entry in check:
                ret.append("\t"*level+str(entry)+" "+str(value)+" ")
            else:
                try:
                    ret.append("\t"*level+str(entry)+' "'+str(value).encode('utf-8')+"\" ")
                except:
                    print 'Couldn\'t parse'
                    print value
        ret.append("\n")
    return "".join(ret)+"".join(last)

if len(sys.argv)%2!=0:
    print 'Something is wrong.Check your parameters again (the count is off)'
    quit()

O=openvgdb.vgdb('openvgdb.sqlite')
print 'Getting the game list.This can take a while...'
header="clrmamepro ( \n\
name \""+O.get_systems()[int(sys.argv[1])]+"\"\n\
)\n"
games=O.get_console(sys.argv[1])
print 'Done.Starting to convert'
#print game
#print parse(par,0)
#f.write(parse(par,0))
#quit()

for i in range(0,(len(sys.argv)-2)/2):
    string=sys.argv[2+i*2]
    par=ast.literal_eval(string)
    f=open(sys.argv[3+i*2],"w")
    print 'writing '+sys.argv[3+i*2]
    f.write(header)
    for game in games:
        f.write(parse(par,0))
    f.close()
print 'Done everything went according to plan'
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys

if __name__ == "__main__":
    os.execv(sys.executable, [sys.executable] + ["-m", "topaz"] + sys.argv[1:])
<commit_msg>fixed code that made no sense<commit_after>#!/usr/bin/env python

import os
import sys

if __name__ == "__main__":
    os.execv(sys.executable, [sys.executable, "-m", "topaz"] + sys.argv[1:])
<|endoftext|>