<commit_before>"""
The following software has been copied from the following website:

https://developers.google.com/sheets/api/quickstart/python

I have made modifications to some parts of it
"""
from __future__ import print_function
import httplib2
import os
import create_log
import dbexec

from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage

try:
    import argparse
    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
    flags = None

# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/sheets.googleapis.com-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/spreadsheets.readonly'
CLIENT_SECRET_FILE = 'client_secret.json'
APPLICATION_NAME = 'Google Sheets API Python Quickstart'

"""
 Both the parrain and filleul objects with be filled using
 dictionaries that associate their column letters with
 their actual parameters
 for example B represents the lastName for the filleul sheet
"""

sheetToParam_f = {
    'lastName': 'B',
    'firstName':  'C',
    'age': 'D',
    'sex': 'E',
    'email': 'F',
    'university': 'G',
    'subject': 'H',
    'nationality': 'I',
    'dateArrival': 'N',
    'countryOrigin': 'P',
    'hobbies': 'Q'
}

sheetToParam_p = {
    'lastName': 'B',
    'firstName':  'C',
    'age': 'D',
    'sex': 'E',
    'email': 'F',
    'university': 'G',
    'nationality': 'H',
    'dateAvailable': 'K',
    'languages': 'I',
    'subject': 'M',
    'hobbies': 'O',
}

class Parrain:
    def __init__(self, firstName, lastName, age, sex, email, dateAvailable,
                 university, nationality, subject, hobbies, languages):
        self.firstName = firstName
        self.lastName = lastName
        self.age = age
        self.sex = sex
        self.email = email
        self.dateAvailable =dateAvailable
        self.university = university
        self.nationality = nationality
        self.subject = subject
        self.hobbies = hobbies
        self.languages = languages

    def __repr__(self):
        return "<Parrain FN: %s | LN: %s | AGE: %s" \
            "\nSEX: %s| EM: %s| DA: %s\nUNI: %s |" \
            "NA: %s | SU: %s| HO: %s\nLA: %s" % (self.firstName, self.lastName, self.age, self.sex,
                                                 self.email, self.dateAvailable,
                                                 self.university, self. nationality,
                                                 self.subject, self.hobbies,
                                                 self.languages)

# filleul object contains all useful info about a filleul in a variable

class Filleul:
    def __init__(self, firstName, lastName, age, sex,
                 email, dateArrival, nationality, countryOrigin,
                 university, subject, hobbies):
        self.firstName = firstName
        self.lastName = lastName
        self.age = age
        self.sex = sex
        self.email = email
        self.dateArrival = dateArrival
        self.nationality = nationality
        self.countryOrigin = countryOrigin
        self.university = university
        self.subject = subject
        self.hobbies = hobbies
    def __repr__(self):
        return "<Filleul FN: %s | LN: %s | AGE: %s" \
    "\nSEX: %s| EM: %s| DA: %s\nNA: %s|CO: %s| UNI: %s" \
    "\nSU: %s| HO: %s" % (self.firstName, self.lastName, self.age, self.sex,
                      self.email, self.dateArrival, self.nationality,
                      self.countryOrigin, self.university, self.subject,
                      self.hobbies)

def get_credentials():
    # verification of API key
    home_dir = os.path.expanduser('~')
    credential_dir = os.path.join(home_dir, '.credentials')
    if not os.path.exists(credential_dir):
        os.makedirs(credential_dir)
    credential_path = os.path.join(credential_dir,
                                   'sheets.googleapis.com-python-quickstart.json')

    store = Storage(credential_path)
    credentials = store.get()
    if not credentials or credentials.invalid:
        flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
        flow.user_agent = APPLICATION_NAME
        if flags:
            credentials = tools.run_flow(flow, store, flags)
        else: # Needed only for compatibility with Python 2.6
            credentials = tools.run(flow, store)
        print('Storing credentials to ' + credential_path)
    return credentials

# given a google sheet id and two column letters, returns the values of the sheet
def sheetToList(sheetid, col1, col2):
    # connect with credentials
    credentials = get_credentials()
    http = credentials.authorize(httplib2.Http())
    discoveryUrl = ('https://sheets.googleapis.com/$discovery/rest?'
                    'version=v4')
    service = discovery.build('sheets', 'v4', http=http,
                              discoveryServiceUrl=discoveryUrl)

    rangeName = col1 + ':' + col2
    result = service.spreadsheets().values().get(
        spreadsheetId=sheetid, range=rangeName).execute()
    values = result.get('values', [])

    return values
"""
 given a character converts to it's order in the alphabet (A => 0) (B=> 1)
 plus it's offset

 example:
 charToNum('B') A => -1, B => 0, C => 2, etc.

"""

def charToNum(char, offset):
    offsetVal = ord(offset.lower()) # gets ascii value of the offset
    return ord(char.lower()) - offsetVal

# reads values from row of listed parameters
def listToVal(listParam, row, firstcol, type):
    listVal = []
    for param in listParam:
        if (type == "p"):
            listVal.append(row[charToNum(sheetToParam_p[param], firstcol)])
        elif (type == "f"):
            listVal.append(row[charToNum(sheetToParam_f[param], firstcol)])
        else:
            create_log.add_err_msg(25, "listToVal", "type can only be an 'f' or 'p'")
    return listVal

def main():

    # prompts user if he wants to input manually or from a file

    inputType = 'z'

    while (inputType != 'O' and inputType != 'N'):
        inputType = input('Lecture de fichier? (O pour oui, N pour non) :')

    if (inputType == 'O'):
        fname = input('Donnez le nom du fichier: ')

        # opens file containing spreatsheedID

        fileWID = open(fname, 'r')

        listID  = fileWID.read().splitlines()

        spreadsheetId_p = listID[0]

        col1_p = listID[1]

        col2_p = listID[2]

        spreadsheetId_f = listID[3]

        col1_f = listID[4]

        col2_f = listID[5]

    else:
        spreadsheetId_P = input('Veuillez entrer l\'identifiant de votre document Google Sheets pour les parrains.'
                                'Vous pourriez le trouver apres le /d/ et avant le /edit de l\'url de votre fichier.'
                                'ID GOOGLE SHEET ICI:')

        col1_p = input('Premiere colonne a lire des parrains (lettre):')
        col2_p = input('Deuxieme colonne a lire des parrains (lettre):')

        spreadsheetId_f = input('Veuillez entrer l\'identifiant de votre document Google Sheets pour les filleuls.'
                                'Vous pourriez le trouver apres le /d/ et avant le /edit de l\'url de votre fichier.'
                                'ID GOOGLE SHEET ICI:')

        col1_f = input('Premiere colonne a lire des filleuls (lettre):')
        col2_f = input('Deuxieme colonne a lire des filleuls (lettre):')

    parrainsVal = sheetToList(spreadsheetId_p, col1_p, col2_p)
    filleulsVal = sheetToList(spreadsheetId_f, col1_f, col2_f)

    if not parrainsVal:
        create_log.add_err_msg(10, "main",
                               "Aucune donnée trouvée pour le fichier de parrains");
    else:
         for row in parrainsVal:
            counter = 0
            if (len(row) < 14):
                create_log.add_err_msg(11, "main", "Parrain de nom %s " \
                                      "n'a pas bien" \
                            " rempli le formulaire" % row[0])
            else:
                listParam = ['firstName', 'lastName', 'age',
                             'sex', 'email', 'dateAvailable',
                             'university', 'nationality',
                             'subject', 'hobbies',
                             'languages']

                # reads values of parameters listed from document
                valParam = listToVal(listParam, row, col1_p, "p")
                parrain = Parrain(valParam[0], valParam[1],
                                  valParam[2], valParam[3],
                                  valParam[4], valParam[5],
                                  valParam[6], valParam[7],
                                  valParam[8], valParam[9],
                                  valParam[10])

                print(repr(parrain))
                print('\n\n\n')

    if not filleulsVal:
         create_log.add_err_msg(10, "main",
                     "Aucune donnée trouvée pour le fichier de filleuls")
    else:
        for row in filleulsVal:
            counter = 0
            if (len(row) < 16):
                create_log.add_err_msg(11, "main", "Filleul de nom %s " \
                                      "n'a pas bien" \
                            " rempli le formulaire" % row[0])
            else:
                listParam = ['firstName', 'lastName', 'age',
                             'sex', 'email',
                             'dateArrival', 'nationality',
                             'countryOrigin',
                             'university',
                             'subject', 'hobbies']
                            # reads values of parameters listed from document
                valParam = listToVal(listParam, row, col1_f, "f")
                filleul = Filleul(valParam[0], valParam[1],
                                  valParam[2],
                                  valParam[3], valParam[4],
                                  valParam[5],
                                  valParam[6], valParam[7],
                                  valParam[8],
                                  valParam[9], valParam[10])

                print(repr(filleul))
                print('\n\n\n')

    print(dbexec.getid('caca'))

if __name__ == '__main__':
    main()

<commit_msg>Delete test in main<commit_after>"""
The following software has been copied from the following website:

https://developers.google.com/sheets/api/quickstart/python

I have made modifications to some parts of it
"""
from __future__ import print_function
import httplib2
import os
import create_log
import dbexec

from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage

try:
    import argparse
    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
    flags = None

# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/sheets.googleapis.com-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/spreadsheets.readonly'
CLIENT_SECRET_FILE = 'client_secret.json'
APPLICATION_NAME = 'Google Sheets API Python Quickstart'

"""
 Both the parrain and filleul objects with be filled using
 dictionaries that associate their column letters with
 their actual parameters
 for example B represents the lastName for the filleul sheet
"""

sheetToParam_f = {
    'lastName': 'B',
    'firstName':  'C',
    'age': 'D',
    'sex': 'E',
    'email': 'F',
    'university': 'G',
    'subject': 'H',
    'nationality': 'I',
    'dateArrival': 'N',
    'countryOrigin': 'P',
    'hobbies': 'Q'
}

sheetToParam_p = {
    'lastName': 'B',
    'firstName':  'C',
    'age': 'D',
    'sex': 'E',
    'email': 'F',
    'university': 'G',
    'nationality': 'H',
    'dateAvailable': 'K',
    'languages': 'I',
    'subject': 'M',
    'hobbies': 'O',
}

class Parrain:
    def __init__(self, firstName, lastName, age, sex, email, dateAvailable,
                 university, nationality, subject, hobbies, languages):
        self.firstName = firstName
        self.lastName = lastName
        self.age = age
        self.sex = sex
        self.email = email
        self.dateAvailable =dateAvailable
        self.university = university
        self.nationality = nationality
        self.subject = subject
        self.hobbies = hobbies
        self.languages = languages

    def __repr__(self):
        return "<Parrain FN: %s | LN: %s | AGE: %s" \
            "\nSEX: %s| EM: %s| DA: %s\nUNI: %s |" \
            "NA: %s | SU: %s| HO: %s\nLA: %s" % (self.firstName, self.lastName, self.age, self.sex,
                                                 self.email, self.dateAvailable,
                                                 self.university, self. nationality,
                                                 self.subject, self.hobbies,
                                                 self.languages)

# filleul object contains all useful info about a filleul in a variable

class Filleul:
    def __init__(self, firstName, lastName, age, sex,
                 email, dateArrival, nationality, countryOrigin,
                 university, subject, hobbies):
        self.firstName = firstName
        self.lastName = lastName
        self.age = age
        self.sex = sex
        self.email = email
        self.dateArrival = dateArrival
        self.nationality = nationality
        self.countryOrigin = countryOrigin
        self.university = university
        self.subject = subject
        self.hobbies = hobbies
    def __repr__(self):
        return "<Filleul FN: %s | LN: %s | AGE: %s" \
    "\nSEX: %s| EM: %s| DA: %s\nNA: %s|CO: %s| UNI: %s" \
    "\nSU: %s| HO: %s" % (self.firstName, self.lastName, self.age, self.sex,
                      self.email, self.dateArrival, self.nationality,
                      self.countryOrigin, self.university, self.subject,
                      self.hobbies)

def get_credentials():
    # verification of API key
    home_dir = os.path.expanduser('~')
    credential_dir = os.path.join(home_dir, '.credentials')
    if not os.path.exists(credential_dir):
        os.makedirs(credential_dir)
    credential_path = os.path.join(credential_dir,
                                   'sheets.googleapis.com-python-quickstart.json')

    store = Storage(credential_path)
    credentials = store.get()
    if not credentials or credentials.invalid:
        flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
        flow.user_agent = APPLICATION_NAME
        if flags:
            credentials = tools.run_flow(flow, store, flags)
        else: # Needed only for compatibility with Python 2.6
            credentials = tools.run(flow, store)
        print('Storing credentials to ' + credential_path)
    return credentials

# given a google sheet id and two column letters, returns the values of the sheet
def sheetToList(sheetid, col1, col2):
    # connect with credentials
    credentials = get_credentials()
    http = credentials.authorize(httplib2.Http())
    discoveryUrl = ('https://sheets.googleapis.com/$discovery/rest?'
                    'version=v4')
    service = discovery.build('sheets', 'v4', http=http,
                              discoveryServiceUrl=discoveryUrl)

    rangeName = col1 + ':' + col2
    result = service.spreadsheets().values().get(
        spreadsheetId=sheetid, range=rangeName).execute()
    values = result.get('values', [])

    return values
"""
 given a character converts to it's order in the alphabet (A => 0) (B=> 1)
 plus it's offset

 example:
 charToNum('B') A => -1, B => 0, C => 2, etc.

"""

def charToNum(char, offset):
    offsetVal = ord(offset.lower()) # gets ascii value of the offset
    return ord(char.lower()) - offsetVal

# reads values from row of listed parameters
def listToVal(listParam, row, firstcol, type):
    listVal = []
    for param in listParam:
        if (type == "p"):
            listVal.append(row[charToNum(sheetToParam_p[param], firstcol)])
        elif (type == "f"):
            listVal.append(row[charToNum(sheetToParam_f[param], firstcol)])
        else:
            create_log.add_err_msg(25, "listToVal", "type can only be an 'f' or 'p'")
    return listVal

def main():

    # prompts user if he wants to input manually or from a file

    inputType = 'z'

    while (inputType != 'O' and inputType != 'N'):
        inputType = input('Lecture de fichier? (O pour oui, N pour non) :')

    if (inputType == 'O'):
        fname = input('Donnez le nom du fichier: ')

        # opens file containing spreatsheedID

        fileWID = open(fname, 'r')

        listID  = fileWID.read().splitlines()

        spreadsheetId_p = listID[0]

        col1_p = listID[1]

        col2_p = listID[2]

        spreadsheetId_f = listID[3]

        col1_f = listID[4]

        col2_f = listID[5]

    else:
        spreadsheetId_P = input('Veuillez entrer l\'identifiant de votre document Google Sheets pour les parrains.'
                                'Vous pourriez le trouver apres le /d/ et avant le /edit de l\'url de votre fichier.'
                                'ID GOOGLE SHEET ICI:')

        col1_p = input('Premiere colonne a lire des parrains (lettre):')
        col2_p = input('Deuxieme colonne a lire des parrains (lettre):')

        spreadsheetId_f = input('Veuillez entrer l\'identifiant de votre document Google Sheets pour les filleuls.'
                                'Vous pourriez le trouver apres le /d/ et avant le /edit de l\'url de votre fichier.'
                                'ID GOOGLE SHEET ICI:')

        col1_f = input('Premiere colonne a lire des filleuls (lettre):')
        col2_f = input('Deuxieme colonne a lire des filleuls (lettre):')

    parrainsVal = sheetToList(spreadsheetId_p, col1_p, col2_p)
    filleulsVal = sheetToList(spreadsheetId_f, col1_f, col2_f)

    if not parrainsVal:
        create_log.add_err_msg(10, "main",
                               "Aucune donnée trouvée pour le fichier de parrains");
    else:
         for row in parrainsVal:
            counter = 0
            if (len(row) < 14):
                create_log.add_err_msg(11, "main", "Parrain de nom %s " \
                                      "n'a pas bien" \
                            " rempli le formulaire" % row[0])
            else:
                listParam = ['firstName', 'lastName', 'age',
                             'sex', 'email', 'dateAvailable',
                             'university', 'nationality',
                             'subject', 'hobbies',
                             'languages']

                # reads values of parameters listed from document
                valParam = listToVal(listParam, row, col1_p, "p")
                parrain = Parrain(valParam[0], valParam[1],
                                  valParam[2], valParam[3],
                                  valParam[4], valParam[5],
                                  valParam[6], valParam[7],
                                  valParam[8], valParam[9],
                                  valParam[10])

                print(repr(parrain))
                print('\n\n\n')

    if not filleulsVal:
         create_log.add_err_msg(10, "main",
                     "Aucune donnée trouvée pour le fichier de filleuls")
    else:
        for row in filleulsVal:
            counter = 0
            if (len(row) < 16):
                create_log.add_err_msg(11, "main", "Filleul de nom %s " \
                                      "n'a pas bien" \
                            " rempli le formulaire" % row[0])
            else:
                listParam = ['firstName', 'lastName', 'age',
                             'sex', 'email',
                             'dateArrival', 'nationality',
                             'countryOrigin',
                             'university',
                             'subject', 'hobbies']
                            # reads values of parameters listed from document
                valParam = listToVal(listParam, row, col1_f, "f")
                filleul = Filleul(valParam[0], valParam[1],
                                  valParam[2],
                                  valParam[3], valParam[4],
                                  valParam[5],
                                  valParam[6], valParam[7],
                                  valParam[8],
                                  valParam[9], valParam[10])

                print(repr(filleul))
                print('\n\n\n')

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before># Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import logging
import sys

from api import API

INSIGHTS = 'insights'
REVIEWS = 'reviews'
SENTIMENT = 'sentiment'
DIRECTIONS = 'directions'
HOURLY_CALLS = 'hourly_calls'

class Alligator():

  @classmethod
  def sentiment_only(cls, project_id):
    api = API(project_id)
    api.sentiments()

  @classmethod
  def for_account_and_location(cls, project_id, account_id, location_id, flags):
    api = API(project_id)

    location_name = u"accounts/{}/locations/{}".format(account_id, location_id)

    api.locations(u"accounts/{}".format(account_id), location_id=location_name)

    if flags[INSIGHTS]:
      logging.info("Insights...")
      api.insights(location_name)
    if flags[DIRECTIONS]:
      logging.info("Directions...")
      api.directions(location_name)
    if flags[HOURLY_CALLS]:
      logging.info("Hourly calls...")
      api.hourly_calls(location_name)
    if flags[REVIEWS]:
      logging.info("Reviews...")
      api.reviews(location_name)
    if flags[SENTIMENT]:
      logging.info("Sentiments...")
      api.sentiments()

  @classmethod
  def for_account(cls, project_id, account_id, flags):
    api = API(project_id)
    locations = api.locations(u"accounts/{}".format(account_id))

    for location in locations:
      location_name = location.get("name")
      if flags[INSIGHTS]:
        api.insights(location_name)
      if flags[DIRECTIONS]:
        api.directions(location_name)
      if flags[HOURLY_CALLS]:
        api.hourly_calls(location_name)
      if flags[REVIEWS]:
        api.reviews(location_name)

    if flags[SENTIMENT]:
      api.sentiments()

  @classmethod
  def all(cls, project_id, flags):
    api = API(project_id)
    accounts = api.accounts()

    for account in accounts:
      account_name = account.get("name")
      locations = api.locations(account_name)

      for location in locations:
        location_name = location.get("name")
        if flags[INSIGHTS]:
          api.insights(location_name)
        if flags[DIRECTIONS]:
          api.directions(location_name)
        if flags[HOURLY_CALLS]:
          api.hourly_calls(location_name)
        if flags[REVIEWS]:
          api.reviews(location_name)

    if flags[SENTIMENT]:
      api.sentiments()

def main(argv):
  parser = argparse.ArgumentParser()
  parser.add_argument(
      "-p",
      "--project_id",
      required=True,
      type=str,
      help="a Google Cloud Project ID")
  parser.add_argument(
      "-a",
      "--account_id",
      type=str,
      help="retrieve and store all Google My Business reviews for a given Account ID"
  )
  parser.add_argument(
      "-l",
      "--location_id",
      type=str,
      help="retrieve and store all Google My Business reviews for a given Location ID (--account_id is also required)"
  )
  parser.add_argument(
      "--no_insights",
      help="skip the insights processing and storage",
      action="store_true")
  parser.add_argument(
      "--no_reviews",
      help="skip the reviews processing and storage",
      action="store_true")
  parser.add_argument(
      "--no_sentiment",
      help="skip the sentiment processing and storage",
      action="store_true")
  parser.add_argument(
      "--no_directions",
      help="skip the directions processing and storage",
      action="store_true")
  parser.add_argument(
      "--no_hourly_calls",
      help="skip the hourly calls processing and storage",
      action="store_true")
  parser.add_argument(
      "--sentiment_only",
      help="only process and store the sentiment of all available reviews (if --no-sentiment is provided, no action is performed)",
      action="store_true")
  parser.add_argument(
      "-v", "--verbose", help="increase output verbosity", action="store_true")
  args = parser.parse_args()

  project_id = args.project_id
  account_id = args.account_id
  location_id = args.location_id

  flags = {}
  flags[INSIGHTS] = not args.no_insights
  flags[DIRECTIONS] = not args.no_directions
  flags[HOURLY_CALLS] = not args.no_hourly_calls
  flags[REVIEWS] = not args.no_reviews
  flags[SENTIMENT] = not args.no_sentiment

  sentiment_only = args.sentiment_only
  verbose = args.verbose

  sys.argv.clear()

  if verbose:
    logging.basicConfig(level=logging.DEBUG)

  logging.info(u"Project ID:\t%s", project_id)
  logging.info(u"Account ID:\t%s", account_id)
  logging.info(u"Location ID:\t%s", location_id)

  if sentiment_only:
    if flags[SENTIMENT]:
      print("Running sentiment analysis for all reviews in BigQuery...")
      Alligator.sentiment_only(project_id)
    else:
      print(
          "No action will be performed as --no_sentiment and --sentiment_only flags have been provided."
      )
    sys.exit()

  print("Loading Google My Business reviews into BigQuery...")

  if account_id and location_id:
    Alligator.for_account_and_location(project_id, account_id, location_id,
                                       flags)
  elif account_id:
    Alligator.for_account(project_id, account_id, flags)
  else:
    Alligator.all(project_id, flags)

  print("Done.")

if __name__ == "__main__":
  main(sys.argv[1:])
<commit_msg>Remove temporary logging info<commit_after># Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import logging
import sys

from api import API

INSIGHTS = 'insights'
REVIEWS = 'reviews'
SENTIMENT = 'sentiment'
DIRECTIONS = 'directions'
HOURLY_CALLS = 'hourly_calls'

class Alligator():

  @classmethod
  def sentiment_only(cls, project_id):
    api = API(project_id)
    api.sentiments()

  @classmethod
  def for_account_and_location(cls, project_id, account_id, location_id, flags):
    api = API(project_id)

    location_name = u"accounts/{}/locations/{}".format(account_id, location_id)

    api.locations(u"accounts/{}".format(account_id), location_id=location_name)

    if flags[INSIGHTS]:
      api.insights(location_name)
    if flags[DIRECTIONS]:
      api.directions(location_name)
    if flags[HOURLY_CALLS]:
      api.hourly_calls(location_name)
    if flags[REVIEWS]:
      api.reviews(location_name)
    if flags[SENTIMENT]:
      api.sentiments()

  @classmethod
  def for_account(cls, project_id, account_id, flags):
    api = API(project_id)
    locations = api.locations(u"accounts/{}".format(account_id))

    for location in locations:
      location_name = location.get("name")
      if flags[INSIGHTS]:
        api.insights(location_name)
      if flags[DIRECTIONS]:
        api.directions(location_name)
      if flags[HOURLY_CALLS]:
        api.hourly_calls(location_name)
      if flags[REVIEWS]:
        api.reviews(location_name)

    if flags[SENTIMENT]:
      api.sentiments()

  @classmethod
  def all(cls, project_id, flags):
    api = API(project_id)
    accounts = api.accounts()

    for account in accounts:
      account_name = account.get("name")
      locations = api.locations(account_name)

      for location in locations:
        location_name = location.get("name")
        if flags[INSIGHTS]:
          api.insights(location_name)
        if flags[DIRECTIONS]:
          api.directions(location_name)
        if flags[HOURLY_CALLS]:
          api.hourly_calls(location_name)
        if flags[REVIEWS]:
          api.reviews(location_name)

    if flags[SENTIMENT]:
      api.sentiments()

def main(argv):
  parser = argparse.ArgumentParser()
  parser.add_argument(
      "-p",
      "--project_id",
      required=True,
      type=str,
      help="a Google Cloud Project ID")
  parser.add_argument(
      "-a",
      "--account_id",
      type=str,
      help="retrieve and store all Google My Business reviews for a given Account ID"
  )
  parser.add_argument(
      "-l",
      "--location_id",
      type=str,
      help="retrieve and store all Google My Business reviews for a given Location ID (--account_id is also required)"
  )
  parser.add_argument(
      "--no_insights",
      help="skip the insights processing and storage",
      action="store_true")
  parser.add_argument(
      "--no_reviews",
      help="skip the reviews processing and storage",
      action="store_true")
  parser.add_argument(
      "--no_sentiment",
      help="skip the sentiment processing and storage",
      action="store_true")
  parser.add_argument(
      "--no_directions",
      help="skip the directions processing and storage",
      action="store_true")
  parser.add_argument(
      "--no_hourly_calls",
      help="skip the hourly calls processing and storage",
      action="store_true")
  parser.add_argument(
      "--sentiment_only",
      help="only process and store the sentiment of all available reviews (if --no-sentiment is provided, no action is performed)",
      action="store_true")
  parser.add_argument(
      "-v", "--verbose", help="increase output verbosity", action="store_true")
  args = parser.parse_args()

  project_id = args.project_id
  account_id = args.account_id
  location_id = args.location_id

  flags = {}
  flags[INSIGHTS] = not args.no_insights
  flags[DIRECTIONS] = not args.no_directions
  flags[HOURLY_CALLS] = not args.no_hourly_calls
  flags[REVIEWS] = not args.no_reviews
  flags[SENTIMENT] = not args.no_sentiment

  sentiment_only = args.sentiment_only
  verbose = args.verbose

  sys.argv.clear()

  if verbose:
    logging.basicConfig(level=logging.DEBUG)

  logging.info(u"Project ID:\t%s", project_id)
  logging.info(u"Account ID:\t%s", account_id)
  logging.info(u"Location ID:\t%s", location_id)

  if sentiment_only:
    if flags[SENTIMENT]:
      print("Running sentiment analysis for all reviews in BigQuery...")
      Alligator.sentiment_only(project_id)
    else:
      print(
          "No action will be performed as --no_sentiment and --sentiment_only flags have been provided."
      )
    sys.exit()

  print("Loading Google My Business reviews into BigQuery...")

  if account_id and location_id:
    Alligator.for_account_and_location(project_id, account_id, location_id,
                                       flags)
  elif account_id:
    Alligator.for_account(project_id, account_id, flags)
  else:
    Alligator.all(project_id, flags)

  print("Done.")

if __name__ == "__main__":
  main(sys.argv[1:])
<|endoftext|>


<commit_before>from __future__ import print_function
import argparse
import random
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
import os

import models.dcgan as dcgan
import models.mlp as mlp

parser = argparse.ArgumentParser()
parser.add_argument('--dataset', required=True, help='cifar10 | lsun | imagenet | folder | lfw ')
parser.add_argument('--dataroot', required=True, help='path to dataset')
parser.add_argument('--workers', type=int, help='number of data loading workers', default=2)
parser.add_argument('--batchSize', type=int, default=64, help='input batch size')
parser.add_argument('--imageSize', type=int, default=64, help='the height / width of the input image to network')
parser.add_argument('--nz', type=int, default=100, help='size of the latent z vector')
parser.add_argument('--ngf', type=int, default=64)
parser.add_argument('--ndf', type=int, default=64)
parser.add_argument('--niter', type=int, default=25, help='number of epochs to train for')
parser.add_argument('--lrD', type=float, default=0.00005, help='learning rate for Critic, default=0.0002')
parser.add_argument('--lrG', type=float, default=0.00005, help='learning rate for Generator, default=0.0002')
parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for adam. default=0.5')
parser.add_argument('--cuda'  , action='store_true', help='enables cuda')
parser.add_argument('--ngpu'  , type=int, default=1, help='number of GPUs to use')
parser.add_argument('--netG', default='', help="path to netG (to continue training)")
parser.add_argument('--netD', default='', help="path to netD (to continue training)")
parser.add_argument('--clamp_lower', type=float, default=-0.01)
parser.add_argument('--clamp_upper', type=float, default=0.01)
parser.add_argument('--Diters', type=int, default=5, help='number of D iters per each G iter')
parser.add_argument('--noBN', action='store_true', help='use batchnorm or not (only for DCGAN)')
parser.add_argument('--mlp_G', action='store_true', help='use MLP for G')
parser.add_argument('--mlp_D', action='store_true', help='use MLP for D')
parser.add_argument('--grad_bound', type=float, default=1e10, help='Keep training the disc until the norm of its gradient is below this')
parser.add_argument('--n_extra_layers', type=int, default=0, help='Number of extra layers on gen and disc')
parser.add_argument('--experiment', default=None, help='Where to store samples and models')
parser.add_argument('--adam', action='store_true', help='Whether to use adam (default is rmsprop)')
opt = parser.parse_args()
print(opt)

if opt.experiment is None:
    opt.experiment = 'samples'
os.system('mkdir {0}'.format(opt.experiment))

opt.manualSeed = random.randint(1, 10000) # fix seed
print("Random Seed: ", opt.manualSeed)
random.seed(opt.manualSeed)
torch.manual_seed(opt.manualSeed)

cudnn.benchmark = True

if torch.cuda.is_available() and not opt.cuda:
    print("WARNING: You have a CUDA device, so you should probably run with --cuda")

if opt.dataset in ['imagenet', 'folder', 'lfw']:
    # folder dataset
    dataset = dset.ImageFolder(root=opt.dataroot,
                               transform=transforms.Compose([
                                   transforms.Scale(opt.imageSize),
                                   transforms.CenterCrop(opt.imageSize),
                                   transforms.ToTensor(),
                                   transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                               ]))
elif opt.dataset == 'lsun':
    dataset = dset.LSUN(db_path=opt.dataroot, classes=['bedroom_train'],
                        transform=transforms.Compose([
                            transforms.Scale(opt.imageSize),
                            transforms.CenterCrop(opt.imageSize),
                            transforms.ToTensor(),
                            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                        ]))
elif opt.dataset == 'cifar10':
    dataset = dset.CIFAR10(root=opt.dataroot, download=True,
                           transform=transforms.Compose([
                               transforms.Scale(opt.imageSize),
                               transforms.ToTensor(),
                               transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                           ])
    )
assert dataset
dataloader = torch.utils.data.DataLoader(dataset, batch_size=opt.batchSize,
                                         shuffle=True, num_workers=int(opt.workers))

ngpu = int(opt.ngpu)
nz = int(opt.nz)
ngf = int(opt.ngf)
ndf = int(opt.ndf)
nc = 3
n_extra_layers = int(opt.n_extra_layers)

# custom weights initialization called on netG and netD
def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

if opt.noBN:
    netG = dcgan.DCGAN_G_nobn(opt.imageSize, nz, nc, ngf, ngpu, n_extra_layers)
elif opt.mlp_G:
    netG = mlp.MLP_G(opt.imageSize, nz, nc, ngf, ngpu)
else:
    netG = dcgan.DCGAN_G(opt.imageSize, nz, nc, ngf, ngpu, n_extra_layers)

netG.apply(weights_init)
if opt.netG != '': # load checkpoint if needed
    netG.load_state_dict(torch.load(opt.netG))
print(netG)

if opt.noBN:
    netD = dcgan.DCGAN_D_nobn(opt.imageSize, nz, nc, ndf, ngpu, n_extra_layers)
    netD.apply(weights_init)
elif opt.mlp_D:
    netD = mlp.MLP_D(opt.imageSize, nz, nc, ndf, ngpu)
else:
    netD = dcgan.DCGAN_D(opt.imageSize, nz, nc, ndf, ngpu, n_extra_layers)
    netD.apply(weights_init)

if opt.netD != '':
    netD.load_state_dict(torch.load(opt.netD))
print(netD)

input = torch.FloatTensor(opt.batchSize, 3, opt.imageSize, opt.imageSize)
noise = torch.FloatTensor(opt.batchSize, nz, 1, 1)
fixed_noise = torch.FloatTensor(opt.batchSize, nz, 1, 1).normal_(0, 1)
one = torch.FloatTensor([1])
mone = one * -1

if opt.cuda:
    netD.cuda()
    netG.cuda()
    input = input.cuda()
    one, mone = one.cuda(), mone.cuda()
    noise, fixed_noise = noise.cuda(), fixed_noise.cuda()

input = Variable(input)
noise = Variable(noise)
fixed_noise = Variable(fixed_noise)

# setup optimizer
if opt.adam:
    optimizerD = optim.Adam(netD.parameters(), lr=opt.lrD, betas=(opt.beta1, 0.999))
    optimizerG = optim.Adam(netG.parameters(), lr=opt.lrG, betas=(opt.beta1, 0.999))
else:
    optimizerD = optim.RMSprop(netD.parameters(), lr = opt.lrD)
    optimizerG = optim.RMSprop(netG.parameters(), lr = opt.lrG)

gen_iterations = 0
for epoch in range(opt.niter):
    data_iter = iter(dataloader)
    i = 0
    while i < len(dataloader):
        ############################
        # (1) Update D network
        ###########################
        for p in netD.parameters(): # reset requires_grad
            p.requires_grad = True # they are set to False below in netG update

        # train the discriminator Diters times
        if gen_iterations < 25 or gen_iterations % 500 == 0:
            Diters = 100
        else:
            Diters = opt.Diters
        grad_D_norm = 0
        j = 0
        while (j < Diters or grad_D_norm > opt.grad_bound) and i < len(dataloader):
            j += 1

            # clamp parameters to a cube
            for p in netD.parameters():
                p.data.clamp_(opt.clamp_lower, opt.clamp_upper)

            data = data_iter.next()
            i += 1

            # train with real
            real_cpu, _ = data
            netD.zero_grad()
            batch_size = real_cpu.size(0)
            input.data.resize_(real_cpu.size()).copy_(real_cpu)

            errD_real = netD(input)
            errD_real.backward(one)

            # train with fake
            noise.data.resize_(batch_size, nz, 1, 1)
            noise.data.normal_(0, 1)
            fake = netG(noise)
            input.data.copy_(fake.data)
            errD_fake = netD(input)
            errD_fake.backward(mone)
            errD = errD_real - errD_fake
            optimizerD.step()

        ############################
        # (2) Update G network
        ###########################
        for p in netD.parameters():
            p.requires_grad = False # to avoid computation
        netG.zero_grad()
        noise.data.normal_(0, 1)
        fake = netG(noise)
        errG = netD(fake)
        errG.backward(one)
        optimizerG.step()
        gen_iterations += 1

        print('[%d/%d][%d/%d] Loss_D: %f Loss_G: %f Loss_D_real: %f Loss_D_fake %f'
            % (epoch, opt.niter, gen_iterations, len(dataloader),
            errD.data[0], errG.data[0], errD_real.data[0], errD_fake.data[0]))
        if gen_iterations % 500 == 0:
            vutils.save_image(real_cpu, '{0}/real_samples.png'.format(opt.experiment))
            fake = netG(fixed_noise)
            vutils.save_image(fake.data, '{0}/fake_samples_{1}.png'.format(opt.experiment, gen_iterations))

    # do checkpointing
    torch.save(netG.state_dict(), '{0}/netG_epoch_{1}.pth'.format(opt.experiment, epoch))
    torch.save(netD.state_dict(), '{0}/netD_epoch_{1}.pth'.format(opt.experiment, epoch))
<commit_msg>doc update<commit_after>from __future__ import print_function
import argparse
import random
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
import os

import models.dcgan as dcgan
import models.mlp as mlp

parser = argparse.ArgumentParser()
parser.add_argument('--dataset', required=True, help='cifar10 | lsun | imagenet | folder | lfw ')
parser.add_argument('--dataroot', required=True, help='path to dataset')
parser.add_argument('--workers', type=int, help='number of data loading workers', default=2)
parser.add_argument('--batchSize', type=int, default=64, help='input batch size')
parser.add_argument('--imageSize', type=int, default=64, help='the height / width of the input image to network')
parser.add_argument('--nz', type=int, default=100, help='size of the latent z vector')
parser.add_argument('--ngf', type=int, default=64)
parser.add_argument('--ndf', type=int, default=64)
parser.add_argument('--niter', type=int, default=25, help='number of epochs to train for')
parser.add_argument('--lrD', type=float, default=0.00005, help='learning rate for Critic, default=0.00005')
parser.add_argument('--lrG', type=float, default=0.00005, help='learning rate for Generator, default=0.00005')
parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for adam. default=0.5')
parser.add_argument('--cuda'  , action='store_true', help='enables cuda')
parser.add_argument('--ngpu'  , type=int, default=1, help='number of GPUs to use')
parser.add_argument('--netG', default='', help="path to netG (to continue training)")
parser.add_argument('--netD', default='', help="path to netD (to continue training)")
parser.add_argument('--clamp_lower', type=float, default=-0.01)
parser.add_argument('--clamp_upper', type=float, default=0.01)
parser.add_argument('--Diters', type=int, default=5, help='number of D iters per each G iter')
parser.add_argument('--noBN', action='store_true', help='use batchnorm or not (only for DCGAN)')
parser.add_argument('--mlp_G', action='store_true', help='use MLP for G')
parser.add_argument('--mlp_D', action='store_true', help='use MLP for D')
parser.add_argument('--grad_bound', type=float, default=1e10, help='Keep training the disc until the norm of its gradient is below this')
parser.add_argument('--n_extra_layers', type=int, default=0, help='Number of extra layers on gen and disc')
parser.add_argument('--experiment', default=None, help='Where to store samples and models')
parser.add_argument('--adam', action='store_true', help='Whether to use adam (default is rmsprop)')
opt = parser.parse_args()
print(opt)

if opt.experiment is None:
    opt.experiment = 'samples'
os.system('mkdir {0}'.format(opt.experiment))

opt.manualSeed = random.randint(1, 10000) # fix seed
print("Random Seed: ", opt.manualSeed)
random.seed(opt.manualSeed)
torch.manual_seed(opt.manualSeed)

cudnn.benchmark = True

if torch.cuda.is_available() and not opt.cuda:
    print("WARNING: You have a CUDA device, so you should probably run with --cuda")

if opt.dataset in ['imagenet', 'folder', 'lfw']:
    # folder dataset
    dataset = dset.ImageFolder(root=opt.dataroot,
                               transform=transforms.Compose([
                                   transforms.Scale(opt.imageSize),
                                   transforms.CenterCrop(opt.imageSize),
                                   transforms.ToTensor(),
                                   transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                               ]))
elif opt.dataset == 'lsun':
    dataset = dset.LSUN(db_path=opt.dataroot, classes=['bedroom_train'],
                        transform=transforms.Compose([
                            transforms.Scale(opt.imageSize),
                            transforms.CenterCrop(opt.imageSize),
                            transforms.ToTensor(),
                            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                        ]))
elif opt.dataset == 'cifar10':
    dataset = dset.CIFAR10(root=opt.dataroot, download=True,
                           transform=transforms.Compose([
                               transforms.Scale(opt.imageSize),
                               transforms.ToTensor(),
                               transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                           ])
    )
assert dataset
dataloader = torch.utils.data.DataLoader(dataset, batch_size=opt.batchSize,
                                         shuffle=True, num_workers=int(opt.workers))

ngpu = int(opt.ngpu)
nz = int(opt.nz)
ngf = int(opt.ngf)
ndf = int(opt.ndf)
nc = 3
n_extra_layers = int(opt.n_extra_layers)

# custom weights initialization called on netG and netD
def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

if opt.noBN:
    netG = dcgan.DCGAN_G_nobn(opt.imageSize, nz, nc, ngf, ngpu, n_extra_layers)
elif opt.mlp_G:
    netG = mlp.MLP_G(opt.imageSize, nz, nc, ngf, ngpu)
else:
    netG = dcgan.DCGAN_G(opt.imageSize, nz, nc, ngf, ngpu, n_extra_layers)

netG.apply(weights_init)
if opt.netG != '': # load checkpoint if needed
    netG.load_state_dict(torch.load(opt.netG))
print(netG)

if opt.noBN:
    netD = dcgan.DCGAN_D_nobn(opt.imageSize, nz, nc, ndf, ngpu, n_extra_layers)
    netD.apply(weights_init)
elif opt.mlp_D:
    netD = mlp.MLP_D(opt.imageSize, nz, nc, ndf, ngpu)
else:
    netD = dcgan.DCGAN_D(opt.imageSize, nz, nc, ndf, ngpu, n_extra_layers)
    netD.apply(weights_init)

if opt.netD != '':
    netD.load_state_dict(torch.load(opt.netD))
print(netD)

input = torch.FloatTensor(opt.batchSize, 3, opt.imageSize, opt.imageSize)
noise = torch.FloatTensor(opt.batchSize, nz, 1, 1)
fixed_noise = torch.FloatTensor(opt.batchSize, nz, 1, 1).normal_(0, 1)
one = torch.FloatTensor([1])
mone = one * -1

if opt.cuda:
    netD.cuda()
    netG.cuda()
    input = input.cuda()
    one, mone = one.cuda(), mone.cuda()
    noise, fixed_noise = noise.cuda(), fixed_noise.cuda()

input = Variable(input)
noise = Variable(noise)
fixed_noise = Variable(fixed_noise)

# setup optimizer
if opt.adam:
    optimizerD = optim.Adam(netD.parameters(), lr=opt.lrD, betas=(opt.beta1, 0.999))
    optimizerG = optim.Adam(netG.parameters(), lr=opt.lrG, betas=(opt.beta1, 0.999))
else:
    optimizerD = optim.RMSprop(netD.parameters(), lr = opt.lrD)
    optimizerG = optim.RMSprop(netG.parameters(), lr = opt.lrG)

gen_iterations = 0
for epoch in range(opt.niter):
    data_iter = iter(dataloader)
    i = 0
    while i < len(dataloader):
        ############################
        # (1) Update D network
        ###########################
        for p in netD.parameters(): # reset requires_grad
            p.requires_grad = True # they are set to False below in netG update

        # train the discriminator Diters times
        if gen_iterations < 25 or gen_iterations % 500 == 0:
            Diters = 100
        else:
            Diters = opt.Diters
        grad_D_norm = 0
        j = 0
        while (j < Diters or grad_D_norm > opt.grad_bound) and i < len(dataloader):
            j += 1

            # clamp parameters to a cube
            for p in netD.parameters():
                p.data.clamp_(opt.clamp_lower, opt.clamp_upper)

            data = data_iter.next()
            i += 1

            # train with real
            real_cpu, _ = data
            netD.zero_grad()
            batch_size = real_cpu.size(0)
            input.data.resize_(real_cpu.size()).copy_(real_cpu)

            errD_real = netD(input)
            errD_real.backward(one)

            # train with fake
            noise.data.resize_(batch_size, nz, 1, 1)
            noise.data.normal_(0, 1)
            fake = netG(noise)
            input.data.copy_(fake.data)
            errD_fake = netD(input)
            errD_fake.backward(mone)
            errD = errD_real - errD_fake
            optimizerD.step()

        ############################
        # (2) Update G network
        ###########################
        for p in netD.parameters():
            p.requires_grad = False # to avoid computation
        netG.zero_grad()
        noise.data.normal_(0, 1)
        fake = netG(noise)
        errG = netD(fake)
        errG.backward(one)
        optimizerG.step()
        gen_iterations += 1

        print('[%d/%d][%d/%d] Loss_D: %f Loss_G: %f Loss_D_real: %f Loss_D_fake %f'
            % (epoch, opt.niter, gen_iterations, len(dataloader),
            errD.data[0], errG.data[0], errD_real.data[0], errD_fake.data[0]))
        if gen_iterations % 500 == 0:
            vutils.save_image(real_cpu, '{0}/real_samples.png'.format(opt.experiment))
            fake = netG(fixed_noise)
            vutils.save_image(fake.data, '{0}/fake_samples_{1}.png'.format(opt.experiment, gen_iterations))

    # do checkpointing
    torch.save(netG.state_dict(), '{0}/netG_epoch_{1}.pth'.format(opt.experiment, epoch))
    torch.save(netD.state_dict(), '{0}/netD_epoch_{1}.pth'.format(opt.experiment, epoch))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2016, Data Exploration Lab
# Distributed under the terms of the Modified BSD License.

from collections import namedtuple
import errno
import datetime
import json
import logging
import os
import random
import re
import socket
import string
import subprocess

import docker
import girder_client
from dateutil.parser import parse as parse_date
import tornado.ioloop
import tornado.web
from tornado import gen
from tornado.httpclient import HTTPRequest, HTTPError, AsyncHTTPClient

import dockworker

AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")

GIRDER_API_URL = os.environ.get(
    "GIRDER_API_URL", "https://girder.hub.yt/api/v1")
DOCKER_URL = os.environ.get("DOCKER_URL", "unix://var/run/docker.sock")
HOSTDIR = os.environ.get("HOSTDIR", "/host")
MAX_FILE_SIZE = os.environ.get("MAX_FILE_SIZE", 200)

MOUNTS = {}

PooledContainer = namedtuple('PooledContainer', ['id', 'path'])

def sample_with_replacement(a, size):
    '''Get a random path. If Python had sampling with replacement built in,
    I would use that. The other alternative is numpy.random.choice, but
    numpy is overkill for this tiny bit of random pathing.'''
    return "".join([random.SystemRandom().choice(a) for x in range(size)])

def new_user(size):
    return sample_with_replacement(string.ascii_letters + string.digits, size)

def _safe_mkdir(dest):
    try:
        os.mkdir(dest)
    except OSError as e:
        if e.errno != 17:
            raise
        logging.warn("Failed to mkdir {}".format(dest))
        pass

@gen.coroutine
def cull_idle(proxy_url, proxy_token, timeout):
    cull_limit = datetime.datetime.utcnow() \
        - datetime.timedelta(seconds=timeout)

    http_client = AsyncHTTPClient()
    logging.debug("Polling proxy for idle containers")
    req = HTTPRequest(proxy_url + '/api/users',
                      headers={"Authorization": "token %s" % proxy_token})
    try:
        resp = yield http_client.fetch(req)
    except HTTPError as e:
        logging.error("Failed to poll proxy for idle containers: %s", e)

    users = json.loads(resp.body.decode('utf8', 'replace'))
    futures = []
    for user in users:
        last_activity = parse_date(user['last_activity'])
        if user['server'] and last_activity < cull_limit:
            logging.info(
                "Culling %s (inactive since %s)", user['name'], last_activity)
            # req = HTTPRequest(url=url+'/api/users/%s/server' % user['name'],
            #                  method='DELETE',
            #                  headers=auth_header,
            #                  )
            futures.append((user['name'], http_client.fetch(req)))
        elif user['server'] and last_activity > cull_limit:
            logging.debug("Not culling %s (active since %s)",
                          user['name'], last_activity)

    for (name, f) in futures:
        yield f
        logging.debug("Finished culling %s", name)

@gen.coroutine
def _get_api_key(gc):
    api_key = None
    for key in gc.get('/api_key'):
        if key['name'] == 'tmpnb' and key['active']:
            api_key = key['key']

    if api_key is None:
        api_key = gc.post('/api_key',
                          data={'name': 'tmpnb', 'active': True})['key']
    return api_key

@gen.coroutine
def parse_request_body(data):
    gc = girder_client.GirderClient(apiUrl=GIRDER_API_URL)
    gc.token = data['girder_token']
    user = gc.get("/user/me")
    if user is None:
        logging.warn("Bad gider token")
        raise tornado.web.HTTPError(
            401, 'Failed to authenticate with girder'
        )

    # Allow sysop to delete any notebook
    userId = data.get('userId', user['_id'])
    if userId != user['_id'] and user["admin"]:
        user = gc.get("/user/{id}".format(id=userId))
        logging.info("Overriding user %s", user["login"])

    logging.debug("USER = %s", json.dumps(user))
    return gc, user

class MainHandler(tornado.web.RequestHandler):

    @property
    def proxy_token(self):
        return self.settings['proxy_token']

    @property
    def pool_name(self):
        return self.settings['pool_name']

    @property
    def proxy_endpoint(self):
        return self.settings['proxy_endpoint']

    @property
    def container_config(self):
        return self.settings['container_config']

    @property
    def spawner(self):
        return self.settings['spawner']

    @property
    def container_name_pattern(self):
        return self.settings['container_name_pattern']

    @gen.coroutine
    def post(self):
        payload = json.loads(self.request.body.decode("utf-8"))
        gc, user = yield parse_request_body(payload)

        vol_name = "%s_%s" % (payload['folderId'], user['login'])
        cli = docker.Client(base_url=DOCKER_URL)
        volume = cli.create_volume(name=vol_name, driver='local')
        logging.info("Volume: %s created", vol_name)
        logging.info("Mountpoint: %s", volume['Mountpoint'])

        params = {'parentType': 'user', 'parentId': user["_id"],
                  'name': 'Notebooks'}
        homeDir = list(gc.listResource("/folder", params))[0]["_id"]

        items = [item["_id"] for item in gc.listItem(homeDir)
                 if item["name"].endswith("pynb")]
        # TODO: should be done in one go with /resource endpoint
        #  but client doesn't have it yet
        for item in items:
            gc.downloadItem(item, HOSTDIR + volume["Mountpoint"])

        # TODO: read uid/gid from env/config
        for item in os.listdir(HOSTDIR + volume["Mountpoint"]):
            os.chown(os.path.join(HOSTDIR + volume["Mountpoint"], item),
                     1000, 100)

        dest = os.path.join(volume["Mountpoint"], "data")
        _safe_mkdir(HOSTDIR + dest)

        # FUSE is silly and needs to have mirror inside container
        if not os.path.isdir(dest):
            os.makedirs(dest)
        api_key = yield _get_api_key(gc)
        cmd = "girderfs -c direct --api-url {} --api-key {} {} {}".format(
            GIRDER_API_URL, api_key, dest, payload['folderId'])
        logging.info("Calling: %s", cmd)
        subprocess.call(cmd, shell=True)

        # CREATE CONTAINER
        # REGISTER CONTAINER WITH PROXY
        container = yield self._launch_container(volume)

        self.write(dict(mountPoint=volume['Mountpoint'],
                        containerId=container.id,
                        containerPath=container.path))
        self.finish()

    @gen.coroutine
    def _launch_container(self, volume):
        user = new_user(12)
        path = "user/" + user
        container_name = 'tmp.{}.{}'.format(self.pool_name, user)
        volume_bindings = {volume["Name"]: {
            'bind': "/home/jovyan/work", 'mode': 'rw'}}
        if not self.container_name_pattern.match(container_name):
            pattern = self.container_name_pattern.pattern
            raise Exception("[{}] does not match [{}]!".format(container_name,
                                                               pattern))

        logging.info("Launching new notebook server [%s] at path [%s].",
                     container_name, path)
        create_result = yield self.spawner.create_notebook_server(
            base_path=path, container_name=container_name,
            container_config=self.container_config,
            volume_bindings=volume_bindings
        )
        container_id, host_ip, host_port = create_result
        logging.info(
            "Created notebook server [%s] for path [%s] at [%s:%s]",
            container_name, path, host_ip, host_port)

        # Wait for the server to launch within the container before adding it
        # to the pool or serving it to a user.

        yield self._wait_for_server(host_ip, host_port, path)

        http_client = AsyncHTTPClient()
        headers = {"Authorization": "token {}".format(self.proxy_token)}

        proxy_endpoint = "{}/api/routes/{}".format(self.proxy_endpoint, path)
        body = json.dumps({
            "target": "http://{}:{}".format(host_ip, host_port),
            "container_id": container_id,
        })

        logging.debug("Proxying path [%s] to port [%s].", path, host_port)
        req = HTTPRequest(proxy_endpoint, method="POST", headers=headers,
                          body=body)
        try:
            yield http_client.fetch(req)
            logging.info("Proxied path [%s] to port [%s].", path, host_port)
        except HTTPError as e:
            logging.error("Failed to create proxy route to [%s]: %s", path, e)

        container = PooledContainer(id=container_id, path=path)
        raise gen.Return(container)

    @gen.coroutine
    def _wait_for_server(self, ip, port, path, timeout=10, wait_time=0.2):
        '''Wait for a server to show up within a newly launched container.'''

        logging.info("Waiting for a container to launch at [%s:%s].", ip, port)
        loop = tornado.ioloop.IOLoop.current()
        tic = loop.time()

        # Docker starts listening on a socket before the container is fully
        # launched. Wait for that, first.

        while loop.time() - tic < timeout:
            try:
                socket.create_connection((ip, port))
            except socket.error as e:
                logging.warn("Socket error on boot: %s", e)
                if e.errno != errno.ECONNREFUSED:
                    logging.warn("Error attempting to connect to [%s:%i]: %s",
                                 ip, port, e)
                yield gen.Task(loop.add_timeout, loop.time() + wait_time)
            else:
                break

        # Fudge factor of IPython notebook bootup.
        # TODO: Implement a webhook in IPython proper to call out when the
        # notebook server is booted.
        yield gen.Task(loop.add_timeout, loop.time() + .5)

        # Now, make sure that we can reach the Notebook server.
        http_client = AsyncHTTPClient()
        req = HTTPRequest("http://{}:{}/{}".format(ip, port, path))

        while loop.time() - tic < timeout:
            try:
                yield http_client.fetch(req)
            except HTTPError as http_error:
                code = http_error.code
                logging.info(
                    "Booting server at [%s], getting HTTP status [%s]",
                    path, code)
                yield gen.Task(loop.add_timeout, loop.time() + wait_time)
            else:
                break

        logging.info("Server [%s] at address [%s:%s] has booted! Have at it.",
                     path, ip, port)

    @gen.coroutine
    def _proxy_remove(self, path):
        '''Remove a path from the proxy.'''

        url = "{}/api/routes/{}".format(self.proxy_endpoint, path.lstrip('/'))
        headers = {"Authorization": "token {}".format(self.proxy_token)}
        req = HTTPRequest(url, method="DELETE", headers=headers)
        http_client = AsyncHTTPClient()

        try:
            yield http_client.fetch(req)
        except HTTPError as e:
            logging.error("Failed to delete route [%s]: %s", path, e)

    @gen.coroutine
    def delete(self):
        payload = json.loads(self.request.body.decode('utf-8'))

        try:
            container = PooledContainer(id=payload["containerId"],
                                        path=payload["containerPath"])
        except KeyError:
            raise tornado.web.HTTPError(
                500, 'Got incomplete request from Girder')

        gc, user = yield parse_request_body(payload)

        try:
            logging.info("Releasing container [%s].", container)
            yield [
                self.spawner.shutdown_notebook_server(container.id),
                self._proxy_remove(container.path)
            ]
            logging.info("Container [%s] has been released.", container)
        except Exception as e:
            logging.error("Unable to release container [%s]: %s", container, e)
            raise tornado.web.HTTPError(
                500, "Unable to remove container, contact admin")

        vol_name = "%s_%s" % (payload['folderId'], user['login'])
        dest = os.path.join(payload['mountPoint'], 'data')
        logging.info("Unmounting %s", dest)
        subprocess.call("umount %s" % dest, shell=True)

        # upload notebooks
        user_id = gc.get("/user/me")["_id"]
        params = {'parentType': 'user', 'parentId': user_id,
                  'name': 'Private'}
        homeDir = list(gc.listResource("/folder", params))[0]["_id"]
        gc.blacklist.append("data")
        try:
            gc.upload('{}/*.ipynb'.format(HOSTDIR + payload["mountPoint"]),
                      homeDir, reuse_existing=True)
        except girder_client.HttpError:
            logging.warn("Something went wrong with data upload"
                         ", should backup data")
            pass  # upload failed, keep going

        cli = docker.Client(base_url=DOCKER_URL)
        try:
            logging.info("Removing volume: %s", vol_name)
            cli.remove_volume(vol_name)
        except Exception as e:
            logging.error("Unable to remove volume [%s]: %s", vol_name, e)
            pass

    @gen.coroutine
    def get(self):
        http_client = AsyncHTTPClient()
        logging.debug('Polling proxy for idle containers')
        headers = {'Authorization': 'token %s' % self.proxy_token}
        req = HTTPRequest(self.proxy_endpoint + '/api/routes', headers=headers)
        try:
            resp = yield http_client.fetch(req)
        except HTTPError as e:
            logging.error("Failed to poll proxy for idle containers: %s", e)
            raise tornado.web.HTTPError(
                400, 'Failed to connect to proxy'
            )

        proxy_entries = json.loads(resp.body.decode('utf8', 'replace'))
        result = {}
        for entry in proxy_entries.values():
            try:
                result[entry['container_id']] = entry['last_activity']
            except KeyError:
                pass
        self.write(json.dumps(result))
        self.finish()

if __name__ == "__main__":
    logging.getLogger().setLevel(logging.INFO)
    handlers = [
        (r"/", MainHandler),
    ]

    docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock')

    command_default = (
        'jupyter notebook --no-browser'
        ' --port {port} --ip=0.0.0.0'
        ' --NotebookApp.base_url=/{base_path}'
        ' --NotebookApp.port_retries=0'
    )

    # TODO: read from env / config file
    container_config = dockworker.ContainerConfig(
        command=command_default,
        image="xarthisius/singleanonuser",
        mem_limit="1024m",
        cpu_shares=None,
        container_ip='172.17.0.1',
        container_port='8888',
        container_user='jovyan',
        host_network=False,
        host_directories=None,
        extra_hosts=[]
    )

    spawner = dockworker.DockerSpawner(docker_host,
                                       timeout=30,
                                       version="auto",
                                       max_workers=4,
                                       assert_hostname=False,
                                       )

    settings = dict(
        spawner=spawner,
        container_name_pattern=re.compile('tmp\.([^.]+)\.(.+)\Z'),
        pool_name="tmpnb",
        container_config=container_config,
        proxy_token=os.environ.get('CONFIGPROXY_AUTH_TOKEN', "devtoken"),
        proxy_endpoint=os.environ.get(
            'CONFIGPROXY_ENDPOINT', "http://127.0.0.1:8001"),
    )
    app = tornado.web.Application(handlers, **settings)
    app.listen(9005)
    tornado.ioloop.IOLoop.current().start()
<commit_msg>Read some container config options from env<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2016, Data Exploration Lab
# Distributed under the terms of the Modified BSD License.

from collections import namedtuple
import errno
import datetime
import json
import logging
import os
import random
import re
import socket
import string
import subprocess

import docker
import girder_client
from dateutil.parser import parse as parse_date
import tornado.ioloop
import tornado.web
from tornado import gen
from tornado.httpclient import HTTPRequest, HTTPError, AsyncHTTPClient

import dockworker

AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")

GIRDER_API_URL = os.environ.get(
    "GIRDER_API_URL", "https://girder.hub.yt/api/v1")
DOCKER_URL = os.environ.get("DOCKER_URL", "unix://var/run/docker.sock")
HOSTDIR = os.environ.get("HOSTDIR", "/host")
MAX_FILE_SIZE = os.environ.get("MAX_FILE_SIZE", 200)

MOUNTS = {}

PooledContainer = namedtuple('PooledContainer', ['id', 'path'])

def sample_with_replacement(a, size):
    '''Get a random path. If Python had sampling with replacement built in,
    I would use that. The other alternative is numpy.random.choice, but
    numpy is overkill for this tiny bit of random pathing.'''
    return "".join([random.SystemRandom().choice(a) for x in range(size)])

def new_user(size):
    return sample_with_replacement(string.ascii_letters + string.digits, size)

def _safe_mkdir(dest):
    try:
        os.mkdir(dest)
    except OSError as e:
        if e.errno != 17:
            raise
        logging.warn("Failed to mkdir {}".format(dest))
        pass

@gen.coroutine
def cull_idle(proxy_url, proxy_token, timeout):
    cull_limit = datetime.datetime.utcnow() \
        - datetime.timedelta(seconds=timeout)

    http_client = AsyncHTTPClient()
    logging.debug("Polling proxy for idle containers")
    req = HTTPRequest(proxy_url + '/api/users',
                      headers={"Authorization": "token %s" % proxy_token})
    try:
        resp = yield http_client.fetch(req)
    except HTTPError as e:
        logging.error("Failed to poll proxy for idle containers: %s", e)

    users = json.loads(resp.body.decode('utf8', 'replace'))
    futures = []
    for user in users:
        last_activity = parse_date(user['last_activity'])
        if user['server'] and last_activity < cull_limit:
            logging.info(
                "Culling %s (inactive since %s)", user['name'], last_activity)
            # req = HTTPRequest(url=url+'/api/users/%s/server' % user['name'],
            #                  method='DELETE',
            #                  headers=auth_header,
            #                  )
            futures.append((user['name'], http_client.fetch(req)))
        elif user['server'] and last_activity > cull_limit:
            logging.debug("Not culling %s (active since %s)",
                          user['name'], last_activity)

    for (name, f) in futures:
        yield f
        logging.debug("Finished culling %s", name)

@gen.coroutine
def _get_api_key(gc):
    api_key = None
    for key in gc.get('/api_key'):
        if key['name'] == 'tmpnb' and key['active']:
            api_key = key['key']

    if api_key is None:
        api_key = gc.post('/api_key',
                          data={'name': 'tmpnb', 'active': True})['key']
    return api_key

@gen.coroutine
def parse_request_body(data):
    gc = girder_client.GirderClient(apiUrl=GIRDER_API_URL)
    gc.token = data['girder_token']
    user = gc.get("/user/me")
    if user is None:
        logging.warn("Bad gider token")
        raise tornado.web.HTTPError(
            401, 'Failed to authenticate with girder'
        )

    # Allow sysop to delete any notebook
    userId = data.get('userId', user['_id'])
    if userId != user['_id'] and user["admin"]:
        user = gc.get("/user/{id}".format(id=userId))
        logging.info("Overriding user %s", user["login"])

    logging.debug("USER = %s", json.dumps(user))
    return gc, user

class MainHandler(tornado.web.RequestHandler):

    @property
    def proxy_token(self):
        return self.settings['proxy_token']

    @property
    def pool_name(self):
        return self.settings['pool_name']

    @property
    def proxy_endpoint(self):
        return self.settings['proxy_endpoint']

    @property
    def container_config(self):
        return self.settings['container_config']

    @property
    def spawner(self):
        return self.settings['spawner']

    @property
    def container_name_pattern(self):
        return self.settings['container_name_pattern']

    @gen.coroutine
    def post(self):
        payload = json.loads(self.request.body.decode("utf-8"))
        gc, user = yield parse_request_body(payload)

        vol_name = "%s_%s" % (payload['folderId'], user['login'])
        cli = docker.Client(base_url=DOCKER_URL)
        volume = cli.create_volume(name=vol_name, driver='local')
        logging.info("Volume: %s created", vol_name)
        logging.info("Mountpoint: %s", volume['Mountpoint'])

        params = {'parentType': 'user', 'parentId': user["_id"],
                  'name': 'Notebooks'}
        homeDir = list(gc.listResource("/folder", params))[0]["_id"]

        items = [item["_id"] for item in gc.listItem(homeDir)
                 if item["name"].endswith("pynb")]
        # TODO: should be done in one go with /resource endpoint
        #  but client doesn't have it yet
        for item in items:
            gc.downloadItem(item, HOSTDIR + volume["Mountpoint"])

        # TODO: read uid/gid from env/config
        for item in os.listdir(HOSTDIR + volume["Mountpoint"]):
            os.chown(os.path.join(HOSTDIR + volume["Mountpoint"], item),
                     1000, 100)

        dest = os.path.join(volume["Mountpoint"], "data")
        _safe_mkdir(HOSTDIR + dest)

        # FUSE is silly and needs to have mirror inside container
        if not os.path.isdir(dest):
            os.makedirs(dest)
        api_key = yield _get_api_key(gc)
        cmd = "girderfs -c direct --api-url {} --api-key {} {} {}".format(
            GIRDER_API_URL, api_key, dest, payload['folderId'])
        logging.info("Calling: %s", cmd)
        subprocess.call(cmd, shell=True)

        # CREATE CONTAINER
        # REGISTER CONTAINER WITH PROXY
        container = yield self._launch_container(volume)

        self.write(dict(mountPoint=volume['Mountpoint'],
                        containerId=container.id,
                        containerPath=container.path))
        self.finish()

    @gen.coroutine
    def _launch_container(self, volume):
        user = new_user(12)
        path = "user/" + user
        container_name = 'tmp.{}.{}'.format(self.pool_name, user)
        volume_bindings = {volume["Name"]: {
            'bind': "/home/jovyan/work", 'mode': 'rw'}}
        if not self.container_name_pattern.match(container_name):
            pattern = self.container_name_pattern.pattern
            raise Exception("[{}] does not match [{}]!".format(container_name,
                                                               pattern))

        logging.info("Launching new notebook server [%s] at path [%s].",
                     container_name, path)
        create_result = yield self.spawner.create_notebook_server(
            base_path=path, container_name=container_name,
            container_config=self.container_config,
            volume_bindings=volume_bindings
        )
        container_id, host_ip, host_port = create_result
        logging.info(
            "Created notebook server [%s] for path [%s] at [%s:%s]",
            container_name, path, host_ip, host_port)

        # Wait for the server to launch within the container before adding it
        # to the pool or serving it to a user.

        yield self._wait_for_server(host_ip, host_port, path)

        http_client = AsyncHTTPClient()
        headers = {"Authorization": "token {}".format(self.proxy_token)}

        proxy_endpoint = "{}/api/routes/{}".format(self.proxy_endpoint, path)
        body = json.dumps({
            "target": "http://{}:{}".format(host_ip, host_port),
            "container_id": container_id,
        })

        logging.debug("Proxying path [%s] to port [%s].", path, host_port)
        req = HTTPRequest(proxy_endpoint, method="POST", headers=headers,
                          body=body)
        try:
            yield http_client.fetch(req)
            logging.info("Proxied path [%s] to port [%s].", path, host_port)
        except HTTPError as e:
            logging.error("Failed to create proxy route to [%s]: %s", path, e)

        container = PooledContainer(id=container_id, path=path)
        raise gen.Return(container)

    @gen.coroutine
    def _wait_for_server(self, ip, port, path, timeout=10, wait_time=0.2):
        '''Wait for a server to show up within a newly launched container.'''

        logging.info("Waiting for a container to launch at [%s:%s].", ip, port)
        loop = tornado.ioloop.IOLoop.current()
        tic = loop.time()

        # Docker starts listening on a socket before the container is fully
        # launched. Wait for that, first.

        while loop.time() - tic < timeout:
            try:
                socket.create_connection((ip, port))
            except socket.error as e:
                logging.warn("Socket error on boot: %s", e)
                if e.errno != errno.ECONNREFUSED:
                    logging.warn("Error attempting to connect to [%s:%i]: %s",
                                 ip, port, e)
                yield gen.Task(loop.add_timeout, loop.time() + wait_time)
            else:
                break

        # Fudge factor of IPython notebook bootup.
        # TODO: Implement a webhook in IPython proper to call out when the
        # notebook server is booted.
        yield gen.Task(loop.add_timeout, loop.time() + .5)

        # Now, make sure that we can reach the Notebook server.
        http_client = AsyncHTTPClient()
        req = HTTPRequest("http://{}:{}/{}".format(ip, port, path))

        while loop.time() - tic < timeout:
            try:
                yield http_client.fetch(req)
            except HTTPError as http_error:
                code = http_error.code
                logging.info(
                    "Booting server at [%s], getting HTTP status [%s]",
                    path, code)
                yield gen.Task(loop.add_timeout, loop.time() + wait_time)
            else:
                break

        logging.info("Server [%s] at address [%s:%s] has booted! Have at it.",
                     path, ip, port)

    @gen.coroutine
    def _proxy_remove(self, path):
        '''Remove a path from the proxy.'''

        url = "{}/api/routes/{}".format(self.proxy_endpoint, path.lstrip('/'))
        headers = {"Authorization": "token {}".format(self.proxy_token)}
        req = HTTPRequest(url, method="DELETE", headers=headers)
        http_client = AsyncHTTPClient()

        try:
            yield http_client.fetch(req)
        except HTTPError as e:
            logging.error("Failed to delete route [%s]: %s", path, e)

    @gen.coroutine
    def delete(self):
        payload = json.loads(self.request.body.decode('utf-8'))

        try:
            container = PooledContainer(id=payload["containerId"],
                                        path=payload["containerPath"])
        except KeyError:
            raise tornado.web.HTTPError(
                500, 'Got incomplete request from Girder')

        gc, user = yield parse_request_body(payload)

        try:
            logging.info("Releasing container [%s].", container)
            yield [
                self.spawner.shutdown_notebook_server(container.id),
                self._proxy_remove(container.path)
            ]
            logging.info("Container [%s] has been released.", container)
        except Exception as e:
            logging.error("Unable to release container [%s]: %s", container, e)
            raise tornado.web.HTTPError(
                500, "Unable to remove container, contact admin")

        vol_name = "%s_%s" % (payload['folderId'], user['login'])
        dest = os.path.join(payload['mountPoint'], 'data')
        logging.info("Unmounting %s", dest)
        subprocess.call("umount %s" % dest, shell=True)

        # upload notebooks
        user_id = gc.get("/user/me")["_id"]
        params = {'parentType': 'user', 'parentId': user_id,
                  'name': 'Private'}
        homeDir = list(gc.listResource("/folder", params))[0]["_id"]
        gc.blacklist.append("data")
        try:
            gc.upload('{}/*.ipynb'.format(HOSTDIR + payload["mountPoint"]),
                      homeDir, reuse_existing=True)
        except girder_client.HttpError:
            logging.warn("Something went wrong with data upload"
                         ", should backup data")
            pass  # upload failed, keep going

        cli = docker.Client(base_url=DOCKER_URL)
        try:
            logging.info("Removing volume: %s", vol_name)
            cli.remove_volume(vol_name)
        except Exception as e:
            logging.error("Unable to remove volume [%s]: %s", vol_name, e)
            pass

    @gen.coroutine
    def get(self):
        http_client = AsyncHTTPClient()
        logging.debug('Polling proxy for idle containers')
        headers = {'Authorization': 'token %s' % self.proxy_token}
        req = HTTPRequest(self.proxy_endpoint + '/api/routes', headers=headers)
        try:
            resp = yield http_client.fetch(req)
        except HTTPError as e:
            logging.error("Failed to poll proxy for idle containers: %s", e)
            raise tornado.web.HTTPError(
                400, 'Failed to connect to proxy'
            )

        proxy_entries = json.loads(resp.body.decode('utf8', 'replace'))
        result = {}
        for entry in proxy_entries.values():
            try:
                result[entry['container_id']] = entry['last_activity']
            except KeyError:
                pass
        self.write(json.dumps(result))
        self.finish()

if __name__ == "__main__":
    logging.getLogger().setLevel(logging.INFO)
    handlers = [
        (r"/", MainHandler),
    ]

    docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock')

    command_default = (
        'jupyter notebook --no-browser'
        ' --port {port} --ip=0.0.0.0'
        ' --NotebookApp.base_url=/{base_path}'
        ' --NotebookApp.port_retries=0'
    )
    container_image = os.environ.get(
        'DOCKER_IMAGE', 'xarthisius/singleanonuser')
    container_memlimit = os.environ.get(
        'DOCKER_MEMLIMIT', '2048m')
    container_ip = os.environ.get(
        'DOCKER_GATEWAY', '172.17.0.1')

    # TODO: read from env / config file
    container_config = dockworker.ContainerConfig(
        command=command_default,
        image=container_image,
        mem_limit=container_memlimit,
        cpu_shares=None,
        container_ip=container_ip,
        container_port='8888',
        container_user='jovyan',
        host_network=False,
        host_directories=None,
        extra_hosts=[]
    )

    spawner = dockworker.DockerSpawner(docker_host,
                                       timeout=30,
                                       version="auto",
                                       max_workers=4,
                                       assert_hostname=False,
                                       )

    settings = dict(
        spawner=spawner,
        container_name_pattern=re.compile('tmp\.([^.]+)\.(.+)\Z'),
        pool_name="tmpnb",
        container_config=container_config,
        proxy_token=os.environ.get('CONFIGPROXY_AUTH_TOKEN', "devtoken"),
        proxy_endpoint=os.environ.get(
            'CONFIGPROXY_ENDPOINT', "http://127.0.0.1:8001"),
    )
    app = tornado.web.Application(handlers, **settings)
    app.listen(9005)
    tornado.ioloop.IOLoop.current().start()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import csv
import datetime
class Transaction:
    def getDate(self):
        return self.date
    def getAmount(self):
        return self.amount
    def getPaymode(self):
        return self.paymode if hasattr(self,"paymode") else 0
    def getInfo(self):
        return self.info if hasattr(self,"info") else ""
    def getPayee(self):
        return self.payee if hasattr(self,"payee") else ""
    def getMemo(self):
        return self.memo if hasattr(self,"memo") else self.getInfo()
    def getCatetory(self):
        return self.category if hasattr(self,"category") else ""
    def getTagsString(self):
        return " ".join(self.tags) if hasattr(self,"tags") else ""
    def __repr__(self):
        return "Transaction(date='{}', amount='{}', info='{}'".format(self.date,self.amount,self.info)

def loadSkandiabanken(path):
    print("skandiabanken")
    paymodes={"Overføring":4, "Avtalegiro":8}
    r=[]
    with open(path, newline='',encoding="iso-8859-1") as csvfile:
        reader = csv.reader(csvfile, delimiter="\t", quotechar='"')
        for row in reader:
            print(row)
            if(len(row)>=6 and row[0] != "" and row[1] != "RENTEDATO"):
              print(row)
              t=Transaction()
              t.date=datetime.datetime.strptime(row[0],"%Y-%m-%d")
              if row[5] != "":
                t.amount=float(row[5]) * -1.0
              if row[6] != "":
                t.amount=float(row[6]) * 1.0
              t.info=row[4]
              t.category=row[3]
              if t.category in paymodes:
                t.paymode=paymodes[t.category]
              r.append(t)
    return r

def loadFanaSparebank(path):
    print("Fana Sparebank")
    paymodes={"GEBYR":10,"GIRO":8,"VARER":1,"OVERFØRT":8,"OVFNETTB":4}
    r=[]
    with open(path, newline='',encoding="iso-8859-1") as csvfile:
        reader = csv.reader(csvfile, delimiter="\t")
        mode="siste"
        for row in reader:
            if row[0] == "Dato":
                mode = "siste"
            elif row[1] == "Rentedato":
                mode = "arkiv"
            elif mode == "siste":
                print(row)
                t=Transaction()
                t.date=datetime.datetime.strptime(row[0],"%d.%m.%Y")
                t.info=row[1]
                if row[3] != "":
                  t.amount=row[3].replace(",",".")
                else:
                  t.amount="-"+row[2].replace(",",".")
                r.append(t)
            elif mode == "arkiv":
                print(row)
                t=Transaction()
                t.date=datetime.datetime.strptime(row[0],"%d.%m.%Y")
                t.info=row[3]
                t.amount = row[4].replace(",",".")
                t.category=row[2]
                r.append(t)
                if t.category in paymodes:
                  t.paymode=paymodes[t.category]
    return r

def loadSpv(path):
    print("Sparebanken Vest")
    r=[]
    with open(path, newline='',encoding="iso-8859-1") as csvfile:
        reader = csv.reader(csvfile, delimiter="\t", quotechar='"')
        for row in reader:
            if row[0] != "Bokføringsdato" and row[0] != "Dato":
                print(row)
                t=Transaction()
                t.date=datetime.datetime.strptime(row[0],"%d.%m.%Y")
                t.info=row[1]
                t.amount=row[5]
                r.append(t)
    return r

def writeHomeBank(transactions, path):
    with open(path, 'w', newline='') as f:
        writer = csv.writer(f,delimiter=";")
        for t in transactions:
            ta=[]
            ta.append(t.getDate().strftime("%m-%d-%y"))
            ta.append(t.getPaymode())
            ta.append(t.getInfo())
            ta.append(t.getPayee())
            ta.append(t.getMemo())
            ta.append(t.getAmount())
            ta.append(t.getCatetory())
            ta.append(t.getTagsString())
            writer.writerow(ta)

handlers={
    "skandiabanken":loadSkandiabanken,
    "skb":loadSkandiabanken,
    "sparebankenvest":loadSpv,
    "spv":loadSpv,
    "fsb":loadFanaSparebank,
    "fanasparebank":loadFanaSparebank
    }

if __name__ == "__main__":
    import sys
    (_,bank,src,dst) = sys.argv
    if bank in handlers:
        reader=handlers[bank]
        writeHomeBank(reader(src),dst)
    else:
        print("Sorry, no reader defined for \"{}\"".format(bank))
<commit_msg>added a somewhat generic parser<commit_after>#!/usr/bin/env python3
import csv
import datetime
class Transaction:
    def getDate(self):
        return self.date
    def getAmount(self):
        return self.amount
    def getPaymode(self):
        return self.paymode if hasattr(self,"paymode") else 0
    def getInfo(self):
        return self.info if hasattr(self,"info") else ""
    def getPayee(self):
        return self.payee if hasattr(self,"payee") else ""
    def getMemo(self):
        return self.memo if hasattr(self,"memo") else self.getInfo()
    def getCatetory(self):
        return self.category if hasattr(self,"category") else ""
    def getTagsString(self):
        return " ".join(self.tags) if hasattr(self,"tags") else ""
    def __repr__(self):
        return "Transaction(date='{}', amount='{}', info='{}'".format(self.date,self.amount,self.info)

aliases={
    "date":["BOKFØRINGSDATO","Bokføringsdato","Dato","Bokført"],
    "paymode":["TYPE","Tekstkode"],
    "info":["TEKST","Spesifikasjon","Forklaring","Beskrivelse"],
    "amount":["Beløp NOK","Beløp","INN PÅ KONTO","Inn på konto","Innskudd"],
    "amount_neg":["UT FRA KONTO","Ut av konto","Uttak"]}

def find_value(key,record):
    if key == "amount":
        for realkey in aliases["amount"]:
            if realkey in record and record[realkey] != "":
                return record[realkey].replace(",",".")
        for realkey in aliases["amount_neg"]:
            if realkey in record and record[realkey] != "":
                return "-"+record[realkey].replace(",",".")
    for realkey in aliases[key]:
        if realkey in record:
            return record[realkey]

def loadGeneric(path):
    with open(path, newline='',encoding="iso-8859-1") as csvfile:
        delimiter="\t"
        if ";" in csvfile.readline():
            delimiter=";"
        csvfile.seek(0)
        reader = csv.reader(csvfile, delimiter=delimiter, quotechar='"')
        alldata=[e for e in reader if len(e)>0 and e[0] !=""]
        print(alldata[0])
        data=[dict(zip(alldata[0],e)) for e in alldata[1:]]
        r=[]
        for e in data:
            t=Transaction()
            t.amount=find_value("amount",e)
            t.info=find_value("info",e)
            t.category=find_value("paymode",e)
            if "-" in find_value("date",e):
                t.date=datetime.datetime.strptime(find_value("date",e),"%Y-%m-%d")
            else:
                t.date=datetime.datetime.strptime(find_value("date",e),"%d.%m.%Y")
            r.append(t)
        return r

def loadSkandiabanken(path):
    print("skandiabanken")
    paymodes={"Overføring":4, "Avtalegiro":8}
    r=[]
    with open(path, newline='',encoding="iso-8859-1") as csvfile:
        reader = csv.reader(csvfile, delimiter="\t", quotechar='"')
        for row in reader:
            print(row)
            if(len(row)>=6 and row[0] != "" and row[1] != "RENTEDATO"):
              print(row)
              t=Transaction()
              t.date=datetime.datetime.strptime(row[0],"%Y-%m-%d")
              if row[5] != "":
                t.amount=float(row[5]) * -1.0
              if row[6] != "":
                t.amount=float(row[6]) * 1.0
              t.info=row[4]
              t.category=row[3]
              if t.category in paymodes:
                t.paymode=paymodes[t.category]
              r.append(t)
    return r

def loadFanaSparebank(path):
    print("Fana Sparebank")
    paymodes={"GEBYR":10,"GIRO":8,"VARER":1,"OVERFØRT":8,"OVFNETTB":4}
    r=[]
    with open(path, newline='',encoding="iso-8859-1") as csvfile:
        reader = csv.reader(csvfile, delimiter="\t")
        mode="siste"
        for row in reader:
            if row[0] == "Dato":
                mode = "siste"
            elif row[1] == "Rentedato":
                mode = "arkiv"
            elif mode == "siste":
                print(row)
                t=Transaction()
                t.date=datetime.datetime.strptime(row[0],"%d.%m.%Y")
                t.info=row[1]
                if row[3] != "":
                  t.amount=row[3].replace(",",".")
                else:
                  t.amount="-"+row[2].replace(",",".")
                r.append(t)
            elif mode == "arkiv":
                print(row)
                t=Transaction()
                t.date=datetime.datetime.strptime(row[0],"%d.%m.%Y")
                t.info=row[3]
                t.amount = row[4].replace(",",".")
                t.category=row[2]
                r.append(t)
                if t.category in paymodes:
                  t.paymode=paymodes[t.category]
    return r

def loadSpv(path):
    print("Sparebanken Vest")
    r=[]
    with open(path, newline='',encoding="iso-8859-1") as csvfile:
        reader = csv.reader(csvfile, delimiter="\t", quotechar='"')
        for row in reader:
            if row[0] != "Bokføringsdato" and row[0] != "Dato":
                print(row)
                t=Transaction()
                t.date=datetime.datetime.strptime(row[0],"%d.%m.%Y")
                t.info=row[1]
                t.amount=row[5]
                r.append(t)
    return r

def writeHomeBank(transactions, path):
    with open(path, 'w', newline='') as f:
        writer = csv.writer(f,delimiter=";")
        for t in transactions:
            ta=[]
            ta.append(t.getDate().strftime("%m-%d-%y"))
            ta.append(t.getPaymode())
            ta.append(t.getInfo())
            ta.append(t.getPayee())
            ta.append(t.getMemo())
            ta.append(t.getAmount())
            ta.append(t.getCatetory())
            ta.append(t.getTagsString())
            writer.writerow(ta)

handlers={
    "skandiabanken":loadSkandiabanken,
    "skb":loadSkandiabanken,
    "sparebankenvest":loadSpv,
    "spv":loadSpv,
    "fsb":loadFanaSparebank,
    "fanasparebank":loadFanaSparebank,
    "generic":loadGeneric
    }

if __name__ == "__main__":
    import sys
    (_,bank,src,dst) = sys.argv
    if bank in handlers:
        reader=handlers[bank]
        writeHomeBank(reader(src),dst)
    else:
        print("Sorry, no reader defined for \"{}\"".format(bank))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import grovepi
import grove_i2c_motor_driver
import itg3200
import time

us_ranger_f = 8
us_ranger_b = 2
us_ranger_l = 7
us_ranger_r = 3
gyro = itg3200.SensorITG3200(1, 0x68) # update with your bus number and address
gyro.default_init()
lr_s = [100, 100]
fb_s = [100, 100]

def normalize_s(xx_s):
    if (xx_s[1] <= xx_s[0]):
        xx_s[1] *= 100.0/xx_s[0]
        xx_s[0] = 100
    elif (xx_s[0] <= xx_s[1]):
        xx_s[0] *= 100.0/xx_s[1]
        xx_s[1] = 100
    print xx_s

def forward(mlr, lr_s, gz):
    #FORWARD
    print("Forward")
    if gz < -10: #turning closewise
        lr_s[0] *= 95/100.0
        lr_s[1] *= 100/95.0

    elif gz > 10: #turning counter-clockwise
        lr_s[0] *= 100/95.0
        lr_s[1] *= 95/100.0
    normalize_s(lr_s)
    mlr.MotorSpeedSetAB(lr_s[0], lr_s[1])    #defines the speed of motor 1 and motor 2;
    mlr.MotorDirectionSet(0b0110)    #"0b1010" defines the output polarity, "10" means the M+ is "positive" while the M- is "negtive"

def backward(mlr, s):
    #BACKWARD
    print("Backward")
    mlr.MotorSpeedSetAB(s,s)
    mlr.MotorDirectionSet(0b1001)    #0b0101    Rotating in the opposite direction

def leftward(mfb, s):
    #LEFTWARD
    print("Leftward")
    mfb.MotorSpeedSetAB(s,s)
    mfb.MotorDirectionSet(0b0110)    #0b0101    Rotating in the opposite direction

def rightward(mfb, s):
    #RIGHTWARD
    print("RIGHTWARD")
    mfb.MotorSpeedSetAB(s,s)
    mfb.MotorDirectionSet(0b1001)    #0b0101    Rotating in the opposite direction

def stop(mxx):
    #STOP
    print("Stop")
    mxx.MotorSpeedSetAB(0,0)
    time.sleep(1)

try:
    try:
        mfb = grove_i2c_motor_driver.motor_driver(address=0x0f)
        mlr = grove_i2c_motor_driver.motor_driver(address=0x0a)

        #forward(mlr, 50);
        #time.sleep(2);
        while 1:
            time.sleep(0.5)
            gz = gyro.read_data()[2]+40
            print gz
            forward(mlr, lr_s, gz)
            time.sleep(0.1)

    except IOError:
        print("Unable to find the motor driver, check the addrees and press reset on the motor driver and try again")

    while True:
        try:
            # Read distance value from Ultrasonic
            dist_f = grovepi.ultrasonicRead(us_ranger_f)
            dist_b = grovepi.ultrasonicRead(us_ranger_b)
            dist_l = grovepi.ultrasonicRead(us_ranger_l)
            dist_r = grovepi.ultrasonicRead(us_ranger_r)
            print(dist_f, dist_r, dist_b, dist_l)
        except TypeError:
            print ("Error")
        except IOError:
            print ("Error")
except KeyboardInterrupt: # stop motors before exit
    stop(mlr)
    stop(mfb)
    sys.exit()
<commit_msg>gyro assit added for all directions<commit_after>#!/usr/bin/env python

import sys
import grovepi
import grove_i2c_motor_driver
import itg3200
import time

# pin numbers for ultrasonic rangers
us_ranger_f = 8
us_ranger_b = 2
us_ranger_l = 7
us_ranger_r = 3
# update with your bus number and address
gyro = itg3200.SensorITG3200(1, 0x68)
gyro.default_init()
# lists storing speeds
lr_s = [100, 100]
fb_s = [100, 100]

# normalize the speed by setting the faster wheel to ms
# and scale the other wheel speed accordingly
def normalize_s(xx_s, ms):
    # set ms to 100 if out of boundary
    if (ms > 100 or ms < 0):
        ms = 100.0
    else:
        ms += 0.0

    if (xx_s[1] <= xx_s[0]):
        xx_s[1] *= ms/xx_s[0]
        xx_s[0] = ms
    elif (xx_s[0] <= xx_s[1]):
        xx_s[0] *= ms/xx_s[1]
        xx_s[1] = ms
    print xx_s

def forward(mlr, lr_s, gz):
    #FORWARD
    print("Forward")
    if gz < -10: #turning closewise
        lr_s[0] *= 95/100.0
        lr_s[1] *= 100/95.0

    elif gz > 10: #turning counter-clockwise
        lr_s[0] *= 100/95.0
        lr_s[1] *= 95/100.0
    normalize_s(lr_s, 50)
    mlr.MotorSpeedSetAB(lr_s[0], lr_s[1])    #defines the speed of motor 1 and motor 2;
    mlr.MotorDirectionSet(0b0110)    #"0b1010" defines the output polarity, "10" means the M+ is "positive" while the M- is "negtive"

def backward(mlr, lr_s, gz):
    #BACKWARD
    print("Backward")
    if gz < -10: #turning closewise
        lr_s[0] *= 100/95.0
        lr_s[1] *= 95/100.0

    elif gz > 10: #turning counter-clockwise
        lr_s[0] *= 95/100.0
        lr_s[1] *= 100/95.0
    normalize_s(lr_s, 50)
    mlr.MotorSpeedSetAB(lr_s[0], lr_s[1])    #defines the speed of motor 1 and motor 2;
    mlr.MotorDirectionSet(0b1001)    #0b0101    Rotating in the opposite direction

def leftward(mfb, fb_s, gz):
    #LEFTWARD
    print("Leftward")
    forward(mfb, fb_s, gz)

def rightward(mfb, fb_s, gz):
    #RIGHTWARD
    print("RIGHTWARD")
    backward(mfb, fb_s, gz)

def stop(mxx):
    #STOP
    print("Stop")
    mxx.MotorSpeedSetAB(0,0)
    time.sleep(1)

try:
    try:
        mfb = grove_i2c_motor_driver.motor_driver(address=0x0f)
        mlr = grove_i2c_motor_driver.motor_driver(address=0x0a)

    except IOError:
        print("Unable to find the motor driver, check the addrees and press reset on the motor driver and try again")

    while True:
        try:
            time.sleep(0.1)
            # storing the z-axis gyro value
            gz = gyro.read_data()[2]+40
            print gz
            # Read distance value from Ultrasonic
            dist_f = grovepi.ultrasonicRead(us_ranger_f)
            dist_b = grovepi.ultrasonicRead(us_ranger_b)
            dist_l = grovepi.ultrasonicRead(us_ranger_l)
            dist_r = grovepi.ultrasonicRead(us_ranger_r)
            print(dist_f, dist_r, dist_b, dist_l)

            if (dist_f > 100):
                forward(mlr, lr_s, gz)
            elif (dist_f < 95):
                backward(mlr, lr_s, gz)

            if (dist_l > 100):
                forward(mfb, fb_s, gz)
            elif (dist_l < 95):
                backward(mfb, fb_s, gz)

        except TypeError:
            print ("Error")
        except IOError:
            print ("Error")
except KeyboardInterrupt: # stop motors before exit
    stop(mlr)
    stop(mfb)
    sys.exit()
<|endoftext|>