<commit_before>#!/usr/bin/python
import os
import sys

command_list = {

"addr": (
"See Bitcoin address of a private key.",

"""\
Usage: sx addr
Read private key from STDIN and output Bitcoin address.\
"""
),

"broadcast-tx": (
"Broadcast tx to network.",

"""\
Usage: sx broadcast-tx FILENAME
Broadcast the transaction to the Bitcoin network.

  $ sx broadcast-tx txfile.tx
\
"""
),

"genpriv": (
"Generate a private key deterministically from a seed.",

"""\
Usage: sx genpriv N [CHANGE]
Generate private keys from a wallet seed.

  $ cat wallet.seed | sx genpriv 0
  5Jmb4EYzEqj63rkPwADFY7WyGV2kga3YB1HfDAzg9dHNG57NMPu
  $ cat wallet.seed | sx genpriv 1
  5KjCYpPyxU2e88S57b1naKUsJ1JNjCudkFSQPxqcYyBYgzzahNe
\
"""
),

"genpub": (
"Generate a Bitcoin address deterministically from a wallet seed or "\
"master public key.",

"""\
Usage: sx genpub N [CHANGE]
Generate Bitcoin addresses from a wallet seed or master public key.

  $ cat wallet.seed | sx genpub 0
  1a4b47AC4ydSnAAcTNH1qozHq2pwJb644
\
"""
),

"qrcode": (
"Generate Bitcoin QR codes offline.",

"""\
Usage: sx qrcode

Make sure you have the program 'qrencode' installed first.

  $ sudo apt-get install qrencode
  $ sx qrcode 13Ft7SkreJY9D823NPm4t6D1cBqLYTJtAe qrcode.png
\
"""
),

"balance": (
"Show balance of a Bitcoin address.",

"""\
Usage: sx balance ADDRESS

The balance tool uses a network connection to make requests against the
load balancer backend.

  $ echo 134HfD2fdeBTohfx8YANxEpsYXsv5UoWyz | sx balance
  100000
\
"""
),

"history": (
"Get list of output points, values, and their spends for an address. grep can "
"filter for just unspent outputs which can be fed into mktx.",

"""\
Usage: sx history ADDRESS

The history tool uses a network connection to make requests against the
load balancer backend.

  $ echo 134HfD2fdeBTohfx8YANxEpsYXsv5UoWyz | sx history
  Output Hash:Index                           Output Height   Value (Satoshis) Spend Hash:Index                Spend Height
  97e06e49dfdd26c5a904670971ccf4c7fe7d9da53cb379bf9b442fc9427080b3:1  247683 100000      Unspent                                 230529504

\
"""
),

"mktx": (
"Create an unsigned tx.",

"""\
Usage: sx mktx FILENAME [-i TXHASH:INDEX]... [-o ADDRESS:VALUE]

Construct the transaction:

  $ sx mktx txfile.tx -i 97e06e49dfdd26c5a904670971ccf4c7fe7d9da53cb379bf9b442fc9427080b3:1 -o 13Ft7SkreJY9D823NPm4t6D1cBqLYTJtAe:90000

The VALUE field is in Satoshis.
\
"""
),

"mpk": (
"Extract a master public key from a deterministic wallet seed.",

"""\
Usage: sx mpk

Extract a master public key from a deterministic wallet seed.

  $ sx newseed > wallet.seed
  $ cat wallet.seed
  b220b5bd2909df1d74b71c9e664233bf
  $ cat wallet.seed | sx mpk > master_public.key
\
"""
),

"newkey": (
"Create a new private key.",

"""\
Usage: sx newkey

  $ sx newkey
  5KPFsatiYrJcvCSRrDbtx61822vZjeGGGx3wu38pQDHRF8eVJ8H
\
"""
),

"newseed": (
"Create a new deterministic wallet seed.",

"""\
Usage: sx newseed

  $ sx newseed
  b220b5bd2909df1d74b71c9e664233bf
\
"""
),

"sendtx": (
"Send transaction to a single node.",

"""\
Usage: sx sendtx FILENAME

Send transaction to one Bitcoin node on localhost port 4009:

  $ sx sendtx txfile.tx localhost 4009
\
"""
),

"showtx": (
"Show the details of a transaction.",

"""\
Usage: sx showtx FILENAME

'showtx' allows inspecting of tx files.

  $ sx showtx txfile.tx
  hash: 4d25b18ed094ad68f75f21692d8540f45ceb90b240a521b8f191e95d8b6b8bb0
  version: 1  locktime: 0
  Input:
    previous output:
  97e06e49dfdd26c5a904670971ccf4c7fe7d9da53cb379bf9b442fc9427080b3:0
    script:   sequence: 4294967295
  Output:
    value: 90000
    script: dup hash160 [ 18c0bd8d1818f1bf99cb1df2269c645318ef7b73 ] equalverify
  checksig
    address: 13Ft7SkreJY9D823NPm4t6D1cBqLYTJtAe
\
"""
),

"sign-input": (
"Sign a transaction input.",

"""\
Usage: sx sign-input FILENAME INDEX

Sign a transaction input.

Note how the input script in the following transaction is empty.

  $ sx showtx txfile.tx
  hash: 4d25b18ed094ad68f75f21692d8540f45ceb90b240a521b8f191e95d8b6b8bb0
  version: 1  locktime: 0
  Input:
    previous output:
  97e06e49dfdd26c5a904670971ccf4c7fe7d9da53cb379bf9b442fc9427080b3:0
    script:   sequence: 4294967295
  Output:
    value: 90000
    script: dup hash160 [ 18c0bd8d1818f1bf99cb1df2269c645318ef7b73 ] equalverify
  checksig
    address: 13Ft7SkreJY9D823NPm4t6D1cBqLYTJtAe

We will now sign the first input using our private key.

  $ cat private.key | sx sign-input txfile.tx 0
  $ sx showtx txfile.tx
  hash: 4a8be467fb75f0f757649348dbb05762142236ec236ac9e55e4683d7083ffca2
  version: 1  locktime: 0
  Input:
    previous output:
  97e06e49dfdd26c5a904670971ccf4c7fe7d9da53cb379bf9b442fc9427080b3:0
    script: [
  304602210083f8b5131f6399cdc01ed8368f6137f288f771e40834a94d65cee6b380fec873022100da5ed6d1d283f228752a88c2d6f629587b2a6fbc102202ae7c20f48af7c0533c01
  ] [
  04e3af7cdfaab6ba2f14001a7ea0a490f40c6ba43607be92a05111d2e8fc1eecf6fa2c248b3e25ddf7db081684f6f49dc83fd7b9b5a3a88a2e6b83b918f8972351
  ]  sequence: 4294967295
    address: 134HfD2fdeBTohfx8YANxEpsYXsv5UoWyz
  Output:
    value: 90000
    script: dup hash160 [ 18c0bd8d1818f1bf99cb1df2269c645318ef7b73 ] equalverify
  checksig
    address: 13Ft7SkreJY9D823NPm4t6D1cBqLYTJtAe

Now the input script is prepared, and the transaction is signed.
\
"""
),

"mnemonic": (
"Work with Electrum compatible mnemonics (12 words wallet seed).",

"""\
Usage: sx mnemonic

Electrum compatible 12 word seeds.

  $ echo 148f0a1d77e20dbaee3ff920ca40240d | sx mnemonic
  people blonde admit dart couple different truth common alas stumble time cookie
  $ echo "people blonde admit dart couple different truth common alas
stumble time cookie" | sx mnemonic
  148f0a1d77e20dbaee3ff920ca40240d
\
"""
),

"watchtx": (
"Watch transactions from the network searching for a certain hash.",

"""\
Usage: sx watchtx [TXHASH]...

Watch transactions from the network searching for a certain hash.
\
"""
),

}

def display_usage():
    print "Usage: sx COMMAND [ARGS]..."
    print
    print "The most commonly used sx commands are:"
    print
    for cmd in sorted(command_list.iterkeys()):
        short_desc = command_list[cmd][0]
        line = "   %s" % cmd
        line += " " * (15 - len(cmd))
        line += short_desc
        print line
    print
    print "See 'sx help COMMAND' for more information on a specific command."
    print
    print "SpesmiloXchange home page: <http://sx.dyne.org/>"

def display_help(command):
    assert command in command_list
    long_desc = command_list[command][1]
    print long_desc
    return 0

def display_bad(command):
    print "sx: '%s' is not a sx command. See 'sx --help'." % command
    return 1

def main(argv):
    if len(argv) == 1:
        display_usage()
        return 1
    command = argv[1]
    # args as one string we can pass to the sx sub-command
    args = " ".join(argv[2:])
    if command == "help" or command == "--help" or command == "-h":
        if not args:
            display_usage()
            return 0
        return display_help(args)
    elif command in command_list:
        # make will come and substitute @corebindir@
        execstr = "@corebindir@/sx-%s %s" % (command, args)
        return os.system(execstr)
    else:
        return display_bad(command)
    return 0

if __name__ == "__main__":
    sys.exit(main(sys.argv))

<commit_msg>removed ugly trailing spaces on some commands help.<commit_after>#!/usr/bin/python
import os
import sys

command_list = {

"addr": (
"See Bitcoin address of a private key.",

"""\
Usage: sx addr
Read private key from STDIN and output Bitcoin address.\
"""
),

"broadcast-tx": (
"Broadcast tx to network.",

"""\
Usage: sx broadcast-tx FILENAME
Broadcast the transaction to the Bitcoin network.

  $ sx broadcast-tx txfile.tx
\
"""
),

"genpriv": (
"Generate a private key deterministically from a seed.",

"""\
Usage: sx genpriv N [CHANGE]
Generate private keys from a wallet seed.

  $ cat wallet.seed | sx genpriv 0
  5Jmb4EYzEqj63rkPwADFY7WyGV2kga3YB1HfDAzg9dHNG57NMPu
  $ cat wallet.seed | sx genpriv 1
  5KjCYpPyxU2e88S57b1naKUsJ1JNjCudkFSQPxqcYyBYgzzahNe
\
"""
),

"genpub": (
"Generate a Bitcoin address deterministically from a wallet seed or "\
"master public key.",

"""\
Usage: sx genpub N [CHANGE]
Generate Bitcoin addresses from a wallet seed or master public key.

  $ cat wallet.seed | sx genpub 0
  1a4b47AC4ydSnAAcTNH1qozHq2pwJb644
\
"""
),

"qrcode": (
"Generate Bitcoin QR codes offline.",

"""\
Usage: sx qrcode

Make sure you have the program 'qrencode' installed first.

  $ sudo apt-get install qrencode
  $ sx qrcode 13Ft7SkreJY9D823NPm4t6D1cBqLYTJtAe qrcode.png
\
"""
),

"balance": (
"Show balance of a Bitcoin address.",

"""\
Usage: sx balance ADDRESS

The balance tool uses a network connection to make requests against the
load balancer backend.

  $ echo 134HfD2fdeBTohfx8YANxEpsYXsv5UoWyz | sx balance
  100000
\
"""
),

"history": (
"Get list of output points, values, and their spends for an address. grep can "
"filter for just unspent outputs which can be fed into mktx.",

"""\
Usage: sx history ADDRESS

The history tool uses a network connection to make requests against the
load balancer backend.

  $ echo 134HfD2fdeBTohfx8YANxEpsYXsv5UoWyz | sx history
  Output Hash:Index                           Output Height   Value (Satoshis) Spend Hash:Index                Spend Height
  97e06e49dfdd26c5a904670971ccf4c7fe7d9da53cb379bf9b442fc9427080b3:1  247683 100000      Unspent                                 230529504
\
"""
),

"mktx": (
"Create an unsigned tx.",

"""\
Usage: sx mktx FILENAME [-i TXHASH:INDEX]... [-o ADDRESS:VALUE]

Construct the transaction:

  $ sx mktx txfile.tx -i 97e06e49dfdd26c5a904670971ccf4c7fe7d9da53cb379bf9b442fc9427080b3:1 -o 13Ft7SkreJY9D823NPm4t6D1cBqLYTJtAe:90000

The VALUE field is in Satoshis.\
"""
),

"mpk": (
"Extract a master public key from a deterministic wallet seed.",

"""\
Usage: sx mpk

Extract a master public key from a deterministic wallet seed.

  $ sx newseed > wallet.seed
  $ cat wallet.seed
  b220b5bd2909df1d74b71c9e664233bf
  $ cat wallet.seed | sx mpk > master_public.key
\
"""
),

"newkey": (
"Create a new private key.",

"""\
Usage: sx newkey

  $ sx newkey
  5KPFsatiYrJcvCSRrDbtx61822vZjeGGGx3wu38pQDHRF8eVJ8H
\
"""
),

"newseed": (
"Create a new deterministic wallet seed.",

"""\
Usage: sx newseed

  $ sx newseed
  b220b5bd2909df1d74b71c9e664233bf
\
"""
),

"sendtx": (
"Send transaction to a single node.",

"""\
Usage: sx sendtx FILENAME

Send transaction to one Bitcoin node on localhost port 4009:

  $ sx sendtx txfile.tx localhost 4009
\
"""
),

"showtx": (
"Show the details of a transaction.",

"""\
Usage: sx showtx FILENAME

'showtx' allows inspecting of tx files.

  $ sx showtx txfile.tx
  hash: 4d25b18ed094ad68f75f21692d8540f45ceb90b240a521b8f191e95d8b6b8bb0
  version: 1  locktime: 0
  Input:
    previous output:
  97e06e49dfdd26c5a904670971ccf4c7fe7d9da53cb379bf9b442fc9427080b3:0
    script:   sequence: 4294967295
  Output:
    value: 90000
    script: dup hash160 [ 18c0bd8d1818f1bf99cb1df2269c645318ef7b73 ] equalverify
  checksig
    address: 13Ft7SkreJY9D823NPm4t6D1cBqLYTJtAe
\
"""
),

"sign-input": (
"Sign a transaction input.",

"""\
Usage: sx sign-input FILENAME INDEX

Sign a transaction input.

Note how the input script in the following transaction is empty.

  $ sx showtx txfile.tx
  hash: 4d25b18ed094ad68f75f21692d8540f45ceb90b240a521b8f191e95d8b6b8bb0
  version: 1  locktime: 0
  Input:
    previous output:
  97e06e49dfdd26c5a904670971ccf4c7fe7d9da53cb379bf9b442fc9427080b3:0
    script:   sequence: 4294967295
  Output:
    value: 90000
    script: dup hash160 [ 18c0bd8d1818f1bf99cb1df2269c645318ef7b73 ] equalverify
  checksig
    address: 13Ft7SkreJY9D823NPm4t6D1cBqLYTJtAe

We will now sign the first input using our private key.

  $ cat private.key | sx sign-input txfile.tx 0
  $ sx showtx txfile.tx
  hash: 4a8be467fb75f0f757649348dbb05762142236ec236ac9e55e4683d7083ffca2
  version: 1  locktime: 0
  Input:
    previous output:
  97e06e49dfdd26c5a904670971ccf4c7fe7d9da53cb379bf9b442fc9427080b3:0
    script: [
  304602210083f8b5131f6399cdc01ed8368f6137f288f771e40834a94d65cee6b380fec873022100da5ed6d1d283f228752a88c2d6f629587b2a6fbc102202ae7c20f48af7c0533c01
  ] [
  04e3af7cdfaab6ba2f14001a7ea0a490f40c6ba43607be92a05111d2e8fc1eecf6fa2c248b3e25ddf7db081684f6f49dc83fd7b9b5a3a88a2e6b83b918f8972351
  ]  sequence: 4294967295
    address: 134HfD2fdeBTohfx8YANxEpsYXsv5UoWyz
  Output:
    value: 90000
    script: dup hash160 [ 18c0bd8d1818f1bf99cb1df2269c645318ef7b73 ] equalverify
  checksig
    address: 13Ft7SkreJY9D823NPm4t6D1cBqLYTJtAe

Now the input script is prepared, and the transaction is signed.\
"""
),

"mnemonic": (
"Work with Electrum compatible mnemonics (12 words wallet seed).",

"""\
Usage: sx mnemonic

Electrum compatible 12 word seeds.

  $ echo 148f0a1d77e20dbaee3ff920ca40240d | sx mnemonic
  people blonde admit dart couple different truth common alas stumble time cookie
  $ echo "people blonde admit dart couple different truth common alas
stumble time cookie" | sx mnemonic
  148f0a1d77e20dbaee3ff920ca40240d
\
"""
),

"watchtx": (
"Watch transactions from the network searching for a certain hash.",

"""\
Usage: sx watchtx [TXHASH]...

Watch transactions from the network searching for a certain hash.\
"""
),

}

def display_usage():
    print "Usage: sx COMMAND [ARGS]..."
    print
    print "The most commonly used sx commands are:"
    print
    for cmd in sorted(command_list.iterkeys()):
        short_desc = command_list[cmd][0]
        line = "   %s" % cmd
        line += " " * (15 - len(cmd))
        line += short_desc
        print line
    print
    print "See 'sx help COMMAND' for more information on a specific command."
    print
    print "SpesmiloXchange home page: <http://sx.dyne.org/>"

def display_help(command):
    assert command in command_list
    long_desc = command_list[command][1]
    print long_desc
    return 0

def display_bad(command):
    print "sx: '%s' is not a sx command. See 'sx --help'." % command
    return 1

def main(argv):
    if len(argv) == 1:
        display_usage()
        return 1
    command = argv[1]
    # args as one string we can pass to the sx sub-command
    args = " ".join(argv[2:])
    if command == "help" or command == "--help" or command == "-h":
        if not args:
            display_usage()
            return 0
        return display_help(args)
    elif command in command_list:
        # make will come and substitute @corebindir@
        execstr = "@corebindir@/sx-%s %s" % (command, args)
        return os.system(execstr)
    else:
        return display_bad(command)
    return 0

if __name__ == "__main__":
    sys.exit(main(sys.argv))

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

class RoutesMixin(object):
    """
    专门做路由管理
    """

    rule_map = None
    events = None

    def __init__(self):
        self.rule_map = dict()

    def add_route_rule(self, cmd=None, view_func=None, **options):
        assert view_func is not None, 'expected view func if cmd is not provided.'

        cmd = cmd or view_func.__name__

        if cmd in self.rule_map and view_func != self.rule_map[cmd]:
            raise Exception, 'duplicate view_func for cmd: %(cmd)s, old_view_func:%(old_view_func)s, new_view_func: %(new_view_func)s' % dict(
                cmd=cmd,
                old_view_func=self.rule_map[cmd],
                new_view_func=view_func,
            )

        self.rule_map[cmd] = view_func

    def route(self, cmd=None, **options):
        def decorator(f):
            self.add_route_rule(cmd, f, **options)
            return f
        return decorator

    def get_route_view_func(self, cmd):
        return self.rule_map.get(cmd)
<commit_msg>该删掉的删掉<commit_after># -*- coding: utf-8 -*-

class RoutesMixin(object):
    """
    专门做路由管理
    """

    rule_map = None

    def __init__(self):
        self.rule_map = dict()

    def add_route_rule(self, cmd=None, view_func=None, **options):
        assert view_func is not None, 'expected view func if cmd is not provided.'

        cmd = cmd or view_func.__name__

        if cmd in self.rule_map and view_func != self.rule_map[cmd]:
            raise Exception, 'duplicate view_func for cmd: %(cmd)s, old_view_func:%(old_view_func)s, new_view_func: %(new_view_func)s' % dict(
                cmd=cmd,
                old_view_func=self.rule_map[cmd],
                new_view_func=view_func,
            )

        self.rule_map[cmd] = view_func

    def route(self, cmd=None, **options):
        def decorator(f):
            self.add_route_rule(cmd, f, **options)
            return f
        return decorator

    def get_route_view_func(self, cmd):
        return self.rule_map.get(cmd)
<|endoftext|>


<commit_before>__author__ = 'jcorbett'

from flask import render_template, Response, request
from flask_gzip import Gzip

from slickqaweb.app import app
from slickqaweb.slicklogging import initialize_logging
from slickqaweb import compiledResources

import os
import api
import logging

app.config.from_object('slickqaweb.settings')
if 'SLICK_SETTINGS' in os.environ:
    app.config.from_envvar("SLICK_SETTINGS")
initialize_logging(app.config)

logger = logging.getLogger("slickqaweb.main")
logger.info("Slick Web App Server starting...")
#Gzip(app)

compiledResources.main_css.seek(0, os.SEEK_END)
logger.debug("main.css is %d bytes in length.", compiledResources.main_css.tell())
for module in api.modules:
    logger.debug("Found module slickqaweb.api.%s", module)

from mongoengine import connect
mongo_hostname = app.config['MONGODB_HOSTNAME']
mongo_dbname = app.config['MONGODB_DBNAME']
logger.debug("Connecting to mongo database '%s' on host '%s'.", mongo_dbname, mongo_hostname)
connect(host=mongo_hostname, db=mongo_dbname)

if app.debug:
    request_logger = logging.getLogger("access")
    @app.after_request
    def write_access_log(response):
        request_logger.debug("%s = %d", request.path, response.status_code)
        return response

# initialize with other apis
@app.route('/', defaults={'path': ''})
@app.route('/<path:path>')
def catch_all(path):
    print("Default catch-all round called with path: {}".format(path))
    base = app.config['APPLICATION_ROOT']
    if base is None:
        base = "/"
    return render_template('index.html', base=base)

<commit_msg>switching to logging in main.py<commit_after>__author__ = 'jcorbett'

from flask import render_template, Response, request
from flask_gzip import Gzip
from mongoengine import connect

from slickqaweb.app import app
from slickqaweb.slicklogging import initialize_logging
from slickqaweb import compiledResources

import os
import sys
import api
import logging

app.config.from_object('slickqaweb.settings')
if 'SLICK_SETTINGS' in os.environ:
    app.config.from_envvar("SLICK_SETTINGS")
initialize_logging(app.config)

logger = logging.getLogger("slickqaweb.main")
logger.info("Slick Web App Server starting...")
#Gzip(app)

if app.debug:
    compiledResources.main_css.seek(0, os.SEEK_END)
    logger.debug("main.css is %d bytes in length.", compiledResources.main_css.tell())
    for module in api.modules:
        logger.debug("Found module slickqaweb.api.%s", module)

mongo_hostname = app.config['MONGODB_HOSTNAME']
mongo_dbname = app.config['MONGODB_DBNAME']
logger.debug("Connecting to mongo database '%s' on host '%s'.", mongo_dbname, mongo_hostname)
try:
    connect(host=mongo_hostname, db=mongo_dbname)
except:
    logger.fatal("Error connecting to database: ", exc_info=sys.exc_info())
    logger.fatal("Unable to connect to mongodb database '%s' on host '%s'.", )

if app.debug:
    request_logger = logging.getLogger("access")
    @app.after_request
    def write_access_log(response):
        request_logger.debug("%s = %d", request.path, response.status_code)
        return response

# initialize with other apis
@app.route('/', defaults={'path': ''})
@app.route('/<path:path>')
def catch_all(path):
    base = app.config['APPLICATION_ROOT']
    if base is None:
        base = "/"
    return render_template('index.html', base=base)

<|endoftext|>


<commit_before>"""
Author: Jaisen Mathai <jaisen@jmathai.com>
Photo package that handles all photo operations
"""

# load modules
from datetime import datetime
from sys import argv

import imghdr
import mimetypes
import LatLon
import os
import pyexiv2
import re
import subprocess
import time

from media import Media
from elodie import geolocation

"""
Photo class for general photo operations
"""
class Photo(Media):
    __name__ = 'Photo'
    extensions = ('jpg', 'jpeg', 'nef', 'dng', 'gif')

    """
    @param, source, string, The fully qualified path to the photo file
    """
    def __init__(self, source=None):
        super(Photo, self).__init__(source)

        # We only want to parse EXIF once so we store it here
        self.exif = None

    """
    Get the duration of a photo in seconds.
    Uses ffmpeg/ffprobe

    @returns, string or None for a non-photo file
    """
    def get_duration(self):
        if(not self.is_valid()):
            return None

        source = self.source
        result = subprocess.Popen(['ffprobe', source],
            stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
        for key in result.stdout.readlines():
            if 'Duration' in key:
                return re.search('(\d{2}:\d{2}.\d{2})', key).group(1).replace('.', ':')
        return None

    """
    Get latitude or longitude of photo from EXIF

    @returns, float or None if not present in EXIF or a non-photo file
    """
    def get_coordinate(self, type='latitude'):
        if(not self.is_valid()):
            return None

        key = self.exif_map['longitude'] if type == 'longitude' else self.exif_map['latitude']
        exif = self.get_exif()

        if(key not in exif):
            return None

        try:
            # this is a hack to get the proper direction by negating the values for S and W
            latdir = 1
            if(key == self.exif_map['latitude'] and str(exif[self.exif_map['latitude_ref']].value) == 'S'):
                latdir = -1
            londir = 1
            if(key == self.exif_map['longitude'] and str(exif[self.exif_map['longitude_ref']].value) == 'W'):
                londir = -1

            coords = exif[key].value
            if(key == 'latitude'):
                return float(str(LatLon.Latitude(degree=coords[0], minute=coords[1], second=coords[2]))) * latdir
            else:
                return float(str(LatLon.Longitude(degree=coords[0], minute=coords[1], second=coords[2]))) * londir
        except KeyError:
            return None

    """
    Get the date which the photo was taken.
    The date value returned is defined by the min() of mtime and ctime.

    @returns, time object or None for non-photo files or 0 timestamp
    """
    def get_date_taken(self):
        if(not self.is_valid()):
            return None

        source = self.source
        seconds_since_epoch = min(os.path.getmtime(source), os.path.getctime(source))
        # We need to parse a string from EXIF into a timestamp.
        # EXIF DateTimeOriginal and EXIF DateTime are both stored in %Y:%m:%d %H:%M:%S format
        # we use date.strptime -> .timetuple -> time.mktime to do the conversion in the local timezone
        # EXIF DateTime is already stored as a timestamp
        # Sourced from https://github.com/photo/frontend/blob/master/src/libraries/models/Photo.php#L500
        exif = self.get_exif()
        for key in self.exif_map['date_taken']:
            try:
                if(key in exif):
                    if(re.match('\d{4}(-|:)\d{2}(-|:)\d{2}', str(exif[key].value)) is not None):
                        seconds_since_epoch = time.mktime(exif[key].value.timetuple())
                        break;
            except BaseException as e:
                if(constants.debug == True):
                    print e
                pass

        if(seconds_since_epoch == 0):
            return None

        return time.gmtime(seconds_since_epoch)

    """
    Check the file extension against valid file extensions as returned by self.extensions

    @returns, boolean
    """
    def is_valid(self):
        source = self.source

        # gh-4 This checks if the source file is an image.
        # It doesn't validate against the list of supported types.
        if(imghdr.what(source) is None):
            return False;

        return os.path.splitext(source)[1][1:].lower() in self.extensions

    """
    Set the date/time a photo was taken

    @param, time, datetime, datetime object of when the photo was taken

    @returns, boolean
    """
    def set_date_taken(self, time):
        if(time is None):
            return False

        source = self.source
        exif_metadata = pyexiv2.ImageMetadata(source)
        exif_metadata.read()

        exif_metadata['Exif.Photo.DateTimeOriginal'].value = time
        exif_metadata['Exif.Image.DateTime'].value = time

        exif_metadata.write()
        return True

    """
    Set lat/lon for a photo

    @param, latitude, float, Latitude of the file
    @param, longitude, float, Longitude of the file

    @returns, boolean
    """
    def set_location(self, latitude, longitude):
        if(latitude is None or longitude is None):
            return False

        source = self.source
        exif_metadata = pyexiv2.ImageMetadata(source)
        exif_metadata.read()

        exif_metadata['Exif.GPSInfo.GPSLatitude'] = geolocation.decimal_to_dms(latitude)
        exif_metadata['Exif.GPSInfo.GPSLatitudeRef'] = pyexiv2.ExifTag('Exif.GPSInfo.GPSLatitudeRef', 'N' if latitude >= 0 else 'S')
        exif_metadata['Exif.GPSInfo.GPSLongitude'] = geolocation.decimal_to_dms(longitude)
        exif_metadata['Exif.GPSInfo.GPSLongitudeRef'] = pyexiv2.ExifTag('Exif.GPSInfo.GPSLongitudeRef', 'E' if longitude >= 0 else 'W')

        exif_metadata.write()
        return True

    """
    Set title for a photo

    @param, latitude, float, Latitude of the file
    @param, longitude, float, Longitude of the file

    @returns, boolean
    """
    def set_title(self, title):
        if(title is None):
            return False

        source = self.source
        exif_metadata = pyexiv2.ImageMetadata(source)
        exif_metadata.read()

        exif_metadata['Xmp.dc.title'] = title

        exif_metadata.write()
        return True

    """
    Static method to access static __valid_extensions variable.

    @returns, tuple
    """
    @classmethod
    def get_valid_extensions(Photo):
        return Photo.extensions
<commit_msg>Closes gh-38 Add import for constants module in photo.py<commit_after>"""
Author: Jaisen Mathai <jaisen@jmathai.com>
Photo package that handles all photo operations
"""

# load modules
from datetime import datetime
from sys import argv

import imghdr
import mimetypes
import LatLon
import os
import pyexiv2
import re
import subprocess
import time

from elodie import constants
from media import Media
from elodie import geolocation

"""
Photo class for general photo operations
"""
class Photo(Media):
    __name__ = 'Photo'
    extensions = ('jpg', 'jpeg', 'nef', 'dng', 'gif')

    """
    @param, source, string, The fully qualified path to the photo file
    """
    def __init__(self, source=None):
        super(Photo, self).__init__(source)

        # We only want to parse EXIF once so we store it here
        self.exif = None

    """
    Get the duration of a photo in seconds.
    Uses ffmpeg/ffprobe

    @returns, string or None for a non-photo file
    """
    def get_duration(self):
        if(not self.is_valid()):
            return None

        source = self.source
        result = subprocess.Popen(['ffprobe', source],
            stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
        for key in result.stdout.readlines():
            if 'Duration' in key:
                return re.search('(\d{2}:\d{2}.\d{2})', key).group(1).replace('.', ':')
        return None

    """
    Get latitude or longitude of photo from EXIF

    @returns, float or None if not present in EXIF or a non-photo file
    """
    def get_coordinate(self, type='latitude'):
        if(not self.is_valid()):
            return None

        key = self.exif_map['longitude'] if type == 'longitude' else self.exif_map['latitude']
        exif = self.get_exif()

        if(key not in exif):
            return None

        try:
            # this is a hack to get the proper direction by negating the values for S and W
            latdir = 1
            if(key == self.exif_map['latitude'] and str(exif[self.exif_map['latitude_ref']].value) == 'S'):
                latdir = -1
            londir = 1
            if(key == self.exif_map['longitude'] and str(exif[self.exif_map['longitude_ref']].value) == 'W'):
                londir = -1

            coords = exif[key].value
            if(key == 'latitude'):
                return float(str(LatLon.Latitude(degree=coords[0], minute=coords[1], second=coords[2]))) * latdir
            else:
                return float(str(LatLon.Longitude(degree=coords[0], minute=coords[1], second=coords[2]))) * londir
        except KeyError:
            return None

    """
    Get the date which the photo was taken.
    The date value returned is defined by the min() of mtime and ctime.

    @returns, time object or None for non-photo files or 0 timestamp
    """
    def get_date_taken(self):
        if(not self.is_valid()):
            return None

        source = self.source
        seconds_since_epoch = min(os.path.getmtime(source), os.path.getctime(source))
        # We need to parse a string from EXIF into a timestamp.
        # EXIF DateTimeOriginal and EXIF DateTime are both stored in %Y:%m:%d %H:%M:%S format
        # we use date.strptime -> .timetuple -> time.mktime to do the conversion in the local timezone
        # EXIF DateTime is already stored as a timestamp
        # Sourced from https://github.com/photo/frontend/blob/master/src/libraries/models/Photo.php#L500
        exif = self.get_exif()
        for key in self.exif_map['date_taken']:
            try:
                if(key in exif):
                    if(re.match('\d{4}(-|:)\d{2}(-|:)\d{2}', str(exif[key].value)) is not None):
                        seconds_since_epoch = time.mktime(exif[key].value.timetuple())
                        break;
            except BaseException as e:
                if(constants.debug == True):
                    print e
                pass

        if(seconds_since_epoch == 0):
            return None

        return time.gmtime(seconds_since_epoch)

    """
    Check the file extension against valid file extensions as returned by self.extensions

    @returns, boolean
    """
    def is_valid(self):
        source = self.source

        # gh-4 This checks if the source file is an image.
        # It doesn't validate against the list of supported types.
        if(imghdr.what(source) is None):
            return False;

        return os.path.splitext(source)[1][1:].lower() in self.extensions

    """
    Set the date/time a photo was taken

    @param, time, datetime, datetime object of when the photo was taken

    @returns, boolean
    """
    def set_date_taken(self, time):
        if(time is None):
            return False

        source = self.source
        exif_metadata = pyexiv2.ImageMetadata(source)
        exif_metadata.read()

        exif_metadata['Exif.Photo.DateTimeOriginal'].value = time
        exif_metadata['Exif.Image.DateTime'].value = time

        exif_metadata.write()
        return True

    """
    Set lat/lon for a photo

    @param, latitude, float, Latitude of the file
    @param, longitude, float, Longitude of the file

    @returns, boolean
    """
    def set_location(self, latitude, longitude):
        if(latitude is None or longitude is None):
            return False

        source = self.source
        exif_metadata = pyexiv2.ImageMetadata(source)
        exif_metadata.read()

        exif_metadata['Exif.GPSInfo.GPSLatitude'] = geolocation.decimal_to_dms(latitude)
        exif_metadata['Exif.GPSInfo.GPSLatitudeRef'] = pyexiv2.ExifTag('Exif.GPSInfo.GPSLatitudeRef', 'N' if latitude >= 0 else 'S')
        exif_metadata['Exif.GPSInfo.GPSLongitude'] = geolocation.decimal_to_dms(longitude)
        exif_metadata['Exif.GPSInfo.GPSLongitudeRef'] = pyexiv2.ExifTag('Exif.GPSInfo.GPSLongitudeRef', 'E' if longitude >= 0 else 'W')

        exif_metadata.write()
        return True

    """
    Set title for a photo

    @param, latitude, float, Latitude of the file
    @param, longitude, float, Longitude of the file

    @returns, boolean
    """
    def set_title(self, title):
        if(title is None):
            return False

        source = self.source
        exif_metadata = pyexiv2.ImageMetadata(source)
        exif_metadata.read()

        exif_metadata['Xmp.dc.title'] = title

        exif_metadata.write()
        return True

    """
    Static method to access static __valid_extensions variable.

    @returns, tuple
    """
    @classmethod
    def get_valid_extensions(Photo):
        return Photo.extensions
<|endoftext|>


<commit_before>"""
    Copyright (c) 2015-2020 Raj Patel(raj454raj@gmail.com), StopStalk

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    THE SOFTWARE.
"""

from .init import *

class Profile(object):
    """
        Class containing methods for retrieving
        submissions of user
    """
    site_name = "CodeForces"

    # --------------------------------------------------------------------------
    def __init__(self, handle=""):
        """
            @param handle (String): Codeforces Handle
        """

        self.site = Profile.site_name
        self.handle = handle

    # -------------------------------------------------------------------------
    @staticmethod
    def is_valid_url(url):
        return url.__contains__("codeforces.com/")

    # -------------------------------------------------------------------------
    @staticmethod
    def is_website_down():
        """
            @return (Boolean): If the website is down
        """
        return (Profile.site_name in current.REDIS_CLIENT.smembers("disabled_retrieval"))

    # -------------------------------------------------------------------------
    @staticmethod
    def get_contest_id(problem_link):
        """
            Get contest ID from a problem_link

            @param problem_link (String): Problem URL
            @return (Integer): Contest ID from the URL
        """
        return problem_link.split("/")[-2]

    # -------------------------------------------------------------------------
    @staticmethod
    def get_url_parts(problem_link=None, parts=None):
        if parts is None:
            parts = problem_link.split("/")[-2:]

        return "problem_" + "_".join(parts)

    # -------------------------------------------------------------------------
    @staticmethod
    def make_codeforces_request(url):
        return get_request(url,
                           headers=current.codeforces_headers,
                           cookies=current.codeforces_cookies)

    # -------------------------------------------------------------------------
    @staticmethod
    def get_editorial_link(problem_link):
        """
            @param problem_link(String): Problem link of the page

            @return (String/None): Editorial link
        """
        editorial_link = None
        response = Profile.make_codeforces_request(problem_link)
        if response in REQUEST_FAILURES:
            return editorial_link

        soup = BeautifulSoup(response.text, "lxml")
        all_as = soup.find_all("a")
        for link in all_as:
            if len(link.contents) == 0:
                continue
            url = link.contents[0]
            if url.__contains__("Tutorial"):
                # Some problems have complete url -_-
                # Example: http://codeforces.com/problemset/problem/358/C
                if link["href"][0] == "/":
                    editorial_link = "http://www.codeforces.com" + link["href"]
                else:
                    editorial_link = link["href"]
                break

        return editorial_link

    # -------------------------------------------------------------------------
    @staticmethod
    def get_tags(problem_link):
        """
            @param problem_link(String): Problem link of the page

            @return (List): List of tags
        """

        value = current.REDIS_CLIENT.hgetall("codeforces_problem_tag_mapping")
        if len(value) == 0:
            response = get_request("https://codeforces.com/api/problemset.problems")
            if response in REQUEST_FAILURES:
                return []

            result = response.json()["result"]["problems"]
            redis_dict = {}
            for problem in result:
                redis_key = Profile.get_url_parts(parts=[str(problem["contestId"]),
                                                         problem["index"]])
                redis_dict[redis_key] = problem["tags"]

            current.REDIS_CLIENT.hmset("codeforces_problem_tag_mapping",
                                       redis_dict)

            value = redis_dict

        redis_key = Profile.get_url_parts(problem_link=problem_link)
        if redis_key in value:
            from ast import literal_eval
            return literal_eval(value[redis_key])
        else:
            return []

    # -------------------------------------------------------------------------
    @staticmethod
    def get_problem_setters(problem_link):
        """
            @param problem_link(String): Problem link of the page

            @return (List/None): Problem authors or None
        """
        import json
        mappings = current.REDIS_CLIENT.get(CODEFORCES_PROBLEM_SETTERS_KEY)
        if mappings is None:
            file_path = "./applications/stopstalk/problem_setters/codeforces_metadata.json"
            try:
                file_obj = open(file_path, "r")
            except IOError:
                return None

            mappings = json.loads(file_obj.read())
            current.REDIS_CLIENT.set(CODEFORCES_PROBLEM_SETTERS_KEY,
                                     json.dumps(mappings))
            file_obj.close()

        mappings = json.loads(mappings)
        contest_id = Profile.get_contest_id(problem_link)
        if contest_id in mappings["normal"]:
            return mappings["normal"][contest_id]
        elif contest_id in mappings["gym"]:
            return mappings["gym"][contest_id]
        else:
            return None

    # -------------------------------------------------------------------------
    @staticmethod
    def get_problem_details(**args):
        """
            Get problem_details given a problem link

            @param args (Dict): Dict containing problem_link
            @return (Dict): Details of the problem returned in a dictionary
        """

        all_tags = []
        editorial_link = None
        problem_link = args["problem_link"]
        problem_setters = None
        if problem_link.__contains__("gymProblem"):
            return dict(editorial_link=editorial_link,
                        tags=all_tags,
                        problem_setters=problem_setters)

        if "problem_setters" in args["update_things"]:
            problem_setters = Profile.get_problem_setters(problem_link)

        if "editorial_link" in args["update_things"]:
            editorial_link = Profile.get_editorial_link(problem_link)

        if "tags" in args["update_things"]:
            all_tags = Profile.get_tags(problem_link)

        return dict(editorial_link=editorial_link,
                    tags=all_tags,
                    problem_setters=problem_setters)

    # -------------------------------------------------------------------------
    @staticmethod
    def is_invalid_handle(handle):
        response = get_request("http://www.codeforces.com/api/user.status?handle=" + \
                               handle + "&from=1&count=2")

        if response in REQUEST_FAILURES:
            return True

        response = response.json()

        if response["status"] != "ok":
            return True
        else:
            return False

        return False

    # -------------------------------------------------------------------------
    @staticmethod
    def download_submission(view_link):
        if Profile.is_website_down():
            return -1

        response = Profile.make_codeforces_request(view_link)
        if response in REQUEST_FAILURES:
            return -1

        try:
            return BeautifulSoup(response.text, "lxml").find("pre").text
        except:
            return -1

    # -------------------------------------------------------------------------
    @staticmethod
    def rating_graph_data(handle):
        website = "http://www.codeforces.com/"

        url = "%sapi/contest.list" % website
        response = get_request(url)
        if response in REQUEST_FAILURES:
            return response

        contest_list = response.json()["result"]
        all_contests = {}

        for contest in contest_list:
            all_contests[contest["id"]] = contest

        url = "%scontests/with/%s" % (website, handle)
        response = Profile.make_codeforces_request(url)

        if response in REQUEST_FAILURES:
            return response

        soup = BeautifulSoup(response.text, "lxml")
        try:
            tbody = soup.find("table", class_="tablesorter").find("tbody")
        except AttributeError:
            print "AttributeError for Codeforces handle: " + handle
            return SERVER_FAILURE

        contest_data = {}
        for tr in tbody.find_all("tr"):
            all_tds = tr.find_all("td")
            contest_id = int(all_tds[1].find("a")["href"].split("/")[-1])
            rank = int(all_tds[3].text.strip())
            solved_count = int(all_tds[4].find("a").contents[0].strip())
            rating_change = int(all_tds[5].find("span").contents[0].strip())
            new_rating = int(all_tds[6].contents[0].strip())
            contest = all_contests[contest_id]
            time_stamp = str(datetime.datetime.fromtimestamp(contest["startTimeSeconds"]))
            contest_data[time_stamp] = {"name": contest["name"],
                                        "url": "%scontest/%d" % (website,
                                                                 contest_id),
                                        "rating": str(new_rating),
                                        "ratingChange": rating_change,
                                        "rank": rank,
                                        "solvedCount": solved_count}

        return [{"title": "Codeforces",
                 "data": contest_data}]

    # -------------------------------------------------------------------------
    def get_submissions(self, last_retrieved, is_daily_retrieval):
        """
            Retrieve CodeForces submissions after last retrieved timestamp

            @param last_retrieved (DateTime): Last retrieved timestamp for the user
            @param is_daily_retrieval (Boolean): If this call is from daily retrieval cron

            @return (Dict): Dictionary of submissions containing all the
                            information about the submissions
        """

        handle = self.handle
        url = "http://www.codeforces.com/api/user.status?handle=" + \
              handle + "&from=1"
        # Timeout for new user submission retrieval
        timeout = 50
        if time.strftime("%Y-%m-%d %H:%M:%S", last_retrieved) != current.INITIAL_DATE:
            # Daily retrieval script to limit submissions to 500
            # A daily submitter of more than 500 submissions is really
            # supposed to contact us to prove he/she is a human :p
            url += "&count=500"
            timeout = 10
        else:
            url += "&count=50000"

        tmp = get_request(url,
                          timeout=timeout,
                          is_daily_retrieval=is_daily_retrieval)

        if tmp in REQUEST_FAILURES:
            return tmp

        # This was not supposed to be done here but optimization :p
        db = current.db
        ptable = db.problem
        today = datetime.datetime.now().strftime("%Y-%m-%d")

        submissions = []
        all_submissions = tmp.json()
        if all_submissions["status"] != "OK":
            if "not found" in all_submissions["comment"]:
                return NOT_FOUND
            else:
                return SERVER_FAILURE
        all_submissions = all_submissions["result"]

        for row in all_submissions:

            curr = time.gmtime(row["creationTimeSeconds"] + 330 * 60)
            if curr <= last_retrieved:
                return submissions

            if row.has_key("contestId") == False:
                try:
                    problem_link = "http://www.codeforces.com/problemsets/" + \
                                   row["problem"]["problemsetName"] + \
                                   "/problem/99999/" + str(row["problem"]["index"])
                except Exception as e:
                    print "Unable to create problem_link for", row
                    continue
            else:
                arg = "problem/"
                if int(row["contestId"]) > 90000:
                    arg = "gymProblem/"

                # Problem Name/URL
                problem_link = "http://www.codeforces.com/problemset/" + arg + \
                               str(row["contestId"]) + "/" + \
                               row["problem"]["index"]

            problem_name = row["problem"]["name"]

            # Problem status
            try:
                status = row["verdict"]
            except KeyError:
                status = "OTHER"
            submission_status = "AC"
            if status == "OK":
                submission_status = "AC"
            elif status == "WRONG_ANSWER":
                submission_status = "WA"
            elif status == "COMPILATION_ERROR":
                submission_status = "CE"
            elif status == "SKIPPED":
                submission_status = "SK"
            elif status == "RUNTIME_ERROR":
                submission_status = "RE"
            elif status == "TIME_LIMIT_EXCEEDED":
                submission_status = "TLE"
            elif status == "CHALLENGED":
                submission_status = "HCK"
            elif status == "MEMORY_LIMIT_EXCEEDED":
                submission_status = "MLE"
            else:
                submission_status = "OTH"

            # Points
            if submission_status == "AC":
                points = "100"
            elif submission_status == "HCK":
                points = "-50"
            else:
                points = "0"

            # View code link
            if problem_link.__contains__("gymProblem") or \
               row.has_key("contestId") == False:
                view_link = ""
            else:
                view_link = "http://www.codeforces.com/contest/" + \
                            str(row["contestId"]) + \
                            "/submission/" + \
                            str(row["id"])

            submissions.append((str(time.strftime("%Y-%m-%d %H:%M:%S", curr)),
                                problem_link,
                                problem_name,
                                submission_status,
                                points,
                                row["programmingLanguage"],
                                view_link))

        return submissions

# =============================================================================
<commit_msg>minor fix<commit_after>"""
    Copyright (c) 2015-2020 Raj Patel(raj454raj@gmail.com), StopStalk

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    THE SOFTWARE.
"""

from .init import *

class Profile(object):
    """
        Class containing methods for retrieving
        submissions of user
    """
    site_name = "CodeForces"

    # --------------------------------------------------------------------------
    def __init__(self, handle=""):
        """
            @param handle (String): Codeforces Handle
        """

        self.site = Profile.site_name
        self.handle = handle

    # -------------------------------------------------------------------------
    @staticmethod
    def is_valid_url(url):
        return url.__contains__("codeforces.com/")

    # -------------------------------------------------------------------------
    @staticmethod
    def is_website_down():
        """
            @return (Boolean): If the website is down
        """
        return (Profile.site_name in current.REDIS_CLIENT.smembers("disabled_retrieval"))

    # -------------------------------------------------------------------------
    @staticmethod
    def get_contest_id(problem_link):
        """
            Get contest ID from a problem_link

            @param problem_link (String): Problem URL
            @return (Integer): Contest ID from the URL
        """
        return problem_link.split("/")[-2]

    # -------------------------------------------------------------------------
    @staticmethod
    def get_url_parts(problem_link=None, parts=None):
        if parts is None:
            parts = problem_link.split("/")[-2:]

        return "problem_" + "_".join(parts)

    # -------------------------------------------------------------------------
    @staticmethod
    def make_codeforces_request(url):
        return get_request(url,
                           headers=current.codeforces_headers,
                           cookies=current.codeforces_cookies)

    # -------------------------------------------------------------------------
    @staticmethod
    def get_editorial_link(problem_link):
        """
            @param problem_link(String): Problem link of the page

            @return (String/None): Editorial link
        """
        editorial_link = None
        response = Profile.make_codeforces_request(problem_link)
        if response in REQUEST_FAILURES:
            return editorial_link

        soup = BeautifulSoup(response.text, "lxml")
        all_as = soup.find_all("a")
        for link in all_as:
            if len(link.contents) == 0:
                continue
            url = link.contents[0]
            if url.__contains__("Tutorial"):
                # Some problems have complete url -_-
                # Example: http://codeforces.com/problemset/problem/358/C
                if link["href"][0] == "/":
                    editorial_link = "http://www.codeforces.com" + link["href"]
                else:
                    editorial_link = link["href"]
                break

        return editorial_link

    # -------------------------------------------------------------------------
    @staticmethod
    def get_tags(problem_link):
        """
            @param problem_link(String): Problem link of the page

            @return (List): List of tags
        """

        value = current.REDIS_CLIENT.hgetall("codeforces_problem_tag_mapping")
        if len(value) == 0:
            response = get_request("https://codeforces.com/api/problemset.problems")
            if response in REQUEST_FAILURES:
                return []

            result = response.json()["result"]["problems"]
            redis_dict = {}
            for problem in result:
                redis_key = Profile.get_url_parts(parts=[str(problem["contestId"]),
                                                         problem["index"]])
                redis_dict[redis_key] = problem["tags"]

            current.REDIS_CLIENT.hmset("codeforces_problem_tag_mapping",
                                       redis_dict)

            value = redis_dict

        redis_key = Profile.get_url_parts(problem_link=problem_link)
        if redis_key in value:
            from ast import literal_eval
            return literal_eval(value[redis_key])
        else:
            return []

    # -------------------------------------------------------------------------
    @staticmethod
    def get_problem_setters(problem_link):
        """
            @param problem_link(String): Problem link of the page

            @return (List/None): Problem authors or None
        """
        import json
        mappings = current.REDIS_CLIENT.get(CODEFORCES_PROBLEM_SETTERS_KEY)
        if mappings is None:
            file_path = "./applications/stopstalk/problem_setters/codeforces_metadata.json"
            try:
                file_obj = open(file_path, "r")
            except IOError:
                return None

            mappings = json.loads(file_obj.read())
            current.REDIS_CLIENT.set(CODEFORCES_PROBLEM_SETTERS_KEY,
                                     json.dumps(mappings))
            file_obj.close()

        mappings = json.loads(mappings)
        contest_id = Profile.get_contest_id(problem_link)
        if contest_id in mappings["normal"]:
            return mappings["normal"][contest_id]
        elif contest_id in mappings["gym"]:
            return mappings["gym"][contest_id]
        else:
            return None

    # -------------------------------------------------------------------------
    @staticmethod
    def get_problem_details(**args):
        """
            Get problem_details given a problem link

            @param args (Dict): Dict containing problem_link
            @return (Dict): Details of the problem returned in a dictionary
        """

        all_tags = []
        editorial_link = None
        problem_link = args["problem_link"]
        problem_setters = None
        if problem_link.__contains__("gymProblem"):
            return dict(editorial_link=editorial_link,
                        tags=all_tags,
                        problem_setters=problem_setters)

        if "problem_setters" in args["update_things"]:
            problem_setters = Profile.get_problem_setters(problem_link)

        if "editorial_link" in args["update_things"]:
            editorial_link = Profile.get_editorial_link(problem_link)

        if "tags" in args["update_things"]:
            all_tags = Profile.get_tags(problem_link)

        return dict(editorial_link=editorial_link,
                    tags=all_tags,
                    problem_setters=problem_setters)

    # -------------------------------------------------------------------------
    @staticmethod
    def is_invalid_handle(handle):
        response = get_request("http://www.codeforces.com/api/user.status?handle=" + \
                               handle + "&from=1&count=2")

        if response in REQUEST_FAILURES:
            return True

        response = response.json()

        if response["status"] != "ok":
            return True
        else:
            return False

        return False

    # -------------------------------------------------------------------------
    @staticmethod
    def download_submission(view_link):
        if Profile.is_website_down():
            return -1

        response = Profile.make_codeforces_request(view_link)
        if response in REQUEST_FAILURES:
            return -1

        try:
            return BeautifulSoup(response.text, "lxml").find(id="program-source-text").text
        except:
            return -1

    # -------------------------------------------------------------------------
    @staticmethod
    def rating_graph_data(handle):
        website = "http://www.codeforces.com/"

        url = "%sapi/contest.list" % website
        response = get_request(url)
        if response in REQUEST_FAILURES:
            return response

        contest_list = response.json()["result"]
        all_contests = {}

        for contest in contest_list:
            all_contests[contest["id"]] = contest

        url = "%scontests/with/%s" % (website, handle)
        response = Profile.make_codeforces_request(url)

        if response in REQUEST_FAILURES:
            return response

        soup = BeautifulSoup(response.text, "lxml")
        try:
            tbody = soup.find("table", class_="tablesorter").find("tbody")
        except AttributeError:
            print "AttributeError for Codeforces handle: " + handle
            return SERVER_FAILURE

        contest_data = {}
        for tr in tbody.find_all("tr"):
            all_tds = tr.find_all("td")
            contest_id = int(all_tds[1].find("a")["href"].split("/")[-1])
            rank = int(all_tds[3].text.strip())
            solved_count = int(all_tds[4].find("a").contents[0].strip())
            rating_change = int(all_tds[5].find("span").contents[0].strip())
            new_rating = int(all_tds[6].contents[0].strip())
            contest = all_contests[contest_id]
            time_stamp = str(datetime.datetime.fromtimestamp(contest["startTimeSeconds"]))
            contest_data[time_stamp] = {"name": contest["name"],
                                        "url": "%scontest/%d" % (website,
                                                                 contest_id),
                                        "rating": str(new_rating),
                                        "ratingChange": rating_change,
                                        "rank": rank,
                                        "solvedCount": solved_count}

        return [{"title": "Codeforces",
                 "data": contest_data}]

    # -------------------------------------------------------------------------
    def get_submissions(self, last_retrieved, is_daily_retrieval):
        """
            Retrieve CodeForces submissions after last retrieved timestamp

            @param last_retrieved (DateTime): Last retrieved timestamp for the user
            @param is_daily_retrieval (Boolean): If this call is from daily retrieval cron

            @return (Dict): Dictionary of submissions containing all the
                            information about the submissions
        """

        handle = self.handle
        url = "http://www.codeforces.com/api/user.status?handle=" + \
              handle + "&from=1"
        # Timeout for new user submission retrieval
        timeout = 50
        if time.strftime("%Y-%m-%d %H:%M:%S", last_retrieved) != current.INITIAL_DATE:
            # Daily retrieval script to limit submissions to 500
            # A daily submitter of more than 500 submissions is really
            # supposed to contact us to prove he/she is a human :p
            url += "&count=500"
            timeout = 10
        else:
            url += "&count=50000"

        tmp = get_request(url,
                          timeout=timeout,
                          is_daily_retrieval=is_daily_retrieval)

        if tmp in REQUEST_FAILURES:
            return tmp

        # This was not supposed to be done here but optimization :p
        db = current.db
        ptable = db.problem
        today = datetime.datetime.now().strftime("%Y-%m-%d")

        submissions = []
        all_submissions = tmp.json()
        if all_submissions["status"] != "OK":
            if "not found" in all_submissions["comment"]:
                return NOT_FOUND
            else:
                return SERVER_FAILURE
        all_submissions = all_submissions["result"]

        for row in all_submissions:

            curr = time.gmtime(row["creationTimeSeconds"] + 330 * 60)
            if curr <= last_retrieved:
                return submissions

            if row.has_key("contestId") == False:
                try:
                    problem_link = "http://www.codeforces.com/problemsets/" + \
                                   row["problem"]["problemsetName"] + \
                                   "/problem/99999/" + str(row["problem"]["index"])
                except Exception as e:
                    print "Unable to create problem_link for", row
                    continue
            else:
                arg = "problem/"
                if int(row["contestId"]) > 90000:
                    arg = "gymProblem/"

                # Problem Name/URL
                problem_link = "http://www.codeforces.com/problemset/" + arg + \
                               str(row["contestId"]) + "/" + \
                               row["problem"]["index"]

            problem_name = row["problem"]["name"]

            # Problem status
            try:
                status = row["verdict"]
            except KeyError:
                status = "OTHER"
            submission_status = "AC"
            if status == "OK":
                submission_status = "AC"
            elif status == "WRONG_ANSWER":
                submission_status = "WA"
            elif status == "COMPILATION_ERROR":
                submission_status = "CE"
            elif status == "SKIPPED":
                submission_status = "SK"
            elif status == "RUNTIME_ERROR":
                submission_status = "RE"
            elif status == "TIME_LIMIT_EXCEEDED":
                submission_status = "TLE"
            elif status == "CHALLENGED":
                submission_status = "HCK"
            elif status == "MEMORY_LIMIT_EXCEEDED":
                submission_status = "MLE"
            else:
                submission_status = "OTH"

            # Points
            if submission_status == "AC":
                points = "100"
            elif submission_status == "HCK":
                points = "-50"
            else:
                points = "0"

            # View code link
            if problem_link.__contains__("gymProblem") or \
               row.has_key("contestId") == False:
                view_link = ""
            else:
                view_link = "http://www.codeforces.com/contest/" + \
                            str(row["contestId"]) + \
                            "/submission/" + \
                            str(row["id"])

            submissions.append((str(time.strftime("%Y-%m-%d %H:%M:%S", curr)),
                                problem_link,
                                problem_name,
                                submission_status,
                                points,
                                row["programmingLanguage"],
                                view_link))

        return submissions

# =============================================================================
<|endoftext|>


<commit_before>'''
Dependency management module for LIVV

Created on January 6, 2015

@author: arbennett
'''

#
# TODO : Try to import urllib2 before checking for setuptools.  If it's not available try using something
#        like curl or wget from a subprocess to download setuptools and then add urllib2 to the list of
#        dependencies to check (and obviously install)
#

import os
import sys
import urllib2
import subprocess

'''
Run all of the checks for dependencies required by LIVV

Checks if modules are used, and if they are checks to see
if they are loaded.  Then, checks for the necessary Python
libraries.  If any are missing they are installed via
easy_install.  If easy_install isn't installed, that is
also built.
'''
def check():
    cwd = os.getcwd()

    # The list of nonstandard python libraries that are used
    libraryList = ["jinja2", "netCDF4", "numpy", "matplotlib"]
    binaryList = ["ncdiff"]

    # Create a list to store all of the errors that were found
    depErrors = []

    print(os.linesep + "Beginning Dependency Checks........")

    # If we need to load modules for LCF machines do so now
    checkModules()

    # Make sure all environment variables are set
    if not os.environ.has_key("NCARG_ROOT"):
        depErrors.append("  NCARG_ROOT not found in environment")

    # Check to make sure that binary files are found
    print("    Checking for external programs....")
    for program in binaryList:
        found = False
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            filePath = os.path.join(path, program)
            print filePath
            if os.path.isfile(filePath) and os.access(filePath, os.X_OK):
                found = True
        if not found:
            depErrors.append("  " + program + " could not be found in system path.  Please install or update your path!")

    # For the python dependencies we may need easy_install
    # Make sure we have it, and if not, build it
    try:
        from setuptools.command import easy_install
    except ImportError:
        if not os.path.exists(cwd + os.sep + "deps"):
            os.mkdir(cwd + os.sep + "deps")
            sys.path.append(cwd + os.sep + "deps")
        installSetupTools()
        from setuptools.command import easy_install

    # Make sure all imports are going to work
    # And if they don't build a copy of the ones that are needed
    print("    Checking for external libraries....")
    libsInstalled=[]
    for lib in libraryList:
        try:
            __import__(lib)
            print("      Found " + lib + "!")
        except ImportError:
            print("      Could not find " + lib + ".  Building a copy for you...."),
            if not os.path.exists(cwd + os.sep + "deps"):
                os.mkdir(cwd + os.sep + "deps")
                sys.path.append(cwd + os.sep + "deps")
            easy_install.main(["--user",lib])
            libsInstalled.append(lib)

    if len(libsInstalled) > 0:
        print("------------------------------------------------------------------------------")
        print "  External Python Libraries have been installed!  Libraries installed:"
        for lib in libsInstalled:
            print("    " + lib)
        print("  Run LIVV again to continue.  ")
        print("------------------------------------------------------------------------------")
        exit(0)

    # Show all of the dependency errors that were found
    if len(depErrors) > 0:
        print("Uh oh!")
        print("")
        print("Errors checking dependencies.  Errors found: ")
        for err in depErrors: print(err)
        exit(len(depErrors))
    else:
        print("Okay!" + os.linesep + "Setting up environment....")

'''
Checks if the system running LIVV uses modules to load dependencies.

If the system does use modules, this method goes through and makes
sure that the correct modules are loaded.  Any modules that are needed
but haven't been loaded are added to a module loader script that the
user is prompted to source before running LIVV again.
'''
def checkModules():
    # Check to see if calling 'module list' is a real command
    cwd = os.getcwd()
    print("    Checking if modules need to be loaded"),
    modules = [
           "python/2.7.5",
           "ncl/6.1.0",
           "nco/4.3.9",
           "python_matplotlib/1.3.1",
           "hdf5/1.8.11",
           "netcdf/4.1.3",
           "python_numpy/1.8.0",
           "python_netcdf4/1.0.6"
          ]
    checkCmd = ["bash", "-c", "module list"]
    checkCall = subprocess.Popen(checkCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = checkCall.communicate()

    # Find out if we need to check if modules are loaded
    if "bash: module: command not found" in err or \
            "bash: module: command not found" in out:
        print(" nope.  Continuing....")
        return
    else:
        # We need to check if all of the correct modules are loaded
        if not os.path.exists(cwd + os.sep + "deps"):
            os.mkdir(cwd + os.sep + "deps")
            sys.path.append(cwd + os.sep + "deps")
        f = open(cwd + os.sep + "deps" + os.sep + "modules", 'w')

        # Record the modules needed, the modules that have been loaded, and start a list for what's missing
        moduleListOutput = out.split() + err.split()
        modulesNeeded = []

        # Go through and find out if anything is missing
        for module in modules:
            f.write("module load " + module + "\n")
            if module not in moduleListOutput:
                modulesNeeded.append(module)
        f.close()

        # If anything was missing, tell the user what it was and how to fix it
        if len(modulesNeeded) != 0:
            print("")
            print("  ------------------------------------------")
            print("  | Could not find all necessary modules!  |")
            print("  ------------------------------------------")
            print("    Modules missing:")
            for each in modulesNeeded: print("   * " + each)
            print("")
            print("  ------------------------------------------")
            print("  | Use the command: source deps/modules   |")
            print("  | to load all of the missing modules.    |")
            print("  | then try running LIVV again.           |")
            print("  ------------------------------------------")
            exit(1)
        else:
            print(" found all required modules!")

'''
Installs setuptools under the user python libraries

If setuptools isn't found on a system it is probably the case that other
packages are also not available.   Once setuptools is installed we can
access the easy_install command from inside of LIVV if  any python
dependencies aren't satisfied
'''
def installSetupTools():
    # Specify where to download from
    cwd = os.getcwd()
    url = "https://bootstrap.pypa.io/ez_setup.py"
    fileName = "ez_setup.py"
    u = urllib2.urlopen(url)
    f = open(cwd + os.sep + "deps" + os.sep + fileName, 'wb')

    block=8192
    while True:
        # We are downloading block by block - if we can't get anymore we must be done
        buffer = u.read(block)
        if not buffer: break
        f.write(buffer)
    os.system("python " + cwd + os.sep + "deps" + os.sep + fileName + " --user")
<commit_msg>Removing stray print line.<commit_after>'''
Dependency management module for LIVV

Created on January 6, 2015

@author: arbennett
'''

#
# TODO : Try to import urllib2 before checking for setuptools.  If it's not available try using something
#        like curl or wget from a subprocess to download setuptools and then add urllib2 to the list of
#        dependencies to check (and obviously install)
#

import os
import sys
import urllib2
import subprocess

'''
Run all of the checks for dependencies required by LIVV

Checks if modules are used, and if they are checks to see
if they are loaded.  Then, checks for the necessary Python
libraries.  If any are missing they are installed via
easy_install.  If easy_install isn't installed, that is
also built.
'''
def check():
    cwd = os.getcwd()

    # The list of nonstandard python libraries that are used
    libraryList = ["jinja2", "netCDF4", "numpy", "matplotlib"]
    binaryList = ["ncdiff"]

    # Create a list to store all of the errors that were found
    depErrors = []

    print(os.linesep + "Beginning Dependency Checks........")

    # If we need to load modules for LCF machines do so now
    checkModules()

    # Make sure all environment variables are set
    if not os.environ.has_key("NCARG_ROOT"):
        depErrors.append("  NCARG_ROOT not found in environment")

    # Check to make sure that binary files are found
    print("    Checking for external programs....")
    for program in binaryList:
        found = False
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            filePath = os.path.join(path, program)
            if os.path.isfile(filePath) and os.access(filePath, os.X_OK):
                found = True
        if not found:
            depErrors.append("  " + program + " could not be found in system path.  Please install or update your path!")

    # For the python dependencies we may need easy_install
    # Make sure we have it, and if not, build it
    try:
        from setuptools.command import easy_install
    except ImportError:
        if not os.path.exists(cwd + os.sep + "deps"):
            os.mkdir(cwd + os.sep + "deps")
            sys.path.append(cwd + os.sep + "deps")
        installSetupTools()
        from setuptools.command import easy_install

    # Make sure all imports are going to work
    # And if they don't build a copy of the ones that are needed
    print("    Checking for external libraries....")
    libsInstalled=[]
    for lib in libraryList:
        try:
            __import__(lib)
            print("      Found " + lib + "!")
        except ImportError:
            print("      Could not find " + lib + ".  Building a copy for you...."),
            if not os.path.exists(cwd + os.sep + "deps"):
                os.mkdir(cwd + os.sep + "deps")
                sys.path.append(cwd + os.sep + "deps")
            easy_install.main(["--user",lib])
            libsInstalled.append(lib)

    if len(libsInstalled) > 0:
        print("------------------------------------------------------------------------------")
        print "  External Python Libraries have been installed!  Libraries installed:"
        for lib in libsInstalled:
            print("    " + lib)
        print("  Run LIVV again to continue.  ")
        print("------------------------------------------------------------------------------")
        exit(0)

    # Show all of the dependency errors that were found
    if len(depErrors) > 0:
        print("Uh oh!")
        print("")
        print("Errors checking dependencies.  Errors found: ")
        for err in depErrors: print(err)
        exit(len(depErrors))
    else:
        print("Okay!" + os.linesep + "Setting up environment....")

'''
Checks if the system running LIVV uses modules to load dependencies.

If the system does use modules, this method goes through and makes
sure that the correct modules are loaded.  Any modules that are needed
but haven't been loaded are added to a module loader script that the
user is prompted to source before running LIVV again.
'''
def checkModules():
    # Check to see if calling 'module list' is a real command
    cwd = os.getcwd()
    print("    Checking if modules need to be loaded"),
    modules = [
           "python/2.7.5",
           "ncl/6.1.0",
           "nco/4.3.9",
           "python_matplotlib/1.3.1",
           "hdf5/1.8.11",
           "netcdf/4.1.3",
           "python_numpy/1.8.0",
           "python_netcdf4/1.0.6"
          ]
    checkCmd = ["bash", "-c", "module list"]
    checkCall = subprocess.Popen(checkCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = checkCall.communicate()

    # Find out if we need to check if modules are loaded
    if "bash: module: command not found" in err or \
            "bash: module: command not found" in out:
        print(" nope.  Continuing....")
        return
    else:
        # We need to check if all of the correct modules are loaded
        if not os.path.exists(cwd + os.sep + "deps"):
            os.mkdir(cwd + os.sep + "deps")
            sys.path.append(cwd + os.sep + "deps")
        f = open(cwd + os.sep + "deps" + os.sep + "modules", 'w')

        # Record the modules needed, the modules that have been loaded, and start a list for what's missing
        moduleListOutput = out.split() + err.split()
        modulesNeeded = []

        # Go through and find out if anything is missing
        for module in modules:
            f.write("module load " + module + "\n")
            if module not in moduleListOutput:
                modulesNeeded.append(module)
        f.close()

        # If anything was missing, tell the user what it was and how to fix it
        if len(modulesNeeded) != 0:
            print("")
            print("  ------------------------------------------")
            print("  | Could not find all necessary modules!  |")
            print("  ------------------------------------------")
            print("    Modules missing:")
            for each in modulesNeeded: print("   * " + each)
            print("")
            print("  ------------------------------------------")
            print("  | Use the command: source deps/modules   |")
            print("  | to load all of the missing modules.    |")
            print("  | then try running LIVV again.           |")
            print("  ------------------------------------------")
            exit(1)
        else:
            print(" found all required modules!")

'''
Installs setuptools under the user python libraries

If setuptools isn't found on a system it is probably the case that other
packages are also not available.   Once setuptools is installed we can
access the easy_install command from inside of LIVV if  any python
dependencies aren't satisfied
'''
def installSetupTools():
    # Specify where to download from
    cwd = os.getcwd()
    url = "https://bootstrap.pypa.io/ez_setup.py"
    fileName = "ez_setup.py"
    u = urllib2.urlopen(url)
    f = open(cwd + os.sep + "deps" + os.sep + fileName, 'wb')

    block=8192
    while True:
        # We are downloading block by block - if we can't get anymore we must be done
        buffer = u.read(block)
        if not buffer: break
        f.write(buffer)
    os.system("python " + cwd + os.sep + "deps" + os.sep + fileName + " --user")
<|endoftext|>


<commit_before><commit_msg>docstring update.<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 Google Inc. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Argument-less script to select what to run on the buildbots."""

import os
import shutil
import subprocess
import sys

if sys.platform in ['win32', 'cygwin']:
  EXE_SUFFIX = '.exe'
else:
  EXE_SUFFIX = ''

BUILDBOT_DIR = os.path.dirname(os.path.abspath(__file__))
TRUNK_DIR = os.path.dirname(BUILDBOT_DIR)
ROOT_DIR = os.path.dirname(TRUNK_DIR)
OUT_DIR = os.path.join(TRUNK_DIR, 'out')
NINJA_PATH = os.path.join(TRUNK_DIR, 'ninja' + EXE_SUFFIX)
NINJA_WORK_DIR = os.path.join(ROOT_DIR, 'ninja_work')

def InstallNinja():
  """Install + build ninja.

  Returns:
    0 for success, 1 for failure.
  """
  print '@@@BUILD_STEP install ninja@@@'
  # Delete old version if any.
  try:
    shutil.rmtree(NINJA_WORK_DIR, ignore_errors=True)
  except:
    pass
  # Sync new copy from git.
  subprocess.check_call(
      'git clone https://github.com/martine/ninja.git ' + NINJA_WORK_DIR,
      shell=True)
  # Bootstrap.
  subprocess.check_call('./bootstrap.py', cwd=NINJA_WORK_DIR, shell=True)
  # Copy out ninja.
  shutil.copyfile(os.path.join(NINJA_WORK_DIR, 'ninja' + EXE_SUFFIX),
                  NINJA_PATH)
  os.chmod(NINJA_PATH, 0777)

def GypTestFormat(title, format=None, msvs_version=None):
  """Run the gyp tests for a given format, emitting annotator tags.

  See annotator docs at:
    https://sites.google.com/a/chromium.org/dev/developers/testing/chromium-build-infrastructure/buildbot-annotations
  Args:
    format: gyp format to test.
  Returns:
    0 for sucesss, 1 for failure.
  """
  if not format:
    format = title

  # Install ninja if needed.
  # NOTE: as ninja gets installed each time, regressions to ninja can come
  # either from changes to ninja itself, or changes to gyp.
  if format == 'ninja':
    try:
      InstallNinja()
    except Exception, e:
      print '@@@STEP_FAILURE@@@'
      print str(e)
      return 1

  print '@@@BUILD_STEP ' + title + '@@@'
  sys.stdout.flush()
  env = os.environ.copy()
  # TODO(bradnelson): remove this when this issue is resolved:
  #     http://code.google.com/p/chromium/issues/detail?id=108251
  if format == 'ninja':
    env['NOGOLD'] = '1'
  if msvs_version:
    env['GYP_MSVS_VERSION'] = msvs_version
  retcode = subprocess.call(' '.join(
      [sys.executable, 'trunk/gyptest.py',
       '--all',
       '--passed',
       '--format', format,
       '--chdir', 'trunk',
       '--path', '../scons']),
      cwd=ROOT_DIR, env=env, shell=True)
  if retcode:
    # Emit failure tag, and keep going.
    print '@@@STEP_FAILURE@@@'
    return 1
  return 0

def GypBuild():
  # Dump out/ directory.
  print '@@@BUILD_STEP cleanup@@@'
  print 'Removing %s...' % OUT_DIR
  shutil.rmtree(OUT_DIR, ignore_errors=True)
  print 'Removing %s...' % NINJA_WORK_DIR
  shutil.rmtree(NINJA_WORK_DIR, ignore_errors=True)
  print 'Removing %s...' % NINJA_PATH
  shutil.rmtree(NINJA_PATH, ignore_errors=True)
  print 'Done.'

  retcode = 0
  if sys.platform.startswith('linux'):
    retcode += GypTestFormat('ninja')
    retcode += GypTestFormat('scons')
    retcode += GypTestFormat('make')
  elif sys.platform == 'darwin':
    retcode += GypTestFormat('ninja')
    retcode += GypTestFormat('xcode')
    retcode += GypTestFormat('make')
  elif sys.platform == 'win32':
    retcode += GypTestFormat('msvs-2008', format='msvs', msvs_version='2008')
    if os.environ['BUILDBOT_BUILDERNAME'] == 'gyp-win64':
      retcode += GypTestFormat('msvs-2010', format='msvs', msvs_version='2010')
  else:
    raise Exception('Unknown platform')
  if retcode:
    # TODO(bradnelson): once the annotator supports a postscript (section for
    #     after the build proper that could be used for cumulative failures),
    #     use that instead of this. This isolates the final return value so
    #     that it isn't misattributed to the last stage.
    print '@@@BUILD_STEP failures@@@'
    sys.exit(retcode)

if __name__ == '__main__':
  GypBuild()
<commit_msg>Fixing presubmit warning.<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 Google Inc. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Argument-less script to select what to run on the buildbots."""

import os
import shutil
import subprocess
import sys

if sys.platform in ['win32', 'cygwin']:
  EXE_SUFFIX = '.exe'
else:
  EXE_SUFFIX = ''

BUILDBOT_DIR = os.path.dirname(os.path.abspath(__file__))
TRUNK_DIR = os.path.dirname(BUILDBOT_DIR)
ROOT_DIR = os.path.dirname(TRUNK_DIR)
OUT_DIR = os.path.join(TRUNK_DIR, 'out')
NINJA_PATH = os.path.join(TRUNK_DIR, 'ninja' + EXE_SUFFIX)
NINJA_WORK_DIR = os.path.join(ROOT_DIR, 'ninja_work')

def InstallNinja():
  """Install + build ninja.

  Returns:
    0 for success, 1 for failure.
  """
  print '@@@BUILD_STEP install ninja@@@'
  # Delete old version if any.
  try:
    shutil.rmtree(NINJA_WORK_DIR, ignore_errors=True)
  except:
    pass
  # Sync new copy from git.
  subprocess.check_call(
      'git clone https://github.com/martine/ninja.git ' + NINJA_WORK_DIR,
      shell=True)
  # Bootstrap.
  subprocess.check_call('./bootstrap.py', cwd=NINJA_WORK_DIR, shell=True)
  # Copy out ninja.
  shutil.copyfile(os.path.join(NINJA_WORK_DIR, 'ninja' + EXE_SUFFIX),
                  NINJA_PATH)
  os.chmod(NINJA_PATH, 0777)

def GypTestFormat(title, format=None, msvs_version=None):
  """Run the gyp tests for a given format, emitting annotator tags.

  See annotator docs at:
    https://sites.google.com/a/chromium.org/dev/developers/testing/chromium-build-infrastructure/buildbot-annotations
  Args:
    format: gyp format to test.
  Returns:
    0 for sucesss, 1 for failure.
  """
  if not format:
    format = title

  # Install ninja if needed.
  # NOTE: as ninja gets installed each time, regressions to ninja can come
  # either from changes to ninja itself, or changes to gyp.
  if format == 'ninja':
    try:
      InstallNinja()
    except (shutil.Error, IOError, OSError, CalledProcessError), e:
      print '@@@STEP_FAILURE@@@'
      print str(e)
      return 1

  print '@@@BUILD_STEP ' + title + '@@@'
  sys.stdout.flush()
  env = os.environ.copy()
  # TODO(bradnelson): remove this when this issue is resolved:
  #     http://code.google.com/p/chromium/issues/detail?id=108251
  if format == 'ninja':
    env['NOGOLD'] = '1'
  if msvs_version:
    env['GYP_MSVS_VERSION'] = msvs_version
  retcode = subprocess.call(' '.join(
      [sys.executable, 'trunk/gyptest.py',
       '--all',
       '--passed',
       '--format', format,
       '--chdir', 'trunk',
       '--path', '../scons']),
      cwd=ROOT_DIR, env=env, shell=True)
  if retcode:
    # Emit failure tag, and keep going.
    print '@@@STEP_FAILURE@@@'
    return 1
  return 0

def GypBuild():
  # Dump out/ directory.
  print '@@@BUILD_STEP cleanup@@@'
  print 'Removing %s...' % OUT_DIR
  shutil.rmtree(OUT_DIR, ignore_errors=True)
  print 'Removing %s...' % NINJA_WORK_DIR
  shutil.rmtree(NINJA_WORK_DIR, ignore_errors=True)
  print 'Removing %s...' % NINJA_PATH
  shutil.rmtree(NINJA_PATH, ignore_errors=True)
  print 'Done.'

  retcode = 0
  if sys.platform.startswith('linux'):
    retcode += GypTestFormat('ninja')
    retcode += GypTestFormat('scons')
    retcode += GypTestFormat('make')
  elif sys.platform == 'darwin':
    retcode += GypTestFormat('ninja')
    retcode += GypTestFormat('xcode')
    retcode += GypTestFormat('make')
  elif sys.platform == 'win32':
    retcode += GypTestFormat('msvs-2008', format='msvs', msvs_version='2008')
    if os.environ['BUILDBOT_BUILDERNAME'] == 'gyp-win64':
      retcode += GypTestFormat('msvs-2010', format='msvs', msvs_version='2010')
  else:
    raise Exception('Unknown platform')
  if retcode:
    # TODO(bradnelson): once the annotator supports a postscript (section for
    #     after the build proper that could be used for cumulative failures),
    #     use that instead of this. This isolates the final return value so
    #     that it isn't misattributed to the last stage.
    print '@@@BUILD_STEP failures@@@'
    sys.exit(retcode)

if __name__ == '__main__':
  GypBuild()
<|endoftext|>


<commit_before>import signal

from pprint import pprint as pp
from numbers import Number
from logbook import Logger

class Timeout(Exception):

    def __init__(self, frame):
        self.frame = frame

class timeout(object):
    """
    Decorator to make a function raise TimeoutException if it spends
    more than a specified number of seconds executing.
    """

    def __init__(self, seconds):
        self.seconds = seconds
        assert isinstance(seconds, Number), "Failed to specify a timeout."
        assert seconds > 0, "Timeout must be greater than 0"

    def handler(self, signum, frame):
        raise Timeout(frame)

    def __call__(self, fn):

        def wrapped(*args, **kwargs):
            # Set the alarm.
            signal.signal(signal.SIGALRM, self.handler)
            signal.alarm(self.seconds)
            try:
                outval = fn(*args, **kwargs)

            # Deactivate the alarm once we're done so that the
            # decorator doesn't have unexpected side-effects later.
            # Note that this will still raise TimeoutException if the
            # call to fn takes too long.
            finally:
                signal.alarm(0)
                signal.signal(signal.SIGALRM, signal.SIG_DFL)

            # Return the value of fn if it finished before the alarm.  This
            # won't execute if the Timeout was raised.
            return outval
        return wrapped

class heartbeat(object):
    """
    Decorator to perform pseudo-heartbeat checks on a single-threaded
    function. Calls frame_handler on the current stack frame of the
    decorated function every ``interval`` seconds.  After ``max_interval``
    intervals, raises MaxHeartBeats
    """

    def __init__(self, interval, max_intervals, frame_handler=None):
        self.count = 0
        self.interval = interval
        self.max_intervals = max_intervals
        self.frame_handler = frame_handler

    def handler(self, signum, frame):
        self.count += 1
        if self.frame_handler:
            self.frame_handler(frame)

        if self.count > self.max_intervals:
            raise Timeout(frame)

    def __call__(self, fn):
        def wrapped(*args, **kwargs):
            # Set a timer to call our handler every N seconds.
            signal.signal(signal.SIGALRM, self.handler)
            signal.setitimer(signal.ITIMER_REAL, self.interval, self.interval)
            try:
                outval = fn(*args, **kwargs)

            # Deactivate the timer once we're done so that the
            # decorator doesn't have unexpected side-effects later.
            finally:
                signal.setitimer(signal.ITIMER_REAL, 0, 0)
                signal.signal(signal.SIGALRM, signal.SIG_DFL)

            # Return the value of fn if it finished without tripping
            # an exception.  This won't execute if the Timeout or any
            # other exception was raised by self.handle.
            return outval
        return wrapped

if __name__ == "__main__":
    import time

    def pframe_g(frame):
        print frame.f_globals

    @heartbeat(1, 10, pframe_g)
    def foo():
        for i in xrange(10000):
            time.sleep(.1)
            print i
    foo()
<commit_msg>heartbeat and timeout usable as context managers<commit_after>import signal

from functools import wraps

from pprint import pprint as pp
from numbers import Number
from logbook import Logger

class Timeout(Exception):

    def __init__(self, frame, message=''):
        self.frame = frame
        self.message = message

class timeout(object):
    """
    Utility to make a function raise TimeoutException if it spends
    more than a specified number of seconds executing. Can be used
    as a decorator to apply a static timeout to a function, or as
    a context manager to dynamically add a timeout to a code block.
    """

    def __init__(self, seconds, message=''):
        self.seconds = seconds
        self.message = message
        assert isinstance(seconds, Number), "Failed to specify a timeout."
        assert seconds > 0, "Timeout must be greater than 0"

    def handler(self, signum, frame):
        raise Timeout(frame, self.message)

    def __call__(self, fn):

        @wraps(fn)
        def call_fn_with_timeout(*args, **kwargs):
            # Set the alarm.
            signal.signal(signal.SIGALRM, self.handler)
            signal.setitimer(signal.ITIMER_REAL, self.seconds, 0)
            try:
                outval = fn(*args, **kwargs)

            # Deactivate the alarm once we're done so that the
            # decorator doesn't have unexpected side-effects later.
            # Note that this will still raise Timeout if the
            # call to fn takes too long.
            finally:
                signal.setitimer(signal.ITIMER_REAL, 0, 0)
                signal.signal(signal.SIGALRM, signal.SIG_DFL)

            # Return the value of fn if it finished before the alarm.  This
            # won't execute if the Timeout was raised.
            return outval
        return call_fn_with_timeout

    def __enter__(self):
        # Set the alarm on entrance.
        signal.signal(signal.SIGALRM, self.handler)
        signal.setitimer(signal.ITIMER_REAL, self.seconds, 0)

    def __exit__(self, type, value, traceback):
        # Deactivate the alarm on exit. This will re-raise
        # any exceptions raised inside the with block.
        signal.signal(signal.SIGALRM, self.handler)
        signal.setitimer(signal.ITIMER_REAL, self.seconds, 0)

class heartbeat(object):
    """
    Utility to perform pseudo-heartbeat checks on a single-threaded
    function. Calls frame_handler on the current stack frame of the
    wrapped function every ``interval`` seconds.  After ``max_interval``
    intervals, raises Timeout.  Can be used either as a decorator or
    a context manager.
    """
    def __init__(self,
                 interval,
                 max_intervals,
                 frame_handler=None,
                 timeout_message=''):

        self.interval = interval
        self.max_intervals = max_intervals
        self.frame_handler = frame_handler
        self.timeout_message = timeout_message
        self.count = 0

    def handler(self, signum, frame):
        self.count += 1
        if self.frame_handler:
            self.frame_handler(self.count, frame)

        if self.count >= self.max_intervals:
            raise Timeout(frame, self.timeout_message)

    def __call__(self, fn):

        @wraps(fn)
        def call_fn_with_heartbeat(*args, **kwargs):
            # Set a timer to call our handler every ``interval`` seconds.
            signal.signal(signal.SIGALRM, self.handler)
            signal.setitimer(signal.ITIMER_REAL, self.interval, self.interval)
            try:
                outval = fn(*args, **kwargs)

            finally:
                # Deactivate the timer once we're done so that the
                # decorator doesn't have unexpected side-effects later.
                signal.setitimer(signal.ITIMER_REAL, 0, 0)
                signal.signal(signal.SIGALRM, signal.SIG_DFL)

            # Return the value of fn if it finished without tripping
            # an exception.  This won't execute if the Timeout or any
            # other exception was raised by self.handle.
            return outval
        return call_fn_with_heartbeat

    def __enter__(self):
        # Set a timer to call our handler every N seconds.
        signal.signal(signal.SIGALRM, self.handler)
        signal.setitimer(signal.ITIMER_REAL, self.interval, self.interval)

    def __exit__(self, type, value, traceback):
        # Turn off the timer on exit.  This will re-raise any exception raised
        # during execution of the with-block
        signal.setitimer(signal.ITIMER_REAL, 0, 0)
        signal.signal(signal.SIGALRM, signal.SIG_DFL)
<|endoftext|>