<commit_before>"""
Keep track of which LBRY Files are downloading and store their LBRY File specific metadata
"""

import logging
import os

from twisted.enterprise import adbapi
from twisted.internet import defer, task, reactor
from twisted.python.failure import Failure

from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.cryptstream.client.CryptStreamDownloader import AlreadyStoppedError, CurrentlyStoppingError
from lbrynet.core.sqlite_helpers import rerun_if_locked

log = logging.getLogger(__name__)

class LBRYFileManager(object):
    """
    Keeps track of currently opened LBRY Files, their options, and their LBRY File specific metadata.
    """

    def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None):
        self.session = session
        self.stream_info_manager = stream_info_manager
        self.sd_identifier = sd_identifier
        self.lbry_files = []
        self.sql_db = None
        if download_directory:
            self.download_directory = download_directory
        else:
            self.download_directory = os.getcwd()
        log.debug("Download directory for LBRYFileManager: %s", str(self.download_directory))

    def setup(self):
        d = self._open_db()
        d.addCallback(lambda _: self._add_to_sd_identifier())
        d.addCallback(lambda _: self._start_lbry_files())
        return d

    def get_lbry_file_status(self, lbry_file):
        return self._get_lbry_file_status(lbry_file.rowid)

    def set_lbry_file_data_payment_rate(self, lbry_file, new_rate):
        return self._set_lbry_file_payment_rate(lbry_file.rowid, new_rate)

    def change_lbry_file_status(self, lbry_file, status):
        log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
        return self._change_file_status(lbry_file.rowid, status)

    def get_lbry_file_status_reports(self):
        ds = []

        for lbry_file in self.lbry_files:
            ds.append(lbry_file.status())

        dl = defer.DeferredList(ds)

        def filter_failures(status_reports):
            return [status_report for success, status_report in status_reports if success is True]

        dl.addCallback(filter_failures)
        return dl

    def _add_to_sd_identifier(self):
        downloader_factory = ManagedLBRYFileDownloaderFactory(self)
        self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, downloader_factory)

    def _start_lbry_files(self):

        def set_options_and_restore(rowid, stream_hash, options):
            payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
            d = self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
                                     blob_data_rate=options)
            d.addCallback(lambda downloader: downloader.restore())
            return d

        def log_error(err):
            log.error("An error occurred while starting a lbry file: %s", err.getErrorMessage())

        def start_lbry_files(lbry_files_and_options):
            for rowid, stream_hash, options in lbry_files_and_options:
                d = set_options_and_restore(rowid, stream_hash, options)
                d.addErrback(log_error)
            return True

        d = self._get_all_lbry_files()
        d.addCallback(start_lbry_files)
        return d

    def start_lbry_file(self, rowid, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
                                                                        download_directory=None, file_name=None):
        if not download_directory:
            download_directory = self.download_directory
        payment_rate_manager.min_blob_data_payment_rate = blob_data_rate
        lbry_file_downloader = ManagedLBRYFileDownloader(rowid, stream_hash,
                                                         self.session.peer_finder,
                                                         self.session.rate_limiter,
                                                         self.session.blob_manager,
                                                         self.stream_info_manager, self,
                                                         payment_rate_manager, self.session.wallet,
                                                         download_directory,
                                                         upload_allowed,
                                                         file_name=file_name)
        self.lbry_files.append(lbry_file_downloader)
        d = lbry_file_downloader.set_stream_info()
        d.addCallback(lambda _: lbry_file_downloader)
        return d

    def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
                                                                download_directory=None, file_name=None):
        d = self._save_lbry_file(stream_hash, blob_data_rate)
        d.addCallback(lambda rowid: self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
                                                         blob_data_rate, upload_allowed, download_directory, file_name))
        return d

    def delete_lbry_file(self, lbry_file):
        for l in self.lbry_files:
            if l == lbry_file:
                lbry_file = l
                break
        else:
            return defer.fail(Failure(ValueError("Could not find that LBRY file")))

        def wait_for_finished(count=2):
            if count <= 0 or lbry_file.saving_status is False:
                return True
            else:
                return task.deferLater(reactor, 1, wait_for_finished, count=count - 1)

        def ignore_stopped(err):
            err.trap(AlreadyStoppedError, CurrentlyStoppingError)
            return wait_for_finished()

        d = lbry_file.stop()
        d.addErrback(ignore_stopped)

        def remove_from_list():
            self.lbry_files.remove(lbry_file)

        d.addCallback(lambda _: remove_from_list())
        d.addCallback(lambda _: self._delete_lbry_file_options(lbry_file.rowid))
        return d

    def toggle_lbry_file_running(self, lbry_file):
        """Toggle whether a stream reader is currently running"""
        for l in self.lbry_files:
            if l == lbry_file:
                return l.toggle_running()
        else:
            return defer.fail(Failure(ValueError("Could not find that LBRY file")))

    def stop(self):

        ds = []

        def wait_for_finished(lbry_file, count=2):
            if count <= 0 or lbry_file.saving_status is False:
                return True
            else:
                return task.deferLater(reactor, 1, wait_for_finished, lbry_file, count=count - 1)

        def ignore_stopped(err, lbry_file):
            err.trap(AlreadyStoppedError, CurrentlyStoppingError)
            return wait_for_finished(lbry_file)

        for lbry_file in self.lbry_files:
            d = lbry_file.stop(change_status=False)
            d.addErrback(ignore_stopped, lbry_file)
            ds.append(d)
        dl = defer.DeferredList(ds)

        def close_db():
            self.db = None

        dl.addCallback(lambda _: close_db())
        return dl

    def get_count_for_stream_hash(self, stream_hash):
        return self._get_count_for_stream_hash(stream_hash)

    ######### database calls #########

    def _open_db(self):
        # check_same_thread=False is solely to quiet a spurious error that appears to be due
        # to a bug in twisted, where the connection is closed by a different thread than the
        # one that opened it. The individual connections in the pool are not used in multiple
        # threads.
        self.sql_db = adbapi.ConnectionPool("sqlite3", os.path.join(self.session.db_dir, "lbryfile_info.db"),
                                            check_same_thread=False)
        return self.sql_db.runQuery("create table if not exists lbry_file_options (" +
                                    "    blob_data_rate real, " +
                                    "    status text," +
                                    "    stream_hash text,"
                                    "    foreign key(stream_hash) references lbry_files(stream_hash)" +
                                    ")")

    @rerun_if_locked
    def _save_lbry_file(self, stream_hash, data_payment_rate):
        def do_save(db_transaction):
            db_transaction.execute("insert into lbry_file_options values (?, ?, ?)",
                                  (data_payment_rate, ManagedLBRYFileDownloader.STATUS_STOPPED,
                                   stream_hash))
            return db_transaction.lastrowid
        return self.sql_db.runInteraction(do_save)

    @rerun_if_locked
    def _delete_lbry_file_options(self, rowid):
        return self.sql_db.runQuery("delete from lbry_file_options where rowid = ?",
                                    (rowid,))

    @rerun_if_locked
    def _set_lbry_file_payment_rate(self, rowid, new_rate):
        return self.sql_db.runQuery("update lbry_file_options set blob_data_rate = ? where rowid = ?",
                                    (new_rate, rowid))

    @rerun_if_locked
    def _get_all_lbry_files(self):
        d = self.sql_db.runQuery("select rowid, stream_hash, blob_data_rate from lbry_file_options")
        return d

    @rerun_if_locked
    def _change_file_status(self, rowid, new_status):
        return self.sql_db.runQuery("update lbry_file_options set status = ? where rowid = ?",
                                    (new_status, rowid))

    @rerun_if_locked
    def _get_lbry_file_status(self, rowid):
        d = self.sql_db.runQuery("select status from lbry_file_options where rowid = ?",
                                 (rowid,))
        d.addCallback(lambda r: r[0][0] if len(r) else ManagedLBRYFileDownloader.STATUS_STOPPED)
        return d

    @rerun_if_locked
    def _get_count_for_stream_hash(self, stream_hash):
        return self.sql_db.runQuery("select count(*) from lbry_file_options where stream_hash = ?",
                                     (stream_hash,))<commit_msg>log conditions of lbry_file failing to start<commit_after>"""
Keep track of which LBRY Files are downloading and store their LBRY File specific metadata
"""

import logging
import os

from twisted.enterprise import adbapi
from twisted.internet import defer, task, reactor
from twisted.python.failure import Failure

from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.cryptstream.client.CryptStreamDownloader import AlreadyStoppedError, CurrentlyStoppingError
from lbrynet.core.sqlite_helpers import rerun_if_locked

log = logging.getLogger(__name__)

class LBRYFileManager(object):
    """
    Keeps track of currently opened LBRY Files, their options, and their LBRY File specific metadata.
    """

    def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None):
        self.session = session
        self.stream_info_manager = stream_info_manager
        self.sd_identifier = sd_identifier
        self.lbry_files = []
        self.sql_db = None
        if download_directory:
            self.download_directory = download_directory
        else:
            self.download_directory = os.getcwd()
        log.debug("Download directory for LBRYFileManager: %s", str(self.download_directory))

    def setup(self):
        d = self._open_db()
        d.addCallback(lambda _: self._add_to_sd_identifier())
        d.addCallback(lambda _: self._start_lbry_files())
        return d

    def get_lbry_file_status(self, lbry_file):
        return self._get_lbry_file_status(lbry_file.rowid)

    def set_lbry_file_data_payment_rate(self, lbry_file, new_rate):
        return self._set_lbry_file_payment_rate(lbry_file.rowid, new_rate)

    def change_lbry_file_status(self, lbry_file, status):
        log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
        return self._change_file_status(lbry_file.rowid, status)

    def get_lbry_file_status_reports(self):
        ds = []

        for lbry_file in self.lbry_files:
            ds.append(lbry_file.status())

        dl = defer.DeferredList(ds)

        def filter_failures(status_reports):
            return [status_report for success, status_report in status_reports if success is True]

        dl.addCallback(filter_failures)
        return dl

    def _add_to_sd_identifier(self):
        downloader_factory = ManagedLBRYFileDownloaderFactory(self)
        self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, downloader_factory)

    def _start_lbry_files(self):

        def set_options_and_restore(rowid, stream_hash, options):
            payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
            d = self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
                                     blob_data_rate=options)
            d.addCallback(lambda downloader: downloader.restore())
            return d

        def log_error(err, rowid, stream_hash, options):
            log.error("An error occurred while starting a lbry file: %s", err.getErrorMessage())
            log.error(rowid)
            log.error(stream_hash)
            log.error(options)

        def start_lbry_files(lbry_files_and_options):
            for rowid, stream_hash, options in lbry_files_and_options:
                d = set_options_and_restore(rowid, stream_hash, options)
                d.addErrback(lambda err: log_error(err, rowid, stream_hash, options))
            return True

        d = self._get_all_lbry_files()
        d.addCallback(start_lbry_files)
        return d

    def start_lbry_file(self, rowid, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
                                                                        download_directory=None, file_name=None):
        if not download_directory:
            download_directory = self.download_directory
        payment_rate_manager.min_blob_data_payment_rate = blob_data_rate
        lbry_file_downloader = ManagedLBRYFileDownloader(rowid, stream_hash,
                                                         self.session.peer_finder,
                                                         self.session.rate_limiter,
                                                         self.session.blob_manager,
                                                         self.stream_info_manager, self,
                                                         payment_rate_manager, self.session.wallet,
                                                         download_directory,
                                                         upload_allowed,
                                                         file_name=file_name)
        self.lbry_files.append(lbry_file_downloader)
        d = lbry_file_downloader.set_stream_info()
        d.addCallback(lambda _: lbry_file_downloader)
        return d

    def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
                                                                download_directory=None, file_name=None):
        d = self._save_lbry_file(stream_hash, blob_data_rate)
        d.addCallback(lambda rowid: self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
                                                         blob_data_rate, upload_allowed, download_directory, file_name))
        return d

    def delete_lbry_file(self, lbry_file):
        for l in self.lbry_files:
            if l == lbry_file:
                lbry_file = l
                break
        else:
            return defer.fail(Failure(ValueError("Could not find that LBRY file")))

        def wait_for_finished(count=2):
            if count <= 0 or lbry_file.saving_status is False:
                return True
            else:
                return task.deferLater(reactor, 1, wait_for_finished, count=count - 1)

        def ignore_stopped(err):
            err.trap(AlreadyStoppedError, CurrentlyStoppingError)
            return wait_for_finished()

        d = lbry_file.stop()
        d.addErrback(ignore_stopped)

        def remove_from_list():
            self.lbry_files.remove(lbry_file)

        d.addCallback(lambda _: remove_from_list())
        d.addCallback(lambda _: self._delete_lbry_file_options(lbry_file.rowid))
        return d

    def toggle_lbry_file_running(self, lbry_file):
        """Toggle whether a stream reader is currently running"""
        for l in self.lbry_files:
            if l == lbry_file:
                return l.toggle_running()
        else:
            return defer.fail(Failure(ValueError("Could not find that LBRY file")))

    def stop(self):

        ds = []

        def wait_for_finished(lbry_file, count=2):
            if count <= 0 or lbry_file.saving_status is False:
                return True
            else:
                return task.deferLater(reactor, 1, wait_for_finished, lbry_file, count=count - 1)

        def ignore_stopped(err, lbry_file):
            err.trap(AlreadyStoppedError, CurrentlyStoppingError)
            return wait_for_finished(lbry_file)

        for lbry_file in self.lbry_files:
            d = lbry_file.stop(change_status=False)
            d.addErrback(ignore_stopped, lbry_file)
            ds.append(d)
        dl = defer.DeferredList(ds)

        def close_db():
            self.db = None

        dl.addCallback(lambda _: close_db())
        return dl

    def get_count_for_stream_hash(self, stream_hash):
        return self._get_count_for_stream_hash(stream_hash)

    ######### database calls #########

    def _open_db(self):
        # check_same_thread=False is solely to quiet a spurious error that appears to be due
        # to a bug in twisted, where the connection is closed by a different thread than the
        # one that opened it. The individual connections in the pool are not used in multiple
        # threads.
        self.sql_db = adbapi.ConnectionPool("sqlite3", os.path.join(self.session.db_dir, "lbryfile_info.db"),
                                            check_same_thread=False)
        return self.sql_db.runQuery("create table if not exists lbry_file_options (" +
                                    "    blob_data_rate real, " +
                                    "    status text," +
                                    "    stream_hash text,"
                                    "    foreign key(stream_hash) references lbry_files(stream_hash)" +
                                    ")")

    @rerun_if_locked
    def _save_lbry_file(self, stream_hash, data_payment_rate):
        def do_save(db_transaction):
            db_transaction.execute("insert into lbry_file_options values (?, ?, ?)",
                                  (data_payment_rate, ManagedLBRYFileDownloader.STATUS_STOPPED,
                                   stream_hash))
            return db_transaction.lastrowid
        return self.sql_db.runInteraction(do_save)

    @rerun_if_locked
    def _delete_lbry_file_options(self, rowid):
        return self.sql_db.runQuery("delete from lbry_file_options where rowid = ?",
                                    (rowid,))

    @rerun_if_locked
    def _set_lbry_file_payment_rate(self, rowid, new_rate):
        return self.sql_db.runQuery("update lbry_file_options set blob_data_rate = ? where rowid = ?",
                                    (new_rate, rowid))

    @rerun_if_locked
    def _get_all_lbry_files(self):
        d = self.sql_db.runQuery("select rowid, stream_hash, blob_data_rate from lbry_file_options")
        return d

    @rerun_if_locked
    def _change_file_status(self, rowid, new_status):
        return self.sql_db.runQuery("update lbry_file_options set status = ? where rowid = ?",
                                    (new_status, rowid))

    @rerun_if_locked
    def _get_lbry_file_status(self, rowid):
        d = self.sql_db.runQuery("select status from lbry_file_options where rowid = ?",
                                 (rowid,))
        d.addCallback(lambda r: r[0][0] if len(r) else ManagedLBRYFileDownloader.STATUS_STOPPED)
        return d

    @rerun_if_locked
    def _get_count_for_stream_hash(self, stream_hash):
        return self.sql_db.runQuery("select count(*) from lbry_file_options where stream_hash = ?",
                                     (stream_hash,))<|endoftext|>


<commit_before>import subprocess
import glob
import constants_and_names as cn
import datetime
import os
import re
import pandas as pd
from osgeo import gdal

# Prints the date as YYYYmmdd
d = datetime.datetime.today()
date = d.strftime('%Y%m%d')

# Creates chunks of tiles for processing
def chunks(l, n):
    # Yield successive n-sized chunks from l
    for i in xrange(0, len(l), n):
        yield l[i:i + n]

# Gets the tile id from the full tile name using a regular expression
def get_tile_id(tile_name):

    # based on https://stackoverflow.com/questions/20003025/find-1-letter-and-2-numbers-using-regex and https://docs.python.org/3.4/howto/regex.html
    tile_id = re.search("[0-9]{2}[A-Z][_][0-9]{3}[A-Z]", tile_name).group()

    return tile_id

# Creates a list of all the biomass tiles (WHRC non-mangrove and mangrove)
def read_biomass_tile_list():

    file_list = []

    with open('{}{}'.format(cn.biomass_tile_list_dir, cn.pattern_biomass_tile_list), 'r') as tiles:
        for tile in tiles:
            file_list.append(tile)

    return file_list

# Lists the tiles in a folder in s3
def tile_list(source):

    print "Creating list of tiles..."

    ## For an s3 folder in a bucket using AWSCLI
    # Captures the list of the files in the folder
    out = subprocess.Popen(['aws', 's3', 'ls', source], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()

    # Writes the output string to a text file for easier interpretation
    biomass_tiles = open("tiles.txt", "w")
    biomass_tiles.write(stdout)
    biomass_tiles.close()

    file_list = []

    # Iterates through the text file to get the names of the tiles and appends them to list
    with open("tiles.txt", 'r') as tile:
        for line in tile:
            num = len(line.strip('\n').split(" "))
            tile_name = line.strip('\n').split(" ")[num - 1]

            # Only tifs will be in the tile list
            if '.tif' in tile_name:

                tile_id = get_tile_id(tile_name)
                file_list.append(tile_id)

    return file_list

# Lists the tiles on the spot machine
def tile_list_spot_machine(source, pattern):

    ## For an s3 folder in a bucket using AWSCLI
    # Captures the list of the files in the folder
    out = subprocess.Popen(['ls', source], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()

    # Writes the output string to a text file for easier interpretation
    biomass_tiles = open("tiles.txt", "w")
    biomass_tiles.write(stdout)
    biomass_tiles.close()

    file_list = []

    # Iterates through the text file to get the names of the tiles and appends them to list
    with open("tiles.txt", 'r') as tile:
        for line in tile:
            num = len(line.strip('\n').split(" "))
            tile_name = line.strip('\n').split(" ")[num - 1]

            # Only files with the specified pattern will be in the tile list
            if pattern in tile_name:

                file_list.append(tile_name)

    return file_list

# Creates a list of all biomass 2000 tiles-- those from WHRC and those only in the mangrove set
def create_combined_tile_list(set1, set2, set3=None):

    print "Making a combined tile list..."

    out = subprocess.Popen(['aws', 's3', 'ls', set1], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()
    # Writes the output string to a text file for easier interpretation
    set1_tiles = open("set1.txt", "w")
    set1_tiles.write(stdout)
    set1_tiles.close()

    out = subprocess.Popen(['aws', 's3', 'ls', set2], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout2, stderr2 = out.communicate()
    # Writes the output string to a text file for easier interpretation
    set2_tiles = open("set2.txt", "w")
    set2_tiles.write(stdout2)
    set2_tiles.close()

    # Empty lists for filling with biomass tile ids
    file_list_set1 = []
    file_list_set2 = []

    # Iterates through the Woods Hole biomass text file to get the names of the tiles and appends them to list
    with open("set1.txt", 'r') as tile:

        for line in tile:
            num = len(line.strip('\n').split(" "))
            tile_name = line.strip('\n').split(" ")[num - 1]

            # Only tifs will be in the tile list
            if '.tif' in tile_name:

                tile_id = get_tile_id(tile_name)
                file_list_set1.append(tile_id)

    # Iterates through the mangrove biomass text file to get the names of the tiles and appends them to list
    with open("set2.txt", 'r') as tile:

        for line in tile:
            num = len(line.strip('\n').split(" "))
            tile_name = line.strip('\n').split(" ")[num - 1]

            # Only tifs will be in the tile list
            if '.tif' in tile_name:

                tile_id = get_tile_id(tile_name)
                file_list_set2.append(tile_id)

    print "There are {} tiles in {}".format(len(file_list_set1), set1)
    print "There are {} tiles in {}".format(len(file_list_set2), set2)

    if set3 != None:

        print "Third set of tiles input. Adding to first two sets of tiles..."

        out = subprocess.Popen(['aws', 's3', 'ls', set3], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        stdout3, stderr3 = out.communicate()
        # Writes the output string to a text file for easier interpretation
        set3_tiles = open("set3.txt", "w")
        set3_tiles.write(stdout3)
        set3_tiles.close()

        file_list_set3 = []

        # Iterates through the mangrove biomass text file to get the names of the tiles and appends them to list
        with open("set3.txt", 'r') as tile:

            for line in tile:
                num = len(line.strip('\n').split(" "))
                tile_name = line.strip('\n').split(" ")[num - 1]

                # Only tifs will be in the tile list
                if '.tif' in tile_name:
                    tile_id = get_tile_id(tile_name)
                    file_list_set3.append(tile_id)

        print "There are {} tiles in {}".format(len(file_list_set3), set3)

    # Combines both tile lists
    all_tiles = file_list_set1 + file_list_set2

    if set3 != None:

        all_tiles = all_tiles + file_list_set3

    # Tile list with tiles found in both lists removed, so only the unique tiles remain
    unique_tiles = list(set(all_tiles))

    # Converts the set to a pandas dataframe to put the tiles in the correct order
    df = pd.DataFrame(unique_tiles, columns=['tile_id'])
    df = df.sort_values(by=['tile_id'])
    # print "Tile list is:", df

    # Converts the pandas dataframe to a Python list so that it can be written to a txt
    unique_tiles_ordered_list = df.tile_id.tolist()

    # # Writes the biomass tile list to a txt
    # with open(cn.pattern_biomass_tile_list, 'w') as f:
    #     for item in unique_tiles_ordered_list:
    #         f.write("%s, " % item)
    #
    # # Copies that list to s3
    # cmd = ['aws', 's3', 'cp', cn.pattern_biomass_tile_list, '{0}{1}'.format(cn.biomass_tile_list_dir, cn.pattern_biomass_tile_list)]
    # subprocess.check_call(cmd)

    set_txt = glob.glob("set*.txt")
    print set_txt

    for set in set_txt:

        os.remove(set)

    # os.remove("set1.txt")
    # os.remove("set2.txt")
    #
    # if set3 != None:
    #
    #     os.remove("set3.txt")

    return unique_tiles_ordered_list

# Counts the number of tiles in a folder in s3
def count_tiles_s3(source):

    ## For an s3 folder in a bucket using AWSCLI
    # Captures the list of the files in the folder
    out = subprocess.Popen(['aws', 's3', 'ls', source], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()

    # Writes the output string to a text file for easier interpretation
    tile_file = open("tiles.txt", "w")
    tile_file.write(stdout)
    tile_file.close()

    i=0
    with open(tile_file) as f:
        for i, l in enumerate(f):
            pass
    return i + 1

# Gets the bounding coordinates of a tile
def coords(tile_id):
    NS = tile_id.split("_")[0][-1:]
    EW = tile_id.split("_")[1][-1:]

    if NS == 'S':
        ymax =-1*int(tile_id.split("_")[0][:2])
    else:
        ymax = int(str(tile_id.split("_")[0][:2]))

    if EW == 'W':
        xmin = -1*int(str(tile_id.split("_")[1][:3]))
    else:
        xmin = int(str(tile_id.split("_")[1][:3]))

    ymin = str(int(ymax) - 10)
    xmax = str(int(xmin) + 10)

    return xmin, ymin, xmax, ymax

def s3_folder_download(source, dest):
    cmd = ['aws', 's3', 'cp', source, dest, '--recursive', '--exclude', '*', '--include', '*.tif']
    subprocess.check_call(cmd)

def s3_file_download(source, dest):
    cmd = ['aws', 's3', 'cp', source, dest]
    subprocess.check_call(cmd)

# Uploads a set of tiles that have been created in a chunk to a specified location on s3
def upload_chunk_set(upload_dir, pattern):

    print "Uploading chunk of tiles..."

    cmd = ['aws', 's3', 'cp', '.', upload_dir, '--exclude', '*', '--include', '*{}*'.format(pattern),
           '--exclude', '*/*', '--recursive']

    # Copies the tiles to s3
    subprocess.check_call(cmd)

    # Moves the copied tiles to a folder of already copied tiles on spot machine
    os.system('''mv *_{0}.tif {1}/'''.format(pattern, cn.already_copied))

# Uploads all tiles of a pattern to specified location
def upload_final_set(upload_dir, pattern):

    cmd = ['aws', 's3', 'cp', '.', upload_dir, '--exclude', '*', '--include', '*{}*tif'.format(pattern), '--recursive']

    try:
        subprocess.check_call(cmd)
    except:
        print "Error uploading output tile"

# Uploads tile to specified location
def upload_final(upload_dir, tile_id, pattern):

    file = '{}_{}.tif'.format(tile_id, pattern)

    print "Uploading {}".format(file)
    cmd = ['aws', 's3', 'cp', file, upload_dir]

    try:
        subprocess.check_call(cmd)
    except:
        print "Error uploading output tile"

def check_for_data(out_tile):

    # Source: http://gis.stackexchange.com/questions/90726
    # Opens raster and chooses band to find min, max
    gtif = gdal.Open(out_tile)
    srcband = gtif.GetRasterBand(1)
    stats = srcband.GetStatistics(True, True)
    print "  Tile stats =  Minimum=%.3f, Maximum=%.3f, Mean=%.3f, StdDev=%.3f" % (stats[0], stats[1], stats[2], stats[3])

    return stats

# Prints the number of tiles that have been processed so far
def count_completed_tiles(pattern):

    mypath = os.getcwd()
    completed = len(glob.glob1(mypath, '*{}*'.format(pattern)))

    # completed = str(os.system('''ls -l *{}*.tif | wc -l'''.format(pattern)))
    print "Number of completed tiles:", completed

# Returns the NoData value of a raster
def get_raster_nodata_value(tile):

    # Source: http://gis.stackexchange.com/questions/90726
    # Opens raster we're getting statistics on
    focus_tile = gdal.Open(tile)

    # Extracts the NoData value for the tile so it can be ignored
    srcband = focus_tile.GetRasterBand(1)
    nodata = srcband.GetNoDataValue()

    return nodata

# Prints information about the tile that was just processed: how long it took and how many tiles have been completed
def end_of_fx_summary(start, tile_id, pattern):

    end = datetime.datetime.now()
    elapsed_time = end-start
    print "Processing time for tile", tile_id, ":", elapsed_time
    count_completed_tiles(pattern)<commit_msg>Working on script to merge all annual and cumulative gain tiles.<commit_after>import subprocess
import glob
import constants_and_names as cn
import datetime
import os
import re
import pandas as pd
from osgeo import gdal

# Prints the date as YYYYmmdd
d = datetime.datetime.today()
date = d.strftime('%Y%m%d')

# Creates chunks of tiles for processing
def chunks(l, n):
    # Yield successive n-sized chunks from l
    for i in xrange(0, len(l), n):
        yield l[i:i + n]

# Gets the tile id from the full tile name using a regular expression
def get_tile_id(tile_name):

    # based on https://stackoverflow.com/questions/20003025/find-1-letter-and-2-numbers-using-regex and https://docs.python.org/3.4/howto/regex.html
    tile_id = re.search("[0-9]{2}[A-Z][_][0-9]{3}[A-Z]", tile_name).group()

    return tile_id

# Creates a list of all the biomass tiles (WHRC non-mangrove and mangrove)
def read_biomass_tile_list():

    file_list = []

    with open('{}{}'.format(cn.biomass_tile_list_dir, cn.pattern_biomass_tile_list), 'r') as tiles:
        for tile in tiles:
            file_list.append(tile)

    return file_list

# Lists the tiles in a folder in s3
def tile_list(source):

    print "Creating list of tiles..."

    ## For an s3 folder in a bucket using AWSCLI
    # Captures the list of the files in the folder
    out = subprocess.Popen(['aws', 's3', 'ls', source], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()

    # Writes the output string to a text file for easier interpretation
    biomass_tiles = open("tiles.txt", "w")
    biomass_tiles.write(stdout)
    biomass_tiles.close()

    file_list = []

    # Iterates through the text file to get the names of the tiles and appends them to list
    with open("tiles.txt", 'r') as tile:
        for line in tile:
            num = len(line.strip('\n').split(" "))
            tile_name = line.strip('\n').split(" ")[num - 1]

            # Only tifs will be in the tile list
            if '.tif' in tile_name:

                tile_id = get_tile_id(tile_name)
                file_list.append(tile_id)

    return file_list

# Lists the tiles on the spot machine
def tile_list_spot_machine(source, pattern):

    ## For an s3 folder in a bucket using AWSCLI
    # Captures the list of the files in the folder
    out = subprocess.Popen(['ls', source], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()

    # Writes the output string to a text file for easier interpretation
    biomass_tiles = open("tiles.txt", "w")
    biomass_tiles.write(stdout)
    biomass_tiles.close()

    file_list = []

    # Iterates through the text file to get the names of the tiles and appends them to list
    with open("tiles.txt", 'r') as tile:
        for line in tile:
            num = len(line.strip('\n').split(" "))
            tile_name = line.strip('\n').split(" ")[num - 1]

            # Only files with the specified pattern will be in the tile list
            if pattern in tile_name:

                file_list.append(tile_name)

    return file_list

# Creates a list of all biomass 2000 tiles-- those from WHRC and those only in the mangrove set
def create_combined_tile_list(set1, set2, set3=None):

    print "Making a combined tile list..."

    out = subprocess.Popen(['aws', 's3', 'ls', set1], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()
    # Writes the output string to a text file for easier interpretation
    set1_tiles = open("set1.txt", "w")
    set1_tiles.write(stdout)
    set1_tiles.close()

    out = subprocess.Popen(['aws', 's3', 'ls', set2], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout2, stderr2 = out.communicate()
    # Writes the output string to a text file for easier interpretation
    set2_tiles = open("set2.txt", "w")
    set2_tiles.write(stdout2)
    set2_tiles.close()

    # Empty lists for filling with biomass tile ids
    file_list_set1 = []
    file_list_set2 = []

    # Iterates through the Woods Hole biomass text file to get the names of the tiles and appends them to list
    with open("set1.txt", 'r') as tile:

        for line in tile:
            num = len(line.strip('\n').split(" "))
            tile_name = line.strip('\n').split(" ")[num - 1]

            # Only tifs will be in the tile list
            if '.tif' in tile_name:

                tile_id = get_tile_id(tile_name)
                file_list_set1.append(tile_id)

    # Iterates through the mangrove biomass text file to get the names of the tiles and appends them to list
    with open("set2.txt", 'r') as tile:

        for line in tile:
            num = len(line.strip('\n').split(" "))
            tile_name = line.strip('\n').split(" ")[num - 1]

            # Only tifs will be in the tile list
            if '.tif' in tile_name:

                tile_id = get_tile_id(tile_name)
                file_list_set2.append(tile_id)

    print "There are {} tiles in {}".format(len(file_list_set1), set1)
    print "There are {} tiles in {}".format(len(file_list_set2), set2)

    if set3 != None:

        print "Third set of tiles input. Adding to first two sets of tiles..."

        out = subprocess.Popen(['aws', 's3', 'ls', set3], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        stdout3, stderr3 = out.communicate()
        # Writes the output string to a text file for easier interpretation
        set3_tiles = open("set3.txt", "w")
        set3_tiles.write(stdout3)
        set3_tiles.close()

        file_list_set3 = []

        # Iterates through the mangrove biomass text file to get the names of the tiles and appends them to list
        with open("set3.txt", 'r') as tile:

            for line in tile:
                num = len(line.strip('\n').split(" "))
                tile_name = line.strip('\n').split(" ")[num - 1]

                # Only tifs will be in the tile list
                if '.tif' in tile_name:
                    tile_id = get_tile_id(tile_name)
                    file_list_set3.append(tile_id)

        print "There are {} tiles in {}".format(len(file_list_set3), set3)

    # Combines both tile lists
    all_tiles = file_list_set1 + file_list_set2

    if set3 != None:

        all_tiles = all_tiles + file_list_set3

    # Tile list with tiles found in both lists removed, so only the unique tiles remain
    unique_tiles = list(set(all_tiles))

    # Converts the set to a pandas dataframe to put the tiles in the correct order
    df = pd.DataFrame(unique_tiles, columns=['tile_id'])
    df = df.sort_values(by=['tile_id'])
    # print "Tile list is:", df

    # Converts the pandas dataframe to a Python list so that it can be written to a txt
    unique_tiles_ordered_list = df.tile_id.tolist()

    # # Writes the biomass tile list to a txt
    # with open(cn.pattern_biomass_tile_list, 'w') as f:
    #     for item in unique_tiles_ordered_list:
    #         f.write("%s, " % item)
    #
    # # Copies that list to s3
    # cmd = ['aws', 's3', 'cp', cn.pattern_biomass_tile_list, '{0}{1}'.format(cn.biomass_tile_list_dir, cn.pattern_biomass_tile_list)]
    # subprocess.check_call(cmd)

    set_txt = glob.glob("set*.txt")
    print set_txt

    for i in set_txt:

        os.remove(i)

    # os.remove("set1.txt")
    # os.remove("set2.txt")
    #
    # if set3 != None:
    #
    #     os.remove("set3.txt")

    return unique_tiles_ordered_list

# Counts the number of tiles in a folder in s3
def count_tiles_s3(source):

    ## For an s3 folder in a bucket using AWSCLI
    # Captures the list of the files in the folder
    out = subprocess.Popen(['aws', 's3', 'ls', source], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()

    # Writes the output string to a text file for easier interpretation
    tile_file = open("tiles.txt", "w")
    tile_file.write(stdout)
    tile_file.close()

    i=0
    with open(tile_file) as f:
        for i, l in enumerate(f):
            pass
    return i + 1

# Gets the bounding coordinates of a tile
def coords(tile_id):
    NS = tile_id.split("_")[0][-1:]
    EW = tile_id.split("_")[1][-1:]

    if NS == 'S':
        ymax =-1*int(tile_id.split("_")[0][:2])
    else:
        ymax = int(str(tile_id.split("_")[0][:2]))

    if EW == 'W':
        xmin = -1*int(str(tile_id.split("_")[1][:3]))
    else:
        xmin = int(str(tile_id.split("_")[1][:3]))

    ymin = str(int(ymax) - 10)
    xmax = str(int(xmin) + 10)

    return xmin, ymin, xmax, ymax

def s3_folder_download(source, dest):
    cmd = ['aws', 's3', 'cp', source, dest, '--recursive', '--exclude', '*', '--include', '*.tif']
    subprocess.check_call(cmd)

def s3_file_download(source, dest):
    cmd = ['aws', 's3', 'cp', source, dest]
    subprocess.check_call(cmd)

# Uploads a set of tiles that have been created in a chunk to a specified location on s3
def upload_chunk_set(upload_dir, pattern):

    print "Uploading chunk of tiles..."

    cmd = ['aws', 's3', 'cp', '.', upload_dir, '--exclude', '*', '--include', '*{}*'.format(pattern),
           '--exclude', '*/*', '--recursive']

    # Copies the tiles to s3
    subprocess.check_call(cmd)

    # Moves the copied tiles to a folder of already copied tiles on spot machine
    os.system('''mv *_{0}.tif {1}/'''.format(pattern, cn.already_copied))

# Uploads all tiles of a pattern to specified location
def upload_final_set(upload_dir, pattern):

    cmd = ['aws', 's3', 'cp', '.', upload_dir, '--exclude', '*', '--include', '*{}*tif'.format(pattern), '--recursive']

    try:
        subprocess.check_call(cmd)
    except:
        print "Error uploading output tile"

# Uploads tile to specified location
def upload_final(upload_dir, tile_id, pattern):

    file = '{}_{}.tif'.format(tile_id, pattern)

    print "Uploading {}".format(file)
    cmd = ['aws', 's3', 'cp', file, upload_dir]

    try:
        subprocess.check_call(cmd)
    except:
        print "Error uploading output tile"

def check_for_data(out_tile):

    # Source: http://gis.stackexchange.com/questions/90726
    # Opens raster and chooses band to find min, max
    gtif = gdal.Open(out_tile)
    srcband = gtif.GetRasterBand(1)
    stats = srcband.GetStatistics(True, True)
    print "  Tile stats =  Minimum=%.3f, Maximum=%.3f, Mean=%.3f, StdDev=%.3f" % (stats[0], stats[1], stats[2], stats[3])

    return stats

# Prints the number of tiles that have been processed so far
def count_completed_tiles(pattern):

    mypath = os.getcwd()
    completed = len(glob.glob1(mypath, '*{}*'.format(pattern)))

    # completed = str(os.system('''ls -l *{}*.tif | wc -l'''.format(pattern)))
    print "Number of completed tiles:", completed

# Returns the NoData value of a raster
def get_raster_nodata_value(tile):

    # Source: http://gis.stackexchange.com/questions/90726
    # Opens raster we're getting statistics on
    focus_tile = gdal.Open(tile)

    # Extracts the NoData value for the tile so it can be ignored
    srcband = focus_tile.GetRasterBand(1)
    nodata = srcband.GetNoDataValue()

    return nodata

# Prints information about the tile that was just processed: how long it took and how many tiles have been completed
def end_of_fx_summary(start, tile_id, pattern):

    end = datetime.datetime.now()
    elapsed_time = end-start
    print "Processing time for tile", tile_id, ":", elapsed_time
    count_completed_tiles(pattern)<|endoftext|>


<commit_before>from collections import OrderedDict
import io
import re

from setuptools import Command, find_packages, setup
from setuptools.command.develop import develop as BaseDevelop
from setuptools.command.sdist import sdist as BaseSDist

try:
    from wheel.bdist_wheel import bdist_wheel as BaseBDistWheel
except ImportError:
    BaseBDistWheel = None

with io.open("README.rst", "rt", encoding="utf8") as f:
    readme = f.read()

with io.open("wtforms/__init__.py", "rt", encoding="utf8") as f:
    version = re.search(r"__version__ = \'(.*?)\'", f.read()).group(1)

class CompileCatalogMixin(object):
    """Compile MO files with Babel's ``compile_catalog`` command. This
    happens after installing in development mode, or before building
    sdist and wheel.
    """

    def __init__(self, dist, **kw):
        # underlying commands are old-style classes on Python 2 :-(
        Command.__init__(self, dist, **kw)

    def run(self):
        is_develop = isinstance(self, Develop)

        if not is_develop:
            self.run_command("compile_catalog")

        super(CompileCatalogMixin, self).run()

        if is_develop and not self.uninstall:
            self.run_command("compile_catalog")

class Develop(CompileCatalogMixin, BaseDevelop):
    pass

class SDist(CompileCatalogMixin, BaseSDist):
    pass

command_classes = {"develop": Develop, "sdist": SDist}

if BaseBDistWheel:

    class BDistWheel(CompileCatalogMixin, BaseBDistWheel):
        pass

    command_classes["bdist_wheel"] = BDistWheel

setup(
    name="WTForms",
    version=version,
    url="https://wtforms.readthedocs.io/",
    project_urls=OrderedDict(
        (
            ("Documentation", "https://wtforms.readthedocs.io/"),
            ("Code", "https://github.com/wtforms/wtforms"),
            ("Issue tracker", "https://github.com/wtforms/wtforms/issues"),
        )
    ),
    license="BSD",
    maintainer="WTForms team",
    maintainer_email="davidism+wtforms@gmail.com",
    description=(
        "A flexible forms validation and rendering library for Python"
        " web development."
    ),
    long_description=readme,
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Environment :: Web Environment",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: BSD License",
        "Operating System :: OS Independent",
        "Programming Language :: Python",
        "Programming Language :: Python :: 2",
        "Programming Language :: Python :: 2.7",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.4",
        "Programming Language :: Python :: 3.5",
        "Programming Language :: Python :: 3.6",
        "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
        "Topic :: Software Development :: Libraries :: Python Modules",
    ],
    packages=find_packages(exclude=("tests",)),
    include_package_data=True,
    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*",
    setup_requires=["Babel"],
    install_requires=["MarkupSafe"],
    extras_require={"locale": ["Babel"]},
    cmdclass=command_classes,
)
<commit_msg>Change a regexp<commit_after>from collections import OrderedDict
import io
import re

from setuptools import Command, find_packages, setup
from setuptools.command.develop import develop as BaseDevelop
from setuptools.command.sdist import sdist as BaseSDist

try:
    from wheel.bdist_wheel import bdist_wheel as BaseBDistWheel
except ImportError:
    BaseBDistWheel = None

with io.open("README.rst", "rt", encoding="utf8") as f:
    readme = f.read()

with io.open("wtforms/__init__.py", "rt", encoding="utf8") as f:
    version = re.search(r"__version__ = \"(.*?)\"", f.read()).group(1)

class CompileCatalogMixin(object):
    """Compile MO files with Babel's ``compile_catalog`` command. This
    happens after installing in development mode, or before building
    sdist and wheel.
    """

    def __init__(self, dist, **kw):
        # underlying commands are old-style classes on Python 2 :-(
        Command.__init__(self, dist, **kw)

    def run(self):
        is_develop = isinstance(self, Develop)

        if not is_develop:
            self.run_command("compile_catalog")

        super(CompileCatalogMixin, self).run()

        if is_develop and not self.uninstall:
            self.run_command("compile_catalog")

class Develop(CompileCatalogMixin, BaseDevelop):
    pass

class SDist(CompileCatalogMixin, BaseSDist):
    pass

command_classes = {"develop": Develop, "sdist": SDist}

if BaseBDistWheel:

    class BDistWheel(CompileCatalogMixin, BaseBDistWheel):
        pass

    command_classes["bdist_wheel"] = BDistWheel

setup(
    name="WTForms",
    version=version,
    url="https://wtforms.readthedocs.io/",
    project_urls=OrderedDict(
        (
            ("Documentation", "https://wtforms.readthedocs.io/"),
            ("Code", "https://github.com/wtforms/wtforms"),
            ("Issue tracker", "https://github.com/wtforms/wtforms/issues"),
        )
    ),
    license="BSD",
    maintainer="WTForms team",
    maintainer_email="davidism+wtforms@gmail.com",
    description=(
        "A flexible forms validation and rendering library for Python"
        " web development."
    ),
    long_description=readme,
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Environment :: Web Environment",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: BSD License",
        "Operating System :: OS Independent",
        "Programming Language :: Python",
        "Programming Language :: Python :: 2",
        "Programming Language :: Python :: 2.7",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.4",
        "Programming Language :: Python :: 3.5",
        "Programming Language :: Python :: 3.6",
        "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
        "Topic :: Software Development :: Libraries :: Python Modules",
    ],
    packages=find_packages(exclude=("tests",)),
    include_package_data=True,
    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*",
    setup_requires=["Babel"],
    install_requires=["MarkupSafe"],
    extras_require={"locale": ["Babel"]},
    cmdclass=command_classes,
)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import json

from .exceptions import ValidationError

class Parameter:

    def __init__(self, type, required=False, recommended=False,
                 default=None, editable=True):
        self.type = type
        self.required = required
        self.recommended = recommended
        self.default = default
        self.editable = editable

        self._check_type(self, 'value of argument default', self.default)

    def new(self, name):
        if self.editable:
            return Variable(self, name)
        else:
            return Constant(self, name)

    def validate(self, owner, name, value, required=True):
        chain = []
        if required:
            chain.append(lambda: self._check_required(owner, name,  value))
        chain.append(lambda: self._check_type(owner, name, value))

        return all(f() for f in chain)

    def _check_required(self, owner, name, value):
        required =\
            self.required(owner) if callable(self.required) else self.required
        if value is None and required:
            raise ValidationError('%s is required' % (name, ))

        return True

    def _check_type(self, owner, name, value):
        if self.type is not None\
                and not isinstance(value, (self.type, type(None))):
            raise ValidationError('%s must be an instance of %r' % (
                name, self.type,
            ))

        return True

class Variable:

    def __init__(self, param, name):
        assert isinstance(param, Parameter)

        self.param = param
        self.name = name

    def __get__(self, inst, owner):
        if inst is None:
            return self.param

        assert isinstance(inst, Message)

        if self.name not in inst.__dict__:
            self.__set__(inst, self.param.default)

        return inst.__dict__[self.name]

    def __set__(self, inst, value):
        assert isinstance(inst, Message)

        inst.__dict__[self.name] = value

    def __delete__(self, instance):
        if self.name in instance.__dict__:
            del instance.__dict__[self.name]

class Constant(Variable):

    def __get__(self, inst, owner):
        if inst is None:
            return self.param

        assert isinstance(inst, (Message, RequestError))
        return self.param.default

    def __set__(self, inst, value):
        assert isinstance(inst, Message)

        raise ValidationError('{0!s} is not editable'.format(self.name))

    def __delete__(self, instance):
        raise ValidationError('{0!s} is not editable'.format(self.name))

class MessageMeta(type):

    def __new__(cls, name, bases, namespace):
        params = {}
        for k, v in namespace.items():
            if not isinstance(v, Parameter):
                continue

            params[k] = v
            namespace[k] = v.new(k)
        else:
            namespace['__msg_params__'] = params

        return super(MessageMeta, cls).__new__(cls, name, bases, namespace)

class Message(dict, metaclass=MessageMeta):

    def __str__(self):
        return self._to_dict().__str__()

    def __repr__(self):
        return self.__str__()

    def _to_dict(self):
        dct = self.copy()
        dct.update((
            k, getattr(self, k)
        ) for k in self.__msg_params__.keys())

        return dct

    def to_json(self):
        self.validate()

        dct = dict((k, v) for k, v in self._to_dict().items() if v is not None)
        return json.dumps(dct)

    def validate(self):
        for name, param in self.__msg_params__.items():
            param.validate(self, name, getattr(self, name))

        return True

    def _from_dict(self, D):
        for k, v in D.items():
            if k in self.__msg_params__:
                param = self.__msg_params__[k]
                if param.editable:
                    setattr(self, k, v)
                elif param.default == v:
                    continue
                else:
                    raise ValidationError(
                        '{0!s} must be \'{1!s}\''.format(k, param.default))
            else:
                self[k] = v

    @classmethod
    def from_dict(cls, D):
        inst = cls()
        inst._from_dict(D)
        return inst

class Request(Message):
    response = None
    err_response = None

    def answer(self, provider, owner):
        raise NotImplementedError

class Response(Message):

    def __init__(self, request, *args, **kwargs):
        super(Response, self).__init__(self, *args, **kwargs)
        self.request = request

    @classmethod
    def from_dict(cls, request, D):
        inst = cls(request)
        inst._from_dict(D)
        return inst

class AccessTokenResponse(Response):
    access_token = Parameter(str, required=True)
    token_type = Parameter(str, required=True)
    expires_in = Parameter(int, recommended=True)
    refresh_token = Parameter(str)
    scope = Parameter(str)

    @classmethod
    def from_request(cls, request, token):
        D = {
            'access_token': token.get_token(),
            'token_type': token.get_type(),
            'expires_in': token.get_expires_in(),
            'refresh_token': token.get_refresh_token(),
        }
        if hasattr(request, 'scope') and request.scope != token.get_scope():
            D['scope'] = token.get_scope()

        return cls.from_dict(request, D)

class ErrorResponse(Response):
    error = Parameter(str, required=True)
    error_descritpion = Parameter(str)
    error_uri = Parameter(str)

class RequestErrorMeta(type):

    def __new__(cls, name, bases, namespace):
        if not isinstance(bases, tuple):
            bases = (bases, )

        if any(base.__name__ == 'RequestError' for base in bases):
            if 'kind' not in namespace:
                raise AttributeError(
                    'Subclasses of RequestError must have a property `kind`')
            elif not isinstance(namespace['kind'], Parameter):
                raise AttributeError(
                    '`kind` must be a instance of provider.message.Parameter')

            namespace['kind'] = namespace['kind'].new('kind')

        return super(RequestErrorMeta, cls).__new__(cls, name, bases,
                                                    namespace)

RequestError = RequestErrorMeta('RequestError', (ValueError, ), {})

class AccessDenied(RequestError):
    kind = Parameter(str, default='access_denied', editable=False)

class UnauthorizedClient(RequestError):
    kind = Parameter(str, default='unauthorized_client', editable=False)

class ServerError(RequestError):
    kind = Parameter(str, default='server_error', editable=False)
<commit_msg>remove AccessTokenResponse#from_request<commit_after># -*- coding: utf-8 -*-

import json

from .exceptions import ValidationError

class Parameter:

    def __init__(self, type, required=False, recommended=False,
                 default=None, editable=True):
        self.type = type
        self.required = required
        self.recommended = recommended
        self.default = default
        self.editable = editable

        self._check_type(self, 'value of argument default', self.default)

    def new(self, name):
        if self.editable:
            return Variable(self, name)
        else:
            return Constant(self, name)

    def validate(self, owner, name, value, required=True):
        chain = []
        if required:
            chain.append(lambda: self._check_required(owner, name,  value))
        chain.append(lambda: self._check_type(owner, name, value))

        return all(f() for f in chain)

    def _check_required(self, owner, name, value):
        required =\
            self.required(owner) if callable(self.required) else self.required
        if value is None and required:
            raise ValidationError('%s is required' % (name, ))

        return True

    def _check_type(self, owner, name, value):
        if self.type is not None\
                and not isinstance(value, (self.type, type(None))):
            raise ValidationError('%s must be an instance of %r' % (
                name, self.type,
            ))

        return True

class Variable:

    def __init__(self, param, name):
        assert isinstance(param, Parameter)

        self.param = param
        self.name = name

    def __get__(self, inst, owner):
        if inst is None:
            return self.param

        assert isinstance(inst, Message)

        if self.name not in inst.__dict__:
            self.__set__(inst, self.param.default)

        return inst.__dict__[self.name]

    def __set__(self, inst, value):
        assert isinstance(inst, Message)

        inst.__dict__[self.name] = value

    def __delete__(self, instance):
        if self.name in instance.__dict__:
            del instance.__dict__[self.name]

class Constant(Variable):

    def __get__(self, inst, owner):
        if inst is None:
            return self.param

        assert isinstance(inst, (Message, RequestError))
        return self.param.default

    def __set__(self, inst, value):
        assert isinstance(inst, Message)

        raise ValidationError('{0!s} is not editable'.format(self.name))

    def __delete__(self, instance):
        raise ValidationError('{0!s} is not editable'.format(self.name))

class MessageMeta(type):

    def __new__(cls, name, bases, namespace):
        params = {}
        for k, v in namespace.items():
            if not isinstance(v, Parameter):
                continue

            params[k] = v
            namespace[k] = v.new(k)
        else:
            namespace['__msg_params__'] = params

        return super(MessageMeta, cls).__new__(cls, name, bases, namespace)

class Message(dict, metaclass=MessageMeta):

    def __str__(self):
        return self._to_dict().__str__()

    def __repr__(self):
        return self.__str__()

    def _to_dict(self):
        dct = self.copy()
        dct.update((
            k, getattr(self, k)
        ) for k in self.__msg_params__.keys())

        return dct

    def to_json(self):
        self.validate()

        dct = dict((k, v) for k, v in self._to_dict().items() if v is not None)
        return json.dumps(dct)

    def validate(self):
        for name, param in self.__msg_params__.items():
            param.validate(self, name, getattr(self, name))

        return True

    def _from_dict(self, D):
        for k, v in D.items():
            if k in self.__msg_params__:
                param = self.__msg_params__[k]
                if param.editable:
                    setattr(self, k, v)
                elif param.default == v:
                    continue
                else:
                    raise ValidationError(
                        '{0!s} must be \'{1!s}\''.format(k, param.default))
            else:
                self[k] = v

    @classmethod
    def from_dict(cls, D):
        inst = cls()
        inst._from_dict(D)
        return inst

class Request(Message):
    response = None
    err_response = None

    def answer(self, provider, owner):
        raise NotImplementedError

class Response(Message):

    def __init__(self, request, *args, **kwargs):
        super(Response, self).__init__(self, *args, **kwargs)
        self.request = request

    @classmethod
    def from_dict(cls, request, D):
        inst = cls(request)
        inst._from_dict(D)
        return inst

class AccessTokenResponse(Response):
    access_token = Parameter(str, required=True)
    token_type = Parameter(str, required=True)
    expires_in = Parameter(int, recommended=True)
    refresh_token = Parameter(str)
    scope = Parameter(str)

class ErrorResponse(Response):
    error = Parameter(str, required=True)
    error_descritpion = Parameter(str)
    error_uri = Parameter(str)

class RequestErrorMeta(type):

    def __new__(cls, name, bases, namespace):
        if not isinstance(bases, tuple):
            bases = (bases, )

        if any(base.__name__ == 'RequestError' for base in bases):
            if 'kind' not in namespace:
                raise AttributeError(
                    'Subclasses of RequestError must have a property `kind`')
            elif not isinstance(namespace['kind'], Parameter):
                raise AttributeError(
                    '`kind` must be a instance of provider.message.Parameter')

            namespace['kind'] = namespace['kind'].new('kind')

        return super(RequestErrorMeta, cls).__new__(cls, name, bases,
                                                    namespace)

RequestError = RequestErrorMeta('RequestError', (ValueError, ), {})

class AccessDenied(RequestError):
    kind = Parameter(str, default='access_denied', editable=False)

class UnauthorizedClient(RequestError):
    kind = Parameter(str, default='unauthorized_client', editable=False)

class ServerError(RequestError):
    kind = Parameter(str, default='server_error', editable=False)
<|endoftext|>


<commit_before>"""
Automated fitting script.

A minimal run must specify an input.json and a datasets folder containing input files.
"""

import os
import argparse
import logging
import multiprocessing
import sys

from pycalphad import Database

from espei import fit
from espei.utils import ImmediateClient
from espei.datasets import DatasetError, load_datasets, recursive_glob

parser = argparse.ArgumentParser(description=__doc__)

parser.add_argument(
    "-v", "--verbose",
    action="count",
    default=0,
    help="Set the logging verbosity. Stacks up to two times (-vv)."
    )

parser.add_argument(
    "--scheduler",
    metavar="HOST:PORT",
    help="Host and port of dask distributed scheduler or 'MPIPool' to use MPI.")

parser.add_argument(
    "--tracefile",
    metavar="FILE",
    default='chain.txt',
    help="Output file for recording MCMC trace (txt). Defaults to chain.txt")

parser.add_argument(
    "--probfile",
    metavar="FILE",
    default='lnprob.txt',
    help="Output file for recording MCMC log probability (txt) corresponding to chain. Defaults to lnprob.txt")

parser.add_argument(
    "--fit-settings",
    metavar="FILE",
    default="input.json",
    help="Input JSON file with settings for fit")

parser.add_argument(
    "--datasets",
    metavar="PATH",
    default=os.getcwd(),
    help="Path containing input datasets as JSON files. Datasets can be organized into sub-directories.")

parser.add_argument(
    "--input-tdb",
    metavar="FILE",
    default=None,
    help="Input TDB file, with desired degrees of freedom to fit specified as FUNCTIONs starting with 'VV'")

parser.add_argument(
    "--output-tdb",
    metavar="FILE",
    default="out.tdb",
    help="Output TDB file")

parser.add_argument(
    "--mcmc-steps",
    default=1000,
    type=int,
    metavar="",
    help="Number of MCMC steps. Total chain steps is (mcmc steps * DOF).")

parser.add_argument(
    "--save-interval",
    default=100,
    type=int,
    metavar="",
    help="Controls the interval for saving the MCMC chain")

parser.add_argument(
    "--no-mcmc",
    action="store_true",
    help="Turns off MCMC calculation. Useful for first-principles only run.")

parser.add_argument(
    "--check-datasets",
    metavar="PATH",
    default=None,
    help="Check input datasets at the path. Does not run a fit.")

def main():
    args = parser.parse_args(sys.argv[1:])

    # handle verbosity
    verbosity = {0: logging.WARNING,
                 1: logging.INFO,
                 2: logging.DEBUG}
    user_verbosity = args.verbose if args.verbose < 2 else 2
    logging.basicConfig(level=verbosity[user_verbosity])

    # if desired, check datasets and return
    if args.check_datasets:
        dataset_filenames = sorted(recursive_glob(args.check_datasets, '*.json'))
        errors = []
        for dataset in dataset_filenames:
            try:
                load_datasets([dataset])
            except (ValueError, DatasetError) as e:
                errors.append(e)
        if len(errors) > 0:
            print(*errors, sep='\n')
            return 1
        else:
            return 0

    # run ESPEI fitting
    # create the scheduler if not passed
    if args.scheduler == 'MPIPool':
        from emcee.utils import MPIPool
        # code recommended by emcee: if not master, wait for instructions then exit
        client = MPIPool()
        logging.info("Using MPIPool on {} MPI ranks".format(client.rank))
        if not client.is_master():
            logging.warning('MPIPool is not master. Waiting for instructions...')
            client.wait()
            sys.exit(0)
        logging.info('Using MPIPool')
    elif not args.scheduler:
        from distributed import LocalCluster
        args.scheduler = LocalCluster(n_workers=int(multiprocessing.cpu_count()/2), threads_per_worker=1, processes=True)
        client = ImmediateClient(args.scheduler)
        logging.info("Running with dask scheduler: %s [%s cores]" % (args.scheduler, sum(client.ncores().values())))
        try:
            logging.info("bokeh server for dask scheduler at localhost:{}".format(client.scheduler_info()['services']['bokeh']))
        except KeyError:
            logging.info("Install bokeh to use the dask bokeh server.")
    else:
        raise ValueError('Custom schedulers not supported. Use \'MPIPool\' or accept the default Dask LocalCluster.')
    # load datasets and handle i/o
    datasets = load_datasets(sorted(recursive_glob(args.datasets, '*.json')))
    tracefile = args.tracefile if args.tracefile else None
    probfile = args.probfile if args.probfile else None
    run_mcmc = not args.no_mcmc
    if args.input_tdb:
        resume = Database(args.input_tdb)
    else:
        resume = None
    dbf, sampler, parameters = fit(args.fit_settings, datasets, scheduler=client,
                                   tracefile=tracefile, probfile=probfile,
                                   resume=resume, run_mcmc=run_mcmc,
                                   mcmc_steps=args.mcmc_steps, save_interval=args.save_interval)
    dbf.to_file(args.output_tdb, if_exists='overwrite')

if __name__ == '__main__':
    main()
<commit_msg>FIX: Python 2.7 compatibility<commit_after>"""
Automated fitting script.

A minimal run must specify an input.json and a datasets folder containing input files.
"""

from __future__ import print_function

import os
import argparse
import logging
import multiprocessing
import sys

from pycalphad import Database

from espei import fit
from espei.utils import ImmediateClient
from espei.datasets import DatasetError, load_datasets, recursive_glob

parser = argparse.ArgumentParser(description=__doc__)

parser.add_argument(
    "-v", "--verbose",
    action="count",
    default=0,
    help="Set the logging verbosity. Stacks up to two times (-vv)."
    )

parser.add_argument(
    "--scheduler",
    metavar="HOST:PORT",
    help="Host and port of dask distributed scheduler or 'MPIPool' to use MPI.")

parser.add_argument(
    "--tracefile",
    metavar="FILE",
    default='chain.txt',
    help="Output file for recording MCMC trace (txt). Defaults to chain.txt")

parser.add_argument(
    "--probfile",
    metavar="FILE",
    default='lnprob.txt',
    help="Output file for recording MCMC log probability (txt) corresponding to chain. Defaults to lnprob.txt")

parser.add_argument(
    "--fit-settings",
    metavar="FILE",
    default="input.json",
    help="Input JSON file with settings for fit")

parser.add_argument(
    "--datasets",
    metavar="PATH",
    default=os.getcwd(),
    help="Path containing input datasets as JSON files. Datasets can be organized into sub-directories.")

parser.add_argument(
    "--input-tdb",
    metavar="FILE",
    default=None,
    help="Input TDB file, with desired degrees of freedom to fit specified as FUNCTIONs starting with 'VV'")

parser.add_argument(
    "--output-tdb",
    metavar="FILE",
    default="out.tdb",
    help="Output TDB file")

parser.add_argument(
    "--mcmc-steps",
    default=1000,
    type=int,
    metavar="",
    help="Number of MCMC steps. Total chain steps is (mcmc steps * DOF).")

parser.add_argument(
    "--save-interval",
    default=100,
    type=int,
    metavar="",
    help="Controls the interval for saving the MCMC chain")

parser.add_argument(
    "--no-mcmc",
    action="store_true",
    help="Turns off MCMC calculation. Useful for first-principles only run.")

parser.add_argument(
    "--check-datasets",
    metavar="PATH",
    default=None,
    help="Check input datasets at the path. Does not run a fit.")

def main():
    args = parser.parse_args(sys.argv[1:])

    # handle verbosity
    verbosity = {0: logging.WARNING,
                 1: logging.INFO,
                 2: logging.DEBUG}
    user_verbosity = args.verbose if args.verbose < 2 else 2
    logging.basicConfig(level=verbosity[user_verbosity])

    # if desired, check datasets and return
    if args.check_datasets:
        dataset_filenames = sorted(recursive_glob(args.check_datasets, '*.json'))
        errors = []
        for dataset in dataset_filenames:
            try:
                load_datasets([dataset])
            except (ValueError, DatasetError) as e:
                errors.append(e)
        if len(errors) > 0:
            print(*errors, sep='\n')
            return 1
        else:
            return 0

    # run ESPEI fitting
    # create the scheduler if not passed
    if args.scheduler == 'MPIPool':
        from emcee.utils import MPIPool
        # code recommended by emcee: if not master, wait for instructions then exit
        client = MPIPool()
        logging.info("Using MPIPool on {} MPI ranks".format(client.rank))
        if not client.is_master():
            logging.warning('MPIPool is not master. Waiting for instructions...')
            client.wait()
            sys.exit(0)
        logging.info('Using MPIPool')
    elif not args.scheduler:
        from distributed import LocalCluster
        args.scheduler = LocalCluster(n_workers=int(multiprocessing.cpu_count()/2), threads_per_worker=1, processes=True)
        client = ImmediateClient(args.scheduler)
        logging.info("Running with dask scheduler: %s [%s cores]" % (args.scheduler, sum(client.ncores().values())))
        try:
            logging.info("bokeh server for dask scheduler at localhost:{}".format(client.scheduler_info()['services']['bokeh']))
        except KeyError:
            logging.info("Install bokeh to use the dask bokeh server.")
    else:
        raise ValueError('Custom schedulers not supported. Use \'MPIPool\' or accept the default Dask LocalCluster.')
    # load datasets and handle i/o
    datasets = load_datasets(sorted(recursive_glob(args.datasets, '*.json')))
    tracefile = args.tracefile if args.tracefile else None
    probfile = args.probfile if args.probfile else None
    run_mcmc = not args.no_mcmc
    if args.input_tdb:
        resume = Database(args.input_tdb)
    else:
        resume = None
    dbf, sampler, parameters = fit(args.fit_settings, datasets, scheduler=client,
                                   tracefile=tracefile, probfile=probfile,
                                   resume=resume, run_mcmc=run_mcmc,
                                   mcmc_steps=args.mcmc_steps, save_interval=args.save_interval)
    dbf.to_file(args.output_tdb, if_exists='overwrite')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from setuptools import setup, find_packages

import inbox

requirements = [
    "click",
]

setup(
    name="inbox",
    version=inbox.__version__,
    url='TODO',
    description=inbox.__doc__,
    author=inbox.__author__,
    license=inbox.__license__,
    long_description="TODO",
    packages=find_packages(),
    install_requires=requirements,
    entry_points={
        'console_scripts': [
            'inbox = inbox.__main__:inbox'
        ]
    },
    classifiers=[
        "Development Status :: 3 - Alpha",
        "Intended Audience :: Customer Service",
        "Intended Audience :: End Users/Desktop",
        "Intended Audience :: Information Technology",
        "Intended Audience :: Other Audience",
        "Intended Audience :: System Administrators",
        "License :: OSI Approved :: MIT License",
        "Operating System :: POSIX",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.4",
        "Programming Language :: Python :: 3.5",
        "Topic :: Internet",
        "Topic :: Office/Business",
        "Topic :: Utilities",
    ],
    keywords='inbox github notifications',
)
<commit_msg>Add github3.py to the requirements<commit_after>from setuptools import setup, find_packages

import inbox

requirements = [
    "click",
    "github3.py",
]

setup(
    name="inbox",
    version=inbox.__version__,
    url='TODO',
    description=inbox.__doc__,
    author=inbox.__author__,
    license=inbox.__license__,
    long_description="TODO",
    packages=find_packages(),
    install_requires=requirements,
    entry_points={
        'console_scripts': [
            'inbox = inbox.__main__:inbox'
        ]
    },
    classifiers=[
        "Development Status :: 3 - Alpha",
        "Intended Audience :: Customer Service",
        "Intended Audience :: End Users/Desktop",
        "Intended Audience :: Information Technology",
        "Intended Audience :: Other Audience",
        "Intended Audience :: System Administrators",
        "License :: OSI Approved :: MIT License",
        "Operating System :: POSIX",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.4",
        "Programming Language :: Python :: 3.5",
        "Topic :: Internet",
        "Topic :: Office/Business",
        "Topic :: Utilities",
    ],
    keywords='inbox github notifications',
)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Pomodoro countdown on i3bar originally written by @Fandekasp (Adrien Lemaire)
"""

from subprocess import call
from time import time

class Py3status:

    # available configuration parameters
    max_breaks = 4
    timer_break = 5 * 60
    timer_long_break = 15 * 60
    timer_pomodoro = 25 * 60

    def __init__(self):
        self.__setup('stop')
        self.alert = False
        self.run = False

    def on_click(self, i3s_output_list, i3s_config, event):
        """
        Handles click events
        """
        if event['button'] == 1:
            if self.status == 'stop':
                self.status = 'start'
            self.run = True

        elif event['button'] == 2:
            self.__setup('stop')
            self.run = False

        elif event['button'] == 3:
            self.__setup('pause')
            self.run = False

    @property
    def response(self):
        """
        Return the response full_text string
        """
        return {
            'full_text': '{} ({})'.format(self.prefix, self.timer)
        }

    def __setup(self, status):
        """
        Setup a step
        """
        self.status = status
        if status == 'stop':
            self.prefix = 'Pomodoro'
            self.status = 'stop'
            self.timer = self.timer_pomodoro
            self.breaks = 1

        elif status == 'start':
            self.prefix = 'Pomodoro'
            self.timer = self.timer_pomodoro

        elif status == 'pause':
            self.prefix = 'Break #%d' % self.breaks
            if self.breaks > self.max_breaks:
                self.timer = self.timer_long_break
                self.breaks = 1
            else:
                self.breaks += 1
                self.timer = self.timer_break

    def __decrement(self):
        """
        Countdown handler
        """
        self.timer -= 1
        if self.timer < 0:
            self.alert = True
            self.run = False
            self.__i3_nagbar()
            if self.status == 'start':
                self.__setup('pause')
                self.status = 'pause'
            elif self.status == 'pause':
                self.__setup('start')
                self.status = 'start'

    def __i3_nagbar(self, level='warning'):
        """
        Make use of i3-nagbar to display warnings to the user.
        """
        msg = '{} time is up !'.format(self.prefix)
        try:
            call(
                ['i3-nagbar', '-m', msg, '-t', level],
                stdout=open('/dev/null', 'w'),
                stderr=open('/dev/null', 'w')
            )
        except:
            pass

    def pomodoro(self, i3s_output_list, i3s_config):
        """
        Pomodoro response handling and countdown
        """
        if self.run:
            self.__decrement()

        response = self.response
        if self.alert:
            response['urgent'] = True
            self.alert = False

        if self.status == 'start':
            response['color'] = i3s_config['color_good']
        elif self.status == 'pause':
            response['color'] = i3s_config['color_degraded']
        else:
            response['color'] = i3s_config['color_bad']

        response['cached_until'] = time()
        return response

if __name__ == "__main__":
    """
    Test this module by calling it directly.
    """
    from time import sleep
    x = Py3status()
    config = {
        'color_good': '#00FF00',
        'color_bad': '#FF0000',
    }
    while True:
        print(x.pomodoro([], config))
        sleep(1)
<commit_msg>replaced 'pause' by 'break'<commit_after># -*- coding: utf-8 -*-
"""
Pomodoro countdown on i3bar originally written by @Fandekasp (Adrien Lemaire)
"""

from subprocess import call
from time import time

class Py3status:

    # available configuration parameters
    max_breaks = 4
    timer_break = 5 * 60
    timer_long_break = 15 * 60
    timer_pomodoro = 25 * 60

    def __init__(self):
        self.__setup('stop')
        self.alert = False
        self.run = False

    def on_click(self, i3s_output_list, i3s_config, event):
        """
        Handles click events
        """
        if event['button'] == 1:
            if self.status == 'stop':
                self.status = 'start'
            self.run = True

        elif event['button'] == 2:
            self.__setup('stop')
            self.run = False

        elif event['button'] == 3:
            self.__setup('break')
            self.run = False

    @property
    def response(self):
        """
        Return the response full_text string
        """
        return {
            'full_text': '{} ({})'.format(self.prefix, self.timer)
        }

    def __setup(self, status):
        """
        Setup a step
        """
        self.status = status
        if status == 'stop':
            self.prefix = 'Pomodoro'
            self.status = 'stop'
            self.timer = self.timer_pomodoro
            self.breaks = 1

        elif status == 'start':
            self.prefix = 'Pomodoro'
            self.timer = self.timer_pomodoro

        elif status == 'break':
            self.prefix = 'Break #%d' % self.breaks
            if self.breaks > self.max_breaks:
                self.timer = self.timer_long_break
                self.breaks = 1
            else:
                self.breaks += 1
                self.timer = self.timer_break

    def __decrement(self):
        """
        Countdown handler
        """
        self.timer -= 1
        if self.timer < 0:
            self.alert = True
            self.run = False
            self.__i3_nagbar()
            if self.status == 'start':
                self.__setup('break')
                self.status = 'break'
            elif self.status == 'break':
                self.__setup('start')
                self.status = 'start'

    def __i3_nagbar(self, level='warning'):
        """
        Make use of i3-nagbar to display warnings to the user.
        """
        msg = '{} time is up !'.format(self.prefix)
        try:
            call(
                ['i3-nagbar', '-m', msg, '-t', level],
                stdout=open('/dev/null', 'w'),
                stderr=open('/dev/null', 'w')
            )
        except:
            pass

    def pomodoro(self, i3s_output_list, i3s_config):
        """
        Pomodoro response handling and countdown
        """
        if self.run:
            self.__decrement()

        response = self.response
        if self.alert:
            response['urgent'] = True
            self.alert = False

        if self.status == 'start':
            response['color'] = i3s_config['color_good']
        elif self.status == 'break':
            response['color'] = i3s_config['color_degraded']
        else:
            response['color'] = i3s_config['color_bad']

        response['cached_until'] = time()
        return response

if __name__ == "__main__":
    """
    Test this module by calling it directly.
    """
    from time import sleep
    x = Py3status()
    config = {
        'color_good': '#00FF00',
        'color_bad': '#FF0000',
    }
    while True:
        print(x.pomodoro([], config))
        sleep(1)
<|endoftext|>


<commit_before>#coding: utf-8
from django.conf.urls import patterns, include, url
from django.contrib import admin

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'estofadora.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),
    url(r'^', include('estofadora.core.urls', namespace='core')),
    url(r'^cliente/', include('estofadora.client.urls', namespace='client')),
    url(r'^item/', include('estofadora.item.urls', namespace='item')),
    url(r'^login/', include('estofadora.login.urls', namespace='login')),
    url(r'^relatorios/', include('estofadora.statement.urls', namespace='statement')),
    url(r'^contas/', include('estofadora.bills.urls', namespace='bills')),
    url(r'^api/', include('estofadora.api.urls', namespace='api')),
    url(r'^admin/', include(admin.site.urls)),
)

from django.conf import settings
if settings.DEBUG:
	from django.conf.urls.static import static
	urlpatterns += static(
		settings.MEDIA_URL, document_root=settings.MEDIA_ROOT
	)<commit_msg>fix pep8 and remove api urls for now.<commit_after># coding: utf-8
from django.conf.urls import patterns, include, url
from django.contrib import admin
from django.conf import settings

urlpatterns = patterns(
    '',
    # Examples:
    # url(r'^$', 'estofadora.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),
    url(r'^', include('estofadora.core.urls', namespace='core')),
    url(r'^cliente/', include('estofadora.client.urls', namespace='client')),
    url(r'^item/', include('estofadora.item.urls', namespace='item')),
    url(r'^login/', include('estofadora.login.urls', namespace='login')),
    url(
        r'^relatorios/',
        include('estofadora.statement.urls', namespace='statement')
    ),
    url(r'^contas/', include('estofadora.bills.urls', namespace='bills')),
    url(r'^admin/', include(admin.site.urls)),
    # url(r'^api/', include('estofadora.api.urls', namespace='api')),
)

if settings.DEBUG:
    from django.conf.urls.static import static
    urlpatterns += static(
        settings.MEDIA_URL, document_root=settings.MEDIA_ROOT
    )
<|endoftext|>


<commit_before>from setuptools import setup

setup(name='rejected',
      version='3.19.5',
      description='Rejected is a Python RabbitMQ Consumer Framework and '
                  'Controller Daemon',
      long_description=open('README.rst').read(),
      classifiers=[
          'Development Status :: 5 - Production/Stable',
          'Intended Audience :: Developers',
          'Programming Language :: Python :: 2',
          'Programming Language :: Python :: 2.7',
          'Programming Language :: Python :: 3',
          'Programming Language :: Python :: 3.4',
          'Programming Language :: Python :: 3.5',
          'Programming Language :: Python :: Implementation :: CPython',
          'Programming Language :: Python :: Implementation :: PyPy',
          'License :: OSI Approved :: BSD License'
      ],
      keywords='amqp rabbitmq',
      author='Gavin M. Roy',
      author_email='gavinmroy@gmail.com',
      url='https://github.com/gmr/rejected',
      license='BSD',
      packages=['rejected'],
      package_data={'': ['LICENSE', 'README.rst']},
      include_package_data=True,
      install_requires=[
          'helper',
          'pika>=0.10.0',
          'psutil',
          'pyyaml',
          'tornado>=4.2,<4.3'
      ],
      extras_require={
          'html': ['beautifulsoup4'],
          'influxdb': ['sprockets-influxdb'],
          'msgpack': ['u-msgpack-python'],
          'sentry': ['raven'],
          'testing': [
              'beautifulsoup4',
              'coverage',
              'mock',
              'nose',
              'raven',
              'sprockets-influxdb',
              'u-msgpack-python'
          ]
      },
      tests_require=['mock', 'nose', 'coverage'],
      entry_points=dict(console_scripts=['rejected=rejected.controller:main']),
      zip_safe=True)
<commit_msg>Bump the version<commit_after>from setuptools import setup

setup(name='rejected',
      version='3.19.6',
      description='Rejected is a Python RabbitMQ Consumer Framework and '
                  'Controller Daemon',
      long_description=open('README.rst').read(),
      classifiers=[
          'Development Status :: 5 - Production/Stable',
          'Intended Audience :: Developers',
          'Programming Language :: Python :: 2',
          'Programming Language :: Python :: 2.7',
          'Programming Language :: Python :: 3',
          'Programming Language :: Python :: 3.4',
          'Programming Language :: Python :: 3.5',
          'Programming Language :: Python :: Implementation :: CPython',
          'Programming Language :: Python :: Implementation :: PyPy',
          'License :: OSI Approved :: BSD License'
      ],
      keywords='amqp rabbitmq',
      author='Gavin M. Roy',
      author_email='gavinmroy@gmail.com',
      url='https://github.com/gmr/rejected',
      license='BSD',
      packages=['rejected'],
      package_data={'': ['LICENSE', 'README.rst']},
      include_package_data=True,
      install_requires=[
          'helper',
          'pika>=0.10.0',
          'psutil',
          'pyyaml',
          'tornado>=4.2,<4.3'
      ],
      extras_require={
          'html': ['beautifulsoup4'],
          'influxdb': ['sprockets-influxdb'],
          'msgpack': ['u-msgpack-python'],
          'sentry': ['raven'],
          'testing': [
              'beautifulsoup4',
              'coverage',
              'mock',
              'nose',
              'raven',
              'sprockets-influxdb',
              'u-msgpack-python'
          ]
      },
      tests_require=['mock', 'nose', 'coverage'],
      entry_points=dict(console_scripts=['rejected=rejected.controller:main']),
      zip_safe=True)
<|endoftext|>