<commit_before>#!/usr/bin/env python
"""Script to load project info from Lims into the project database in statusdb.

Maya Brandi, Science for Life Laboratory, Stockholm, Sweden.
"""
from __future__ import print_function
from genologics.config import BASEURI, USERNAME, PASSWORD
from genologics.lims import *
from LIMS2DB.objectsDB.functions import *
from optparse import OptionParser
from statusdb.db.utils import *
from LIMS2DB.utils import formatStack

from pprint import pprint

import codecs
import datetime
import LIMS2DB.objectsDB.objectsDB as DB
import logging
import logging.handlers
import multiprocessing as mp
import os
import Queue
import sys
import time
import traceback

class PSUL():
    def __init__(self, proj, samp_db, proj_db, upload_data, man_name, output_f, log):
        self.proj = proj
        self.id = proj.id
        self.udfs = proj.udf
        self.name = proj.name
        self.open_date = proj.open_date
        self.close_date = proj.close_date
        self.samp_db = samp_db
        self.proj_db = proj_db
        self.upload_data = upload_data
        self.man_name = man_name
        self.output_f = output_f
        self.ordered_opened = self.get_ordered_opened()
        self.lims = Lims(BASEURI, USERNAME, PASSWORD)
        self.log=log

    def print_couchdb_obj_to_file(self, obj):
        if self.output_f is not None:
            with open(self.output_f, 'w') as f:
                print(obj, file = f)
        else:
            print(obj, file = sys.stdout)

    def get_ordered_opened(self):
        """Is project registered as opened or ordered?"""

        if self.open_date:
            return self.open_date
        elif 'Order received' in self.proj.udf:
            return self.proj.udf['Order received'].isoformat()
        else:
            return datetime.date.today().strftime("%Y-%m-%d")

    def determine_update(self):
        """Determine wether to and how to update project"""
        opened_after_130630 = comp_dates('2013-06-30', self.ordered_opened)
        log_info = ''
        if not opened_after_130630 :
                start_update = False
                self.log.info('Project is not updated because: '
                    'It was opened or ordered before 2013-06-30 : '
                    '({ord_op})'.format(ord_op = self.ordered_opened))
        else:
            start_update = True

        return start_update

    def update_project(self):
        """Fetch project info and update project in the database."""
        self.log.info('Handling {proj}'.format(proj = self.name))
        project = DB.ProjectDB(self.lims, self.id, self.samp_db, self.log)

        key = find_proj_from_view(self.proj_db, self.name)
        project.obj['_id'] = find_or_make_key(key)
        if self.upload_data:
            info = save_couchdb_obj(self.proj_db, project.obj)
        else:
            info = self.print_couchdb_obj_to_file(project.obj)

        self.log.info("project {name} is handled and {info}: _id = {id}".format(
                           name=self.name, info=info, id=project.obj['_id']))

    def handle_project(self):
        if self.determine_update():
            self.update_project()

def main(options):
    conf = options.conf
    upload_data = options.upload
    output_f = options.output_f
    couch = load_couch_server(conf)
    proj_db = couch['projects']
    samp_db = couch['samples']
    mainlims = Lims(BASEURI, USERNAME, PASSWORD)
    mainlog = logging.getLogger('psullogger')
    mainlog.setLevel(level=logging.INFO)
    mfh = logging.handlers.RotatingFileHandler(options.logfile, maxBytes=209715200, backupCount=5)
    mft = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    mfh.setFormatter(mft)
    mainlog.addHandler(mfh)

    if options.project_name:
        proj = mainlims.get_projects(name = options.project_name)
        if not proj:
            mainlog.warn('No project named {man_name} in Lims'.format(
                        man_name = options.project_name))
        else:
            P = PSUL(proj[0], samp_db, proj_db, options.upload, options.project_name, output_f, mainlog)
            P.handle_project()
    else :
        projects=create_projects_list(options, mainlims, mainlog)
        masterProcess(options,projects, mainlims, mainlog)

def create_projects_list(options, lims, log):
        projects=[]
        if options.all_projects:
            projects = lims.get_projects()
            if options.hours:
                delta=datetime.timedelta(hours=options.hours)
                #timestring for processes is different from timestring for projects. First needs Z, second needs full timezone
                time_string_pc=(datetime.datetime.now()-delta).strftime('%Y-%m-%dT%H:%M:%SZ')
                time_string_pj=(datetime.datetime.now()-delta).strftime('%Y-%m-%dT%H:%M:%SCET')
                valid_projects=lims.get_projects(last_modified=time_string_pj)
                for proj in projects:
                    procs=lims.get_processes(last_modified=time_string_pc, projectname=proj.name)
                    if procs:
                        valid_projects.append(proj)
                log.info("project list : {0}".format(" ".join([p.id for p in valid_projects])))
                return valid_projects
            else:
                log.info("project list : {0}".format(" ".join([p.id for p in projects])))
                return projects

        elif options.input:
            with open(options.input, "r") as input_file:
                for pname in input_file:
                    try:
                        projects.append(lims.get_projects(name=pname.rstrip())[0] )
                    except IndexError:
                        pass

            return projects

def processPSUL(options, queue, logqueue):
    couch = load_couch_server(options.conf)
    proj_db = couch['projects']
    samp_db = couch['samples']
    mylims = Lims(BASEURI, USERNAME, PASSWORD)
    work=True
    procName=mp.current_process().name
    proclog=logging.getLogger(procName)
    proclog.setLevel(level=logging.INFO)
    mfh = QueueHandler(logqueue)
    mft = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    mfh.setFormatter(mft)
    proclog.addHandler(mfh)
    try:
        time.sleep(int(procname[8:]))
    except:
        time.sleep(1)

    while work:
        #grabs project from queue
        try:
            projname = queue.get(block=True, timeout=3)
            proclog.info("Starting work on {} ".format(projname))
            proclog.info("Approximately {} projects left in queue".format(queue.qsize()))
        except Queue.Empty:
            work=False
            proclog.info("exiting gracefully")
            break
        except NotImplementedError:
            #qsize failed, no big deal
            pass
        else:
            #locks the project : cannot be updated more than once.
            lockfile=os.path.join(options.lockdir, projname)
            if not os.path.exists(lockfile):
                try:
                    open(lockfile,'w').close()
                except:
                    proclog.error("cannot create lockfile {}".format(lockfile))
                try:
                    proj=mylims.get_projects(name=projname)[0]
                    P = PSUL(proj, samp_db, proj_db, options.upload, options.project_name, options.output_f, proclog)
                    P.handle_project()
                except :
                    error=sys.exc_info()
                    stack=traceback.extract_tb(error[2])
                    proclog.error("{0}:{1}\n{2}".format(error[0], error[1], formatStack(stack)))

                try:
                    os.remove(lockfile)
                except:
                    proclog.error("cannot remove lockfile {}".format(lockfile))

            #signals to queue job is done
            queue.task_done()

def masterProcess(options,projectList, mainlims, logger):
    projectsQueue=mp.JoinableQueue()
    logQueue=mp.Queue()
    childs=[]
    #Initial step : order projects by sample number:
    logger.info("ordering the project list")
    orderedprojectlist=sorted(projectList, key=lambda x: (mainlims.get_sample_number(projectname=x.name)), reverse=True)
    logger.info("done ordering the project list")
    #spawn a pool of processes, and pass them queue instance
    for i in range(options.processes):
        p = mp.Process(target=processPSUL, args=(options,projectsQueue, logQueue))
        p.start()
        childs.append(p)
    #populate queue with data
    for proj in orderedprojectlist:
        projectsQueue.put(proj.name)

    #wait on the queue until everything has been processed
    notDone=True
    while notDone:
        try:
            log=logQueue.get(False)
            logger.handle(log)
        except Queue.Empty:
            if not stillRunning(childs):
                notDone=False
                break

def stillRunning(processList):
    ret=False
    for p in processList:
        if p.is_alive():
            ret=True

    return ret

class QueueHandler(logging.Handler):
    """
    This handler sends events to a queue. Typically, it would be used together
    with a multiprocessing Queue to centralise logging to file in one process
    (in a multi-process application), so as to avoid file write contention
    between processes.

    This code is new in Python 3.2, but this class can be copy pasted into
    user code for use with earlier Python versions.
    """

    def __init__(self, queue):
        """
        Initialise an instance, using the passed queue.
        """
        logging.Handler.__init__(self)
        self.queue = queue

    def enqueue(self, record):
        """
        Enqueue a record.

        The base implementation uses put_nowait. You may want to override
        this method if you want to use blocking, timeouts or custom queue
        implementations.
        """
        self.queue.put_nowait(record)

    def prepare(self, record):
        """
        Prepares a record for queuing. The object returned by this method is
        enqueued.

        The base implementation formats the record to merge the message
        and arguments, and removes unpickleable items from the record
        in-place.

        You might want to override this method if you want to convert
        the record to a dict or JSON string, or send a modified copy
        of the record while leaving the original intact.
        """
        # The format operation gets traceback text into record.exc_text
        # (if there's exception data), and also puts the message into
        # record.message. We can then use this to replace the original
        # msg + args, as these might be unpickleable. We also zap the
        # exc_info attribute, as it's no longer needed and, if not None,
        # will typically not be pickleable.
        self.format(record)
        record.msg = record.message
        record.args = None
        record.exc_info = None
        return record

    def emit(self, record):
        """
        Emit a record.

        Writes the LogRecord to the queue, preparing it for pickling first.
        """
        try:
            self.enqueue(self.prepare(record))
        except Exception:
            self.handleError(record)

if __name__ == '__main__':
    usage = "Usage:       python project_summary_upload_LIMS.py [options]"
    parser = OptionParser(usage=usage)
    parser.add_option("-p", "--project", dest = "project_name", default = None,
                      help = "eg: M.Uhlen_13_01. Dont use with -a flagg.")
    parser.add_option("-a", "--all_projects", dest = "all_projects", action =
                      "store_true", default = False, help = ("Upload all Lims ",
                      "projects into couchDB. Don't use with -f flagg."))
    parser.add_option("-c", "--conf", dest = "conf", default = os.path.join(
                      os.environ['HOME'],'opt/config/post_process.yaml'), help =
                      "Config file.  Default: ~/opt/config/post_process.yaml")
    parser.add_option("--no_upload", dest = "upload", default = True, action =
                      "store_false", help = ("Use this tag if project objects ",
                      "should not be uploaded, but printed to output_f, or to ",
                      "stdout"))
    parser.add_option("--output_f", dest = "output_f", help = ("Output file",
                      " that will be used only if --no_upload tag is used"), default=None)
    parser.add_option("-m", "--multiprocs", type='int', dest = "processes", default = 4,
                      help = "How many processes will be spawned. Will only work with -a")
    parser.add_option("-l", "--logfile", dest = "logfile", help = ("log file",
                      " that will be used. default is $HOME/lims2db_projects.log "), default=os.path.expanduser("~/lims2db_projects.log"))
    parser.add_option("--lockdir", dest = "lockdir", help = ("directory handling the lock files",
                      " to avoid multiple updating of one project. default is $HOME/psul_locks "), default=os.path.expanduser("~/psul_locks"))
    parser.add_option("-j", "--hours", dest = "hours",type='int', help = ("only handle projects modified in the last X hours"), default=None)
    parser.add_option("-k", "--control", dest = "control", action="store_true", help = ("only perform a dry-run"), default=False)
    parser.add_option("-i", "--input", dest = "input", help = ("path to the input file containing projects to update"), default=None)

    (options, args) = parser.parse_args()
    main(options)

<commit_msg>adding log message for locks<commit_after>#!/usr/bin/env python
"""Script to load project info from Lims into the project database in statusdb.

Maya Brandi, Science for Life Laboratory, Stockholm, Sweden.
"""
from __future__ import print_function
from genologics.config import BASEURI, USERNAME, PASSWORD
from genologics.lims import *
from LIMS2DB.objectsDB.functions import *
from optparse import OptionParser
from statusdb.db.utils import *
from LIMS2DB.utils import formatStack

from pprint import pprint

import codecs
import datetime
import LIMS2DB.objectsDB.objectsDB as DB
import logging
import logging.handlers
import multiprocessing as mp
import os
import Queue
import sys
import time
import traceback

class PSUL():
    def __init__(self, proj, samp_db, proj_db, upload_data, man_name, output_f, log):
        self.proj = proj
        self.id = proj.id
        self.udfs = proj.udf
        self.name = proj.name
        self.open_date = proj.open_date
        self.close_date = proj.close_date
        self.samp_db = samp_db
        self.proj_db = proj_db
        self.upload_data = upload_data
        self.man_name = man_name
        self.output_f = output_f
        self.ordered_opened = self.get_ordered_opened()
        self.lims = Lims(BASEURI, USERNAME, PASSWORD)
        self.log=log

    def print_couchdb_obj_to_file(self, obj):
        if self.output_f is not None:
            with open(self.output_f, 'w') as f:
                print(obj, file = f)
        else:
            print(obj, file = sys.stdout)

    def get_ordered_opened(self):
        """Is project registered as opened or ordered?"""

        if self.open_date:
            return self.open_date
        elif 'Order received' in self.proj.udf:
            return self.proj.udf['Order received'].isoformat()
        else:
            return datetime.date.today().strftime("%Y-%m-%d")

    def determine_update(self):
        """Determine wether to and how to update project"""
        opened_after_130630 = comp_dates('2013-06-30', self.ordered_opened)
        log_info = ''
        if not opened_after_130630 :
                start_update = False
                self.log.info('Project is not updated because: '
                    'It was opened or ordered before 2013-06-30 : '
                    '({ord_op})'.format(ord_op = self.ordered_opened))
        else:
            start_update = True

        return start_update

    def update_project(self):
        """Fetch project info and update project in the database."""
        self.log.info('Handling {proj}'.format(proj = self.name))
        project = DB.ProjectDB(self.lims, self.id, self.samp_db, self.log)

        key = find_proj_from_view(self.proj_db, self.name)
        project.obj['_id'] = find_or_make_key(key)
        if self.upload_data:
            info = save_couchdb_obj(self.proj_db, project.obj)
        else:
            info = self.print_couchdb_obj_to_file(project.obj)

        self.log.info("project {name} is handled and {info}: _id = {id}".format(
                           name=self.name, info=info, id=project.obj['_id']))

    def handle_project(self):
        if self.determine_update():
            self.update_project()

def main(options):
    conf = options.conf
    upload_data = options.upload
    output_f = options.output_f
    couch = load_couch_server(conf)
    proj_db = couch['projects']
    samp_db = couch['samples']
    mainlims = Lims(BASEURI, USERNAME, PASSWORD)
    mainlog = logging.getLogger('psullogger')
    mainlog.setLevel(level=logging.INFO)
    mfh = logging.handlers.RotatingFileHandler(options.logfile, maxBytes=209715200, backupCount=5)
    mft = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    mfh.setFormatter(mft)
    mainlog.addHandler(mfh)

    if options.project_name:
        proj = mainlims.get_projects(name = options.project_name)
        if not proj:
            mainlog.warn('No project named {man_name} in Lims'.format(
                        man_name = options.project_name))
        else:
            P = PSUL(proj[0], samp_db, proj_db, options.upload, options.project_name, output_f, mainlog)
            P.handle_project()
    else :
        projects=create_projects_list(options, mainlims, mainlog)
        masterProcess(options,projects, mainlims, mainlog)

def create_projects_list(options, lims, log):
        projects=[]
        if options.all_projects:
            projects = lims.get_projects()
            if options.hours:
                delta=datetime.timedelta(hours=options.hours)
                #timestring for processes is different from timestring for projects. First needs Z, second needs full timezone
                time_string_pc=(datetime.datetime.now()-delta).strftime('%Y-%m-%dT%H:%M:%SZ')
                time_string_pj=(datetime.datetime.now()-delta).strftime('%Y-%m-%dT%H:%M:%SCET')
                valid_projects=lims.get_projects(last_modified=time_string_pj)
                for proj in projects:
                    procs=lims.get_processes(last_modified=time_string_pc, projectname=proj.name)
                    if procs:
                        valid_projects.append(proj)
                log.info("project list : {0}".format(" ".join([p.id for p in valid_projects])))
                return valid_projects
            else:
                log.info("project list : {0}".format(" ".join([p.id for p in projects])))
                return projects

        elif options.input:
            with open(options.input, "r") as input_file:
                for pname in input_file:
                    try:
                        projects.append(lims.get_projects(name=pname.rstrip())[0] )
                    except IndexError:
                        pass

            return projects

def processPSUL(options, queue, logqueue):
    couch = load_couch_server(options.conf)
    proj_db = couch['projects']
    samp_db = couch['samples']
    mylims = Lims(BASEURI, USERNAME, PASSWORD)
    work=True
    procName=mp.current_process().name
    proclog=logging.getLogger(procName)
    proclog.setLevel(level=logging.INFO)
    mfh = QueueHandler(logqueue)
    mft = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    mfh.setFormatter(mft)
    proclog.addHandler(mfh)
    try:
        time.sleep(int(procname[8:]))
    except:
        time.sleep(1)

    while work:
        #grabs project from queue
        try:
            projname = queue.get(block=True, timeout=3)
            proclog.info("Starting work on {} ".format(projname))
            proclog.info("Approximately {} projects left in queue".format(queue.qsize()))
        except Queue.Empty:
            work=False
            proclog.info("exiting gracefully")
            break
        except NotImplementedError:
            #qsize failed, no big deal
            pass
        else:
            #locks the project : cannot be updated more than once.
            lockfile=os.path.join(options.lockdir, projname)
            if not os.path.exists(lockfile):
                try:
                    open(lockfile,'w').close()
                except:
                    proclog.error("cannot create lockfile {}".format(lockfile))
                try:
                    proj=mylims.get_projects(name=projname)[0]
                    P = PSUL(proj, samp_db, proj_db, options.upload, options.project_name, options.output_f, proclog)
                    P.handle_project()
                except :
                    error=sys.exc_info()
                    stack=traceback.extract_tb(error[2])
                    proclog.error("{0}:{1}\n{2}".format(error[0], error[1], formatStack(stack)))

                try:
                    os.remove(lockfile)
                except:
                    proclog.error("cannot remove lockfile {}".format(lockfile))
            else:
                proclog.info("project {} is locked, skipping.".format(projname))

            #signals to queue job is done
            queue.task_done()

def masterProcess(options,projectList, mainlims, logger):
    projectsQueue=mp.JoinableQueue()
    logQueue=mp.Queue()
    childs=[]
    #Initial step : order projects by sample number:
    logger.info("ordering the project list")
    orderedprojectlist=sorted(projectList, key=lambda x: (mainlims.get_sample_number(projectname=x.name)), reverse=True)
    logger.info("done ordering the project list")
    #spawn a pool of processes, and pass them queue instance
    for i in range(options.processes):
        p = mp.Process(target=processPSUL, args=(options,projectsQueue, logQueue))
        p.start()
        childs.append(p)
    #populate queue with data
    for proj in orderedprojectlist:
        projectsQueue.put(proj.name)

    #wait on the queue until everything has been processed
    notDone=True
    while notDone:
        try:
            log=logQueue.get(False)
            logger.handle(log)
        except Queue.Empty:
            if not stillRunning(childs):
                notDone=False
                break

def stillRunning(processList):
    ret=False
    for p in processList:
        if p.is_alive():
            ret=True

    return ret

class QueueHandler(logging.Handler):
    """
    This handler sends events to a queue. Typically, it would be used together
    with a multiprocessing Queue to centralise logging to file in one process
    (in a multi-process application), so as to avoid file write contention
    between processes.

    This code is new in Python 3.2, but this class can be copy pasted into
    user code for use with earlier Python versions.
    """

    def __init__(self, queue):
        """
        Initialise an instance, using the passed queue.
        """
        logging.Handler.__init__(self)
        self.queue = queue

    def enqueue(self, record):
        """
        Enqueue a record.

        The base implementation uses put_nowait. You may want to override
        this method if you want to use blocking, timeouts or custom queue
        implementations.
        """
        self.queue.put_nowait(record)

    def prepare(self, record):
        """
        Prepares a record for queuing. The object returned by this method is
        enqueued.

        The base implementation formats the record to merge the message
        and arguments, and removes unpickleable items from the record
        in-place.

        You might want to override this method if you want to convert
        the record to a dict or JSON string, or send a modified copy
        of the record while leaving the original intact.
        """
        # The format operation gets traceback text into record.exc_text
        # (if there's exception data), and also puts the message into
        # record.message. We can then use this to replace the original
        # msg + args, as these might be unpickleable. We also zap the
        # exc_info attribute, as it's no longer needed and, if not None,
        # will typically not be pickleable.
        self.format(record)
        record.msg = record.message
        record.args = None
        record.exc_info = None
        return record

    def emit(self, record):
        """
        Emit a record.

        Writes the LogRecord to the queue, preparing it for pickling first.
        """
        try:
            self.enqueue(self.prepare(record))
        except Exception:
            self.handleError(record)

if __name__ == '__main__':
    usage = "Usage:       python project_summary_upload_LIMS.py [options]"
    parser = OptionParser(usage=usage)
    parser.add_option("-p", "--project", dest = "project_name", default = None,
                      help = "eg: M.Uhlen_13_01. Dont use with -a flagg.")
    parser.add_option("-a", "--all_projects", dest = "all_projects", action =
                      "store_true", default = False, help = ("Upload all Lims ",
                      "projects into couchDB. Don't use with -f flagg."))
    parser.add_option("-c", "--conf", dest = "conf", default = os.path.join(
                      os.environ['HOME'],'opt/config/post_process.yaml'), help =
                      "Config file.  Default: ~/opt/config/post_process.yaml")
    parser.add_option("--no_upload", dest = "upload", default = True, action =
                      "store_false", help = ("Use this tag if project objects ",
                      "should not be uploaded, but printed to output_f, or to ",
                      "stdout"))
    parser.add_option("--output_f", dest = "output_f", help = ("Output file",
                      " that will be used only if --no_upload tag is used"), default=None)
    parser.add_option("-m", "--multiprocs", type='int', dest = "processes", default = 4,
                      help = "How many processes will be spawned. Will only work with -a")
    parser.add_option("-l", "--logfile", dest = "logfile", help = ("log file",
                      " that will be used. default is $HOME/lims2db_projects.log "), default=os.path.expanduser("~/lims2db_projects.log"))
    parser.add_option("--lockdir", dest = "lockdir", help = ("directory handling the lock files",
                      " to avoid multiple updating of one project. default is $HOME/psul_locks "), default=os.path.expanduser("~/psul_locks"))
    parser.add_option("-j", "--hours", dest = "hours",type='int', help = ("only handle projects modified in the last X hours"), default=None)
    parser.add_option("-k", "--control", dest = "control", action="store_true", help = ("only perform a dry-run"), default=False)
    parser.add_option("-i", "--input", dest = "input", help = ("path to the input file containing projects to update"), default=None)

    (options, args) = parser.parse_args()
    main(options)

<|endoftext|>


<commit_before>
from celery import Celery
from celerycontrib.sqlalchemyscheduler import model, SQLAlchemyScheduler
from flask import Flask
from flask_admin import Admin
from flask_admin.contrib.sqla import ModelView
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

BROKER_URL = 'amqp://guest@localhost//'
DATABASE_URL = 'sqlite:///example.sqlite'
SECRET_KEY = '0123456789'

flask = Flask(__name__)
admin = Admin(flask, name='Celery Scheduler', template_mode='bootstrap3')
celery = Celery(__name__, broker=BROKER_URL)
session = sessionmaker(bind=create_engine(DATABASE_URL))()

flask.config.update(
    SECRET_KEY=SECRET_KEY,
)
celery.conf.update(
    CELERY_ACCEPT_CONTENT=['json'],
    CELERY_TASK_SERIALIZER='json',
    CELERY_RESULT_SERIALIZER='json',
    CELERY_ENABLE_UTC=True,
)

class PeriodicTaskView(ModelView):
    # inline_models = [model.IntervalSchedule, model.CrontabSchedule]
    pass

admin.add_view(PeriodicTaskView(model.PeriodicTask, session))

# Make a subclass of SQLAlchemyScheduler so we can override database_url
class DatabaseScheduler(SQLAlchemyScheduler):
    database_url = DATABASE_URL
    max_interval = 10
    sync_every = 10

@celery.task()
def add_together(a, b):
    return a + b

if __name__ == '__main__':
    model.Base.metadata.create_all(session.bind)
    flask.run(debug=True)
<commit_msg>improve example app interface<commit_after>
from celery import Celery
from celerycontrib.sqlalchemyscheduler import (
    Base,
    PeriodicTask,
    SQLAlchemyScheduler,
)
from flask import Flask
from flask_admin import Admin
from flask_admin.contrib.sqla import ModelView
from flask_admin.form.fields import Select2Field
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

BROKER_URL = 'amqp://guest@localhost//'
DATABASE_URL = 'sqlite:///example.sqlite'
SECRET_KEY = '0123456789'

flask = Flask(__name__)
celery = Celery(__name__, broker=BROKER_URL)
session = sessionmaker(bind=create_engine(DATABASE_URL))()

flask.config.update(
    SECRET_KEY=SECRET_KEY,
)
celery.conf.update(
    CELERY_ACCEPT_CONTENT=['json'],
    CELERY_TASK_SERIALIZER='json',
    CELERY_RESULT_SERIALIZER='json',
    CELERY_ENABLE_UTC=True,
)

# Make a subclass of SQLAlchemyScheduler so we can override database_url
class DatabaseScheduler(SQLAlchemyScheduler):
    database_url = DATABASE_URL
    max_interval = 10
    sync_every = 10

@celery.task()
def add_together(a, b):
    return a + b

@celery.task()
def multiply(a, b):
    return a * b

# Don't show built-in celery tasks like 'chord', 'map', etc.
task_names = list(filter(
    lambda name: name.startswith('example_app.'),
    (name.replace('__main__.', 'example_app.') for name in celery.tasks.keys())
))

class PeriodicTaskView(ModelView):
    create_modal = True
    edit_modal = True

    form_columns = [
        'name',
        'description',
        'task',
        'args',
        'kwargs',
        'interval_schedule',
        'crontab_schedule',
        'expires',
        'enabled',
        'queue',
        'exchange',
        'routing_key',
    ]
    form_overrides = dict(
        task=Select2Field,
    )
    form_args = dict(
        task=dict(
            choices=list(zip(task_names, task_names)),
        ),
    )

    column_list = [
        'name',
        'task',
        'args',
        'kwargs',
        'schedule',
        'expires',
        'last_run_at',
        'total_run_count',
        'enabled',
    ]

admin = Admin(
    flask,
    name='Celery Task Scheduler',
    template_mode='bootstrap3',
    url='/',
    index_view=PeriodicTaskView(PeriodicTask, session, endpoint='admin'),
)

if __name__ == '__main__':
    Base.metadata.create_all(session.bind)
    flask.run(debug=True)
<|endoftext|>


<commit_before>from datetime import datetime

import odoo
import json

DB_NAME = 'MAXTRACKER'
ODOO_CONF = '/etc/odoo/odoo.conf'
UID = odoo.SUPERUSER_ID

odoo.tools.config.parse_config(['--config=%s' % ODOO_CONF])
with odoo.api.Environment.manage():
    registry = odoo.modules.registry.RegistryManager.get(DB_NAME)
    with registry.cursor() as cr:
        ctx = odoo.api.Environment(cr, UID, {})['res.users'].context_get()
        env = odoo.api.Environment(cr, UID, ctx)
        invoice_proxy = env['account.invoice']
        partner_proxy = env['res.partner']
        invoices = invoice_proxy.search([('state', '=', 'draft'), ('date_due', '!=', False)])
        response = invoices[0].wsfe_request_detail_ids[0].request_received
        response = response.replace("'", '"').replace('False', 'false').replace('None', 'false').replace('L,', ',')
        dump = json.loads(r'{}'.format(response))
        det_resp = dump['FeDetResp']['FECAEDetResponse']
        for invoice in invoices:
            filt = filter(lambda l: l['DocNro'] == int(invoice.partner_id.vat), det_resp)
            resp = filt[0] if filt else None
            invoice.write({
                'cae': resp['CAE'],
                'cae_due_date': datetime.strptime(resp['CAEFchVto'], '%Y%m%d'),
                'name': invoice.pos_ar_id.name.zfill(4) + '-' + str(resp['CbteDesde']).zfill(8),
            })
<commit_msg>Delete script-us-3072-fix-invoices.py<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals
import sys
from contextlib import contextmanager
from collections import namedtuple
from .ast import AST
from .exceptions import FailedParse, FailedCut, FailedLookahead

__all__ = ['ParseInfo', 'ParseContext']

ParseInfo = namedtuple('ParseInfo', ['buffer', 'rule', 'pos', 'endpos'])

class ParseContext(object):
    def __init__(self,
                 buffer=None,
                 semantics=None,
                 parseinfo=False,
                 trace=False,
                 encoding='utf-8',
                 **kwargs):
        super(ParseContext, self).__init__()

        self._buffer = buffer
        self.semantics = semantics if semantics is not None else self
        self.encoding = encoding
        self.parseinfo = parseinfo

        self._ast_stack = []
        self._concrete_stack = [None]
        self._rule_stack = []
        self._cut_stack = [False]
        self._memoization_cache = dict()
        if not trace:
            self._trace = lambda x: ()
            self._trace_event = lambda x: ()
            self._trace_match = lambda x, y: ()

    def _reset_context(self, buffer=None, semantics=None):
        self._buffer = buffer
        self._ast_stack = []
        self._concrete_stack = [None]
        self._rule_stack = []
        self._cut_stack = [False]
        self._memoization_cache = dict()
        if semantics is not None:
            self.semantics = semantics
        if self.semantics is not None:
            set_buffer = getattr(self.semantics, 'set_buffer', None)
            if set_buffer is not None:
                set_buffer(buffer)

    def goto(self, pos):
        self._buffer.goto(pos)

    @property
    def _pos(self):
        return self._buffer.pos

    @property
    def linecount(self):
        return self._buffer.linecount

    def _goto(self, pos):
        self._buffer.goto(pos)

    def _next_token(self):
        self._buffer.next_token()

    @property
    def ast(self):
        return self._ast_stack[-1]

    @ast.setter
    def ast(self, value):
        self._ast_stack[-1] = value

    def _push_ast(self):
        self._push_cst()
        self._ast_stack.append(AST())

    def _pop_ast(self):
        self._pop_cst()
        return self._ast_stack.pop()

    def _add_ast_node(self, name, node, force_list=False):
        if name is not None:  # and node:
            self.ast.add(name, node, force_list)
        return node

    def _update_ast(self, ast):
        for key, value in ast.items():
            if key not in self.ast or not isinstance(value, list):
                self._add_ast_node(key, value)
            else:
                prev = self.ast[key]
                if isinstance(prev, list):
                    prev.extend(value)
                else:
                    self.ast[key] = [prev] + value

    @property
    def cst(self):
        return self._concrete_stack[-1]

    @cst.setter
    def cst(self, cst):
        self._concrete_stack[-1] = cst

    def _push_cst(self):
        self._concrete_stack.append(None)

    def _pop_cst(self):
        return self._concrete_stack.pop()

    def _add_cst_node(self, node):
        if node is None:
            return
        previous = self._concrete_stack[-1]
        if previous is None:
            self._concrete_stack[-1] = node
        elif previous == node:  # FIXME: Don't know how this happens, but it does
            return
        elif isinstance(previous, list):
            previous.append(node)
        else:
            self._concrete_stack[-1] = [previous, node]

    def _extend_cst(self, cst):
        if cst is None:
            return
        if self.cst is None:
            self.cst = cst
        elif isinstance(self.cst, list):
            if isinstance(cst, list):
                self.cst.extend(cst)
            else:
                self.cst.append(cst)
        elif isinstance(cst, list):
            self.cst = [self.cst] + cst
        else:
            self.cst = [self.cst, cst]

    def _is_cut_set(self):
        return self._cut_stack[-1]

    def _cut(self):
        self._cut_stack[-1] = True

        # Kota Mizushima et al say that we can throw away
        # memos for previous positions in the buffer.
        #   http://goo.gl/VaGpj
        cutpos = self._pos
        cache = self._memoization_cache
        cutkeys = [(p, n) for p, n in cache.keys() if p < cutpos]
        for key in cutkeys:
            del cache[key]

    def _push_cut(self):
        self._cut_stack.append(False)

    def _pop_cut(self):
        return self._cut_stack.pop()

    def _rulestack(self):
        stack = '.'.join(self._rule_stack)
        if len(stack) > 60:
            stack = '...' + stack[-60:]
        return stack

    def _find_rule(self, name):
        return None

    def _find_semantic_rule(self, name):
        if self.semantics is None:
            return None
        result = getattr(self.semantics, name, None)
        if result is None or not isinstance(result, type(self._find_semantic_rule)):
            return None
        return result

    def _trace(self, msg, *params):
        print(unicode(msg % params).encode(self.encoding), file=sys.stderr)

    def _trace_event(self, event):
        self._trace('%s   %s \n\t%s', event, self._rulestack(), self._buffer.lookahead())

    def _trace_match(self, token, name=None):
        if self._trace:
            name = name if name else ''
            self._trace('MATCHED <%s> /%s/\n\t%s', token, name, self._buffer.lookahead())

    def _error(self, item, etype=FailedParse):
        raise etype(self._buffer, item)

    @contextmanager
    def _try(self):
        p = self._pos
        self._push_ast()
        try:
            yield
            ast = self.ast
            cst = self.cst
        except:
            self._goto(p)
            raise
        finally:
            self._pop_ast()
        self._update_ast(ast)
        self._add_cst_node(cst)

    @contextmanager
    def _option(self):
        self._push_cut()
        try:
            with self._try():
                yield
        except FailedCut:
            raise
        except FailedParse as e:
            if self._is_cut_set():
                raise FailedCut(e)
        finally:
            self._pop_cut()

    @contextmanager
    def _choice(self):
        try:
            yield
        except FailedCut as e:
            raise e.nested

    @contextmanager
    def _optional(self):
        with self._choice():
            with self._option():
                yield

    @contextmanager
    def _group(self):
        self._push_cst()
        try:
            yield
            cst = self.cst
        finally:
            self._pop_cst()
        self._add_cst_node(cst)

    @contextmanager
    def _if(self):
        p = self._pos
        self._push_ast()
        try:
            yield
        finally:
            self._goto(p)
            self._pop_ast()  # simply discard

    @contextmanager
    def _ifnot(self):
        p = self._pos
        self._push_ast()
        try:
            yield
        except FailedParse:
            pass
        else:
            self._error('', etype=FailedLookahead)
        finally:
            self._goto(p)
            self._pop_ast()  # simply discard

    def _repeater(self, f):
        result = []
        while True:
            self._push_cut()
            try:
                p = self._pos
                with self._try():
                    value = f()
                if value is not None:
                    result.append(value)
                if self._pos == p:
                    self._error('empty closure')
            except FailedCut:
                raise
            except FailedParse as e:
                if self._is_cut_set():
                    raise FailedCut(e)
                return result
            finally:
                self._pop_cut()

    def _repeat(self, f, plus=False):
        self._push_cst()
        try:
            one = []
            if plus:
                with self._try():
                    one = [f()]
            result = one + self._repeater(f)
            cst = self.cst
        finally:
            self._pop_cst()
        self._add_cst_node(cst)
        return result
<commit_msg>linecount should not be a parser property.<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals
import sys
from contextlib import contextmanager
from collections import namedtuple
from .ast import AST
from .exceptions import FailedParse, FailedCut, FailedLookahead

__all__ = ['ParseInfo', 'ParseContext']

ParseInfo = namedtuple('ParseInfo', ['buffer', 'rule', 'pos', 'endpos'])

class ParseContext(object):
    def __init__(self,
                 buffer=None,
                 semantics=None,
                 parseinfo=False,
                 trace=False,
                 encoding='utf-8',
                 **kwargs):
        super(ParseContext, self).__init__()

        self._buffer = buffer
        self.semantics = semantics if semantics is not None else self
        self.encoding = encoding
        self.parseinfo = parseinfo

        self._ast_stack = []
        self._concrete_stack = [None]
        self._rule_stack = []
        self._cut_stack = [False]
        self._memoization_cache = dict()
        if not trace:
            self._trace = lambda x: ()
            self._trace_event = lambda x: ()
            self._trace_match = lambda x, y: ()

    def _reset_context(self, buffer=None, semantics=None):
        self._buffer = buffer
        self._ast_stack = []
        self._concrete_stack = [None]
        self._rule_stack = []
        self._cut_stack = [False]
        self._memoization_cache = dict()
        if semantics is not None:
            self.semantics = semantics
        if self.semantics is not None:
            set_buffer = getattr(self.semantics, 'set_buffer', None)
            if set_buffer is not None:
                set_buffer(buffer)

    def goto(self, pos):
        self._buffer.goto(pos)

    @property
    def _pos(self):
        return self._buffer.pos

    def _goto(self, pos):
        self._buffer.goto(pos)

    def _next_token(self):
        self._buffer.next_token()

    @property
    def ast(self):
        return self._ast_stack[-1]

    @ast.setter
    def ast(self, value):
        self._ast_stack[-1] = value

    def _push_ast(self):
        self._push_cst()
        self._ast_stack.append(AST())

    def _pop_ast(self):
        self._pop_cst()
        return self._ast_stack.pop()

    def _add_ast_node(self, name, node, force_list=False):
        if name is not None:  # and node:
            self.ast.add(name, node, force_list)
        return node

    def _update_ast(self, ast):
        for key, value in ast.items():
            if key not in self.ast or not isinstance(value, list):
                self._add_ast_node(key, value)
            else:
                prev = self.ast[key]
                if isinstance(prev, list):
                    prev.extend(value)
                else:
                    self.ast[key] = [prev] + value

    @property
    def cst(self):
        return self._concrete_stack[-1]

    @cst.setter
    def cst(self, cst):
        self._concrete_stack[-1] = cst

    def _push_cst(self):
        self._concrete_stack.append(None)

    def _pop_cst(self):
        return self._concrete_stack.pop()

    def _add_cst_node(self, node):
        if node is None:
            return
        previous = self._concrete_stack[-1]
        if previous is None:
            self._concrete_stack[-1] = node
        elif previous == node:  # FIXME: Don't know how this happens, but it does
            return
        elif isinstance(previous, list):
            previous.append(node)
        else:
            self._concrete_stack[-1] = [previous, node]

    def _extend_cst(self, cst):
        if cst is None:
            return
        if self.cst is None:
            self.cst = cst
        elif isinstance(self.cst, list):
            if isinstance(cst, list):
                self.cst.extend(cst)
            else:
                self.cst.append(cst)
        elif isinstance(cst, list):
            self.cst = [self.cst] + cst
        else:
            self.cst = [self.cst, cst]

    def _is_cut_set(self):
        return self._cut_stack[-1]

    def _cut(self):
        self._cut_stack[-1] = True

        # Kota Mizushima et al say that we can throw away
        # memos for previous positions in the buffer.
        #   http://goo.gl/VaGpj
        cutpos = self._pos
        cache = self._memoization_cache
        cutkeys = [(p, n) for p, n in cache.keys() if p < cutpos]
        for key in cutkeys:
            del cache[key]

    def _push_cut(self):
        self._cut_stack.append(False)

    def _pop_cut(self):
        return self._cut_stack.pop()

    def _rulestack(self):
        stack = '.'.join(self._rule_stack)
        if len(stack) > 60:
            stack = '...' + stack[-60:]
        return stack

    def _find_rule(self, name):
        return None

    def _find_semantic_rule(self, name):
        if self.semantics is None:
            return None
        result = getattr(self.semantics, name, None)
        if result is None or not isinstance(result, type(self._find_semantic_rule)):
            return None
        return result

    def _trace(self, msg, *params):
        print(unicode(msg % params).encode(self.encoding), file=sys.stderr)

    def _trace_event(self, event):
        self._trace('%s   %s \n\t%s', event, self._rulestack(), self._buffer.lookahead())

    def _trace_match(self, token, name=None):
        if self._trace:
            name = name if name else ''
            self._trace('MATCHED <%s> /%s/\n\t%s', token, name, self._buffer.lookahead())

    def _error(self, item, etype=FailedParse):
        raise etype(self._buffer, item)

    @contextmanager
    def _try(self):
        p = self._pos
        self._push_ast()
        try:
            yield
            ast = self.ast
            cst = self.cst
        except:
            self._goto(p)
            raise
        finally:
            self._pop_ast()
        self._update_ast(ast)
        self._add_cst_node(cst)

    @contextmanager
    def _option(self):
        self._push_cut()
        try:
            with self._try():
                yield
        except FailedCut:
            raise
        except FailedParse as e:
            if self._is_cut_set():
                raise FailedCut(e)
        finally:
            self._pop_cut()

    @contextmanager
    def _choice(self):
        try:
            yield
        except FailedCut as e:
            raise e.nested

    @contextmanager
    def _optional(self):
        with self._choice():
            with self._option():
                yield

    @contextmanager
    def _group(self):
        self._push_cst()
        try:
            yield
            cst = self.cst
        finally:
            self._pop_cst()
        self._add_cst_node(cst)

    @contextmanager
    def _if(self):
        p = self._pos
        self._push_ast()
        try:
            yield
        finally:
            self._goto(p)
            self._pop_ast()  # simply discard

    @contextmanager
    def _ifnot(self):
        p = self._pos
        self._push_ast()
        try:
            yield
        except FailedParse:
            pass
        else:
            self._error('', etype=FailedLookahead)
        finally:
            self._goto(p)
            self._pop_ast()  # simply discard

    def _repeater(self, f):
        result = []
        while True:
            self._push_cut()
            try:
                p = self._pos
                with self._try():
                    value = f()
                if value is not None:
                    result.append(value)
                if self._pos == p:
                    self._error('empty closure')
            except FailedCut:
                raise
            except FailedParse as e:
                if self._is_cut_set():
                    raise FailedCut(e)
                return result
            finally:
                self._pop_cut()

    def _repeat(self, f, plus=False):
        self._push_cst()
        try:
            one = []
            if plus:
                with self._try():
                    one = [f()]
            result = one + self._repeater(f)
            cst = self.cst
        finally:
            self._pop_cst()
        self._add_cst_node(cst)
        return result
<|endoftext|>


<commit_before>import re
import logging
from .util import memoize, simplify, indent, trim
from .rendering import Renderer, render
from .buffering import Buffer
from .exceptions import *  # @UnusedWildImport
from .ast import AST

log = logging.getLogger('grako.parsing')

def check(result):
    assert isinstance(result, _Grammar), str(result)

class Context(object):
    def __init__(self, rules, text):
        self.rules = {rule.name :rule for rule in rules}
        self.buf = Buffer(text)
        self.buf.goto(0)
        self._ast_stack = [AST()]
        self._rule_stack = []

    def goto(self, pos):
        self.buf.goto(pos)

    @property
    def pos(self):
        return self.buf.pos

    def rulestack(self):
        return '.'.join(self._rule_stack)

    @property
    def ast(self):
        return self._ast_stack[-1]

    def push_ast(self):
        self._ast_stack.append(AST())

    def pop_ast(self):
        return self._ast_stack.pop()

    def add_ast_node(self, name, node):
        if name is not None and node:
            self.ast.add(name, node)
        return node

class _Grammar(Renderer):
    def parse(self, ctx):
        return None

class EOFGrammar(_Grammar):
    def parse(self, ctx):
        if not ctx.buf.atend():
            raise FailedParse(ctx.buf, '<EOF>')

    template = 'self._eol()'

class _DecoratorGrammar(_Grammar):
    def __init__(self, exp):
        assert isinstance(exp, _Grammar), str(exp)
        super(_DecoratorGrammar, self).__init__()
        self.exp = exp

    def parse(self, ctx):
        return self.exp.parse(ctx)

    def __str__(self):
        return str(self.exp)

class GroupGrammar(_DecoratorGrammar):
    def __str__(self):
        return '(%s)' % str(self.exp).strip()

    def render_fields(self, fields):
        fields.update(
                      n=self.counter(),
                      exp=indent(render(self.exp))
                      )

    template = '''\
                def group{n}():
                {exp}
                    return exp
                exp = group{n}() '''

class TokenGrammar(_Grammar):
    def __init__(self, token):
        super(TokenGrammar, self).__init__()
        self.token = token

    def parse(self, ctx):
        log.debug('token <%s>\n\t%s', self.token, ctx.buf.lookahead())
        result = ctx.buf.match(self.token)
        if result is None:
            raise FailedToken(ctx.buf, self.token)
        return result

    def __str__(self):
        if "'" in self.token:
            if '"' in self.token:
                return "'%s'" % self.token.encode('string-escape')
            else:
                return '"%s"' % self.token
        return "'%s'" % self.token

    def render_fields(self, fields):
        fields.update(token=self.token.encode('string-escape'))

    template = "exp = self._token('{token}')"

class PatternGrammar(_Grammar):
    def __init__(self, pattern):
        super(PatternGrammar, self).__init__()
        self.pattern = pattern
        self._re = re.compile(pattern)

    def parse(self, ctx):
        log.debug('pattern <%s>\n\t%s', self.pattern, ctx.buf.lookahead())
        result = ctx.buf.matchre(self._re)
        if result is None:
            raise FailedPattern(ctx.buf, self.pattern)
        return result

    def __str__(self):
        return '?/%s/?' % self.pattern

    template = '''\
                exp = self._pattern(r'{pattern}')
                '''

class SequenceGrammar(_Grammar):
    def __init__(self, sequence):
        super(SequenceGrammar, self).__init__()
        assert isinstance(sequence, list), str(sequence)
        self.sequence = sequence

    def parse(self, ctx):
        return simplify(self.parse_seq(ctx, self.sequence))

    def parse_seq(self, ctx, seq):
        log.debug('sequence %s', str([type(s) for s in self.sequence]))
        result = []
        for _i, s in enumerate(seq):
            tree = s.parse(ctx)
            result.append(tree)
#            if not isinstance(s, CutParser):
#                tree = s.parse(ctx)
#                result.append(tree)
#            else:
#                try:
#                    result.extend(self.parse_seq(ctx, seq[i + 1:]))
#                except FailedParse as e:
#                    raise FailedCut(ctx.buf, e)
        return [r for r in result if r is not None]

    def __str__(self):
        return ' '.join(str(s).strip() for s in self.sequence)

    def render_fields(self, fields):
        fields.update(seq='\n'.join(trim(render(s)) for s in self.sequence))

    template = '{seq}'

class ChoiceGrammar(_Grammar):
    def __init__(self, options):
        super(ChoiceGrammar, self).__init__()
        assert isinstance(options, list), repr(options)
        self.options = options

    def parse(self, ctx):
        items = []
        pos = ctx.pos
        for o in self.options:
            ctx.goto(pos)
            try:
                return o.parse(ctx)
            except FailedCut as e:
                raise
            except FailedParse as e:
                items.append(e.item)
        raise FailedParse(ctx.buf, 'one of {%s}' % ','.join(items))

    def __str__(self):
        return ' | '.join(str(o).strip() for o in self.options)

    def render_fields(self, fields):
        template = trim(self.option_template)
        options = [template.format(option=render(o)) for o in self.options]
        options = '\n'.join(o for o in options)
        fields.update(options=indent(options))

    def render(self):
        if len(self.options) == 1:
            return render(self.options[0])
        else:
            return super(ChoiceGrammar, self).render()

    option_template = '''\
                    try:
                        {option}
                        break
                    except FailedCut:
                        raise
                    except FailedParse:
                        pass\
                    '''

    template = '''\
                for _ in ('once',):
                    # break with the first option that succeeds
                {options}
                    raise FailedParse(self.buf, 'no viable option') '''

class RepeatGrammar(_DecoratorGrammar):
    def parse(self, ctx):
        log.debug('repeat %s', str(self.exp))
        result = []
        while True:
            p = ctx.buf.pos
            try:
                tree = self.exp.parse(ctx)
                result.append(tree)
            except FailedCut:
                raise
            except FailedParse:
                ctx.buf.goto(p)
                break
        return simplify(result)

    def __str__(self):
        return '{%s}' % str(self.exp)

    def render_fields(self, fields):
        fields.update(n=self.counter(),
                      innerexp=indent(render(self.exp), 3))

    template = '''
                def repeat{n}():
                    result = []
                    while True:
                        p = self.pos
                        try:
                {innerexp}
                            result.append(exp)
                        except FailedCut:
                            raise
                        except FailedParse:
                            self.goto(p)
                            break
                    return result
                exp = repeat{n}() '''

class RepeatOneGrammar(RepeatGrammar):
    def parse(self, ctx):
        head = self.exp.parse(ctx)
        return simplify([head] + super(RepeatOneGrammar, self).parse(ctx))

    def __str__(self):
        return '{%s}+' % str(self.exp)

    def render_fields(self, fields):
        fields.update(n=self.counter(),
                      exp=indent(render(self.exp)),
                      innerexp=indent(render(self.exp), 3))

    template = '''
                def repeat{n}():
                    result = []
                {exp}
                    while True:
                        p = self.pos
                        try:
                {innerexp}
                            result.append(exp)
                        except FailedCut:
                            raise
                        except FailedParse:
                            self.goto(p)
                            break
                    return result
                exp = repeat{n}() '''

class OptionalGrammar(_DecoratorGrammar):
    def parse(self, ctx):
        p = ctx.pos
        try:
            return self.exp.parse(ctx)
        except FailedParse:
            ctx.goto(p)
            return None

    def __str__(self):
        return '[%s]' % str(self.exp)

    template = '''\
            p = self.pos
            try:
                {exp}
            except FailedParse:
                self.goto(p)
            '''

class CutGrammar(_Grammar):
    def parse(self, ctx):
        return None

    def __str__(self):
        return '!'

class NamedGrammar(_DecoratorGrammar):
    def __init__(self, name, exp):
        super(NamedGrammar, self).__init__(exp)
        assert isinstance(exp, _Grammar), str(exp)
        self.name = name

    def parse(self, ctx):
        value = self.exp.parse(ctx)
        ctx.add_ast_node(self.name, value)
        return value

    def __str__(self):
        return '%s:%s' % (self.name, str(self.exp))

    template = '''{exp}
                    self.ast.{name} += exp '''

class SpecialGrammar(_Grammar):
    def __init__(self, special):
        super(SpecialGrammar, self).__init__()
        self.special = special

    def __str__(self):
        return '?/%s/?' % self.pattern

class RuleRefGrammar(_Grammar):
    def __init__(self, name):
        super(RuleRefGrammar, self).__init__()
        self.name = name

    def parse(self, ctx):
        try:
            rule = ctx.rules[self.name]
            return rule.parse(ctx)
        except KeyError:
            raise FailedRef(ctx.buf, self.name)
        except FailedParse:
            raise

    def __str__(self):
        return self.name

    template = 'exp = self._call("{name}")'

class RuleGrammar(NamedGrammar):
    def parse(self, ctx):
        ctx._rule_stack.append(self.name)
        ctx.push_ast()
        log.debug('%s \n\t%s', ctx.rulestack(), ctx.buf.lookahead())
        try:
            _tree, newpos = self._invoke_rule(self.name, ctx, ctx.pos)
            ctx.goto(newpos)
            log.debug('SUCCESS %s \n\t%s', ctx.rulestack(), ctx.buf.lookahead())
            return ctx.ast
        except FailedParse:
            log.debug('FAIL %s \n\t%s', ctx.rulestack(), ctx.buf.lookahead())
            raise
        finally:
            ctx.pop_ast()
            ctx._rule_stack.pop()

    @memoize
    def _invoke_rule(self, name, ctx, pos):
        ctx.goto(pos)
        return (self.exp.parse(ctx), ctx.pos)

    def __str__(self):
        return '%s = %s ;' % (self.name, str(self.exp).strip())

    def render_fields(self, fields):
        fields.update(exp=indent(render(self.exp)))

    template = '''
                def _{name}_(self):
                {exp}

                '''

class Grammar(Renderer):
    def __init__(self, name, rules):
        super(Grammar, self).__init__()
        assert isinstance(rules, list), str(rules)
        self.name = name
        self.rules = rules

    def parse(self, start, text):
        log.info('enter grammar')
        try:
            try:
                ctx = Context(self.rules, text)
                start_rule = ctx.rules[start]
                tree = start_rule.parse(ctx)
                ctx.add_ast_node(start, tree)
                ctx.buf.eatwhitespace()
                if not ctx.buf.atend():
                    raise FailedParse(ctx.buf, '<EOF>')
                log.info('SUCCESS grammar')
                return ctx.ast
            except FailedCut as e:
                raise e.nested
        except:
            log.warning('failed grammar')
            raise

    def __str__(self):
        return '\n\n'.join(str(rule) for rule in self.rules) + '\n'

    def render_fields(self, fields):
        abstract_template = trim(self.abstract_rule_template)
        abstract_rules = [abstract_template.format(name=rule.name) for rule in self.rules]
        abstract_rules = indent('\n'.join(abstract_rules))
        fields.update(rules=indent(render(self.rules)),
                      abstract_rules=abstract_rules
                      )

    abstract_rule_template = '''
            def {name}(ast):
                return ast
            '''

    template = '''\
                from parsing import *

                class {name}ParserBase(Parser):
                {rules}
                class Abstract{name}Parser({name}ParserBase):
                {abstract_rules}
                '''
<commit_msg>BUG: RepeatOne was omitting the first item.<commit_after>import re
import logging
from .util import memoize, simplify, indent, trim
from .rendering import Renderer, render
from .buffering import Buffer
from .exceptions import *  # @UnusedWildImport
from .ast import AST

log = logging.getLogger('grako.parsing')

def check(result):
    assert isinstance(result, _Grammar), str(result)

class Context(object):
    def __init__(self, rules, text):
        self.rules = {rule.name :rule for rule in rules}
        self.buf = Buffer(text)
        self.buf.goto(0)
        self._ast_stack = [AST()]
        self._rule_stack = []

    def goto(self, pos):
        self.buf.goto(pos)

    @property
    def pos(self):
        return self.buf.pos

    def rulestack(self):
        return '.'.join(self._rule_stack)

    @property
    def ast(self):
        return self._ast_stack[-1]

    def push_ast(self):
        self._ast_stack.append(AST())

    def pop_ast(self):
        return self._ast_stack.pop()

    def add_ast_node(self, name, node):
        if name is not None and node:
            self.ast.add(name, node)
        return node

class _Grammar(Renderer):
    def parse(self, ctx):
        return None

class EOFGrammar(_Grammar):
    def parse(self, ctx):
        if not ctx.buf.atend():
            raise FailedParse(ctx.buf, '<EOF>')

    template = 'self._eol()'

class _DecoratorGrammar(_Grammar):
    def __init__(self, exp):
        assert isinstance(exp, _Grammar), str(exp)
        super(_DecoratorGrammar, self).__init__()
        self.exp = exp

    def parse(self, ctx):
        return self.exp.parse(ctx)

    def __str__(self):
        return str(self.exp)

class GroupGrammar(_DecoratorGrammar):
    def __str__(self):
        return '(%s)' % str(self.exp).strip()

    def render_fields(self, fields):
        fields.update(
                      n=self.counter(),
                      exp=indent(render(self.exp))
                      )

    template = '''\
                def group{n}():
                {exp}
                    return exp
                exp = group{n}() '''

class TokenGrammar(_Grammar):
    def __init__(self, token):
        super(TokenGrammar, self).__init__()
        self.token = token

    def parse(self, ctx):
        log.debug('token <%s>\n\t%s', self.token, ctx.buf.lookahead())
        result = ctx.buf.match(self.token)
        if result is None:
            raise FailedToken(ctx.buf, self.token)
        return result

    def __str__(self):
        if "'" in self.token:
            if '"' in self.token:
                return "'%s'" % self.token.encode('string-escape')
            else:
                return '"%s"' % self.token
        return "'%s'" % self.token

    def render_fields(self, fields):
        fields.update(token=self.token.encode('string-escape'))

    template = "exp = self._token('{token}')"

class PatternGrammar(_Grammar):
    def __init__(self, pattern):
        super(PatternGrammar, self).__init__()
        self.pattern = pattern
        self._re = re.compile(pattern)

    def parse(self, ctx):
        log.debug('pattern <%s>\n\t%s', self.pattern, ctx.buf.lookahead())
        result = ctx.buf.matchre(self._re)
        if result is None:
            raise FailedPattern(ctx.buf, self.pattern)
        return result

    def __str__(self):
        return '?/%s/?' % self.pattern

    template = '''\
                exp = self._pattern(r'{pattern}')
                '''

class SequenceGrammar(_Grammar):
    def __init__(self, sequence):
        super(SequenceGrammar, self).__init__()
        assert isinstance(sequence, list), str(sequence)
        self.sequence = sequence

    def parse(self, ctx):
        return simplify(self.parse_seq(ctx, self.sequence))

    def parse_seq(self, ctx, seq):
        log.debug('sequence %s', str([type(s) for s in self.sequence]))
        result = []
        for _i, s in enumerate(seq):
            tree = s.parse(ctx)
            result.append(tree)
#            if not isinstance(s, CutParser):
#                tree = s.parse(ctx)
#                result.append(tree)
#            else:
#                try:
#                    result.extend(self.parse_seq(ctx, seq[i + 1:]))
#                except FailedParse as e:
#                    raise FailedCut(ctx.buf, e)
        return [r for r in result if r is not None]

    def __str__(self):
        return ' '.join(str(s).strip() for s in self.sequence)

    def render_fields(self, fields):
        fields.update(seq='\n'.join(trim(render(s)) for s in self.sequence))

    template = '{seq}'

class ChoiceGrammar(_Grammar):
    def __init__(self, options):
        super(ChoiceGrammar, self).__init__()
        assert isinstance(options, list), repr(options)
        self.options = options

    def parse(self, ctx):
        items = []
        pos = ctx.pos
        for o in self.options:
            ctx.goto(pos)
            try:
                return o.parse(ctx)
            except FailedCut as e:
                raise
            except FailedParse as e:
                items.append(e.item)
        raise FailedParse(ctx.buf, 'one of {%s}' % ','.join(items))

    def __str__(self):
        return ' | '.join(str(o).strip() for o in self.options)

    def render_fields(self, fields):
        template = trim(self.option_template)
        options = [template.format(option=render(o)) for o in self.options]
        options = '\n'.join(o for o in options)
        fields.update(options=indent(options))

    def render(self):
        if len(self.options) == 1:
            return render(self.options[0])
        else:
            return super(ChoiceGrammar, self).render()

    option_template = '''\
                    try:
                        {option}
                        break
                    except FailedCut:
                        raise
                    except FailedParse:
                        pass\
                    '''

    template = '''\
                for _ in ('once',):
                    # break with the first option that succeeds
                {options}
                    raise FailedParse(self.buf, 'no viable option') '''

class RepeatGrammar(_DecoratorGrammar):
    def parse(self, ctx):
        log.debug('repeat %s', str(self.exp))
        result = []
        while True:
            p = ctx.buf.pos
            try:
                tree = self.exp.parse(ctx)
                result.append(tree)
            except FailedCut:
                raise
            except FailedParse:
                ctx.buf.goto(p)
                break
        return simplify(result)

    def __str__(self):
        return '{%s}' % str(self.exp)

    def render_fields(self, fields):
        fields.update(n=self.counter(),
                      innerexp=indent(render(self.exp), 3))

    template = '''
                def repeat{n}():
                    result = []
                    while True:
                        p = self.pos
                        try:
                {innerexp}
                            result.append(exp)
                        except FailedCut:
                            raise
                        except FailedParse:
                            self.goto(p)
                            break
                    return result
                exp = repeat{n}() '''

class RepeatOneGrammar(RepeatGrammar):
    def parse(self, ctx):
        head = self.exp.parse(ctx)
        return simplify([head] + super(RepeatOneGrammar, self).parse(ctx))

    def __str__(self):
        return '{%s}+' % str(self.exp)

    def render_fields(self, fields):
        fields.update(n=self.counter(),
                      exp=indent(render(self.exp)),
                      innerexp=indent(render(self.exp), 3))

    template = '''
                def repeat{n}():
                {exp}
                    result = [exp]
                    while True:
                        p = self.pos
                        try:
                {innerexp}
                            result.append(exp)
                        except FailedCut:
                            raise
                        except FailedParse:
                            self.goto(p)
                            break
                    return result
                exp = repeat{n}() '''

class OptionalGrammar(_DecoratorGrammar):
    def parse(self, ctx):
        p = ctx.pos
        try:
            return self.exp.parse(ctx)
        except FailedParse:
            ctx.goto(p)
            return None

    def __str__(self):
        return '[%s]' % str(self.exp)

    template = '''\
            p = self.pos
            try:
                {exp}
            except FailedParse:
                self.goto(p)
            '''

class CutGrammar(_Grammar):
    def parse(self, ctx):
        return None

    def __str__(self):
        return '!'

class NamedGrammar(_DecoratorGrammar):
    def __init__(self, name, exp):
        super(NamedGrammar, self).__init__(exp)
        assert isinstance(exp, _Grammar), str(exp)
        self.name = name

    def parse(self, ctx):
        value = self.exp.parse(ctx)
        ctx.add_ast_node(self.name, value)
        return value

    def __str__(self):
        return '%s:%s' % (self.name, str(self.exp))

    template = '''{exp}
                    self.ast.{name} += exp '''

class SpecialGrammar(_Grammar):
    def __init__(self, special):
        super(SpecialGrammar, self).__init__()
        self.special = special

    def __str__(self):
        return '?/%s/?' % self.pattern

class RuleRefGrammar(_Grammar):
    def __init__(self, name):
        super(RuleRefGrammar, self).__init__()
        self.name = name

    def parse(self, ctx):
        try:
            rule = ctx.rules[self.name]
            return rule.parse(ctx)
        except KeyError:
            raise FailedRef(ctx.buf, self.name)
        except FailedParse:
            raise

    def __str__(self):
        return self.name

    template = 'exp = self._call("{name}")'

class RuleGrammar(NamedGrammar):
    def parse(self, ctx):
        ctx._rule_stack.append(self.name)
        ctx.push_ast()
        log.debug('%s \n\t%s', ctx.rulestack(), ctx.buf.lookahead())
        try:
            _tree, newpos = self._invoke_rule(self.name, ctx, ctx.pos)
            ctx.goto(newpos)
            log.debug('SUCCESS %s \n\t%s', ctx.rulestack(), ctx.buf.lookahead())
            return ctx.ast
        except FailedParse:
            log.debug('FAIL %s \n\t%s', ctx.rulestack(), ctx.buf.lookahead())
            raise
        finally:
            ctx.pop_ast()
            ctx._rule_stack.pop()

    @memoize
    def _invoke_rule(self, name, ctx, pos):
        ctx.goto(pos)
        return (self.exp.parse(ctx), ctx.pos)

    def __str__(self):
        return '%s = %s ;' % (self.name, str(self.exp).strip())

    def render_fields(self, fields):
        fields.update(exp=indent(render(self.exp)))

    template = '''
                def _{name}_(self):
                {exp}

                '''

class Grammar(Renderer):
    def __init__(self, name, rules):
        super(Grammar, self).__init__()
        assert isinstance(rules, list), str(rules)
        self.name = name
        self.rules = rules

    def parse(self, start, text):
        log.info('enter grammar')
        try:
            try:
                ctx = Context(self.rules, text)
                start_rule = ctx.rules[start]
                tree = start_rule.parse(ctx)
                ctx.add_ast_node(start, tree)
                ctx.buf.eatwhitespace()
                if not ctx.buf.atend():
                    raise FailedParse(ctx.buf, '<EOF>')
                log.info('SUCCESS grammar')
                return ctx.ast
            except FailedCut as e:
                raise e.nested
        except:
            log.warning('failed grammar')
            raise

    def __str__(self):
        return '\n\n'.join(str(rule) for rule in self.rules) + '\n'

    def render_fields(self, fields):
        abstract_template = trim(self.abstract_rule_template)
        abstract_rules = [abstract_template.format(name=rule.name) for rule in self.rules]
        abstract_rules = indent('\n'.join(abstract_rules))
        fields.update(rules=indent(render(self.rules)),
                      abstract_rules=abstract_rules
                      )

    abstract_rule_template = '''
            def {name}(ast):
                return ast
            '''

    template = '''\
                from parsing import *

                class {name}ParserBase(Parser):
                {rules}
                class Abstract{name}Parser({name}ParserBase):
                {abstract_rules}
                '''
<|endoftext|>


<commit_before>"""Module for running Louvain community detection algorithm.
"""

from __future__ import division, print_function, absolute_import
import six
range = six.moves.range

import numpy as np
import subprocess
import tempfile
import os

import scipy.sparse as sp

def optimize_modularity(conn_mx, rand_init=True, num_runs=1, debug=False):
  """Optimize directed, weighted Newman's modularity
  using the Louvain algorithm.

  This uses C++ implementation from
  https://github.com/raldecoa/SurpriseMe/tree/master/src/CPM

  For example:

  >>> import networkx as nx
  >>> import graphy
  >>> G = nx.karate_club_graph()
  >>> best_membership, q = graphy.louvain.optimize_modularity(nx.to_numpy_matrix(G))
  >>> print(best_membership, q)   # doctest: +SKIP

  Parameters
  ----------
  conn_mx : 2-dimensional np.array or scipy.sparse matrix
    Connectivity matrix.
  rand_init : bool (default True)
    Whether to randomly shuffle order of nodes (makes results non-deterministic)
  num_runs : int (default 1)
    How many runs to perform (highest quality run returned).  Only allow if
    rand_init is True.
  debug : bool (default False)
    If True, prints various debugging information.

  Returns
  -------
  np.array
    Optimal membership vector indicating community membership of each node.
  float
    Modularity value corresponding to the optimal membership vector. Notice that
    because the modularity value is computed by adding up increments over
    many moves, this may only be accurate to a few decimal places.

  """
  # TODO: Implement multithreading?  Code seems to support it already

  if num_runs > 1 and not rand_init:
    raise ValueError('Multiple runs only makes sense when initial order of'
                     'nodes is randomized')

  is_sparse = sp.isspmatrix(conn_mx)
  if is_sparse:
      # transform to list of lists
      conn_mx = conn_mx.tolil()
  else:
      conn_mx = np.asarray(conn_mx)

  if conn_mx.shape[0] != conn_mx.shape[1]:
      raise ValueError('conn mx should be square')

  DIR = tempfile.gettempdir()
  pfx = str(os.getpid()) + '_'
  NETWORK_FILE = os.path.join(DIR, pfx+'net.txt')
  OUTPUT_FILE  = os.path.join(DIR, pfx+'net.bin')
  NODEMAP_FILE = os.path.join(DIR, pfx+'nmap.bin')
  CONF_FILE    = os.path.join(DIR, pfx+'conf.bin')

  with open(NETWORK_FILE, 'w') as f:
      f.write('>\n')
      for ndx in range(conn_mx.shape[0]):
          f.write('%d %d\n' % (ndx, ndx))
      f.write('>\n0 0\n>\n')
      for ndx in range(conn_mx.shape[0]):
          r = conn_mx[ndx,:]
          if is_sparse:
            conns = r.rows[0]
            weights = r.data[0]
          else:
            conns = np.flatnonzero(r)
            weights = r[conns]

          if not len(conns):
              f.write('%d %d %0.5f 0\n' % (ndx, 0, 0))
          for c,w in zip(conns, weights):
              f.write('%d %d %0.5f 0\n' % (ndx, c, w))

  if debug:
    print("**** NETWORK FILE: ****")
    with open(NETWORK_FILE) as f:
      print(f.read())
    print()

  bin_dir = os.path.join(os.path.dirname(__file__),'..','external','SurpriseMeCPM','bin')
  subprocess.call([os.path.join(bin_dir,'slicer'),
                   '-i', NETWORK_FILE,
                   '-o', OUTPUT_FILE,
                   '-n', NODEMAP_FILE,
                   '-c', CONF_FILE], stderr=subprocess.PIPE)

  best_membership, best_q = None, None
  for run_ndx in range(num_runs):
    res = subprocess.Popen([os.path.join(bin_dir,'community'), '-r',
                            OUTPUT_FILE,
                            CONF_FILE],
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    output, errors = map(lambda s: s.decode('ascii'), res.communicate())

    all_files = [NETWORK_FILE, OUTPUT_FILE, NODEMAP_FILE, CONF_FILE]
    if debug:
      print("******* RUN %d *******" % run_ndx)
      print("**** OUTPUT: ****")
      print(output)
      print()
      print("**** STDERR: ****")
      print(errors)
      print("*****************")
      print("Leaving files in place:")
      print(all_files)

    q_value = float(errors.strip().split("\n")[-1].split(" ")[0])
    membership = np.zeros(conn_mx.shape[0], dtype='int')
    ndxs, vals = zip(*[map(int, l.strip().split("\t")) for l in output.strip().split("\n")])
    membership[list(ndxs)] = vals

    if best_q is None or q_value > best_q:
      best_membership, best_q = membership, q_value

  if not debug:
    for fname in all_files:
      os.remove(fname)

  return best_membership, best_q

<commit_msg>Updates to louvain<commit_after>"""Module for running Louvain community detection algorithm.
"""

from __future__ import division, print_function, absolute_import
import six
range = six.moves.range

import numpy as np
import subprocess
import tempfile
import os

import scipy.sparse as sp

def optimize_modularity(conn_mx, rand_init=True, num_runs=1, debug=False):
  """Optimize directed, weighted Newman's modularity
  using the Louvain algorithm.

  This uses C++ implementation from
  https://github.com/raldecoa/SurpriseMe/tree/master/src/CPM

  For example:

  >>> import networkx as nx
  >>> import graphy
  >>> G = nx.karate_club_graph()
  >>> best_membership, q = graphy.louvain.optimize_modularity(nx.to_numpy_matrix(G))
  >>> print(best_membership, q)   # doctest: +SKIP

  Parameters
  ----------
  conn_mx : 2-dimensional np.array or scipy.sparse matrix
    Connectivity matrix.
  rand_init : bool (default True)
    Whether to randomly shuffle order of nodes (makes results non-deterministic)
  num_runs : int (default 1)
    How many runs to perform (highest quality run returned).  Only allow if
    rand_init is True.
  debug : bool (default False)
    If True, prints various debugging information.

  Returns
  -------
  np.array
    Optimal membership vector indicating community membership of each node.
  float
    Modularity value corresponding to the optimal membership vector. Notice that
    because the modularity value is computed by adding up increments over
    many moves, this may only be accurate to a few decimal places.

  """
  # TODO: Implement multithreading?  Code seems to support it already

  if num_runs > 1 and not rand_init:
    raise ValueError('Multiple runs only makes sense when initial order of'
                     'nodes is randomized')

  is_sparse = sp.isspmatrix(conn_mx)
  if is_sparse:
      # transform to list of lists
      conn_mx = conn_mx.tolil()
  else:
      conn_mx = np.asarray(conn_mx)

  if conn_mx.shape[0] != conn_mx.shape[1]:
      raise ValueError('conn mx should be square')

  DIR = tempfile.gettempdir()
  pfx = str(os.getpid()) + '_'
  NETWORK_FILE = os.path.join(DIR, pfx+'net.txt')
  OUTPUT_FILE  = os.path.join(DIR, pfx+'net.bin')
  NODEMAP_FILE = os.path.join(DIR, pfx+'nmap.bin')
  CONF_FILE    = os.path.join(DIR, pfx+'conf.bin')

  with open(NETWORK_FILE, 'w') as f:
      f.write('>\n')
      for ndx in range(conn_mx.shape[0]):
          f.write('%d %d\n' % (ndx, ndx))
      f.write('>\n0 0\n>\n')
      for ndx in range(conn_mx.shape[0]):
          r = conn_mx[ndx,:]
          if is_sparse:
            conns = r.rows[0]
            weights = r.data[0]
          else:
            conns = np.flatnonzero(r)
            weights = r[conns]

          if not len(conns):
              f.write('%d %d %0.5f 0\n' % (ndx, 0, 0))
          for c,w in zip(conns, weights):
              f.write('%d %d %0.5f 0\n' % (ndx, c, w))

  if debug:
    print("**** NETWORK FILE: ****")
    with open(NETWORK_FILE) as f:
      print(f.read())
    print()

  bin_dir = os.path.join(os.path.dirname(__file__),'..','external','SurpriseMeCPM','bin')
  subprocess.call([os.path.join(bin_dir,'slicer'),
                   '-i', NETWORK_FILE,
                   '-o', OUTPUT_FILE,
                   '-n', NODEMAP_FILE,
                   '-c', CONF_FILE], stderr=subprocess.PIPE)

  call_opts = [os.path.join(bin_dir,'community'),]
  if rand_init:
    call_opts.append('-r')
  call_opts.append(OUTPUT_FILE)
  call_opts.append(CONF_FILE)

  best_membership, best_q = None, None
  for run_ndx in range(num_runs):
    res = subprocess.Popen(call_opts, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    output, errors = map(lambda s: s.decode('ascii'), res.communicate())

    all_files = [NETWORK_FILE, OUTPUT_FILE, NODEMAP_FILE, CONF_FILE]
    if debug:
      print("******* RUN %d *******" % run_ndx)
      print("**** OUTPUT: ****")
      print(output)
      print()
      print("**** STDERR: ****")
      print(errors)
      print("*****************")
      print("Leaving files in place:")
      print(all_files)

    q_value = float(errors.strip().split("\n")[-1].split(" ")[0])
    membership = np.zeros(conn_mx.shape[0], dtype='int')
    ndxs, vals = zip(*[map(int, l.strip().split("\t")) for l in output.strip().split("\n")])
    membership[list(ndxs)] = vals

    if best_q is None or q_value > best_q:
      best_membership, best_q = membership, q_value

  if not debug:
    for fname in all_files:
      os.remove(fname)

  return best_membership, best_q

<|endoftext|>


<commit_before>import numpy as np
from StringIO import StringIO
from matplotlib import image as img
import requests

class Map(object):
    def __init__(self, lat, long, satellite=True, zoom=10, size=(400,400), sensor=False):
        base="http://maps.googleapis.com/maps/api/staticmap?"

        params=dict(
            sensor= str(sensor).lower(),
            zoom= zoom,
            size= "x".join(map(str, size)),
            center= ",".join(map(str, (lat, long) )),
            style="feature:all|element:labels|visibility:off"
          )

        if satellite:
            params["maptype"]="satellite"

        self.image = requests.get(base, params=params).content # Fetch our PNG image data
        self.pixels= img.imread(StringIO(self.image)) # Parse our PNG image as a numpy array

    def green(self, threshold):
        # Use NumPy to build an element-by-element logical array
        greener_than_red = self.pixels[:,:,1] > threshold* self.pixels[:,:,0]
        greener_than_blue = self.pixels[:,:,1] > threshold*self.pixels[:,:,2]
        green = np.logical_and(greener_than_red, greener_than_blue)
        return green

    def count_green(self, threshold = 1.1):
        return np.sum(self.green(threshold))

    def show_green(self, threshold = 1.1):
        green = self.green(threshold)
        out = green[:,:,np.newaxis]*np.array([0,1,0])[np.newaxis,np.newaxis,:]
        buffer = StringIO()
        result = img.imsave(buffer, out, format='png')
        return buffer.getvalue()<commit_msg>Added comments to map.py<commit_after>import numpy as np
from StringIO import StringIO
from matplotlib import image as img
import requests

class Map(object):
    def __init__(self, lat, long, satellite=True, zoom=10, size=(400,400), sensor=False):
        """Initialise map object with a <lat> and <long> latitude/longitude coordinates."""
        base="http://maps.googleapis.com/maps/api/staticmap?"

        params=dict(
            sensor= str(sensor).lower(),
            zoom= zoom,
            size= "x".join(map(str, size)),
            center= ",".join(map(str, (lat, long) )),
            style="feature:all|element:labels|visibility:off"
          )

        if satellite:
            params["maptype"]="satellite"

        self.image = requests.get(base, params=params).content # Fetch our PNG image data
        self.pixels= img.imread(StringIO(self.image)) # Parse our PNG image as a numpy array

    def green(self, threshold):
        """Parse our PNG image as a numpy array"""
        # Use NumPy to build an element-by-element logical array
        greener_than_red = self.pixels[:,:,1] > threshold* self.pixels[:,:,0]
        greener_than_blue = self.pixels[:,:,1] > threshold*self.pixels[:,:,2]
        green = np.logical_and(greener_than_red, greener_than_blue)
        return green

    def count_green(self, threshold = 1.1):
        """Count the number of green pixels in an array"""
        return np.sum(self.green(threshold))

    def show_green(self, threshold = 1.1):
        """Save image in PNG format"""
        green = self.green(threshold)
        out = green[:,:,np.newaxis]*np.array([0,1,0])[np.newaxis,np.newaxis,:]
        buffer = StringIO()
        result = img.imsave(buffer, out, format='png')
        return buffer.getvalue()<|endoftext|>