<commit_before>import threading
import time
import string
import random
import json

import master.monitor

# must import logger after initlogging, ugly
from utils.log import initlogging
initlogging("docklet-taskmgr")
from utils.log import logger

# grpc
from concurrent import futures
import grpc
from protos.rpc_pb2 import Task, TaskMsg, Status, Reply, Parameters, Cluster, Command, Image, Mount, Instance
from protos.rpc_pb2_grpc import MasterServicer, add_MasterServicer_to_server, WorkerStub

class TaskReporter(MasterServicer):

    def __init__(self, taskmgr):
        self.taskmgr = taskmgr

    def report(self, request, context):
        self.taskmgr.on_task_report(request)
        return Reply(message=Reply.ACCEPTED)

class TaskMgr(threading.Thread):

    # load task information from etcd
    # initial a task queue and task schedueler
    # taskmgr: a taskmgr instance
    def __init__(self, nodemgr):
        threading.Thread.__init__(self)
        self.thread_stop = False
        self.jobmgr = None
        self.task_queue = []
        self.heart_beat_timeout = 60 # (s)

        # nodes
        self.nodemgr = nodemgr
        self.all_nodes = None
        self.last_nodes_info_update_time = 0
        self.nodes_info_update_interval = 30 # (s)

    def run(self):
        self.serve()
        while not self.thread_stop:
            task, instance_id, worker = self.task_scheduler()
            if task is not None and worker is not None:
                self.task_processor(task, instance_id, worker)
            else:
                time.sleep(2)

    def serve(self):
        self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
        add_MasterServicer_to_server(TaskReporter(self), self.server)
        self.server.add_insecure_port('[::]:50051')
        self.server.start()
        logger.info('[taskmgr_rpc] start rpc server')

    def stop(self):
        self.thread_stop = True
        self.server.stop(0)
        logger.info('[taskmgr_rpc] stop rpc server')

    # this method is called when worker send heart-beat rpc request
    def on_task_report(self, report):
        logger.info('[on_task_report] receive task report: id %s-%d, status %d' % (report.taskid, report.instanceid, report.instanceStatus))
        task = get_task(report.taskid)
        if task == None:
            logger.error('[on_task_report] task not found')
            return

        instance = task.instance_list[report.instanceid]
        if instance['token'] != report.token:
            logger.warning('[on_task_report] wrong token')
            return

        instance['status'] = report.instanceStatus
        if report.instanceStatus == Status.RUNNING:
            instance['last_update_time'] = time.time()
        elif report.instanceStatus == Status.COMPLETED:
            check_task_completed(task)
        elif report.instanceStatus == Status.FAILED || report.instanceStatus == Status.TIMEOUT:
            if instance['try_count'] > task.maxRetryCount:
                check_task_completed(task)
        else:
            logger.error('[on_task_report] receive report from waiting task')

    def check_task_completed(self, task):
        if len(task.instance_list) < task.instanceCount:
            return
        failed = False
        for instance in task.instance_list:
            if instance['status'] == Status.RUNNING || instance['status'] == Status.WAITING:
                return
            if instance['status'] == Status.FAILED || instance['status'] == Status.TIMEOUT:
                if instance['try_count'] > task.maxRetryCount:
                    failed = True
                else:
                    return
        if self.jobmgr is None:
            logger.error('[check_task_completed] jobmgr is None!')
            return
        if failed:
            # TODO tell jobmgr task failed
            task.status = Status.FAILED
        else:
            # TODO tell jobmgr task completed
            task.status = Status.COMPLETED
        logger.info('task %s completed' % task.id)
        self.task_queue.remove(task)

    def task_processor(self, task, instance_id, worker):
        task.status = Status.RUNNING

        # properties for transaction
        task.instanceid = instance_id
        task.token = ''.join(random.sample(string.ascii_letters + string.digits, 8))

        instance = task.instance_list[instance_id]
        instance['status'] = Status.RUNNING
        instance['last_update_time'] = time.time()
        instance['try_count'] += 1
        instance['token'] = task.token

        try:
            logger.info('[task_processor] processing %s' % task.id)
            channel = grpc.insecure_channel('%s:50052' % worker)
            stub = WorkerStub(channel)
            response = stub.process_task(task)
            logger.info('[task_processor] worker response: %d' response.message)
        except Exception as e:
            logger.error('[task_processor] rpc error message: %s' e)
            instance['status'] = Status.FAILED
            instance['try_count'] -= 1

    # return task, worker
    def task_scheduler(self):
        # simple FIFO
        for task in self.task_queue:
            worker = self.find_proper_worker(task)
            if worker is not None:
                # find instance to retry
                for instance, index in enumerate(task.instance_list):
                    if (instance['status'] == Status.FAILED || instance['status'] == Status.TIMEOUT) and instance['try_count'] <= task.maxRetryCount:
                        return task, index, worker
                    elif instance['status'] == Status.RUNNING:
                        if time.time() - instance['last_update_time'] > self.heart_beat_timeout:
                            instance['status'] = Status.FAILED
                            instance['token'] = ''
                            return task, index, worker

                # start new instance
                if len(task.instance_list) < task.instanceCount:
                    instance = {}
                    instance['try_count'] = 0
                    task.instance_list.append(instance)
                    return task, len(task.instance_list) - 1, worker
        return None

    def find_proper_worker(self, task):
        nodes = get_all_nodes()
        if nodes is None or len(nodes) == 0:
            logger.warning('[task_scheduler] running nodes not found')
            return None

        for node in nodes:
            # TODO
            if True:
                return node[0]
        return None

    def get_all_nodes(self):
        # cache running nodes
        if self.all_nodes is not None and time.time() - self.last_nodes_info_update_time < self.nodes_info_update_interval:
            return self.all_nodes
        # get running nodes
        node_ips = self.nodemgr.get_nodeips()
        self.all_nodes = []
        for node_ip in node_ips:
            fetcher = master.monitor.Fetcher(node_ip)
            self.all_nodes.append((node_ip, fetcher.info))
        return self.all_nodes

    def set_jobmgr(self, jobmgr):
        self.jobmgr = jobmgr

    # user: username
    # task: a json string
    # save the task information into database
    # called when jobmgr assign task to taskmgr
    def add_task(self, username, taskid, json_task):
        # decode json string to object defined in grpc
        json_task = json.loads(json_task)
        task = Task(
            id = taskid,
            username = username,
            instanceCount = json_task['instanceCount'],
            maxRetryCount = json_task['maxRetryCount'],
            timeout = json_task['timeout'],
            parameters = Parameters(
                command = Command(
                    commandLine = json_task['parameters']['command']['commandLine'],
                    packagePath = json_task['parameters']['command']['packagePath'],
                    envVars = json_task['parameters']['command']['envVars']),
                stderrRedirectPath = json_task['parameters']['stderrRedirectPath'],
                stdoutRedirectPath = json_task['parameters']['stdoutRedirectPath']),
            cluster = Cluster(
                ,image = Image(
                    name = json_task['cluster']['image']['name'],
                    type = json_task['cluster']['image']['type'],
                    owner = json_task['cluster']['image']['owner']),
                instance = Instance(
                    cpu = json_task['cluster']['instance']['cpu'],
                    memory = json_task['cluster']['instance']['memory'],
                    disk = json_task['cluster']['instance']['disk'],
                    gpu = json_task['cluster']['instance']['gpu'])))
        task.cluster.mount = []
        for mount in json_task['cluster']['mount']:
            task.cluster.mount.append(Mount(localPath=mount['localPath'], remotePath=mount['remotePath']))

        # local properties
        task.status = Status.WAITING
        task.instance_list = []
        self.task_queue.append(task)

    # user: username
    # get the information of a task, including the status, task description and other information
    def get_task(self, taskid):
        for task in self.task_queue:
            if task.id == taskid:
                return task
        return None
<commit_msg>add simple resource manage to taskmgr<commit_after>import threading
import time
import string
import random
import json

import master.monitor

# must import logger after initlogging, ugly
from utils.log import initlogging
initlogging("docklet-taskmgr")
from utils.log import logger

# grpc
from concurrent import futures
import grpc
from protos.rpc_pb2 import Task, TaskMsg, Status, Reply, Parameters, Cluster, Command, Image, Mount, Instance
from protos.rpc_pb2_grpc import MasterServicer, add_MasterServicer_to_server, WorkerStub

class TaskReporter(MasterServicer):

    def __init__(self, taskmgr):
        self.taskmgr = taskmgr

    def report(self, request, context):
        self.taskmgr.on_task_report(request)
        return Reply(status=Reply.ACCEPTED, message='')

class TaskMgr(threading.Thread):

    # load task information from etcd
    # initial a task queue and task schedueler
    # taskmgr: a taskmgr instance
    def __init__(self, nodemgr):
        threading.Thread.__init__(self)
        self.thread_stop = False
        self.jobmgr = None
        self.task_queue = []
        self.heart_beat_timeout = 60 # (s)

        # nodes
        self.nodemgr = nodemgr
        self.cpu_usage = {}
        # self.all_nodes = None
        # self.last_nodes_info_update_time = 0
        # self.nodes_info_update_interval = 30 # (s)

    def run(self):
        self.serve()
        while not self.thread_stop:
            task, instance_id, worker = self.task_scheduler()
            if task is not None and worker is not None:
                self.task_processor(task, instance_id, worker)
            else:
                time.sleep(2)

    def serve(self):
        self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
        add_MasterServicer_to_server(TaskReporter(self), self.server)
        self.server.add_insecure_port('[::]:50051')
        self.server.start()
        logger.info('[taskmgr_rpc] start rpc server')

    def stop(self):
        self.thread_stop = True
        self.server.stop(0)
        logger.info('[taskmgr_rpc] stop rpc server')

    # this method is called when worker send heart-beat rpc request
    def on_task_report(self, report):
        logger.info('[on_task_report] receive task report: id %s-%d, status %d' % (report.taskid, report.instanceid, report.instanceStatus))
        task = get_task(report.taskid)
        if task == None:
            logger.error('[on_task_report] task not found')
            return

        instance = task.instance_list[report.instanceid]
        if instance['token'] != report.token:
            logger.warning('[on_task_report] wrong token')
            return

        instance['status'] = report.instanceStatus

        if report.instanceStatus == Status.RUNNING:
            instance['last_update_time'] = time.time()
        else:
            self.cpu_usage[instance['worker']] -= task.cluster.instance.cpu

        if report.instanceStatus == Status.COMPLETED:
            check_task_completed(task)
        elif report.instanceStatus == Status.FAILED or report.instanceStatus == Status.TIMEOUT:
            if instance['try_count'] > task.maxRetryCount:
                check_task_completed(task)
        else:
            logger.error('[on_task_report] receive report from waiting task')

    def check_task_completed(self, task):
        if len(task.instance_list) < task.instanceCount:
            return
        failed = False
        for instance in task.instance_list:
            if instance['status'] == Status.RUNNING or instance['status'] == Status.WAITING:
                return
            if instance['status'] == Status.FAILED or instance['status'] == Status.TIMEOUT:
                if instance['try_count'] > task.maxRetryCount:
                    failed = True
                else:
                    return
        if self.jobmgr is None:
            logger.error('[check_task_completed] jobmgr is None!')
            return
        if failed:
            # TODO tell jobmgr task failed
            task.status = Status.FAILED
        else:
            # TODO tell jobmgr task completed
            task.status = Status.COMPLETED
        logger.info('task %s completed' % task.id)
        self.task_queue.remove(task)

    def task_processor(self, task, instance_id, worker):
        task.status = Status.RUNNING

        # properties for transaction
        task.instanceid = instance_id
        task.token = ''.join(random.sample(string.ascii_letters + string.digits, 8))

        instance = task.instance_list[instance_id]
        instance['status'] = Status.RUNNING
        instance['last_update_time'] = time.time()
        instance['try_count'] += 1
        instance['token'] = task.token
        instance['worker'] = worker

        self.cpu_usage[worker] += task.cluster.instance.cpu

        try:
            logger.info('[task_processor] processing %s' % task.id)
            channel = grpc.insecure_channel('%s:50052' % worker)
            stub = WorkerStub(channel)
            response = stub.process_task(task)
            if response.status != Reply.ACCEPTED:
                raise Exception(response.message)
        except Exception as e:
            logger.error('[task_processor] rpc error message: %s' % e)
            instance['status'] = Status.FAILED
            instance['try_count'] -= 1

    # return task, worker
    def task_scheduler(self):
        # simple FIFO
        for task in self.task_queue:
            worker = self.find_proper_worker(task)
            if worker is not None:
                # find instance to retry
                for instance, index in enumerate(task.instance_list):
                    if (instance['status'] == Status.FAILED or instance['status'] == Status.TIMEOUT) and instance['try_count'] <= task.maxRetryCount:
                        return task, index, worker
                    elif instance['status'] == Status.RUNNING:
                        if time.time() - instance['last_update_time'] > self.heart_beat_timeout:
                            instance['status'] = Status.FAILED
                            instance['token'] = ''
                            return task, index, worker

                # start new instance
                if len(task.instance_list) < task.instanceCount:
                    instance = {}
                    instance['try_count'] = 0
                    task.instance_list.append(instance)
                    return task, len(task.instance_list) - 1, worker
        return None

    def find_proper_worker(self, task):
        nodes = get_all_nodes()
        if nodes is None or len(nodes) == 0:
            logger.warning('[task_scheduler] running nodes not found')
            return None

        for worker_ip, worker_info in nodes:
            if task.cluster.instance.cpu + get_cpu_usage(worker_ip) > worker_info['cpu']:
                continue
            if task.cluster.instance.memory > worker_info['memory']:
                continue
            if task.cluster.instance.disk > worker_info['disk']:
                continue
            if task.cluster.instance.gpu > worker_info['gpu']:
                continue
            return worker_ip
        return None

    def get_all_nodes(self):
        # cache running nodes
        # if self.all_nodes is not None and time.time() - self.last_nodes_info_update_time < self.nodes_info_update_interval:
        #     return self.all_nodes
        # get running nodes
        node_ips = self.nodemgr.get_nodeips()
        all_nodes = [(node_ip, get_worker_resource_info(node_ip)) for node_ip in node_ips]
        return all_nodes

    def get_worker_resource_info(self, worker_ip):
        fetcher = master.monitor.Fetcher(worker_ip)
        worker_info = fetcher.info
        info = {}
        info['cpu'] = len(worker_info['cpuconfig'])
        info['memory'] = worker_info['meminfo']['free'] / 1024 # (Mb)
        info['disk'] = sum([disk['free'] for disk in worker_info['diskinfo']]) / 1024 / 1024 # (Mb)
        info['gpu'] = 0 # not support yet
        return info

    def get_cpu_usage(self, worker_ip):
        try:
            return self.cpu_usage[worker_ip]
        except:
            self.cpu_usage[worker_ip] = 0
            return 0

    def set_jobmgr(self, jobmgr):
        self.jobmgr = jobmgr

    # user: username
    # task: a json string
    # save the task information into database
    # called when jobmgr assign task to taskmgr
    def add_task(self, username, taskid, json_task):
        # decode json string to object defined in grpc
        json_task = json.loads(json_task)
        task = Task(
            id = taskid,
            username = username,
            instanceCount = json_task['instanceCount'],
            maxRetryCount = json_task['maxRetryCount'],
            timeout = json_task['timeout'],
            parameters = Parameters(
                command = Command(
                    commandLine = json_task['parameters']['command']['commandLine'],
                    packagePath = json_task['parameters']['command']['packagePath'],
                    envVars = json_task['parameters']['command']['envVars']),
                stderrRedirectPath = json_task['parameters']['stderrRedirectPath'],
                stdoutRedirectPath = json_task['parameters']['stdoutRedirectPath']),
            cluster = Cluster(
                image = Image(
                    name = json_task['cluster']['image']['name'],
                    type = json_task['cluster']['image']['type'],
                    owner = json_task['cluster']['image']['owner']),
                instance = Instance(
                    cpu = json_task['cluster']['instance']['cpu'],
                    memory = json_task['cluster']['instance']['memory'],
                    disk = json_task['cluster']['instance']['disk'],
                    gpu = json_task['cluster']['instance']['gpu'])))
        task.cluster.mount = [Mount(localPath=mount['localPath'], remotePath=mount['remotePath'])
                                 for mount in json_task['cluster']['mount']]

        # local properties
        task.status = Status.WAITING
        task.instance_list = []
        self.task_queue.append(task)

    # user: username
    # get the information of a task, including the status, task description and other information
    def get_task(self, taskid):
        for task in self.task_queue:
            if task.id == taskid:
                return task
        return None
<|endoftext|>


<commit_before># inspired from:

# https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
# https://github.com/pypa/warehouse/blob/master/warehouse/__about__.py

__name__ = 'blis'
__title__ = 'blis'
__version__ = '0.0.2'
__summary__ = 'Cython bindings for Blis'
__uri__ = 'https://github.com/spacy-io/cython-blis'
__author__ = 'Matthew Honnibal'
__email__ = 'honnibal@gmail.com'
__license__ = 'MIT'
__release__ = True
<commit_msg>Bump version<commit_after># inspired from:

# https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
# https://github.com/pypa/warehouse/blob/master/warehouse/__about__.py

__name__ = 'blis'
__title__ = 'blis'
__version__ = '0.0.4'
__summary__ = 'Cython bindings for Blis'
__uri__ = 'https://github.com/spacy-io/cython-blis'
__author__ = 'Matthew Honnibal'
__email__ = 'honnibal@gmail.com'
__license__ = 'MIT'
__release__ = True
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# encoding: utf-8
"""
ylmir.py

Created by Karl Dubost on 2011-12-03.
Copyright (c) 2011 Grange. All rights reserved.
"""

import sys
import argparse
from lxml.html import tostring, html5parser
from lxml import etree
from lxml.etree import Element, SubElement

# CONFIG
SITENAME = "Les carnets Web de La Grange"
DOMAIN = "la-grange.net"
SITE = "http://www.%s/" % (DOMAIN)
STATUSLIST = ['draft','pub','acl']
DATETYPELIST = ['created','modified']
LICENSELIST = {'ccby': 'http://creativecommons.org/licenses/by/2.0/fr/',
               'copy': u'©'}
AUTHOR = "Karl Dubost"
AUTHORURL = "http://www.la-grange.net/karl/"
FEEDIDTAG = "tag:la-grange.net,2000-04-12:karl"
FEEDATOMNOM = "feed.atom"
HTMLNS = "http://www.w3.org/1999/xhtml"
ATOMNS = "http://www.w3.org/2005/Atom"
HTML = "{%s}" % HTMLNS
NSMAP = {None : HTMLNS}

# CONFIG with cli (TODO)
STYLESHEET = "/2011/12/01/proto/style/article.css"
STATUS = ""
MAXFEEDITEM = 20
LICENSE = "ccby"

# PATHS

help_message = '''
This script has been entirely created
for processing text files for the site
La Grange http://www.la-grange.net/.
'''

# General processing features

def parserawpost(rawpostpath):
    """Given a path, parse an html file
    TODO check if the file is correct.
    """
    doc = html5parser.parse(rawpostpath).getroot()
    print "INFO: Document parsed"
    return doc

# Extracting information from the blog posts

def getdocstatus(doc):
    """Check the publication status of the document
    returns a string
    if there are multiple meta, returns the first one and issues a warning"""
    findstatus = etree.ETXPath("//{%s}meta[@name='status']" % HTMLNS)
    docstatus = findstatus(doc)
    if len(docstatus) >= 1:
        status = docstatus[0].attrib['content']
        if status in STATUSLIST:
            if len(docstatus) > 1:
                print "WARNING: There are more than one status. Taking the first one : " + status
            else:
                print "INFO: The document status is " + status
            return status
        else:
            raise Exception, "ERROR: No valid status for your document: %s not in %s" % (status, STATUSLIST)
    if len(docstatus) == 0:
        print "WARNING: There is no status for this document."
        status = "undefined"
        return status

def getdocdate(doc, DATETYPE):
    """return the creation date of the document in ISO format YYYY-MM-DD
    Input the document, typeofdate in between created and modified"""
    # TODO: check if the format is correct aka YYYY-MM-DD
    if DATETYPE not in DATETYPELIST:
        sys.exit("ERROR: No valid type for the date: " + DATETYPE)
    finddate = etree.ETXPath("string(//{%s}time[@class=%r]/@datetime)" % (HTMLNS,DATETYPE))
    date = finddate(doc)
    return date

def getcontent(doc):
    """return the full content of an article"""
    findcontent = etree.ETXPath("//{%s}article" % HTMLNS)
    content = findcontent(doc)
    return etree.tostring(content[0], method="html",encoding="utf-8")

def gettitle(doc):
    """return a list of markup and text being the title of the document"""
    findtitle =  etree.ETXPath("//{%s}h1[text()]" % HTMLNS)
    if len(findtitle(doc)) == 0:
        sys.exit("ERROR: The document has no title")
    title = findtitle(doc)[0]
    titlemarkup = etree.tostring(title,encoding="utf-8")
    titletext = etree.tostring(title,encoding="utf-8",method="text")
    return titlemarkup, titletext

def makeblogpost(doc):
    """create a blog post ready to be publish from a raw or already published document"""
    pass

def makefeedentry(url, tagid, posttitle, created, modified, postcontent):
    """create an individual Atom feed entry from a ready to be publish post"""
    # etree.register_namespace("", ATOMNS)
    entry = Element('entry')
    id = SubElement(entry, 'id')
    id.text = tagid
    linkfeedentry = SubElement(entry, 'link')
    linkfeedentry.attrib["rel"] = "alternate"
    # TODO: This should be probably on a case by case.
    linkfeedentry.attrib["type"] = "text/html"
    linkfeedentry.attrib["href"] = url
    title = SubElement(entry, 'title')
    title.text = posttitle
    published = SubElement(entry, 'published')
    published.text = created
    updated = SubElement(entry, 'updated')
    updated.text = modified
    content = SubElement(entry, 'content')
    content.attrib["type"] = "xhtml"
    divcontent = SubElement(content, 'div')
    divcontent.attrib["xmlns"] = HTMLNS
    print postcontent
    divcontent.text = postcontent.decode("utf-8")
    return etree.tostring(entry, pretty_print=True, encoding="utf-8")

def createtagid(urlpath,isodate):
    """Create a unide tagid for a given blog post
    tag:la-grange.net,2012-01-24:2012/01/24/silence"""
    tagid = "tag:%s,%s:%s" % (DOMAIN,isodate,urlpath)
    return tagid

def updatefeed(feedentry):
    """Update the feed with the last individual feed entry"""
    pass

def updateannualindex(feedentry):
    """update the HTML Annual index with the feedendry"""
    pass

def updatemonthlyindex(feedentry):
    """update the HTML Monthly index with the feedendry"""
    pass

def updatearchivemap():
    """update the archive map page for new months and/or new years.
    not sure it is necessary. Manually is kind of cool with less
    dependencies."""
    pass

def createmonthlyindex(month):
    """create a monthly index when it doesn't exist"""
    pass

def createannualindex(year):
    """create an annual index when it doesn't exist"""
    pass

# MAIN

def main():

    # Parsing the cli
    parser = argparse.ArgumentParser(description="Managing Web site blog posts")

    parser.add_argument('rawpost', metavar='FILE', help='file to be processed', action='store', nargs=1, type=argparse.FileType('rt'))
    parser.add_argument('-o', '--output', help='the blog post ready to be sync', nargs=1, dest="output", type=argparse.FileType('wt'))
    atomgroup = parser.add_mutually_exclusive_group()
    atomgroup.add_argument('--atom', help='create an atom feed. DEFAULT', action='store_true', dest="createfeed", default=True)
    atomgroup.add_argument('--noatom', help='do not create the atom feed', action='store_false', dest='createfeed', default=False)

    args = parser.parse_args()
    rawpostpath = args.rawpost[0]

    # Parse the document
    rawpost = parserawpost(rawpostpath)
    # A few tests when developing
    STATUS = getdocstatus(rawpost)
    titlemarkup, title = gettitle(rawpost)
    title = title.decode("utf-8")
    print "TITLE: ", title
    print "TITLEMARKUP: ", titlemarkup
    created = getdocdate(rawpost, 'created')
    modified = getdocdate(rawpost, 'modified')
    print "CREATED:  ", created
    print "MODIFIED: ", modified
    content = getcontent(rawpost)
    urlpath = "2010/12/24/foo"
    url= "%s%s" % (SITE,urlpath)
    tagid =  createtagid(urlpath,created)
    print makefeedentry(url, tagid, title, created, modified, content)

if __name__ == "__main__":
    sys.exit(main())

<commit_msg>content properly added to the feed entry<commit_after>#!/usr/bin/env python2.7
# encoding: utf-8
"""
ylmir.py

Created by Karl Dubost on 2011-12-03.
Copyright (c) 2011 Grange. All rights reserved.
"""

import sys
import argparse
from lxml.html import tostring, html5parser
from lxml import etree
from lxml.etree import Element, SubElement

# CONFIG
SITENAME = "Les carnets Web de La Grange"
DOMAIN = "la-grange.net"
SITE = "http://www.%s/" % (DOMAIN)
STATUSLIST = ['draft','pub','acl']
DATETYPELIST = ['created','modified']
LICENSELIST = {'ccby': 'http://creativecommons.org/licenses/by/2.0/fr/',
               'copy': u'©'}
AUTHOR = "Karl Dubost"
AUTHORURL = "http://www.la-grange.net/karl/"
FEEDIDTAG = "tag:la-grange.net,2000-04-12:karl"
FEEDATOMNOM = "feed.atom"
HTMLNS = "http://www.w3.org/1999/xhtml"
ATOMNS = "http://www.w3.org/2005/Atom"
HTML = "{%s}" % HTMLNS
NSMAP = {None : HTMLNS}

# CONFIG with cli (TODO)
STYLESHEET = "/2011/12/01/proto/style/article.css"
STATUS = ""
MAXFEEDITEM = 20
LICENSE = "ccby"

# PATHS

help_message = '''
This script has been entirely created
for processing text files for the site
La Grange http://www.la-grange.net/.
'''

# General processing features

def parserawpost(rawpostpath):
    """Given a path, parse an html file
    TODO check if the file is correct.
    """
    doc = html5parser.parse(rawpostpath).getroot()
    #TODO find a way to send the element with only the namespace on the first element.
    print "INFO: Document parsed"
    return doc

# Extracting information from the blog posts

def getdocstatus(doc):
    """Check the publication status of the document
    returns a string
    if there are multiple meta, returns the first one and issues a warning"""
    findstatus = etree.ETXPath("//{%s}meta[@name='status']" % HTMLNS)
    docstatus = findstatus(doc)
    if len(docstatus) >= 1:
        status = docstatus[0].attrib['content']
        if status in STATUSLIST:
            if len(docstatus) > 1:
                print "WARNING: There are more than one status. Taking the first one : " + status
            else:
                print "INFO: The document status is " + status
            return status
        else:
            raise Exception, "ERROR: No valid status for your document: %s not in %s" % (status, STATUSLIST)
    if len(docstatus) == 0:
        print "WARNING: There is no status for this document."
        status = "undefined"
        return status

def getdocdate(doc, DATETYPE):
    """return the creation date of the document in ISO format YYYY-MM-DD
    Input the document, typeofdate in between created and modified"""
    # TODO: check if the format is correct aka YYYY-MM-DD
    if DATETYPE not in DATETYPELIST:
        sys.exit("ERROR: No valid type for the date: " + DATETYPE)
    finddate = etree.ETXPath("string(//{%s}time[@class=%r]/@datetime)" % (HTMLNS,DATETYPE))
    date = finddate(doc)
    return date

def getcontent(doc):
    """return the full content of an article"""
    findcontent = etree.ETXPath("//{%s}article" % HTMLNS)
    content = findcontent(doc)
    return content[0]

def gettitle(doc):
    """return a list of markup and text being the title of the document"""
    findtitle =  etree.ETXPath("//{%s}h1[text()]" % HTMLNS)
    if len(findtitle(doc)) == 0:
        sys.exit("ERROR: The document has no title")
    title = findtitle(doc)[0]
    titlemarkup = etree.tostring(title,encoding="utf-8")
    titletext = etree.tostring(title,encoding="utf-8",method="text")
    return titlemarkup, titletext

def makeblogpost(doc):
    """create a blog post ready to be publish from a raw or already published document"""
    pass

def makefeedentry(url, tagid, posttitle, created, modified, postcontent):
    """create an individual Atom feed entry from a ready to be publish post"""
    entry = Element('entry')
    id = SubElement(entry, 'id')
    id.text = tagid
    linkfeedentry = SubElement(entry, 'link')
    linkfeedentry.attrib["rel"] = "alternate"
    # TODO: This should be probably on a case by case.
    linkfeedentry.attrib["type"] = "text/html"
    linkfeedentry.attrib["href"] = url
    title = SubElement(entry, 'title')
    title.text = posttitle
    published = SubElement(entry, 'published')
    published.text = created
    updated = SubElement(entry, 'updated')
    updated.text = modified
    content = SubElement(entry, 'content')
    content.attrib["type"] = "xhtml"
    content.append(postcontent)
    return etree.tostring(entry, pretty_print=True, encoding="utf-8")

def createtagid(urlpath,isodate):
    """Create a unide tagid for a given blog post
    tag:la-grange.net,2012-01-24:2012/01/24/silence"""
    tagid = "tag:%s,%s:%s" % (DOMAIN,isodate,urlpath)
    return tagid

def updatefeed(feedentry):
    """Update the feed with the last individual feed entry"""
    pass

def updateannualindex(feedentry):
    """update the HTML Annual index with the feedendry"""
    pass

def updatemonthlyindex(feedentry):
    """update the HTML Monthly index with the feedendry"""
    pass

def updatearchivemap():
    """update the archive map page for new months and/or new years.
    not sure it is necessary. Manually is kind of cool with less
    dependencies."""
    pass

def createmonthlyindex(month):
    """create a monthly index when it doesn't exist"""
    pass

def createannualindex(year):
    """create an annual index when it doesn't exist"""
    pass

# MAIN

def main():

    # Parsing the cli
    parser = argparse.ArgumentParser(description="Managing Web site blog posts")

    parser.add_argument('rawpost', metavar='FILE', help='file to be processed', action='store', nargs=1, type=argparse.FileType('rt'))
    parser.add_argument('-o', '--output', help='the blog post ready to be sync', nargs=1, dest="output", type=argparse.FileType('wt'))
    atomgroup = parser.add_mutually_exclusive_group()
    atomgroup.add_argument('--atom', help='create an atom feed. DEFAULT', action='store_true', dest="createfeed", default=True)
    atomgroup.add_argument('--noatom', help='do not create the atom feed', action='store_false', dest='createfeed', default=False)

    args = parser.parse_args()
    rawpostpath = args.rawpost[0]

    # Parse the document
    rawpost = parserawpost(rawpostpath)
    # A few tests when developing
    STATUS = getdocstatus(rawpost)
    titlemarkup, title = gettitle(rawpost)
    title = title.decode("utf-8")
    print "TITLE: ", title
    print "TITLEMARKUP: ", titlemarkup
    created = getdocdate(rawpost, 'created')
    modified = getdocdate(rawpost, 'modified')
    print "CREATED:  ", created
    print "MODIFIED: ", modified
    content = getcontent(rawpost)
    urlpath = "2010/12/24/foo"
    url= "%s%s" % (SITE,urlpath)
    tagid =  createtagid(urlpath,created)
    print makefeedentry(url, tagid, title, created, modified, content)

if __name__ == "__main__":
    sys.exit(main())

<|endoftext|>


<commit_before>#!/usr/bin/python

# Transmageddon
# Copyright (C) 2009 Christian Schaller <uraeus@gnome.org>
# Copyright (C) 2009 Edward Hervey <edward.hervey@collabora.co.uk>
#
# Some code in this file came originally from the encode.py file in Pitivi
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import pygst
pygst.require("0.10")
import gst

def list_compat(a1, b1):
   for x1 in a1:
       if not x1 in b1:
           return False
   return True

containermap = { 'Ogg' : "application/ogg",'Matroska' : "video/x-matroska", 'MXF' : "application/mxf", 'AVI' : "video/x-msvideo",
                        'Quicktime' : "video/quicktime", 'MPEG4' : "application/x-iso-mp4", 'MPEG PS' : "ffmux_mpeg",
                        'MPEG TS' : "video/mpegts", 'FLV' : "video/x-flv", '3GPP' : "application/x-3gp" }

csuffixmap =   { 'Ogg' : ".ogg", 'Matroska' : ".mkv", 'MXF' : ".mxf", 'AVI' : ".avi", 'Quicktime' : ".mov",
                        'MPEG4' : ".mp4", 'MPEG PS' : ".mpg", 'MPEG TS' : ".ts", 'FLV' : ".flv", '3GPP' : ".3gp" }

codecmap = {     'vorbis' : "audio/x-vorbis", 'flac' : "audio/x-flac", 'mp3' : "audio/mpeg,mpegversion=1,layer=3",
                        'aac' : "audio/mpeg,mpegversion=[4, 2]", 'ac3' : "audio/x-ac3", 'speex' : "audio/x-speex",
                        'celt' : "audio/x-celt", 'alac' : "audio/x-alac", 'wma2' : "audio/x-wma,wmaversion=2",
                        'theora' : "video/x-theora", 'dirac' : "video/x-dirac", 'h264' : "video/x-h264",
                        'mpeg2' : "video/mpeg,mpegversion=2", 'mpeg4' : "video/mpeg,mpegversion=4",
                        'xvid' : "video/x-xvid", 'dnxhd' : "video/x-dnxhd", 'wmv2' : "video/x-wmv,wmvversion=2",
                        'dnxhd' : "video/x-dnxhd" }

#####
#This code checks for available muxers and return a unique caps string
#for each. It also creates a python dictionary mapping the caps strings
#to concrete element names.
#####

def available_muxers():
   """ return all available muxers except the broken ffmpeg ones """
   flist = gst.registry_get_default().get_feature_list(gst.ElementFactory)
   res = []
   for fact in flist:
       if list_compat(["Codec", "Muxer"], fact.get_klass().split('/')) and not fact.get_name().startswith('ffmux'):
           res.append(fact.get_name())
   return res

def get_muxer_element(containercaps):
   """
   Check all muxers for their caps and create a dictionary mapping caps
   to element names. Then return elementname
   """
   muxers = available_muxers()
   stringcaps = []
   for x in muxers:
       factory = gst.registry_get_default().lookup_feature(str(x))
       sinkcaps = [x.get_caps() for x in factory.get_static_pad_templates() if x.direction == gst.PAD_SRC]
       for caps in sinkcaps:
           stringcaps.append(caps[0].get_name())

   # print stringcaps
   muxerchoice = dict(zip(stringcaps, muxers))
   if muxerchoice.has_key(containercaps):
       elementname = muxerchoice[containercaps]
   else:
       elementname = False
   return elementname

######
#   This code checks for available audio encoders and return a unique caps string for each.
#   It also creates a python dictionary mapping the caps strings to concrete element
#   names.
#####
def available_audio_encoders():
   """ returns all available audio encoders """
   flist = gst.registry_get_default().get_feature_list(gst.ElementFactory)
   res = []
   for fact in flist:
       if list_compat(["Codec", "Encoder", "Audio"], fact.get_klass().split('/')):
           # excluding wavpackenc as the fact that it got two SRC pads mess up the logic of this code
           if fact.get_name() != 'wavpackenc':
               res.append(fact.get_name())
           else:
               print ""
   return res

def get_audio_encoder_element(audioencodercaps):
   """
   Check all audio encoders for their caps and create a dictionary
   mapping caps to element names. Then return elementname.
   """
   audioencoders = available_audio_encoders()
   audiocaps = []
   # blacklist all caps information we do not need to create a unique identifier
   blacklist = ['rate','channels','bitrate','block_align','mode','subbands'
               ,'allocation','framed','bitpool','blocks','width']
   for x in audioencoders:
       factory = gst.registry_get_default().lookup_feature(str(x))
       sinkcaps = [x.get_caps() for x in factory.get_static_pad_templates() if x.direction == gst.PAD_SRC]
       for caps in sinkcaps:
           result = caps[0].get_name();
           for attr in caps[0].keys():
               if attr not in blacklist:
                   result += ","+attr+"="+str(caps[0][attr])
           audiocaps.append(result)
   # print audiocaps
   audioencoderchoice = dict(zip(audiocaps, audioencoders))
   if audioencoderchoice.has_key(audioencodercaps):
       elementname = audioencoderchoice[audioencodercaps]
   else:
       elementname = False
   return elementname

#######
# This code checks for available video encoders and return a unique caps
# string for each. It also creates a python dictionary mapping the caps
# strings to concrete element names.
#######

def available_video_encoders():
   """ returns all available video encoders """
   flist = gst.registry_get_default().get_feature_list(gst.ElementFactory)
   res = []
   for fact in flist:
       if list_compat(["Codec", "Encoder", "Video"], fact.get_klass().split('/')):
           if fact.get_name() != 'ffenc_libtheora':
               res.append(fact.get_name())
           elif 'theoraenc' in res:
               print "adding nothing"
           else:
               res.append(fact.get_name())
       elif list_compat(["Codec", "Encoder", "Image"], fact.get_klass().split('/')):
           res.append(fact.get_name())
   return res

def get_video_encoder_element(videoencodercaps):
   """
   Check all video encoders for their caps and create a dictionary
   mapping caps to element names. Then return elementname.
   """
   encoders = available_video_encoders()
   videocaps = []
   # blacklist all caps information we do not need to create a unique identifier
   blacklist = ['height','width','framerate','systemstream','depth']
   for x in encoders:
       factory = gst.registry_get_default().lookup_feature(str(x))
       sinkcaps = [x.get_caps() for x in factory.get_static_pad_templates() if x.direction == gst.PAD_SRC]
       for caps in sinkcaps:
           result = caps[0].get_name();
           for attr in caps[0].keys():
               if attr not in blacklist:
                   result += ","+attr+"="+str(caps[0][attr])
           videocaps.append(result)
   # print videocaps
   videoencoderchoice = dict(zip(videocaps, encoders))
   # print videoencoderchoice
   if videoencoderchoice.has_key(videoencodercaps):
       elementname = videoencoderchoice[videoencodercaps]
   else:
       elementname = False
   return elementname

<commit_msg>Switch the Quicktime variant mimetypes over the to the new names defined in GStreamer bug 580005<commit_after>#!/usr/bin/python

# Transmageddon
# Copyright (C) 2009 Christian Schaller <uraeus@gnome.org>
# Copyright (C) 2009 Edward Hervey <edward.hervey@collabora.co.uk>
#
# Some code in this file came originally from the encode.py file in Pitivi
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import pygst
pygst.require("0.10")
import gst

def list_compat(a1, b1):
   for x1 in a1:
       if not x1 in b1:
           return False
   return True

containermap = { 'Ogg' : "application/ogg",'Matroska' : "video/x-matroska", 'MXF' : "application/mxf", 'AVI' : "video/x-msvideo",
                        'Quicktime' : "video/quicktime,variant=apple", 'MPEG4' : "video/quicktime,variant=iso", 'MPEG PS' : "ffmux_mpeg",
                        'MPEG TS' : "video/mpegts", 'FLV' : "video/x-flv", '3GPP' : "video/quicktime,variant=3gpp" }

csuffixmap =   { 'Ogg' : ".ogg", 'Matroska' : ".mkv", 'MXF' : ".mxf", 'AVI' : ".avi", 'Quicktime' : ".mov",
                        'MPEG4' : ".mp4", 'MPEG PS' : ".mpg", 'MPEG TS' : ".ts", 'FLV' : ".flv", '3GPP' : ".3gp" }

codecmap = {     'vorbis' : "audio/x-vorbis", 'flac' : "audio/x-flac", 'mp3' : "audio/mpeg,mpegversion=1,layer=3",
                        'aac' : "audio/mpeg,mpegversion=[4, 2]", 'ac3' : "audio/x-ac3", 'speex' : "audio/x-speex",
                        'celt' : "audio/x-celt", 'alac' : "audio/x-alac", 'wma2' : "audio/x-wma,wmaversion=2",
                        'theora' : "video/x-theora", 'dirac' : "video/x-dirac", 'h264' : "video/x-h264",
                        'mpeg2' : "video/mpeg,mpegversion=2", 'mpeg4' : "video/mpeg,mpegversion=4",
                        'xvid' : "video/x-xvid", 'dnxhd' : "video/x-dnxhd", 'wmv2' : "video/x-wmv,wmvversion=2",
                        'dnxhd' : "video/x-dnxhd" }

#####
#This code checks for available muxers and return a unique caps string
#for each. It also creates a python dictionary mapping the caps strings
#to concrete element names.
#####

def available_muxers():
   """ return all available muxers except the broken ffmpeg ones """
   flist = gst.registry_get_default().get_feature_list(gst.ElementFactory)
   res = []
   for fact in flist:
       if list_compat(["Codec", "Muxer"], fact.get_klass().split('/')) and not fact.get_name().startswith('ffmux'):
           res.append(fact.get_name())
   return res

def get_muxer_element(containercaps):
   """
   Check all muxers for their caps and create a dictionary mapping caps
   to element names. Then return elementname
   """
   muxers = available_muxers()
   stringcaps = []
   for x in muxers:
       factory = gst.registry_get_default().lookup_feature(str(x))
       sinkcaps = [x.get_caps() for x in factory.get_static_pad_templates() if x.direction == gst.PAD_SRC]
       for caps in sinkcaps:
           stringcaps.append(caps[0].get_name())

   # print stringcaps
   muxerchoice = dict(zip(stringcaps, muxers))
   if muxerchoice.has_key(containercaps):
       elementname = muxerchoice[containercaps]
   else:
       elementname = False
   return elementname

######
#   This code checks for available audio encoders and return a unique caps string for each.
#   It also creates a python dictionary mapping the caps strings to concrete element
#   names.
#####
def available_audio_encoders():
   """ returns all available audio encoders """
   flist = gst.registry_get_default().get_feature_list(gst.ElementFactory)
   res = []
   for fact in flist:
       if list_compat(["Codec", "Encoder", "Audio"], fact.get_klass().split('/')):
           # excluding wavpackenc as the fact that it got two SRC pads mess up the logic of this code
           if fact.get_name() != 'wavpackenc':
               res.append(fact.get_name())
           else:
               print ""
   return res

def get_audio_encoder_element(audioencodercaps):
   """
   Check all audio encoders for their caps and create a dictionary
   mapping caps to element names. Then return elementname.
   """
   audioencoders = available_audio_encoders()
   audiocaps = []
   # blacklist all caps information we do not need to create a unique identifier
   blacklist = ['rate','channels','bitrate','block_align','mode','subbands'
               ,'allocation','framed','bitpool','blocks','width']
   for x in audioencoders:
       factory = gst.registry_get_default().lookup_feature(str(x))
       sinkcaps = [x.get_caps() for x in factory.get_static_pad_templates() if x.direction == gst.PAD_SRC]
       for caps in sinkcaps:
           result = caps[0].get_name();
           for attr in caps[0].keys():
               if attr not in blacklist:
                   result += ","+attr+"="+str(caps[0][attr])
           audiocaps.append(result)
   # print audiocaps
   audioencoderchoice = dict(zip(audiocaps, audioencoders))
   if audioencoderchoice.has_key(audioencodercaps):
       elementname = audioencoderchoice[audioencodercaps]
   else:
       elementname = False
   return elementname

#######
# This code checks for available video encoders and return a unique caps
# string for each. It also creates a python dictionary mapping the caps
# strings to concrete element names.
#######

def available_video_encoders():
   """ returns all available video encoders """
   flist = gst.registry_get_default().get_feature_list(gst.ElementFactory)
   res = []
   for fact in flist:
       if list_compat(["Codec", "Encoder", "Video"], fact.get_klass().split('/')):
           if fact.get_name() != 'ffenc_libtheora':
               res.append(fact.get_name())
           elif 'theoraenc' in res:
               print "adding nothing"
           else:
               res.append(fact.get_name())
       elif list_compat(["Codec", "Encoder", "Image"], fact.get_klass().split('/')):
           res.append(fact.get_name())
   return res

def get_video_encoder_element(videoencodercaps):
   """
   Check all video encoders for their caps and create a dictionary
   mapping caps to element names. Then return elementname.
   """
   encoders = available_video_encoders()
   videocaps = []
   # blacklist all caps information we do not need to create a unique identifier
   blacklist = ['height','width','framerate','systemstream','depth']
   for x in encoders:
       factory = gst.registry_get_default().lookup_feature(str(x))
       sinkcaps = [x.get_caps() for x in factory.get_static_pad_templates() if x.direction == gst.PAD_SRC]
       for caps in sinkcaps:
           result = caps[0].get_name();
           for attr in caps[0].keys():
               if attr not in blacklist:
                   result += ","+attr+"="+str(caps[0][attr])
           videocaps.append(result)
   # print videocaps
   videoencoderchoice = dict(zip(videocaps, encoders))
   # print videoencoderchoice
   if videoencoderchoice.has_key(videoencodercaps):
       elementname = videoencoderchoice[videoencodercaps]
   else:
       elementname = False
   return elementname

<|endoftext|>


<commit_before>""" This file explains how we talk to the database. It explains the rules for
    adding blocks and transactions.
"""
import time, copy, custom, tools, networking, transactions

def db_get(n, DB):
    n=str(n)
    try:
        return tools.unpackage(DB['db'].Get(n))
    except:
        db_put(n, {'count':0, 'amount':0}, DB)#everyone defaults with
        #having zero money, and having broadcast zero transcations.
        return db_get(n, DB)

def db_put(key, dic, DB):
    return DB['db'].Put(str(key), tools.package(dic))

def db_delete(key, DB): return DB['db'].Delete(str(key))

def count(address, DB):
    #returns the number of transactions that pubkey has broadcast.

    def zeroth_confirmation_txs(address, DB):
        def func(t): address==tools.make_address(t['pubkeys'], len(t['signatures']))
        return len(filter(func, DB['txs']))

    current=db_get(address, DB)['count']
    return current+zeroth_confirmation_txs(address, DB)

def add_tx(tx, DB):
    #Attempt to add a new transaction into the pool.
    address=tools.make_address(tx['pubkeys'], len(tx['signatures']))

    def verify_count(tx, txs):
        return tx['count']!=count(address, DB)

    def tx_type_check(tx, txs): return type(tx) != type({'a':1})

    def type_check(tx, txs):
        if 'type' not in tx: return True
        if tx['type']=='mint': return True
        return tx['type'] not in transactions.tx_check

    def too_big_block(tx, txs):
        return len(tools.package(txs+[tx]))>networking.MAX_MESSAGE_SIZE-5000

    def verify_tx(tx, txs):
        if type_check(tx, txs): return False
        if tx in txs: return False
        if verify_count(tx, txs): return False
        if too_big_block(tx, txs): return False
        if 'start' in tx and DB['length']<tx['start']: return False
        if 'end' in tx and DB['length']>tx['end']: return False
        return transactions.tx_check[tx['type']](tx, txs, DB)

    if verify_tx(tx, DB['txs']): DB['txs'].append(tx)

targets={}
times={}#stores blocktimes
def recent_blockthings(key, DB, size, length=0):
    #Grabs info from old blocks.
    if key=='time': storage=times
    if key=='target': storage=targets

    def get_val(length):
        leng=str(length)
        if not leng in storage: storage[leng]=db_get(leng, DB)[key]
        return storage[leng]

    if length==0: length=DB['length']
    start= (length-size) if (length-size)>=0 else 0
    return map(get_val, range(start, length))

#sum of numbers expressed as hexidecimal strings
def hexSum(a, b): return tools.buffer_(str(hex(int(a, 16)+int(b, 16)))[2:-1], 64)

#use double-size for division, to reduce information leakage.
def hexInvert(n): return tools.buffer_(str(hex(int('f'*128, 16)/int(n, 16)))[2:-1], 64)

def target(DB, length=0):
    #returns the target difficulty at a paticular blocklength.
    if length==0: length=DB['length']
    if length<4: return '0'*4+'f'*60#use same difficulty for first few blocks.
    if length<=DB['length']: return targets[str(length)]#memoized

    def targetTimesFloat(target, number):
        a=int(str(target), 16)
        b=int(a*number)
        return tools.buffer_(str(hex(b))[2:-1], 64)

    def weights(length): return [custom.inflection**(length-i) for i in range(length)]

    def estimate_target(DB):
        #We are actually interested in the average number of hashes requred
        #to mine a block. number of hashes required is inversely proportional
        #to target. So we average over inverse-targets, and inverse the final
        #answer.
        def sumTargets(l):
            if len(l)<1: return 0
            while len(l)>1:
                l=[hexSum(l[0], l[1])]+l[2:]
            return l[0]

        targets=recent_blockthings('target', DB, custom.history_length)
        w=weights(len(targets))
        tw=sum(w)
        targets=map(hexInvert, targets)
        def weighted_multiply(i): return targetTimesFloat(targets[i], w[i]/tw)
        weighted_targets=[weighted_multiply(i) for i in range(len(targets))]
        return hexInvert(sumTargets(weighted_targets))

    def estimate_time(DB):
        times=recent_blockthings('time', DB, custom.history_length)
        blocklengths=[times[i]-times[i-1] for i in range(1, len(times))]
        w=weights(len(blocklengths))#geometric weighting
        tw=sum(w)#normalization constant
        return sum([w[i]*blocklengths[i]/tw for i in range(len(blocklengths))])

    retarget=estimate_time(DB)/custom.blocktime(length)
    return targetTimesFloat(estimate_target(DB), retarget)

def add_block(block, DB):
    #attempts adding a new block to the blockchain.
    #median is good for weeding out liars, so long as
    def median(mylist): #the liars don't have 51% hashpower.
        if len(mylist)<1: return 0
        return sorted(mylist)[len(mylist) / 2]

    def block_check(block, DB):
        def tx_check(txs):
            start=copy.deepcopy(txs)
            out=[]
            start_copy=[]
            while start!=start_copy:
                if start==[]:
                    return False#block passes this test
                start_copy=copy.deepcopy(start)
                if transactions.tx_check[start[-1]['type']](start[-1], out, DB):
                    out.append(start.pop())
                else:
                    return True#block is invalid
            return True#block is invalid
        if type(block)!=type({'a':1}):
            return False
        if 'error' in block:
            return False
        if 'length' not in block:
            return False
        length=DB['length']
        if int(block['length'])!=int(length)+1:
            return False
        if block['diffLength']!=hexSum(DB['diffLength'],
                                       hexInvert(block['target'])):
            return False
        if length >= 0:
            if tools.det_hash(db_get(length, DB))!=block['prevHash']:
                return False
        a=copy.deepcopy(block)
        a.pop('nonce')
        if u'target' not in block.keys():
            return False
        half_way={u'nonce':block['nonce'], u'halfHash':tools.det_hash(a)}
        if tools.det_hash(half_way)>block['target']:
            return False
        if block['target']!=target(DB, block['length']):
            return False
        earliest=median(recent_blockthings('time', DB, custom.mmm))
        if 'time' not in block:
            return False
        if block['time']>time.time():
            return False
        if block['time']<earliest:
            return False
        if tx_check(block['txs']):
            return False
        return True

    if block_check(block, DB):
        print('add_block: '+str(block))
        db_put(block['length'], block, DB)
        DB['length']=block['length']
        DB['diffLength']=block['diffLength']
        orphans=DB['txs']
        DB['txs']=[]
        for tx in block['txs']:
            transactions.add_block[tx['type']](tx, DB)
        for tx in orphans:
            add_tx(tx, DB)

def delete_block(DB):
    #removes the most recent block from the blockchain.
    if DB['length']<0: return
    try: targets.pop(str(DB['length']))
    except: pass
    try: times.pop(str(DB['length']))
    except: pass
    block=db_get(DB['length'], DB)
    orphans=DB['txs']
    DB['txs']=[]
    for tx in block['txs']:
        orphans.append(tx)
        transactions.delete_block[tx['type']](tx, DB)
    db_delete(DB['length'], DB)
    DB['length']-=1
    if DB['length']==-1:
        DB['diffLength']='0'
    else:
        block=db_get(DB['length'], DB)
        DB['diffLength']=db_get(DB['length'], DB)['diffLength']
    for orphan in sorted(orphans, key=lambda x: x['count']):
        add_tx(tx, DB)
<commit_msg>Moved imports onto seprate lines<commit_after>""" This file explains how we talk to the database. It explains the rules for
    adding blocks and transactions.
"""
import time
import copy
import custom
import tools
import networking
import transactions

def db_get(n, DB):
    n=str(n)
    try:
        return tools.unpackage(DB['db'].Get(n))
    except:
        db_put(n, {'count':0, 'amount':0}, DB)#everyone defaults with
        #having zero money, and having broadcast zero transcations.
        return db_get(n, DB)

def db_put(key, dic, DB):
    return DB['db'].Put(str(key), tools.package(dic))

def db_delete(key, DB): return DB['db'].Delete(str(key))

def count(address, DB):
    #returns the number of transactions that pubkey has broadcast.

    def zeroth_confirmation_txs(address, DB):
        def func(t): address==tools.make_address(t['pubkeys'], len(t['signatures']))
        return len(filter(func, DB['txs']))

    current=db_get(address, DB)['count']
    return current+zeroth_confirmation_txs(address, DB)

def add_tx(tx, DB):
    #Attempt to add a new transaction into the pool.
    address=tools.make_address(tx['pubkeys'], len(tx['signatures']))

    def verify_count(tx, txs):
        return tx['count']!=count(address, DB)

    def tx_type_check(tx, txs): return type(tx) != type({'a':1})

    def type_check(tx, txs):
        if 'type' not in tx: return True
        if tx['type']=='mint': return True
        return tx['type'] not in transactions.tx_check

    def too_big_block(tx, txs):
        return len(tools.package(txs+[tx]))>networking.MAX_MESSAGE_SIZE-5000

    def verify_tx(tx, txs):
        if type_check(tx, txs): return False
        if tx in txs: return False
        if verify_count(tx, txs): return False
        if too_big_block(tx, txs): return False
        if 'start' in tx and DB['length']<tx['start']: return False
        if 'end' in tx and DB['length']>tx['end']: return False
        return transactions.tx_check[tx['type']](tx, txs, DB)

    if verify_tx(tx, DB['txs']): DB['txs'].append(tx)

targets={}
times={}#stores blocktimes
def recent_blockthings(key, DB, size, length=0):
    #Grabs info from old blocks.
    if key=='time': storage=times
    if key=='target': storage=targets

    def get_val(length):
        leng=str(length)
        if not leng in storage: storage[leng]=db_get(leng, DB)[key]
        return storage[leng]

    if length==0: length=DB['length']
    start= (length-size) if (length-size)>=0 else 0
    return map(get_val, range(start, length))

#sum of numbers expressed as hexidecimal strings
def hexSum(a, b): return tools.buffer_(str(hex(int(a, 16)+int(b, 16)))[2:-1], 64)

#use double-size for division, to reduce information leakage.
def hexInvert(n): return tools.buffer_(str(hex(int('f'*128, 16)/int(n, 16)))[2:-1], 64)

def target(DB, length=0):
    #returns the target difficulty at a paticular blocklength.
    if length==0: length=DB['length']
    if length<4: return '0'*4+'f'*60#use same difficulty for first few blocks.
    if length<=DB['length']: return targets[str(length)]#memoized

    def targetTimesFloat(target, number):
        a=int(str(target), 16)
        b=int(a*number)
        return tools.buffer_(str(hex(b))[2:-1], 64)

    def weights(length): return [custom.inflection**(length-i) for i in range(length)]

    def estimate_target(DB):
        #We are actually interested in the average number of hashes requred
        #to mine a block. number of hashes required is inversely proportional
        #to target. So we average over inverse-targets, and inverse the final
        #answer.
        def sumTargets(l):
            if len(l)<1: return 0
            while len(l)>1:
                l=[hexSum(l[0], l[1])]+l[2:]
            return l[0]

        targets=recent_blockthings('target', DB, custom.history_length)
        w=weights(len(targets))
        tw=sum(w)
        targets=map(hexInvert, targets)
        def weighted_multiply(i): return targetTimesFloat(targets[i], w[i]/tw)
        weighted_targets=[weighted_multiply(i) for i in range(len(targets))]
        return hexInvert(sumTargets(weighted_targets))

    def estimate_time(DB):
        times=recent_blockthings('time', DB, custom.history_length)
        blocklengths=[times[i]-times[i-1] for i in range(1, len(times))]
        w=weights(len(blocklengths))#geometric weighting
        tw=sum(w)#normalization constant
        return sum([w[i]*blocklengths[i]/tw for i in range(len(blocklengths))])

    retarget=estimate_time(DB)/custom.blocktime(length)
    return targetTimesFloat(estimate_target(DB), retarget)

def add_block(block, DB):
    #attempts adding a new block to the blockchain.
    #median is good for weeding out liars, so long as
    def median(mylist): #the liars don't have 51% hashpower.
        if len(mylist)<1: return 0
        return sorted(mylist)[len(mylist) / 2]

    def block_check(block, DB):
        def tx_check(txs):
            start=copy.deepcopy(txs)
            out=[]
            start_copy=[]
            while start!=start_copy:
                if start==[]:
                    return False#block passes this test
                start_copy=copy.deepcopy(start)
                if transactions.tx_check[start[-1]['type']](start[-1], out, DB):
                    out.append(start.pop())
                else:
                    return True#block is invalid
            return True#block is invalid
        if type(block)!=type({'a':1}):
            return False
        if 'error' in block:
            return False
        if 'length' not in block:
            return False
        length=DB['length']
        if int(block['length'])!=int(length)+1:
            return False
        if block['diffLength']!=hexSum(DB['diffLength'],
                                       hexInvert(block['target'])):
            return False
        if length >= 0:
            if tools.det_hash(db_get(length, DB))!=block['prevHash']:
                return False
        a=copy.deepcopy(block)
        a.pop('nonce')
        if u'target' not in block.keys():
            return False
        half_way={u'nonce':block['nonce'], u'halfHash':tools.det_hash(a)}
        if tools.det_hash(half_way)>block['target']:
            return False
        if block['target']!=target(DB, block['length']):
            return False
        earliest=median(recent_blockthings('time', DB, custom.mmm))
        if 'time' not in block:
            return False
        if block['time']>time.time():
            return False
        if block['time']<earliest:
            return False
        if tx_check(block['txs']):
            return False
        return True

    if block_check(block, DB):
        print('add_block: '+str(block))
        db_put(block['length'], block, DB)
        DB['length']=block['length']
        DB['diffLength']=block['diffLength']
        orphans=DB['txs']
        DB['txs']=[]
        for tx in block['txs']:
            transactions.add_block[tx['type']](tx, DB)
        for tx in orphans:
            add_tx(tx, DB)

def delete_block(DB):
    #removes the most recent block from the blockchain.
    if DB['length']<0: return
    try: targets.pop(str(DB['length']))
    except: pass
    try: times.pop(str(DB['length']))
    except: pass
    block=db_get(DB['length'], DB)
    orphans=DB['txs']
    DB['txs']=[]
    for tx in block['txs']:
        orphans.append(tx)
        transactions.delete_block[tx['type']](tx, DB)
    db_delete(DB['length'], DB)
    DB['length']-=1
    if DB['length']==-1:
        DB['diffLength']='0'
    else:
        block=db_get(DB['length'], DB)
        DB['diffLength']=db_get(DB['length'], DB)['diffLength']
    for orphan in sorted(orphans, key=lambda x: x['count']):
        add_tx(tx, DB)
<|endoftext|>


<commit_before>import re
import os
import sys
import codecs
import shutil
import string
import argparse
import subprocess

# Mátodo que altera a variável
def changeVar(line, params):
    # Variáveis a serem alteradas
    keys = params.keys();
    for key in keys:
        if key in line:
            #line = re.sub(key, params[key], line)
            line = line.replace(key, params[key])
    return line

# Método que altera os valores nos arquivos
def changeValues(filename, params):
    # Ler o arquivo
    with codecs.open(filename, "r", "utf-8") as f:
        lines = (line.rstrip() for line in f)
        # Alterar o arquivo
        newlines = [ changeVar(line, params) if not re.match("(.*?)[\{\{\}\}](.*?)+", line) == None else line for line in lines]
    # Abrir o arquivo
    file = open(filename)
    #Salvar o arquivo
    with codecs.open(filename, "w", "utf-8") as f:
        f.write('\n'.join(newlines) + '\n')
# Projeto a ser clonado
parser = argparse.ArgumentParser()
parser.add_argument("--dir", help="Diretório onde o projeto será iniciado!")
parser.add_argument("project", help="Node para NodeJS, Cordova para Cordova!")
args = parser.parse_args()
project = args.project.lower()[:3]
# Repositório a ser clonado
repo = None
if project == 'cor':
    repo = "https://github.com/r2ttecnologia/CordovaTemplate.git"
    pass
elif project == 'nod':
    repo = "https://github.com/r2ttecnologia/NodeApiSample.git"
    pass
else:
    print("Parâmetro inválido!")
    sys.exit(-1)

# Lê o nome da pasta onde será criado o template
if args.dir:
    dirpath = args.dir
else:
    dirpath = input("Digite o local do diretório onde será criado o projeto: ")
# Retira aspas
dirpath = dirpath.replace("\"", "")
# Nome do projeto
projectname = input("Nome do projeto: ")
projectname = projectname.replace("-", "")
projectname = projectname.replace(" ", "")
# Descrição do projeto
projectdesc = input("Descrição do projeto: ")
# Repó do projeto
projectrepo = input("Repositório: ")
# Parâmetros a serem modificados nos arquivos
params = {}
params["{{title}}"] = projectname.lower()
params["{{description}}"] = projectdesc
params["{{repo_url}}"] = projectrepo
# Caminho do projeto
projectpath = os.path.join(dirpath, projectname)
# Corrige o caminho
dirpath = os.path.normpath(dirpath)
# Verifica se o diretório informado existe
if os.path.exists(dirpath):
    if(os.path.exists(projectpath)):
        print("Diretório do projeto já existe!")
    else:
        os.mkdir(projectpath)
        print("Clonando " + repo + " em " + projectpath)
        subprocess.call("git clone " + repo + " \"" + projectpath + "\"")
        # Vai para a pasta do projeto
        os.chdir(projectpath)
        # Apaga os objetos de .git
        for dpath, dname, fname in os.walk('.git'):
            for filename in fname:
                p = os.path.join(dpath, filename)
                os.chmod(p, 0o777) # for example
                os.remove(p)
        # Remover a pasta .git
        shutil.rmtree('.git')
        # Alterar os valores dos arquivos
        for dpath, dname, fname in os.walk('.'):
            for filename in fname:
                f = os.path.join(dpath, filename)
                if filename == '.delme':
                    os.remove(f)
                else:
                    changeValues(f, params)
        # Instalar os módulos
        os.chdir(os.path.join(projectpath, 'source'))
        subprocess.check_call("npm install", shell=True)
else:
    print("O diretório informado não existe! Saindo ...")

<commit_msg>Correção erro na chamada git<commit_after>import re
import os
import sys
import codecs
import shutil
import string
import argparse
import subprocess

# Mátodo que altera a variável
def changeVar(line, params):
    # Variáveis a serem alteradas
    keys = params.keys();
    for key in keys:
        if key in line:
            #line = re.sub(key, params[key], line)
            line = line.replace(key, params[key])
    return line

# Método que altera os valores nos arquivos
def changeValues(filename, params):
    # Ler o arquivo
    with codecs.open(filename, "r", "utf-8") as f:
        lines = (line.rstrip() for line in f)
        # Alterar o arquivo
        newlines = [ changeVar(line, params) if not re.match("(.*?)[\{\{\}\}](.*?)+", line) == None else line for line in lines]
    # Abrir o arquivo
    file = open(filename)
    #Salvar o arquivo
    with codecs.open(filename, "w", "utf-8") as f:
        f.write('\n'.join(newlines) + '\n')
# Projeto a ser clonado
parser = argparse.ArgumentParser()
parser.add_argument("--dir", help="Diretório onde o projeto será iniciado!")
parser.add_argument("project", help="Node para NodeJS, Cordova para Cordova!")
args = parser.parse_args()
project = args.project.lower()[:3]
# Repositório a ser clonado
repo = None
if project == 'cor':
    repo = "https://github.com/r2ttecnologia/CordovaTemplate.git"
    pass
elif project == 'nod':
    repo = "https://github.com/r2ttecnologia/NodeApiSample.git"
    pass
else:
    print("Parâmetro inválido!")
    sys.exit(-1)

# Lê o nome da pasta onde será criado o template
if args.dir:
    dirpath = args.dir
else:
    dirpath = input("Digite o local do diretório onde será criado o projeto: ")
# Retira aspas
dirpath = dirpath.replace("\"", "")
# Nome do projeto
projectname = input("Nome do projeto: ")
projectname = projectname.replace("-", "")
projectname = projectname.replace(" ", "")
# Descrição do projeto
projectdesc = input("Descrição do projeto: ")
# Repó do projeto
projectrepo = input("Repositório: ")
# Parâmetros a serem modificados nos arquivos
params = {}
params["{{title}}"] = projectname.lower()
params["{{description}}"] = projectdesc
params["{{repo_url}}"] = projectrepo
# Caminho do projeto
projectpath = os.path.join(dirpath, projectname)
# Corrige o caminho
dirpath = os.path.normpath(dirpath)
# Verifica se o diretório informado existe
if os.path.exists(dirpath):
    if(os.path.exists(projectpath)):
        print("Diretório do projeto já existe!")
    else:
        os.mkdir(projectpath)
        print("Clonando " + repo + " em " + projectpath)
        subprocess.check_call("git clone " + repo + " \"" + projectpath + "\"", shell=True)
        # Vai para a pasta do projeto
        os.chdir(projectpath)
        # Apaga os objetos de .git
        for dpath, dname, fname in os.walk('.git'):
            for filename in fname:
                p = os.path.join(dpath, filename)
                os.chmod(p, 0o777) # for example
                os.remove(p)
        # Remover a pasta .git
        shutil.rmtree('.git')
        # Alterar os valores dos arquivos
        for dpath, dname, fname in os.walk('.'):
            for filename in fname:
                f = os.path.join(dpath, filename)
                if filename == '.delme':
                    os.remove(f)
                else:
                    changeValues(f, params)
        # Instalar os módulos
        os.chdir(os.path.join(projectpath, 'source'))
        subprocess.check_call("npm install", shell=True)
else:
    print("O diretório informado não existe! Saindo ...")

<|endoftext|>


<commit_before>import time
from datetime import datetime
import xbmc
import xbmcaddon
import xbmcvfs
import os
from resources.lib.croniter import croniter

class CronSchedule:
    expression = ''
    name = 'library'
    timer_type = 'xbmc'
    command = 'UpdateLibrary(video)'
    next_run = 0
    on_delay = False  #used to defer processing until after player finishes
    clean_library = False  #if clean library command should be run after this schedule

    def cleanLibrarySchedule(self,selectedIndex):
        if(selectedIndex == 1):
            #once per day
            return "* * *"
        elif (selectedIndex == 2):
            #once per week
            return "* * 0"
        else:
            #once per month
            return "1 * *"

class AutoUpdater:
    addon_id = "service.libraryautoupdate"
    Addon = xbmcaddon.Addon(addon_id)
    datadir = Addon.getAddonInfo('profile')
    addondir = Addon.getAddonInfo('path')
    last_run = 0
    sleep_time = 10
    schedules = []
    settings_update_time = 0

    #setup the timer amounts
    timer_amounts = {}
    timer_amounts['0'] = 1
    timer_amounts['1'] = 2
    timer_amounts['2'] = 4
    timer_amounts['3'] = 6
    timer_amounts['4'] = 12
    timer_amounts['5'] = 24

    def __init__(self):
        self.readLastRun()

        #force and update on startup to create the array
        self.createSchedules(True)

    def runProgram(self):
        #a one-time catch for the startup delay
        if(int(self.Addon.getSetting("startup_delay")) != 0):
            count = 0
            while count < len(self.schedules):
                if(time.time() > self.schedules[count].next_run):
                    #we missed at least one update, fix this
                    self.schedules[count].next_run = time.time() + int(self.Addon.getSetting("startup_delay")) * 60
                count = count + 1

        #program has started, check if we should show a notification
        self.showNotify()

        while(not xbmc.abortRequested):
            self.readLastRun()

            self.evalSchedules()

            time.sleep(self.sleep_time)

    def evalSchedules(self):
        now = time.time()
        self.createSchedules()

        count = 0
        tempLastRun = self.last_run
        while count < len(self.schedules):
            cronJob = self.schedules[count]

            if(cronJob.next_run <= now and now > tempLastRun + 60):
                if(xbmc.Player().isPlaying() == False or self.Addon.getSetting("run_during_playback") == "true"):
                    #check if this scan was delayed due to playback
                    if(cronJob.on_delay == True):
                        #add another minute to the delay
                        self.schedules[count].next_run = now + 60
                        self.schedules[count].on_delay = False
                        self.log(cronJob.name + " paused due to playback")

                    elif(self.scanRunning() == False):
                        #run the command for this job
                        self.log(cronJob.name)

                        if(cronJob.timer_type == 'xbmc'):
                            xbmc.executebuiltin(cronJob.command)
                            self.waitForScan()
                        else:
                            self.cleanLibrary(cronJob)

                        self.last_run = time.time() - (time.time() % 60)
                        self.writeLastRun()

                        #find the next run time
                        cronJob.next_run = self.calcNextRun(cronJob.expression,now)
                        self.schedules[count] = cronJob

                        #show any notifications
                        self.showNotify()

                        #check if we should clean the library too
                        if(cronJob.clean_library):
                            self.cleanLibrary(cronJob)
                else:
                    self.schedules[count].on_delay = True
                    self.log("Player is running, wait until finished")

            count = count + 1

    def createSchedules(self,forceUpdate = False):
        mod_time = self.settings_update_time

        try:
            #get the last modified time of the file
            mod_time = os.path.getmtime(xbmc.translatePath(self.datadir) + "settings.xml")
        except:
            #don't do anything here
            mod_time = self.settings_update_time

        if(mod_time > self.settings_update_time or forceUpdate):
            self.log("update timers")
            self.schedules = []

            clean_after_update = False
            if(self.Addon.getSetting('clean_libraries') == 'true'):
                #create clean schedule (if needed)
                if(int(self.Addon.getSetting("clean_timer")) == 0):
                    clean_after_update = True  #clean after executing updates
                else:
                    #create a separate schedule for cleaning - use right now rather than last_run, never 'catch-up'
                    clean_after_update = False

                    if(self.Addon.getSetting('update_video') == 'true' or self.Addon.getSetting('use_custom_1_path') == 'true'):
                        #video clean schedule starts at 12am
                        aSchedule = CronSchedule()
                        aSchedule.name = 'Clean Video Library'
                        aSchedule.timer_type = self.addon_id
                        aSchedule.command = 'video'
                        aSchedule.expression = "0 0 " + aSchedule.cleanLibrarySchedule(int(self.Addon.getSetting("clean_timer")))
                        aSchedule.next_run = self.calcNextRun(aSchedule.expression,time.time())

                        self.schedules.append(aSchedule)

                    if(self.Addon.getSetting('update_music') == 'true'):
                        #music clean schedule starts at 2am
                        aSchedule = CronSchedule()
                        aSchedule.name = 'Clean Music Library'
                        aSchedule.timer_type = self.addon_id
                        aSchedule.command = 'music'
                        aSchedule.expression = "0 2 " + aSchedule.cleanLibrarySchedule(int(self.Addon.getSetting("clean_timer")))
                        aSchedule.next_run = self.calcNextRun(aSchedule.expression,time.time())

                        self.schedules.append(aSchedule)

            if(self.Addon.getSetting('update_video') == 'true'):
                #create the video schedule
                aSchedule = CronSchedule()
                aSchedule.name = 'Update Video Library'
                aSchedule.command = 'UpdateLibrary(video)'
                aSchedule.expression = self.checkTimer('video')
                aSchedule.next_run = self.calcNextRun(aSchedule.expression,self.last_run)
                aSchedule.clean_library = clean_after_update

                self.schedules.append(aSchedule)

            if(self.Addon.getSetting('update_music') == 'true'):
                #create the music schedule
                aSchedule = CronSchedule()
                aSchedule.name = 'Update Music Library'
                aSchedule.command = 'UpdateLibrary(music)'
                aSchedule.expression = self.checkTimer('music')
                aSchedule.next_run = self.calcNextRun(aSchedule.expression,self.last_run)
                aSchedule.clean_library = clean_after_update

                self.schedules.append(aSchedule)

            if(self.Addon.getSetting('use_custom_1_path') == 'true'):
                #create a custom video path schedule
                aSchedule = CronSchedule()
                aSchedule.name = 'Specific Video Path'
                aSchedule.command = 'UpdateLibrary(video,' + self.Addon.getSetting('custom_1_scan_path') + ')'
                aSchedule.expression = self.checkTimer('custom_1')
                aSchedule.next_run = self.calcNextRun(aSchedule.expression,self.last_run)
                aSchedule.clean_library = clean_after_update

                self.schedules.append(aSchedule)

            #update the mod time for the file
            self.settings_update_time = mod_time

            #show any notifications
            self.showNotify(not forceUpdate)

    def checkTimer(self,settingName):
        result = ''

        #figure out if using standard or advanced timer
        if(self.Addon.getSetting(settingName + '_advanced_timer') == 'true'):
            #copy the expression
            result = self.Addon.getSetting(settingName + "_cron_expression")
        else:
            result = '0 */' + str(self.timer_amounts[self.Addon.getSetting(settingName + "_timer")]) + ' * * *'

        return result

    def calcNextRun(self,cronExp,startTime):

        #create croniter for this expression
        cron = croniter(cronExp,startTime)
        nextRun = cron.get_next(float)

        return nextRun

    def showNotify(self,displayToScreen = True):
        #go through and find the next schedule to run
        next_run_time = CronSchedule()
        for cronJob in self.schedules:
            if(cronJob.next_run < next_run_time.next_run or next_run_time.next_run == 0):
                next_run_time = cronJob

        inWords = self.nextRunCountdown(next_run_time.next_run)
        #show the notification (if applicable)
        if(next_run_time.next_run > time.time() and self.Addon.getSetting('notify_next_run') == 'true' and displayToScreen == True):
            xbmc.executebuiltin("Notification(Library Auto Update," + next_run_time.name + " " + inWords + ",4000," + xbmc.translatePath(self.addondir + "/resources/images/clock.png") + ")")

        return inWords

    def nextRunCountdown(self,nextRun):
        #compare now with next date
        cronDiff = nextRun - time.time()

        if cronDiff < 0:
            return ""

        hours = int((cronDiff / 60) / 60)
        minutes = int((cronDiff / 60) - hours * 60)

        #we always have at least one minute
        if minutes == 0:
            minutes = 1

        result = str(hours) + " h " + str(minutes) + " m"
        if hours == 0:
            result = str(minutes) + " m"
        elif hours > 36:
            #just show the date instead
            result = datetime.fromtimestamp(nextRun).strftime('%m/%d %I:%M%p')
        elif hours > 24:
            days = int(hours / 24)
            hours = hours - days * 24
            result = str(days) + " d " + str(hours) + " h " + str(minutes) + " m"

        return result

    def cleanLibrary(self,cronJob):
        #check which database we're in
        media_type = 'video'
        if(cronJob.command.find('music') != -1):
            media_type = 'music'

        #check if we should verify paths
        if(self.Addon.getSetting("verify_paths") == 'true'):
            response = eval(xbmc.executeJSONRPC('{ "jsonrpc" : "2.0", "method" : "Files.GetSources", "params":{"media":"' + media_type + '"}, "id": 1}'))
            for source in response['result']['sources']:
                if not xbmcvfs.exists(source['file']):
                    #let the user know this failed, if they subscribe to notifications
                    if(self.Addon.getSetting('notify_next_run') == 'true'):
                        xbmc.executebuiltin("Notification(Error Cleaning Databse,Source " + source['label'] + " does not exist,4000," + xbmc.translatePath(self.addondir + "/resources/images/clock.png") + ")")

                    self.log("Path " + source['file'] + " does not exist")
                    return

        #run the clean operation
        self.log("Cleaning Database")
        xbmc.executebuiltin("CleanLibrary(" + media_type + ")")

    def readLastRun(self):

        try:
            f = open(xbmc.translatePath(self.datadir + "last_run.txt"),"r")
            self.last_run = float(f.read())
            f.close()
        except IOError:
            #the file doesn't exist, most likely first time running
            self.last_run = 0

    def writeLastRun(self):
        #create the addon folder if it doesn't exist
        if(not os.path.exists(xbmc.translatePath(self.datadir))):
            os.makedirs(xbmc.translatePath(self.datadir))

        f = open(xbmc.translatePath(self.datadir + "last_run.txt"),"w")

        #write out the value for the last time the program ran
        f.write(str(self.last_run));
        f.close();

    def scanRunning(self):
        #check if any type of scan is currently running
        if(xbmc.getCondVisibility('Library.IsScanningVideo') or xbmc.getCondVisibility('Library.IsScanningMusic')):
            return True
        else:
            return False

    def waitForScan(self):
        #what for scan to start
        time.sleep(2)

        while(self.scanRunning()):
            time.sleep(5)

    def log(self,message):
        xbmc.log('service.libraryautoupdate: ' + message)

<commit_msg>updated service.py for additional custom paths<commit_after>import time
from datetime import datetime
import xbmc
import xbmcaddon
import xbmcvfs
import os
from resources.lib.croniter import croniter

class CronSchedule:
    expression = ''
    name = 'library'
    timer_type = 'xbmc'
    command = 'UpdateLibrary(video)'
    next_run = 0
    on_delay = False  #used to defer processing until after player finishes
    clean_library = False  #if clean library command should be run after this schedule

    def cleanLibrarySchedule(self,selectedIndex):
        if(selectedIndex == 1):
            #once per day
            return "* * *"
        elif (selectedIndex == 2):
            #once per week
            return "* * 0"
        else:
            #once per month
            return "1 * *"

class AutoUpdater:
    addon_id = "service.libraryautoupdate"
    Addon = xbmcaddon.Addon(addon_id)
    datadir = Addon.getAddonInfo('profile')
    addondir = Addon.getAddonInfo('path')
    last_run = 0
    sleep_time = 10
    schedules = []
    settings_update_time = 0

    #setup the timer amounts
    timer_amounts = {}
    timer_amounts['0'] = 1
    timer_amounts['1'] = 2
    timer_amounts['2'] = 4
    timer_amounts['3'] = 6
    timer_amounts['4'] = 12
    timer_amounts['5'] = 24

    def __init__(self):
        self.readLastRun()

        #force and update on startup to create the array
        self.createSchedules(True)

    def runProgram(self):
        #a one-time catch for the startup delay
        if(int(self.Addon.getSetting("startup_delay")) != 0):
            count = 0
            while count < len(self.schedules):
                if(time.time() > self.schedules[count].next_run):
                    #we missed at least one update, fix this
                    self.schedules[count].next_run = time.time() + int(self.Addon.getSetting("startup_delay")) * 60
                count = count + 1

        #program has started, check if we should show a notification
        self.showNotify()

        while(not xbmc.abortRequested):
            self.readLastRun()

            self.evalSchedules()

            time.sleep(self.sleep_time)

    def evalSchedules(self):
        now = time.time()
        self.createSchedules()

        count = 0
        tempLastRun = self.last_run
        while count < len(self.schedules):
            cronJob = self.schedules[count]

            if(cronJob.next_run <= now and now > tempLastRun + 60):
                if(xbmc.Player().isPlaying() == False or self.Addon.getSetting("run_during_playback") == "true"):
                    #check if this scan was delayed due to playback
                    if(cronJob.on_delay == True):
                        #add another minute to the delay
                        self.schedules[count].next_run = now + 60
                        self.schedules[count].on_delay = False
                        self.log(cronJob.name + " paused due to playback")

                    elif(self.scanRunning() == False):
                        #run the command for this job
                        self.log(cronJob.name)

                        if(cronJob.timer_type == 'xbmc'):
                            xbmc.executebuiltin(cronJob.command)
                            self.waitForScan()
                        else:
                            self.cleanLibrary(cronJob)

                        self.last_run = time.time() - (time.time() % 60)
                        self.writeLastRun()

                        #find the next run time
                        cronJob.next_run = self.calcNextRun(cronJob.expression,now)
                        self.schedules[count] = cronJob

                        #show any notifications
                        self.showNotify()

                        #check if we should clean the library too
                        if(cronJob.clean_library):
                            self.cleanLibrary(cronJob)
                else:
                    self.schedules[count].on_delay = True
                    self.log("Player is running, wait until finished")

            count = count + 1

    def createSchedules(self,forceUpdate = False):
        mod_time = self.settings_update_time

        try:
            #get the last modified time of the file
            mod_time = os.path.getmtime(xbmc.translatePath(self.datadir) + "settings.xml")
        except:
            #don't do anything here
            mod_time = self.settings_update_time

        if(mod_time > self.settings_update_time or forceUpdate):
            self.log("update timers")
            self.schedules = []

            clean_after_update = False
            if(self.Addon.getSetting('clean_libraries') == 'true'):
                #create clean schedule (if needed)
                if(int(self.Addon.getSetting("clean_timer")) == 0):
                    clean_after_update = True  #clean after executing updates
                else:
                    #create a separate schedule for cleaning - use right now rather than last_run, never 'catch-up'
                    clean_after_update = False

                    if(self.Addon.getSetting('update_video') == 'true' or (self.Addon.getSetting('use_custom_1_path') == 'true' or self.Addon.getSetting('use_custom_2_path') == 'true' or self.Addon.getSetting('use_custom_3_path') == 'true')):
                        #video clean schedule starts at 12am
                        aSchedule = CronSchedule()
                        aSchedule.name = 'Clean Video Library'
                        aSchedule.timer_type = self.addon_id
                        aSchedule.command = 'video'
                        aSchedule.expression = "0 0 " + aSchedule.cleanLibrarySchedule(int(self.Addon.getSetting("clean_timer")))
                        aSchedule.next_run = self.calcNextRun(aSchedule.expression,time.time())

                        self.schedules.append(aSchedule)

                    if(self.Addon.getSetting('update_music') == 'true'):
                        #music clean schedule starts at 2am
                        aSchedule = CronSchedule()
                        aSchedule.name = 'Clean Music Library'
                        aSchedule.timer_type = self.addon_id
                        aSchedule.command = 'music'
                        aSchedule.expression = "0 2 " + aSchedule.cleanLibrarySchedule(int(self.Addon.getSetting("clean_timer")))
                        aSchedule.next_run = self.calcNextRun(aSchedule.expression,time.time())

                        self.schedules.append(aSchedule)

            if(self.Addon.getSetting('update_video') == 'true'):
                #create the video schedule
                aSchedule = CronSchedule()
                aSchedule.name = 'Update Video Library'
                aSchedule.command = 'UpdateLibrary(video)'
                aSchedule.expression = self.checkTimer('video')
                aSchedule.next_run = self.calcNextRun(aSchedule.expression,self.last_run)
                aSchedule.clean_library = clean_after_update

                self.schedules.append(aSchedule)

            if(self.Addon.getSetting('update_music') == 'true'):
                #create the music schedule
                aSchedule = CronSchedule()
                aSchedule.name = 'Update Music Library'
                aSchedule.command = 'UpdateLibrary(music)'
                aSchedule.expression = self.checkTimer('music')
                aSchedule.next_run = self.calcNextRun(aSchedule.expression,self.last_run)
                aSchedule.clean_library = clean_after_update

                self.schedules.append(aSchedule)

            if(self.Addon.getSetting('use_custom_1_path') == 'true'):
                #create a custom video path schedule
                aSchedule = CronSchedule()
                aSchedule.name = 'Specific Video Path'
                aSchedule.command = 'UpdateLibrary(video,' + self.Addon.getSetting('custom_1_scan_path') + ')'
                aSchedule.expression = self.checkTimer('custom_1')
                aSchedule.next_run = self.calcNextRun(aSchedule.expression,self.last_run)
                aSchedule.clean_library = clean_after_update

                self.schedules.append(aSchedule)

            if(self.Addon.getSetting('use_custom_2_path') == 'true'):
                #create a custom video path schedule
                aSchedule = CronSchedule()
                aSchedule.name = 'Specific Video Path'
                aSchedule.command = 'UpdateLibrary(video,' + self.Addon.getSetting('custom_2_scan_path') + ')'
                aSchedule.expression = self.checkTimer('custom_2')
                aSchedule.next_run = self.calcNextRun(aSchedule.expression,self.last_run)
                aSchedule.clean_library = clean_after_update

                self.schedules.append(aSchedule)

            if(self.Addon.getSetting('use_custom_3_path') == 'true'):
                #create a custom video path schedule
                aSchedule = CronSchedule()
                aSchedule.name = 'Specific Video Path'
                aSchedule.command = 'UpdateLibrary(video,' + self.Addon.getSetting('custom_3_scan_path') + ')'
                aSchedule.expression = self.checkTimer('custom_3')
                aSchedule.next_run = self.calcNextRun(aSchedule.expression,self.last_run)
                aSchedule.clean_library = clean_after_update

                self.schedules.append(aSchedule)

            #update the mod time for the file
            self.settings_update_time = mod_time

            #show any notifications
            self.showNotify(not forceUpdate)

    def checkTimer(self,settingName):
        result = ''

        #figure out if using standard or advanced timer
        if(self.Addon.getSetting(settingName + '_advanced_timer') == 'true'):
            #copy the expression
            result = self.Addon.getSetting(settingName + "_cron_expression")
        else:
            result = '0 */' + str(self.timer_amounts[self.Addon.getSetting(settingName + "_timer")]) + ' * * *'

        return result

    def calcNextRun(self,cronExp,startTime):

        #create croniter for this expression
        cron = croniter(cronExp,startTime)
        nextRun = cron.get_next(float)

        return nextRun

    def showNotify(self,displayToScreen = True):
        #go through and find the next schedule to run
        next_run_time = CronSchedule()
        for cronJob in self.schedules:
            if(cronJob.next_run < next_run_time.next_run or next_run_time.next_run == 0):
                next_run_time = cronJob

        inWords = self.nextRunCountdown(next_run_time.next_run)
        #show the notification (if applicable)
        if(next_run_time.next_run > time.time() and self.Addon.getSetting('notify_next_run') == 'true' and displayToScreen == True):
            xbmc.executebuiltin("Notification(Library Auto Update," + next_run_time.name + " " + inWords + ",4000," + xbmc.translatePath(self.addondir + "/resources/images/clock.png") + ")")

        return inWords

    def nextRunCountdown(self,nextRun):
        #compare now with next date
        cronDiff = nextRun - time.time()

        if cronDiff < 0:
            return ""

        hours = int((cronDiff / 60) / 60)
        minutes = int((cronDiff / 60) - hours * 60)

        #we always have at least one minute
        if minutes == 0:
            minutes = 1

        result = str(hours) + " h " + str(minutes) + " m"
        if hours == 0:
            result = str(minutes) + " m"
        elif hours > 36:
            #just show the date instead
            result = datetime.fromtimestamp(nextRun).strftime('%m/%d %I:%M%p')
        elif hours > 24:
            days = int(hours / 24)
            hours = hours - days * 24
            result = str(days) + " d " + str(hours) + " h " + str(minutes) + " m"

        return result

    def cleanLibrary(self,cronJob):
        #check which database we're in
        media_type = 'video'
        if(cronJob.command.find('music') != -1):
            media_type = 'music'

        #check if we should verify paths
        if(self.Addon.getSetting("verify_paths") == 'true'):
            response = eval(xbmc.executeJSONRPC('{ "jsonrpc" : "2.0", "method" : "Files.GetSources", "params":{"media":"' + media_type + '"}, "id": 1}'))
            for source in response['result']['sources']:
                if not xbmcvfs.exists(source['file']):
                    #let the user know this failed, if they subscribe to notifications
                    if(self.Addon.getSetting('notify_next_run') == 'true'):
                        xbmc.executebuiltin("Notification(Error Cleaning Databse,Source " + source['label'] + " does not exist,4000," + xbmc.translatePath(self.addondir + "/resources/images/clock.png") + ")")

                    self.log("Path " + source['file'] + " does not exist")
                    return

        #run the clean operation
        self.log("Cleaning Database")
        xbmc.executebuiltin("CleanLibrary(" + media_type + ")")

    def readLastRun(self):

        try:
            f = open(xbmc.translatePath(self.datadir + "last_run.txt"),"r")
            self.last_run = float(f.read())
            f.close()
        except IOError:
            #the file doesn't exist, most likely first time running
            self.last_run = 0

    def writeLastRun(self):
        #create the addon folder if it doesn't exist
        if(not os.path.exists(xbmc.translatePath(self.datadir))):
            os.makedirs(xbmc.translatePath(self.datadir))

        f = open(xbmc.translatePath(self.datadir + "last_run.txt"),"w")

        #write out the value for the last time the program ran
        f.write(str(self.last_run));
        f.close();

    def scanRunning(self):
        #check if any type of scan is currently running
        if(xbmc.getCondVisibility('Library.IsScanningVideo') or xbmc.getCondVisibility('Library.IsScanningMusic')):
            return True
        else:
            return False

    def waitForScan(self):
        #what for scan to start
        time.sleep(2)

        while(self.scanRunning()):
            time.sleep(5)

    def log(self,message):
        xbmc.log('service.libraryautoupdate: ' + message)

<|endoftext|>