<commit_before>#!/usr/bin/python

import ConfigParser, os, re
import facebook
import pprint
import feedparser
from bs4 import BeautifulSoup
from bs4 import NavigableString
from bs4 import Tag

config = ConfigParser.ConfigParser()

config.read([os.path.expanduser('~/.rssBlogs')])

print "Configured blogs:"

i=1
for section in config.sections():
	print str(i), ')', section, config.get(section, "rssFeed")
	i = i + 1

if (int(i)>1):
	i = raw_input ('Select one: ')
else:
	i = 1

print "You have chosen ", config.get("Blog"+str(i), "rssFeed")

rssFeed = config.get("Blog"+str(i), "rssFeed")
pageFB = config.get("Blog"+str(i), "pageFB")

print "*"+rssFeed+"*"

feed = feedparser.parse(rssFeed)

print feed

i = 0

soup = BeautifulSoup(feed.entries[i].title)
theTitle = soup.get_text()
theLink =  feed.entries[i].link

soup = BeautifulSoup(feed.entries[i].summary)

# Now the links

j = 0
linksTxt = ""
for link in soup("a"):
	# print type(link.contents[0])
	if not isinstance(link.contents[0], Tag):
		# We want to avoid embdeded tags (mainly <img ... )
		if not re.search(r"delicious.*fernand0.*rei", link['href']):
			# We want to avoid tags in fernand0.blogalia.com
			link.append(" ["+str(j)+"]")
			j =  j + 1
			linksTxt = linksTxt + "["+str(j)+"] " + link.contents[0] + "\n"
			linksTxt = linksTxt + "    " + link['href'] + "\n"

theSummary = theTitle+"\n"
theSummary = theSummary + soup.get_text()
theSummary = theSummary+"\n\n"

if linksTxt != "":
	theSummary = theSummary + linksTxt
theSummary = theSummary+"\n\n"

print theSummary

config.read([os.path.expanduser('~/.rssFacebook')])
oauth_access_token= config.get("Facebook", "oauth_access_token")

graph = facebook.GraphAPI(oauth_access_token)
pages = graph.get_connections("me", "accounts")

for i in range(len(pages['data'])):
	if (pages['data'][i]['name'] == pageFB):
		print "Writing in... ", pages['data'][i]['name']
		graph2 = facebook.GraphAPI(pages['data'][i]['access_token'])
		graph2.put_object(pages['data'][i]['id'],
			"feed", message = 'Nueva entrada:\n\n'+theSummary,
			link=theLink,
			#picture = 'http://fernand0.github.io/',
			# You could select a picture
			name=theTitle, caption='', description=theSummary)

<commit_msg>Adding a brief explanation<commit_after>#!/usr/bin/python
# encoding: utf-8
#
# Very simple Python program to publish the last RSS entry of a feed in
# a Facebook Page. It shows the blogs available and allows to select
# one of them.
#
# It has a configuration file with a number of blogs with:
#	- The RSS feed of the blog
#	- The Twitter account where the news will be published
#	- The Facebook page where the news will be published
# It uses a configuration file that has two sections:
#  	- The oauth access token
#

import ConfigParser, os, re
import facebook
import pprint
import feedparser
from bs4 import BeautifulSoup
from bs4 import NavigableString
from bs4 import Tag

config = ConfigParser.ConfigParser()

config.read([os.path.expanduser('~/.rssBlogs')])

print "Configured blogs:"

i=1
for section in config.sections():
	print str(i), ')', section, config.get(section, "rssFeed")
	i = i + 1

if (int(i)>1):
	i = raw_input ('Select one: ')
else:
	i = 1

print "You have chosen ", config.get("Blog"+str(i), "rssFeed")

rssFeed = config.get("Blog"+str(i), "rssFeed")
pageFB = config.get("Blog"+str(i), "pageFB")

print "*"+rssFeed+"*"

feed = feedparser.parse(rssFeed)

print feed

i = 0

soup = BeautifulSoup(feed.entries[i].title)
theTitle = soup.get_text()
theLink =  feed.entries[i].link

soup = BeautifulSoup(feed.entries[i].summary)

# Now the links

j = 0
linksTxt = ""
for link in soup("a"):
	# print type(link.contents[0])
	if not isinstance(link.contents[0], Tag):
		# We want to avoid embdeded tags (mainly <img ... )
		if not re.search(r"delicious.*fernand0.*rei", link['href']):
			# We want to avoid tags in fernand0.blogalia.com
			link.append(" ["+str(j)+"]")
			j =  j + 1
			linksTxt = linksTxt + "["+str(j)+"] " + link.contents[0] + "\n"
			linksTxt = linksTxt + "    " + link['href'] + "\n"

theSummary = theTitle+"\n"
theSummary = theSummary + soup.get_text()
theSummary = theSummary+"\n\n"

if linksTxt != "":
	theSummary = theSummary + linksTxt
theSummary = theSummary+"\n\n"

print theSummary

config.read([os.path.expanduser('~/.rssFacebook')])
oauth_access_token= config.get("Facebook", "oauth_access_token")

graph = facebook.GraphAPI(oauth_access_token)
pages = graph.get_connections("me", "accounts")

for i in range(len(pages['data'])):
	if (pages['data'][i]['name'] == pageFB):
		print "Writing in... ", pages['data'][i]['name']
		graph2 = facebook.GraphAPI(pages['data'][i]['access_token'])
		graph2.put_object(pages['data'][i]['id'],
			"feed", message = 'Nueva entrada:\n\n'+theSummary,
			link=theLink,
			#picture = 'http://fernand0.github.io/',
			# You could select a picture
			name=theTitle, caption='', description=theSummary)

<|endoftext|>


<commit_before>import sys, logging, httplib2
import feedparser
from calendar import timegm
from time import gmtime
from lxml import etree

#To-Do:
# 1. More authoritative/less hacky method for determining authenticity of RSS url
# 2. Regex matching for more reliable determination of what key,value pair from the feed to use as the timestamp
#    field
# 3. Add link-mining capability.

def rssdownload(username, feedurl, last_reference=0, mode=0):
    #Added some test cases of known failing cases. You may want to add more.
    '''
    >>> rssdownload('http://www.cnn.com', timegm(gmtime())-15000)
    'The URL provided does not appear to be a valid RSS feed.'
    >>> rssdownload('http://www.cnn.com', timegm(gmtime())+15000)
    'The URL provided does not appear to be a valid RSS feed.'
    >>> rssdownload('http://feeds.feedburner.com/chrisbrogandotcom', timegm(gmtime())-15000)

    >>> rssdownload('http://feeds.feedburner.com/chrisbrogandotcom', timegm(gmtime())+15000)
    'There do not seem to be any new links since the last update.'
    >>> rssdownload('http://rss.cnn.com/rss/cnn_topstories', timegm(gmtime())-15000)

    >>> rssdownload('http://rss.cnn.com/rss/cnn_topstories', timegm(gmtime())+15000)
    'There do not seem to be any new links since the last update.'
    '''

    messages = []
    feed = feedparser.parse(feedurl)

    logger = logging.getLogger('proxy.rss')
    logger.debug("User %s's update URL is %s" % (username, feedurl))

    if not feed.feed.has_key['title']:
        logger.error('User %s supplied a URL that does not seem to be a valid RSS feed (%s)' % (username, feedurl))
        return {'messages':[],'last_reference':last_ref}
    else:
        for item in feed.entries:
            if timegm(item.updated_parsed) > last_reference:
                messages.append({'url':item.link,
                                 'timestamp':item.updated_parsed,
                                 'description':item.title,
                                 'extra':feed.feed.link,
                                 'refer':''})

    if mode == 1:
        mlinks = [i['url'] for i in messages]
        deeplinks = linkminer(mlinks)

    if len(messages) == 0:
        logger.warning("%s doesn't have anything new for us." % feed.feed.title)
        return 'There do not seem to be any new links since the last update.' #String for rendering in browser

    last_ref = timegm(messages[len(messages)-1]['timestamp'])

    return{'messages':messages,
           'last_reference':last_ref,
           'protected':False}

def linkminer(mlinks):
    http = httplib2.Http()
    response, content = http.request(item, 'GET')
    html = etree.HTML(content)
    result = etree.tostring(html, pretty_print=True, method="html")

    for item in mlinks:

if __name__ == "__main__":

    def testgo(g):
        if isinstance(g, dict):
            return 'The sampled feed is %d items long.' % len(g['messages'])
        else:
            return g

    test_params = ['-doc', '-cust', None]
    if sys.argv[1] in test_params:
        if sys.argv[1] == '-doc':
            import doctest
            doctest.testmod()
        if sys.argv[1] == '-cust' or '':
            url = raw_input('Paste URL to test: ')
            while True:
                try:
                    timeframe = raw_input('Set last_ref to past or future [P/f]? ')
                    assert timeframe.lower() in ('', 'p', 'f')
                    break
                except AssertionError:
                    pass
            if timeframe.lower() in ('', 'p'):
                with rssdownload(url, timegm(gmtime())-15000) as test1:
                    print test1
            else:
                with rssdownload(url, timegm(gmtime())+15000) as test1:
                    print test1
            result = testgo(test1)
            print result
    else:
        print 'Valid arguments are -doc (for doctest.testmod() testing) and -cust (for custom URL/last_ref input).'

<commit_msg>uncrappifying testing<commit_after>import sys, logging, httplib2
import feedparser
from calendar import timegm
from time import gmtime
from lxml import etree

#To-Do:
# 1. More authoritative/less hacky method for determining authenticity of RSS url
# 2. Regex matching for more reliable determination of what key,value pair from the feed to use as the timestamp
#    field
# 3. Add link-mining capability.
# 4. Improve doctesting by adding better test cases.
# 5. Improve command-line testing.
# 6. Implement "mode 1" (deep link-mining) capabilities

def rssdownload(username, feedurl, last_reference=0):

    messages = []
    feed = feedparser.parse(feedurl)

    logger = logging.getLogger('proxy.rss')
    logger.debug("User %s's update URL is %s" % (username, feedurl))

    if not feed.feed.has_key['title']:
        logger.error('User %s supplied a URL that does not seem to be a valid RSS feed (%s)' % (username, feedurl))
        return {'messages':[],'last_reference':last_ref}
    else:
        for item in feed.entries:
            if timegm(item.updated_parsed) > last_reference:
                messages.append({'url':item.link,
                                 'timestamp':item.updated_parsed,
                                 'description':item.title,
                                 'extra':feed.feed.link,
                                 'refer':''})
##
##    if mode == 1:
##        mlinks = [i['url'] for i in messages]
##        deeplinks = linkminer(mlinks)

    if len(messages) == 0:
        logger.warning("%s doesn't have anything new for us." % feed.feed.title)
        return 'There do not seem to be any new links since the last update.' #String for rendering in browser

    last_ref = timegm(messages[len(messages)-1]['timestamp'])

    return{'messages':messages,
           'last_reference':last_ref,
           'protected':False}

##def linkminer(mlinks):
##    http = httplib2.Http()
##    response, content = http.request(item, 'GET')
##    html = etree.HTML(content)
##    result = etree.tostring(html, pretty_print=True, method="html")
##
##    for item in mlinks:

if __name__ == "__main__":

<|endoftext|>


<commit_before>#!/usr/bin/env python
# rtrsub - A RTR Substitution
#
# Copyright (C) 2016 Job Snijders <job@instituut.net>
#
# This file is part of rtrsub
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from __future__ import print_function

from ipaddress import ip_network
from operator import itemgetter
import jinja2
import json
import pprint
import os
import sys

try:
    import argparse
except ImportError:
    print("ERROR: install argparse manually")
    print("HINT: sudo pip install argparse")
    sys.exit(2)

try:
    import requests
except ImportError:
    print("ERROR: requests missing")
    print("HINT: pip install requests")
    sys.exit(2)

def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('-c', dest='cache',
                        default="http://localcert.ripe.net:8088/export.json",
                        type=str,
                        help="""Location of the RPKI Cache in JSON format
(default: http://localcert.ripe.net:8088/export.json)""")

    parser.add_argument('--afi', dest='afi', type=str, required=True,
                        help="[ ipv4 | ipv6 | mixed ]")

    parser.add_argument('-t', dest='template', type=str,
                        default="-", help='Template file (default: STDIN)')

    parser.add_argument('-o', dest='output', type=str,
                        default='-',
                        help='Output file (default: STDOUT)')

    parser.add_argument('-v', dest='version', action='store_true',
                        help="Display rtrsub version")

    parser.add_argument('-r', dest='ruby', action='store_true',
                        help='Use other delimiters')

    args = parser.parse_args()

    if args.afi not in ["ipv4", "ipv6", "mixed"]:
        print("ERROR: afi must be 'ipv4', 'ipv6' or 'mixed'")
        sys.exit(2)

    if args.ruby:
        env = jinja2.Environment(
            block_start_string='<%',
            block_end_string='%>',
            variable_start_string='<%=',
            variable_end_string='%>',
            comment_start_string='<#',
            comment_end_string='#>',
            line_statement_prefix="%%")
    else:
        env = jinja2.Environment()

    if args.template == "-":
        template_stdin = sys.stdin.read()
        template = env.from_string(template_stdin)
    else:
        template = env.from_string(open(os.path.abspath(args.template), 'r').read())

    if 'http' in args.cache:
        r = requests.get(args.cache)
        validator_export = r.json()
    else:
        validator_export = json.load(open(args.cache, "r"))

    data = load_pfx_dict(args.afi, validator_export)
    data['afi'] = args.afi

    if args.output == "-":
        print(template.render(**data))
    else:
        f = open(args.output, 'w')
        f.write(template.render(data=data))
        f.close

def load_pfx_dict(afi, export):
    """
    :param afi:     which address family to filter for
    :param export:  the JSON blob with all ROAs
    """
    pfx_dict = {}
    origin_dict = {}
    pfx_list = []

    """ each roa has these fields:
        asn, prefix, maxLength, ta
    """

    for roa in export['roas']:
        prefix_obj = ip_network(roa['prefix'])
        if afi == "ipv4":
            if prefix_obj.version == 6:
                continue
        elif afi == "ipv6":
            if prefix_obj.version == 4:
                continue

        try:
            asn = int(roa['asn'].replace("AS", ""))
            if not 0 <= asn < 4294967296:
                raise ValueError
        except ValueError:
            print("ERROR: ASN malformed", file=sys.stderr)
            print(pprint.pformat(roa, indent=4), file=sys.stderr)
            continue

        prefix = str(prefix_obj)
        prefixlen = prefix_obj.prefixlen
        maxlength = int(roa['maxLength'])

        if prefix not in pfx_dict:
            pfx_dict[prefix] = {}
            pfx_dict[prefix]['origins'] = [asn]
        else:
            if asn not in pfx_dict[prefix]['origins']:
                pfx_dict[prefix]['origins'] += [asn]

        pfx_dict[prefix]['maxlength'] = maxlength
        pfx_dict[prefix]['prefixlen'] = prefixlen
        pfx_list.append((prefix, prefixlen))

        if asn not in origin_dict:
            origin_dict[asn] = {}

        origin_dict[asn][prefix] = {'maxlength': maxlength,
                                    'length': prefixlen}

    pfx_list = set(map(lambda x: x[0], sorted(pfx_list, key=itemgetter(1))))

    return {"pfx_dict": pfx_dict, "origin_dict": origin_dict,
            "pfx_list": pfx_list}

if __name__ == '__main__':
    main()
<commit_msg>Fix bug<commit_after>#!/usr/bin/env python
# rtrsub - A RTR Substitution
#
# Copyright (C) 2016 Job Snijders <job@instituut.net>
#
# This file is part of rtrsub
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from __future__ import print_function

from collections import OrderedDict
from ipaddress import ip_network
from operator import itemgetter
import jinja2
import json
import pprint
import os
import sys

try:
    import argparse
except ImportError:
    print("ERROR: install argparse manually")
    print("HINT: sudo pip install argparse")
    sys.exit(2)

try:
    import requests
except ImportError:
    print("ERROR: requests missing")
    print("HINT: pip install requests")
    sys.exit(2)

def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('-c', dest='cache',
                        default="http://localcert.ripe.net:8088/export.json",
                        type=str,
                        help="""Location of the RPKI Cache in JSON format
(default: http://localcert.ripe.net:8088/export.json)""")

    parser.add_argument('--afi', dest='afi', type=str, required=True,
                        help="[ ipv4 | ipv6 | mixed ]")

    parser.add_argument('-t', dest='template', type=str,
                        default="-", help='Template file (default: STDIN)')

    parser.add_argument('-o', dest='output', type=str,
                        default='-',
                        help='Output file (default: STDOUT)')

    parser.add_argument('-v', dest='version', action='store_true',
                        help="Display rtrsub version")

    parser.add_argument('-r', dest='ruby', action='store_true',
                        help='Use other delimiters')

    args = parser.parse_args()

    if args.afi not in ["ipv4", "ipv6", "mixed"]:
        print("ERROR: afi must be 'ipv4', 'ipv6' or 'mixed'")
        sys.exit(2)

    if args.ruby:
        env = jinja2.Environment(
            block_start_string='<%',
            block_end_string='%>',
            variable_start_string='<%=',
            variable_end_string='%>',
            comment_start_string='<#',
            comment_end_string='#>',
            line_statement_prefix="%%")
    else:
        env = jinja2.Environment()

    if args.template == "-":
        template_stdin = sys.stdin.read()
        template = env.from_string(template_stdin)
    else:
        template = env.from_string(open(os.path.abspath(args.template), 'r').read())

    if 'http' in args.cache:
        r = requests.get(args.cache)
        validator_export = r.json()
    else:
        validator_export = json.load(open(args.cache, "r"))

    data = load_pfx_dict(args.afi, validator_export)
    data['afi'] = args.afi

    if args.output == "-":
        print(template.render(**data))
    else:
        f = open(args.output, 'w')
        f.write(template.render(data=data))
        f.close

def load_pfx_dict(afi, export):
    """
    :param afi:     which address family to filter for
    :param export:  the JSON blob with all ROAs
    """
    pfx_dict = {}
    origin_dict = {}
    pfx_list = []

    """ each roa has these fields:
        asn, prefix, maxLength, ta
    """

    for roa in export['roas']:
        prefix_obj = ip_network(roa['prefix'])
        if afi == "ipv4":
            if prefix_obj.version == 6:
                continue
        elif afi == "ipv6":
            if prefix_obj.version == 4:
                continue

        try:
            asn = int(roa['asn'].replace("AS", ""))
            if not 0 <= asn < 4294967296:
                raise ValueError
        except ValueError:
            print("ERROR: ASN malformed", file=sys.stderr)
            print(pprint.pformat(roa, indent=4), file=sys.stderr)
            continue

        prefix = str(prefix_obj)
        prefixlen = prefix_obj.prefixlen
        maxlength = int(roa['maxLength'])

        if prefix not in pfx_dict:
            pfx_dict[prefix] = {}
            pfx_dict[prefix]['origins'] = [asn]
        else:
            if asn not in pfx_dict[prefix]['origins']:
                pfx_dict[prefix]['origins'] += [asn]

        pfx_dict[prefix]['maxlength'] = maxlength
        pfx_dict[prefix]['prefixlen'] = prefixlen
        pfx_list.append((prefix, prefixlen))

        if asn not in origin_dict:
            origin_dict[asn] = {}

        origin_dict[asn][prefix] = {'maxlength': maxlength,
                                    'length': prefixlen}

    # order the list of prefixes by prefix length
    pfx_list = map(lambda x: x[0], sorted(pfx_list, key=itemgetter(1)))
    # deduplicate the list and maintain the order
    pfx_list = list(OrderedDict.fromkeys(pfx_list))

    return {"pfx_dict": pfx_dict, "origin_dict": origin_dict,
            "pfx_list": pfx_list}

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/bin/env python
__author__ = "John Hover"
__copyright__ = "2017 John Hover"
__credits__ = []
__license__ = "GPL"
__version__ = "0.9.0"
__maintainer__ = "John Hover"
__email__ = "jhover@bnl.gov"
__status__ = "Production"

import argparse
import logging
import os
import subprocess
import sys
import time
from ConfigParser import ConfigParser

def _runtimedcommand(cmd):
    log = logging.getLogger()
    before = time.time()
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    out = None
    (out, err) = p.communicate()
    delta = time.time() - before
    log.debug('%s seconds to perform command' %delta)
    if p.returncode == 0:
        log.debug('Leaving with OK return code.')
    else:
        log.warning('Leaving with bad return code. rc=%s err=%s out=%s' %(p.returncode, err, out ))
        out = None
    return out

class GemHandler(object):

    handledgems = set()

    def __init__(self, cp , gemname):
        self.log = logging.getLogger()
        self.log.info("Handling gem %s" % gemname)
        self.config = cp
        self.gemname = gemname
        self.rpmbuilddir = os.path.expanduser(cp.get('global','rpmbuilddir'))
        self.gemtemplate = os.path.expanduser(cp.get('global','gemtemplate'))
        self.tempdir = os.path.expanduser(cp.get('global','tempdir'))
        #self.packagelog = os.path.expanduser(cp.get('packagelog'))

    def setupDirs(self):
        '''
        setup rpmbuild RPMS SOURCES SPECS
        '''
        dirs = [ '%s/SOURCES' % self.rpmbuilddir,
                 '%s/SPECS' % self.rpmbuilddir,
                 '%s/BUILD' % self.rpmbuilddir,
                 '%s/RPMS/noarch' % self.rpmbuilddir,
                 '%s/RPMS/x86_64' % self.rpmbuilddir,
                 '%s/BUILDROOT' % self.rpmbuilddir,
                 '%s/%s' % (self.tempdir, self.gemname),
            ]
        for d in dirs:
            try:
                os.makedirs(d)
            except OSError, e:
                pass
        self.log.debug("Various dirs made OK.")

    def fetchGem(self):
        '''
        '''
        cmd =  "cd %s/SOURCES ;  gem fetch %s " % (self.rpmbuilddir, self.gemname)
        self.log.debug("Command is %s" % cmd )
        o = _runtimedcommand(cmd)
        if o is not None:
            self.log.debug("Out is %s" % o)
            fields = o.split()
            nv = fields[1]
            self.version = nv[len(self.gemname)+1:]
            self.log.debug("Version is %s" % self.version)

    def makeSpec(self):
        '''
        gem2rpm -t $TEMPLATE  $gem-[0-9]*.gem > $RPMBUILDDIR/SPECS/rubygem-$gem.spec

        '''
        cmd =  "gem2rpm -t %s %s-%s.gem > %s/SPECS/rubygem-%s.spec " % (self.gemtemplate,
                                                                        self.gemname,
                                                                        self.version,
                                                                        self.rpmbuilddir,
                                                                        self.gemname)
        self.log.debug("Command is %s" % cmd )
        o = _runtimedcommand(cmd)
        self.log.debug("Created rubygem-%s.spec " % self.gemname)

    def fixSpec(self):
        '''
        '''

    def parseDeps(self):
        '''

        '''
        self.deps = []
        depset = set()
        cmd =  "cd %s/SOURCES ;  gem2rpm -d %s-%s.gem" % (self.rpmbuilddir, self.gemname, self.version)
        self.log.debug("Command is %s" % cmd )
        o = _runtimedcommand(cmd)
        if o is not None:
            self.log.debug("Out is %s" % o)
            o = o.strip()
            if len(o) > 3:
                lines = o.split('\n')
                for line in lines:
                    if len(line) > 3:
                        fields = line.split()
                        dep = fields[0]
                        if len(fields) > 2:
                            op = fields[1]
                            ver = fields[2]
                        self.log.debug("Dep is %s" % dep)
                if dep.startswith('rubygem'):
                    depname = dep[8:-1]
                    depset.add(depname)
            else:
                self.log.debug("No dependencies.")
        self.deps = list(depset)

    def buildRPM(self):
        '''
    rpmbuild -bb $RPMBUILDDIR/SPECS/rubygem-$gem.spec
        '''
        self.log.debug("Building gem %s" % self.gemname)
        cmd =  "rpmbuild -bb %s/SPECS/rubygem-%s.spec" % (self.rpmbuilddir,
                                                          self.gemname)
        self.log.debug("Command is %s" % cmd )
        o = _runtimedcommand(cmd)
        if o is not None:
            self.log.debug("RPM for rubygem-%s built OK." % self.gemname)
        else:
            self.log.error("Problem building RPM for rubygem-%s." % self.gemname)

    def handleDeps(self):
        '''

        '''
        for dep in self.deps:
            self.log.debug('Processing dep %s' % dep)
            if dep not in GemHandler.handledgems:
                gh = GemHandler(self.config, dep)
                gh.handleGem()
            else:
                self.log.debug("Gem %s already done." % dep)
        self.log.debug("Finished with deps for %s" % self.gemname)

    def handleGem(self):
        self.setupDirs()
        self.fetchGem()
        self.makeSpec()
        self.fixSpec()
        self.buildRPM()
        self.log.debug("Adding gem %s to done list." % self.gemname)
        GemHandler.handledgems.add(self.gemname)
        self.parseDeps()
        self.handleDeps()

class GemRPMCLI(object):

    def __init__(self):
        self.parseopts()
        self.setuplogging()

    def setuplogging(self):
        self.log = logging.getLogger()
        FORMAT='%(asctime)s (UTC) [ %(levelname)s ] %(name)s %(filename)s:%(lineno)d %(funcName)s(): %(message)s'
        formatter = logging.Formatter(FORMAT)
        #formatter.converter = time.gmtime  # to convert timestamps to UTC
        logStream = logging.StreamHandler()
        logStream.setFormatter(formatter)
        self.log.addHandler(logStream)

        self.log.setLevel(logging.WARN)
        if self.results.debug:
            self.log.setLevel(logging.DEBUG)
        # adding a new Handler for the console,
        # to be used only for DEBUG and INFO modes.
        #if self.options.logLevel in [logging.DEBUG, logging.INFO]:
        #    if self.options.console:
        #        console = logging.StreamHandler(sys.stdout)
        #        console.setFormatter(formatter)
        #        console.setLevel(self.options.logLevel)
        #        self.log.addHandler(console)
        #self.log.setLevel(self.options.logLevel)
        self.log.info('Logging initialized.')

    def parseopts(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c', '--config',
                            action="store",
                            dest='configpath',
                            default='~/etc/rubygemrpm.conf',
                            help='configuration file path.')

        parser.add_argument('-d', '--debug',
                            action="store_true",
                            dest='debug',
                            help='debug logging')

        parser.add_argument('gemname',
                             action="store")

        self.results= parser.parse_args()
        #print(self.results)

    def invoke(self):
        cp = ConfigParser()
        ns = self.results
        self.log.info("Config is %s" % ns.configpath)
        cp.read(os.path.expanduser(ns.configpath))
        gh = GemHandler(cp, ns.gemname)
        gh.handleGem()

if __name__ == '__main__':
    rgcli = GemRPMCLI()
    rgcli.invoke()
<commit_msg>Update. <commit_after>#!/bin/env python
__author__ = "John Hover"
__copyright__ = "2017 John Hover"
__credits__ = []
__license__ = "GPL"
__version__ = "0.9.0"
__maintainer__ = "John Hover"
__email__ = "jhover@bnl.gov"
__status__ = "Production"

import argparse
import logging
import os
import subprocess
import sys
import time
from ConfigParser import ConfigParser

def _runtimedcommand(cmd):
    log = logging.getLogger()
    before = time.time()
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    out = None
    (out, err) = p.communicate()
    delta = time.time() - before
    log.debug('%s seconds to perform command' %delta)
    if p.returncode == 0:
        log.debug('Leaving with OK return code.')
    else:
        log.warning('Leaving with bad return code. rc=%s err=%s out=%s' %(p.returncode, err, out ))
        out = None
    return out

class GemHandler(object):

    handledgems = set()

    def __init__(self, cp , gemname):
        self.log = logging.getLogger()
        self.log.info("Handling gem %s" % gemname)
        self.config = cp
        self.gemname = gemname
        self.rpmbuilddir = os.path.expanduser(cp.get('global','rpmbuilddir'))
        self.gemtemplate = os.path.expanduser(cp.get('global','gemtemplate'))
        self.tempdir = os.path.expanduser(cp.get('global','tempdir'))
        #self.packagelog = os.path.expanduser(cp.get('packagelog'))

    def setupDirs(self):
        '''
        setup rpmbuild RPMS SOURCES SPECS
        '''
        dirs = [ '%s/SOURCES' % self.rpmbuilddir,
                 '%s/SPECS' % self.rpmbuilddir,
                 '%s/BUILD' % self.rpmbuilddir,
                 '%s/RPMS/noarch' % self.rpmbuilddir,
                 '%s/RPMS/x86_64' % self.rpmbuilddir,
                 '%s/BUILDROOT' % self.rpmbuilddir,
                 '%s/%s' % (self.tempdir, self.gemname),
            ]
        for d in dirs:
            try:
                os.makedirs(d)
            except OSError, e:
                pass
        self.log.debug("Various dirs made OK.")

    def fetchGem(self):
        '''
        '''
        cmd =  "cd %s/SOURCES ;  gem fetch %s " % (self.rpmbuilddir, self.gemname)
        self.log.debug("Command is %s" % cmd )
        o = _runtimedcommand(cmd)
        if o is not None:
            self.log.debug("Out is %s" % o)
            fields = o.split()
            nv = fields[1]
            self.version = nv[len(self.gemname)+1:]
            self.log.debug("Version is %s" % self.version)

    def makeSpec(self):
        '''
        gem2rpm -t $TEMPLATE  $gem-[0-9]*.gem > $RPMBUILDDIR/SPECS/rubygem-$gem.spec

        '''
        cmd =  "gem2rpm -t %s %s/SPECS%s-%s.gem > %s/SPECS/rubygem-%s.spec " % (
                                                                        self.gemtemplate,
                                                                         self.rpmbuilddir,
                                                                        self.gemname,
                                                                        self.version,
                                                                        self.rpmbuilddir,
                                                                        self.gemname)
        self.log.debug("Command is %s" % cmd )
        o = _runtimedcommand(cmd)
        self.log.debug("Created rubygem-%s.spec " % self.gemname)

    def fixSpec(self):
        '''
        '''

    def parseDeps(self):
        '''

        '''
        self.deps = []
        depset = set()
        cmd =  "cd %s/SOURCES ;  gem2rpm -d %s-%s.gem" % (self.rpmbuilddir, self.gemname, self.version)
        self.log.debug("Command is %s" % cmd )
        o = _runtimedcommand(cmd)
        if o is not None:
            self.log.debug("Out is %s" % o)
            o = o.strip()
            if len(o) > 3:
                lines = o.split('\n')
                for line in lines:
                    if len(line) > 3:
                        fields = line.split()
                        dep = fields[0]
                        if len(fields) > 2:
                            op = fields[1]
                            ver = fields[2]
                        self.log.debug("Dep is %s" % dep)
                if dep.startswith('rubygem'):
                    depname = dep[8:-1]
                    depset.add(depname)
            else:
                self.log.debug("No dependencies.")
        self.deps = list(depset)

    def buildRPM(self):
        '''
    rpmbuild -bb $RPMBUILDDIR/SPECS/rubygem-$gem.spec
        '''
        self.log.debug("Building gem %s" % self.gemname)
        cmd =  "rpmbuild -bb %s/SPECS/rubygem-%s.spec" % (self.rpmbuilddir,
                                                          self.gemname)
        self.log.debug("Command is %s" % cmd )
        o = _runtimedcommand(cmd)
        if o is not None:
            self.log.debug("RPM for rubygem-%s built OK." % self.gemname)
        else:
            self.log.error("Problem building RPM for rubygem-%s." % self.gemname)

    def handleDeps(self):
        '''

        '''
        for dep in self.deps:
            self.log.debug('Processing dep %s' % dep)
            if dep not in GemHandler.handledgems:
                gh = GemHandler(self.config, dep)
                gh.handleGem()
            else:
                self.log.debug("Gem %s already done." % dep)
        self.log.debug("Finished with deps for %s" % self.gemname)

    def handleGem(self):
        self.setupDirs()
        self.fetchGem()
        self.makeSpec()
        self.fixSpec()
        self.buildRPM()
        self.log.debug("Adding gem %s to done list." % self.gemname)
        GemHandler.handledgems.add(self.gemname)
        self.parseDeps()
        self.handleDeps()

class GemRPMCLI(object):

    def __init__(self):
        self.parseopts()
        self.setuplogging()

    def setuplogging(self):
        self.log = logging.getLogger()
        FORMAT='%(asctime)s (UTC) [ %(levelname)s ] %(name)s %(filename)s:%(lineno)d %(funcName)s(): %(message)s'
        formatter = logging.Formatter(FORMAT)
        #formatter.converter = time.gmtime  # to convert timestamps to UTC
        logStream = logging.StreamHandler()
        logStream.setFormatter(formatter)
        self.log.addHandler(logStream)

        self.log.setLevel(logging.WARN)
        if self.results.debug:
            self.log.setLevel(logging.DEBUG)
        # adding a new Handler for the console,
        # to be used only for DEBUG and INFO modes.
        #if self.options.logLevel in [logging.DEBUG, logging.INFO]:
        #    if self.options.console:
        #        console = logging.StreamHandler(sys.stdout)
        #        console.setFormatter(formatter)
        #        console.setLevel(self.options.logLevel)
        #        self.log.addHandler(console)
        #self.log.setLevel(self.options.logLevel)
        self.log.info('Logging initialized.')

    def parseopts(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c', '--config',
                            action="store",
                            dest='configpath',
                            default='~/etc/rubygemrpm.conf',
                            help='configuration file path.')

        parser.add_argument('-d', '--debug',
                            action="store_true",
                            dest='debug',
                            help='debug logging')

        parser.add_argument('gemname',
                             action="store")

        self.results= parser.parse_args()
        #print(self.results)

    def invoke(self):
        cp = ConfigParser()
        ns = self.results
        self.log.info("Config is %s" % ns.configpath)
        cp.read(os.path.expanduser(ns.configpath))
        gh = GemHandler(cp, ns.gemname)
        gh.handleGem()

if __name__ == '__main__':
    rgcli = GemRPMCLI()
    rgcli.invoke()
<|endoftext|>


<commit_before>def init_laser_data(bus):
    bus.write_byte_data(0x62, 0x11, 0xff) #sets laser to read forever
    bus.write_byte_data(0x62, 0x00, 0x04) #sets laser to start reading

def read_laser_data(bus):
    try:
        hi = bus.read_byte_data(0x62, 0x14)
        lo = bus.read_byte_data(0x62, 0x15)
        data = (hi << 8) | lo
        return data
    except:
        return read_laser_data(bus)
<commit_msg>changed laser read address<commit_after>def init_laser_data(bus):
    bus.write_byte_data(0x62, 0x11, 0xff) #sets laser to read forever
    bus.write_byte_data(0x62, 0x00, 0x04) #sets laser to start reading

def read_laser_data(bus):
    try:
        hi = bus.read_byte_data(0x62, 0x0f)
        lo = bus.read_byte_data(0x62, 0x10)
        data = (hi << 8) | lo
        return data
    except:
        return read_laser_data(bus)
<|endoftext|>


<commit_before>from bus import LASER_ADDR
from eventbus import EventBus

class Laser:
    @staticmethod
    def initialize():
        EventBus.bus.bus.write_byte_data(LASER_ADDR, 0x00, 0x00) #Resets FPGA registers
        EventBus.bus.bus.write_byte_data(LASER_ADDR, 0x11, 0xff) #sets laser to read forever
        EventBus.bus.bus.write_byte_data(LASER_ADDR, 0x00, 0x04) #sets laser to start reading

    @staticmethod
    def read_data():
        try:
            hi = EventBus.bus.bus.read_byte_data(LASER_ADDR, 0x0f)
            lo = EventBus.bus.bus.read_byte_data(LASER_ADDR, 0x10)
            data = (hi << 8) | lo
            return data * 10
        except:
            return -1
<commit_msg>added sleep during init<commit_after>from bus import LASER_ADDR
from eventbus import EventBus
from time import sleep

class Laser:
    @staticmethod
    def initialize():
        EventBus.bus.bus.write_byte_data(LASER_ADDR, 0x00, 0x00) #Resets FPGA registers
        sleep(1)
        EventBus.bus.bus.write_byte_data(LASER_ADDR, 0x11, 0xff) #sets laser to read forever
        EventBus.bus.bus.write_byte_data(LASER_ADDR, 0x00, 0x04) #sets laser to start reading

    @staticmethod
    def read_data():
        try:
            hi = EventBus.bus.bus.read_byte_data(LASER_ADDR, 0x0f)
            lo = EventBus.bus.bus.read_byte_data(LASER_ADDR, 0x10)
            data = (hi << 8) | lo
            return data * 10
        except:
            return -1
<|endoftext|>


<commit_before># encoding=utf8
import datetime
from distutils.version import StrictVersion
import hashlib
import os.path
import random
from seesaw.config import realize, NumberConfigValue
from seesaw.item import ItemInterpolation, ItemValue
from seesaw.task import SimpleTask, LimitConcurrent
from seesaw.tracker import GetItemFromTracker, PrepareStatsForTracker, \
    UploadWithTracker, SendDoneToTracker
import shutil
import socket
import subprocess
import sys
import time

import seesaw
from seesaw.externalprocess import WgetDownload
from seesaw.pipeline import Pipeline
from seesaw.project import Project
from seesaw.util import find_executable

# check the seesaw version
if StrictVersion(seesaw.__version__) < StrictVersion("0.1.5"):
    raise Exception("This pipeline needs seesaw version 0.1.5 or higher.")

###########################################################################
# Find a useful Wget+Lua executable.
#
# WGET_LUA will be set to the first path that
# 1. does not crash with --version, and
# 2. prints the required version string
WGET_LUA = find_executable(
    "Wget+Lua",
    ["GNU Wget 1.14.lua.20130523-9a5c"],
    [
        "./wget-lua",
        "./wget-lua-warrior",
        "./wget-lua-local",
        "../wget-lua",
        "../../wget-lua",
        "/home/warrior/wget-lua",
        "/usr/bin/wget-lua"
    ]
)

if not WGET_LUA:
    raise Exception("No usable Wget+Lua found.")

###########################################################################
# The version number of this pipeline definition.
#
# Update this each time you make a non-cosmetic change.
# It will be added to the WARC files and reported to the tracker.
VERSION = "20140821.03"
USER_AGENT = 'ArchiveTeam'
TRACKER_ID = 'swipnet'
TRACKER_HOST = 'tracker.archiveteam.org'

###########################################################################
# This section defines project-specific tasks.
#
# Simple tasks (tasks that do not need any concurrency) are based on the
# SimpleTask class and have a process(item) method that is called for
# each item.
class CheckIP(SimpleTask):
    def __init__(self):
        SimpleTask.__init__(self, "CheckIP")
        self._counter = 0

    def process(self, item):
        # NEW for 2014! Check if we are behind firewall/proxy

        if self._counter <= 0:
            item.log_output('Checking IP address.')
            ip_set = set()

            ip_set.add(socket.gethostbyname('twitter.com'))
            ip_set.add(socket.gethostbyname('facebook.com'))
            ip_set.add(socket.gethostbyname('youtube.com'))
            ip_set.add(socket.gethostbyname('microsoft.com'))
            ip_set.add(socket.gethostbyname('icanhas.cheezburger.com'))
            ip_set.add(socket.gethostbyname('archiveteam.org'))

            if len(ip_set) != 6:
                item.log_output('Got IP addresses: {0}'.format(ip_set))
                item.log_output(
                    'Are you behind a firewall/proxy? That is a big no-no!')
                raise Exception(
                    'Are you behind a firewall/proxy? That is a big no-no!')

        # Check only occasionally
        if self._counter <= 0:
            self._counter = 10
        else:
            self._counter -= 1

class PrepareDirectories(SimpleTask):
    def __init__(self, warc_prefix):
        SimpleTask.__init__(self, "PrepareDirectories")
        self.warc_prefix = warc_prefix

    def process(self, item):
        item_name = item["item_name"]
        escaped_item_name = item_name.replace(':', '_').replace('/', '_').replace('~', '_')
        dirname = "/".join((item["data_dir"], escaped_item_name))

        if os.path.isdir(dirname):
            shutil.rmtree(dirname)

        os.makedirs(dirname)

        item["item_dir"] = dirname
        item["warc_file_base"] = "%s-%s-%s" % (self.warc_prefix, escaped_item_name,
            time.strftime("%Y%m%d-%H%M%S"))

        open("%(item_dir)s/%(warc_file_base)s.warc.gz" % item, "w").close()

class MoveFiles(SimpleTask):
    def __init__(self):
        SimpleTask.__init__(self, "MoveFiles")

    def process(self, item):
        # NEW for 2014! Check if wget was compiled with zlib support
        if os.path.exists("%(item_dir)s/%(warc_file_base)s.warc"):
            raise Exception('Please compile wget with zlib support!')

        os.rename("%(item_dir)s/%(warc_file_base)s.warc.gz" % item,
              "%(data_dir)s/%(warc_file_base)s.warc.gz" % item)

        shutil.rmtree("%(item_dir)s" % item)

def get_hash(filename):
    with open(filename, 'rb') as in_file:
        return hashlib.sha1(in_file.read()).hexdigest()

CWD = os.getcwd()
PIPELINE_SHA1 = get_hash(os.path.join(CWD, 'pipeline.py'))
LUA_SHA1 = get_hash(os.path.join(CWD, 'swipnet.lua'))

def stats_id_function(item):
    # NEW for 2014! Some accountability hashes and stats.
    d = {
        'pipeline_hash': PIPELINE_SHA1,
        'lua_hash': LUA_SHA1,
        'python_version': sys.version,
    }

    return d

class WgetArgs(object):
    def realize(self, item):
        wget_args = [
            WGET_LUA,
            "-U", USER_AGENT,
            "-nv",
            "--lua-script", "swipnet.lua",
            "-o", ItemInterpolation("%(item_dir)s/wget.log"),
            "--no-check-certificate",
            "--output-document", ItemInterpolation("%(item_dir)s/wget.tmp"),
            "--truncate-output",
            "-e", "robots=off",
            "--no-cookies",
            "--rotate-dns",
            "--recursive", "--level=inf",
            "--no-parent",
            "--page-requisites",
            "--timeout", "30",
            "--tries", "inf",
            "--span-hosts",
            "--waitretry", "30",
            "--domains", "swipnet.se",
            "--warc-file", ItemInterpolation("%(item_dir)s/%(warc_file_base)s"),
            "--warc-header", "operator: Archive Team",
            "--warc-header", "swipnet-dld-script-version: " + VERSION,
            "--warc-header", ItemInterpolation("swipnet-user: %(item_name)s"),
        ]

        item_name = item['item_name']
        assert ':' in item_name
        item_type, item_value = item_name.split(':', 1)

        assert item_type
        assert item_value

        item['item_type'] = item_type
        item['item_value'] = item_value

        wget_args.append('http://{0}.swipnet.se/{1}/'.format(item_type, item_value))

        # wget_args.append('http://home.swipnet.se/{0}/'.format(item_name))

        if 'bind_address' in globals():
            wget_args.extend(['--bind-address', globals()['bind_address']])
            print('')
            print('*** Wget will bind address at {0} ***'.format(
                globals()['bind_address']))
            print('')

        return realize(wget_args, item)

###########################################################################
# Initialize the project.
#
# This will be shown in the warrior management panel. The logo should not
# be too big. The deadline is optional.
project = Project(
    title="Swipnet",
    project_html="""

home.swipnet.se Website · Leaderboard

Archiving Swedish websites from home.swipnet.se.

    """,
    utc_deadline=datetime.datetime(2014, 8, 31, 23, 59, 0)
)

pipeline = Pipeline(
    CheckIP(),
    GetItemFromTracker("http://%s/%s" % (TRACKER_HOST, TRACKER_ID), downloader,
        VERSION),
    PrepareDirectories(warc_prefix="swipnet"),
    WgetDownload(
        WgetArgs(),
        max_tries=10,
#        max_tries=2,
        accept_on_exit_code=[0, 8],
        env={
            "item_dir": ItemValue("item_dir"),
            "item_value": ItemValue("item_value"),
            "item_type": ItemValue("item_type"),
        }
    ),
    PrepareStatsForTracker(
        defaults={"downloader": downloader, "version": VERSION},
        file_groups={
            "data": [
                ItemInterpolation("%(item_dir)s/%(warc_file_base)s.warc.gz")
            ]
        },
        id_function=stats_id_function,
    ),
    MoveFiles(),
    LimitConcurrent(NumberConfigValue(min=1, max=4, default="1",
        name="shared:rsync_threads", title="Rsync threads",
        description="The maximum number of concurrent uploads."),
        UploadWithTracker(
            "http://%s/%s" % (TRACKER_HOST, TRACKER_ID),
            downloader=downloader,
            version=VERSION,
            files=[
                ItemInterpolation("%(data_dir)s/%(warc_file_base)s.warc.gz")
            ],
            rsync_target_source_path=ItemInterpolation("%(data_dir)s/"),
            rsync_extra_args=[
                "--recursive",
                "--partial",
                "--partial-dir", ".rsync-tmp",
            ]
            ),
    ),
    SendDoneToTracker(
        tracker_url="http://%s/%s" % (TRACKER_HOST, TRACKER_ID),
        stats=ItemValue("stats")
    )
)
<commit_msg>pipeline.py: bump version<commit_after># encoding=utf8
import datetime
from distutils.version import StrictVersion
import hashlib
import os.path
import random
from seesaw.config import realize, NumberConfigValue
from seesaw.item import ItemInterpolation, ItemValue
from seesaw.task import SimpleTask, LimitConcurrent
from seesaw.tracker import GetItemFromTracker, PrepareStatsForTracker, \
    UploadWithTracker, SendDoneToTracker
import shutil
import socket
import subprocess
import sys
import time

import seesaw
from seesaw.externalprocess import WgetDownload
from seesaw.pipeline import Pipeline
from seesaw.project import Project
from seesaw.util import find_executable

# check the seesaw version
if StrictVersion(seesaw.__version__) < StrictVersion("0.1.5"):
    raise Exception("This pipeline needs seesaw version 0.1.5 or higher.")

###########################################################################
# Find a useful Wget+Lua executable.
#
# WGET_LUA will be set to the first path that
# 1. does not crash with --version, and
# 2. prints the required version string
WGET_LUA = find_executable(
    "Wget+Lua",
    ["GNU Wget 1.14.lua.20130523-9a5c"],
    [
        "./wget-lua",
        "./wget-lua-warrior",
        "./wget-lua-local",
        "../wget-lua",
        "../../wget-lua",
        "/home/warrior/wget-lua",
        "/usr/bin/wget-lua"
    ]
)

if not WGET_LUA:
    raise Exception("No usable Wget+Lua found.")

###########################################################################
# The version number of this pipeline definition.
#
# Update this each time you make a non-cosmetic change.
# It will be added to the WARC files and reported to the tracker.
VERSION = "20140823.01"
USER_AGENT = 'ArchiveTeam'
TRACKER_ID = 'swipnet'
TRACKER_HOST = 'tracker.archiveteam.org'

###########################################################################
# This section defines project-specific tasks.
#
# Simple tasks (tasks that do not need any concurrency) are based on the
# SimpleTask class and have a process(item) method that is called for
# each item.
class CheckIP(SimpleTask):
    def __init__(self):
        SimpleTask.__init__(self, "CheckIP")
        self._counter = 0

    def process(self, item):
        # NEW for 2014! Check if we are behind firewall/proxy

        if self._counter <= 0:
            item.log_output('Checking IP address.')
            ip_set = set()

            ip_set.add(socket.gethostbyname('twitter.com'))
            ip_set.add(socket.gethostbyname('facebook.com'))
            ip_set.add(socket.gethostbyname('youtube.com'))
            ip_set.add(socket.gethostbyname('microsoft.com'))
            ip_set.add(socket.gethostbyname('icanhas.cheezburger.com'))
            ip_set.add(socket.gethostbyname('archiveteam.org'))

            if len(ip_set) != 6:
                item.log_output('Got IP addresses: {0}'.format(ip_set))
                item.log_output(
                    'Are you behind a firewall/proxy? That is a big no-no!')
                raise Exception(
                    'Are you behind a firewall/proxy? That is a big no-no!')

        # Check only occasionally
        if self._counter <= 0:
            self._counter = 10
        else:
            self._counter -= 1

class PrepareDirectories(SimpleTask):
    def __init__(self, warc_prefix):
        SimpleTask.__init__(self, "PrepareDirectories")
        self.warc_prefix = warc_prefix

    def process(self, item):
        item_name = item["item_name"]
        escaped_item_name = item_name.replace(':', '_').replace('/', '_').replace('~', '_')
        dirname = "/".join((item["data_dir"], escaped_item_name))

        if os.path.isdir(dirname):
            shutil.rmtree(dirname)

        os.makedirs(dirname)

        item["item_dir"] = dirname
        item["warc_file_base"] = "%s-%s-%s" % (self.warc_prefix, escaped_item_name,
            time.strftime("%Y%m%d-%H%M%S"))

        open("%(item_dir)s/%(warc_file_base)s.warc.gz" % item, "w").close()

class MoveFiles(SimpleTask):
    def __init__(self):
        SimpleTask.__init__(self, "MoveFiles")

    def process(self, item):
        # NEW for 2014! Check if wget was compiled with zlib support
        if os.path.exists("%(item_dir)s/%(warc_file_base)s.warc"):
            raise Exception('Please compile wget with zlib support!')

        os.rename("%(item_dir)s/%(warc_file_base)s.warc.gz" % item,
              "%(data_dir)s/%(warc_file_base)s.warc.gz" % item)

        shutil.rmtree("%(item_dir)s" % item)

def get_hash(filename):
    with open(filename, 'rb') as in_file:
        return hashlib.sha1(in_file.read()).hexdigest()

CWD = os.getcwd()
PIPELINE_SHA1 = get_hash(os.path.join(CWD, 'pipeline.py'))
LUA_SHA1 = get_hash(os.path.join(CWD, 'swipnet.lua'))

def stats_id_function(item):
    # NEW for 2014! Some accountability hashes and stats.
    d = {
        'pipeline_hash': PIPELINE_SHA1,
        'lua_hash': LUA_SHA1,
        'python_version': sys.version,
    }

    return d

class WgetArgs(object):
    def realize(self, item):
        wget_args = [
            WGET_LUA,
            "-U", USER_AGENT,
            "-nv",
            "--lua-script", "swipnet.lua",
            "-o", ItemInterpolation("%(item_dir)s/wget.log"),
            "--no-check-certificate",
            "--output-document", ItemInterpolation("%(item_dir)s/wget.tmp"),
            "--truncate-output",
            "-e", "robots=off",
            "--no-cookies",
            "--rotate-dns",
            "--recursive", "--level=inf",
            "--no-parent",
            "--page-requisites",
            "--timeout", "30",
            "--tries", "inf",
            "--span-hosts",
            "--waitretry", "30",
            "--domains", "swipnet.se",
            "--warc-file", ItemInterpolation("%(item_dir)s/%(warc_file_base)s"),
            "--warc-header", "operator: Archive Team",
            "--warc-header", "swipnet-dld-script-version: " + VERSION,
            "--warc-header", ItemInterpolation("swipnet-user: %(item_name)s"),
        ]

        item_name = item['item_name']
        assert ':' in item_name
        item_type, item_value = item_name.split(':', 1)

        assert item_type
        assert item_value

        item['item_type'] = item_type
        item['item_value'] = item_value

        wget_args.append('http://{0}.swipnet.se/{1}/'.format(item_type, item_value))

        # wget_args.append('http://home.swipnet.se/{0}/'.format(item_name))

        if 'bind_address' in globals():
            wget_args.extend(['--bind-address', globals()['bind_address']])
            print('')
            print('*** Wget will bind address at {0} ***'.format(
                globals()['bind_address']))
            print('')

        return realize(wget_args, item)

###########################################################################
# Initialize the project.
#
# This will be shown in the warrior management panel. The logo should not
# be too big. The deadline is optional.
project = Project(
    title="Swipnet",
    project_html="""

home.swipnet.se Website · Leaderboard

Archiving Swedish websites from home.swipnet.se.

    """,
    utc_deadline=datetime.datetime(2014, 8, 31, 23, 59, 0)
)

pipeline = Pipeline(
    CheckIP(),
    GetItemFromTracker("http://%s/%s" % (TRACKER_HOST, TRACKER_ID), downloader,
        VERSION),
    PrepareDirectories(warc_prefix="swipnet"),
    WgetDownload(
        WgetArgs(),
        max_tries=10,
#        max_tries=2,
        accept_on_exit_code=[0, 8],
        env={
            "item_dir": ItemValue("item_dir"),
            "item_value": ItemValue("item_value"),
            "item_type": ItemValue("item_type"),
        }
    ),
    PrepareStatsForTracker(
        defaults={"downloader": downloader, "version": VERSION},
        file_groups={
            "data": [
                ItemInterpolation("%(item_dir)s/%(warc_file_base)s.warc.gz")
            ]
        },
        id_function=stats_id_function,
    ),
    MoveFiles(),
    LimitConcurrent(NumberConfigValue(min=1, max=4, default="1",
        name="shared:rsync_threads", title="Rsync threads",
        description="The maximum number of concurrent uploads."),
        UploadWithTracker(
            "http://%s/%s" % (TRACKER_HOST, TRACKER_ID),
            downloader=downloader,
            version=VERSION,
            files=[
                ItemInterpolation("%(data_dir)s/%(warc_file_base)s.warc.gz")
            ],
            rsync_target_source_path=ItemInterpolation("%(data_dir)s/"),
            rsync_extra_args=[
                "--recursive",
                "--partial",
                "--partial-dir", ".rsync-tmp",
            ]
            ),
    ),
    SendDoneToTracker(
        tracker_url="http://%s/%s" % (TRACKER_HOST, TRACKER_ID),
        stats=ItemValue("stats")
    )
)
<|endoftext|>


<commit_before>#-------------------------------------------------------------------------------
import logging
import sys
import multiprocessing
import os
from os import path
import subprocess
import re
import time
import config
import database
import smtplib
from email.mime.text import MIMEText

sys.path.append(config.LAUNCHER_PATH)

try:
    import launcher
except ImportError:
    print "Error loading Launcher library"
    print "Check LAUNCHER_PATH at config file"
    exit(-1)

#-------------------------------------------------------------------------------
remotehost = config.REMOTEHOST
remotehome = config.REMOTEHOME
pipe_launch = config.PIPE_LAUNCH
data_dir = config.DATADIR

reSLURMLINE = re.compile(r"slurmids: (?P<slurmids>\d+(,\d+)*)")
reSLURMID = re.compile(r"\,")

#-------------------------------------------------------------------------------
def startJob( user, var1 ):
    jobid = database.createJob( user )

    p = multiprocessing.Process( target=runjob, args=(user, jobid, var1) )
    logging.debug( str(var1) )
    p.start()

#-------------------------------------------------------------------------------
def cancelJob( user, jobid ):
    logging.info( "CANCELING JOB %d", jobid )
    jobinfo = database.getJobInfo( jobid )
    logging.debug( str(jobinfo) )

    if jobinfo['state'] == database.JOB_COMPLETED or jobinfo['state'] == database.JOB_CANCELED:
        logging.warning( "job %d already canceled", jobid )
        return true

    # Canceling Jobid:
    logging.info( str(jobinfo['slurmids']) )
    for slurmid in jobinfo['slurmids']:
        logging.info( "canceling slurm job %d", slurmid["slurmid"] )

        command = ["ssh", remotehost, "mncancel", str(slurmid["slurmid"]) ]
        proc = subprocess.Popen( command, stdout=subprocess.PIPE )
        output = proc.communicate()[0]
        logging.info( str(output) )

    # Removing job folders:
        # Web and server job names should be corresponding before activating this
    # jname = jobinfo[ 'name' ]
    # print "Removing outputs from Job named: ", jname
    # command =  [ 'ssh', remotehost,
    #              'cd', data_dir + user,
    #              'rm -r', jname,
    #              'touch', "DONE"]
    # proc = subprocess.Popen( command, stdout=subprocess.PIPE )
    #output = proc.communicate()[0]

    database.setJobCanceled( jobid )
    return True

#-------------------------------------------------------------------------------
def getSlurmIds( output ):
    mm = reSLURMLINE.search( output )
    sids = []
    if mm:
        slurmline = mm.group('slurmids')
        slurmids = reSLURMID.split( slurmline )
        sids = map(int,slurmids)

    return sids

#-------------------------------------------------------------------------------
def runjob( user, jobid, var1):
    logging.info( "RUNNING JOB %d", jobid )

# stagein
    if "file" in var1:
        fileid1 = int(var1["file"])
        localfile1 = database.getFileFullName( fileid1 )
        remotefile1 = os.path.join( remotehome, localfile1 )
        database.addJobFile( jobid, fileid1, database.FILEIN )
        var1['file_read1'] = remotefile1

    if "file2" in var1:
        fileid2 = int(var1["file2"])
        localfile2 = database.getFileFullName( fileid2 )
        remotefile2 = os.path.join( remotehome, localfile2 )
        database.addJobFile( jobid, fileid2, database.FILEIN )
        var1['file_read2'] = remotefile2

    if "file3" in var1:
        fileid3 = int(var1["file3"])
        localfile3 = database.getFileFullName( fileid3 )
        remotefile3 = os.path.join( remotehome, localfile3 )
        database.addJobFile( jobid, fileid3, database.FILEIN )
        var1['file_ass'] = remotefile3

        # submit

    jinfo = database.getJobInfo( jobid )
    juid =  jinfo['juid']

    if var1["input_type"] == "single":
        command = [ pipe_launch, user, str(var1), remotefile1, str(juid) ]
    elif var1["input_type"] == "paired":
        command = [ pipe_launch, user, str(var1), remotefile1, remotefile2, str(juid) ]
    elif var1.input_type =="contigs":
        command = [ pipe_launch, user, str(var1), remotefile3, str(juid) ]
    elif var1.input_type =="contigs_with_single":
        command = [ pipe_launch, user, str(var1), remotefile1, remotefile3,
                    str(juid) ]
    elif var1.input_type =="contigs_with_paired":
        command = [ pipe_launch, user, str(var1), remotefile1, remotefile2,
                    remotefile3, str(juid) ]

    logging.debug( str(command) )
    logging.debug( str(var1) )

    proc = subprocess.Popen( command, stdout=subprocess.PIPE )
    output = proc.communicate()[0]

    slurmids = getSlurmIds( output )

    if len(slurmids) > 0:
        database.setJobSubmitted( jobid )
    else:
        logging.warning( "Task without slurm ids" )

    for si in slurmids:
        database.addJobSlurmRef( jobid, si )

#-------------------------------------------------------------------------------
def run():
    p = multiprocessing.Process( target=pipelineLoop )
    p.start()
    return p

#-------------------------------------------------------------------------------
def checkSlurmJob( slurmids ):
    if len(slurmids) > 0:
        idsstr = ",".join(map(str,slurmids))
        command = ["ssh", remotehost, "mnq", "--job", idsstr ]
        proc = subprocess.Popen( command, stdout=subprocess.PIPE )
        output = proc.communicate()[0]

        if len(output.splitlines()) <= 1:
            return database.JOB_COMPLETED

        mm = re.search( "RUNNING", output )
        if mm is not None:
            return database.JOB_RUNNING

    return database.JOB_SUBMITTED

#-------------------------------------------------------------------------------
def sendJobCompletedEmail(jobid, uid):

    usermail = database.getUserEmail(uid)
    jdata = database.getJobInfo(jobid)

    body = ""
    with open( config.EMAIL_JOB_COMPLETE, 'r' ) as f:
        body = f.read()

    msg = MIMEText( body.format(jdata['name'], jdata['juid']))

    msg['Subject'] = config.EMAIL_JOB_COMPLETE_SUBJECT.format(jdata['name'])
    msg['From'] = config.EMAIL_SENDER
    msg['To'] = usermail

    print msg

    s = smtplib.SMTP( config.EMAIL_SMTP )
    s.sendmail( config.EMAIL_SENDER, usermail, msg.as_string())
    s.quit()

#-------------------------------------------------------------------------------
def pipelineLoop():
    try:
        logging.info( "Start pipeline loop" )
        while (1 == 1):
            time.sleep( 60 )
            # check
            zjobs = database.getJustCreatedJobs()
            if len(zjobs) > 0:
                zjids = map( lambda js: js['jid'], zjobs )
                logging.warning( "%d zombie jobs: %s", len(zjobs), str(zjids) )

            jobs = database.getActiveJobs()
            if len(jobs) > 0:
                logging.info( "Checking %d job/s", len(jobs) )

            for job in jobs:
                jobid = job['jid']
                newstate = checkSlurmJob( job['slurmids'] )
                if newstate == database.JOB_RUNNING and job['state'] != database.JOB_RUNNING:
                    logging.info( "Job %d start RUNNING", jobid )
                    database.setJobRunning( jobid )

                if newstate == database.JOB_COMPLETED:
            #         userid = job['uid']
            #         slurmid = job['slurmid']
            #         # stageout
            #         outs = ["jor_"+str(slurmid)+".out", "jor_"+str(slurmid)+".err"]
            #         for fileoutname in outs:
            #             fileout = database.createFile( userid, fileoutname )
            #             database.addJobFile( jobid, fileout, database.FILEOUT )
            #             localfile = database.getFileFullName( fileout )
            #             (localdir, localbase) = os.path.split( localfile )
            #             remotedir = os.path.join( remotehome, localdir )
            #             remotefile = os.path.join( remotehome, localfile )
            #             os.system('scp "%s:%s" "%s"' % (remotehost, remotefile, localfile) )

                    logging.info( "Job %d COMPLETED", jobid )
                    database.setJobCompleted( jobid )

                    #sendJobCompletedEmail(job['jid'], job['uid'])

    except KeyboardInterrupt:
        logging.info( "Ending pipeline loop" )

#-------------------------------------------------------------------------------
<commit_msg>rename launcher VARS<commit_after>#-------------------------------------------------------------------------------
import logging
import sys
import multiprocessing
import os
from os import path
import subprocess
import re
import time
import config
import database
import smtplib
from email.mime.text import MIMEText

sys.path.append(config.LAUNCHER_LIB)

try:
    import launcher
except ImportError:
    print "Error loading Launcher library"
    print "Check LAUNCHER_LIB at config file"
    exit(-1)

#-------------------------------------------------------------------------------
remotehost = config.REMOTEHOST
remotehome = config.REMOTEHOME
pipe_launch = config.LAUNCHER_TOOL
data_dir = config.DATADIR

reSLURMLINE = re.compile(r"slurmids: (?P<slurmids>\d+(,\d+)*)")
reSLURMID = re.compile(r"\,")

#-------------------------------------------------------------------------------
def startJob( user, var1 ):
    jobid = database.createJob( user )

    p = multiprocessing.Process( target=runjob, args=(user, jobid, var1) )
    logging.debug( str(var1) )
    p.start()

#-------------------------------------------------------------------------------
def cancelJob( user, jobid ):
    logging.info( "CANCELING JOB %d", jobid )
    jobinfo = database.getJobInfo( jobid )
    logging.debug( str(jobinfo) )

    if jobinfo['state'] == database.JOB_COMPLETED or jobinfo['state'] == database.JOB_CANCELED:
        logging.warning( "job %d already canceled", jobid )
        return true

    # Canceling Jobid:
    logging.info( str(jobinfo['slurmids']) )
    for slurmid in jobinfo['slurmids']:
        logging.info( "canceling slurm job %d", slurmid["slurmid"] )

        command = ["ssh", remotehost, "mncancel", str(slurmid["slurmid"]) ]
        proc = subprocess.Popen( command, stdout=subprocess.PIPE )
        output = proc.communicate()[0]
        logging.info( str(output) )

    # Removing job folders:
        # Web and server job names should be corresponding before activating this
    # jname = jobinfo[ 'name' ]
    # print "Removing outputs from Job named: ", jname
    # command =  [ 'ssh', remotehost,
    #              'cd', data_dir + user,
    #              'rm -r', jname,
    #              'touch', "DONE"]
    # proc = subprocess.Popen( command, stdout=subprocess.PIPE )
    #output = proc.communicate()[0]

    database.setJobCanceled( jobid )
    return True

#-------------------------------------------------------------------------------
def getSlurmIds( output ):
    mm = reSLURMLINE.search( output )
    sids = []
    if mm:
        slurmline = mm.group('slurmids')
        slurmids = reSLURMID.split( slurmline )
        sids = map(int,slurmids)

    return sids

#-------------------------------------------------------------------------------
def runjob( user, jobid, var1):
    logging.info( "RUNNING JOB %d", jobid )

# stagein
    if "file" in var1:
        fileid1 = int(var1["file"])
        localfile1 = database.getFileFullName( fileid1 )
        remotefile1 = os.path.join( remotehome, localfile1 )
        database.addJobFile( jobid, fileid1, database.FILEIN )
        var1['file_read1'] = remotefile1

    if "file2" in var1:
        fileid2 = int(var1["file2"])
        localfile2 = database.getFileFullName( fileid2 )
        remotefile2 = os.path.join( remotehome, localfile2 )
        database.addJobFile( jobid, fileid2, database.FILEIN )
        var1['file_read2'] = remotefile2

    if "file3" in var1:
        fileid3 = int(var1["file3"])
        localfile3 = database.getFileFullName( fileid3 )
        remotefile3 = os.path.join( remotehome, localfile3 )
        database.addJobFile( jobid, fileid3, database.FILEIN )
        var1['file_ass'] = remotefile3

        # submit

    jinfo = database.getJobInfo( jobid )
    juid =  jinfo['juid']

    if var1["input_type"] == "single":
        command = [ pipe_launch, user, str(var1), remotefile1, str(juid) ]
    elif var1["input_type"] == "paired":
        command = [ pipe_launch, user, str(var1), remotefile1, remotefile2, str(juid) ]
    elif var1.input_type =="contigs":
        command = [ pipe_launch, user, str(var1), remotefile3, str(juid) ]
    elif var1.input_type =="contigs_with_single":
        command = [ pipe_launch, user, str(var1), remotefile1, remotefile3,
                    str(juid) ]
    elif var1.input_type =="contigs_with_paired":
        command = [ pipe_launch, user, str(var1), remotefile1, remotefile2,
                    remotefile3, str(juid) ]

    logging.debug( str(command) )
    logging.debug( str(var1) )

    proc = subprocess.Popen( command, stdout=subprocess.PIPE )
    output = proc.communicate()[0]

    slurmids = getSlurmIds( output )

    if len(slurmids) > 0:
        database.setJobSubmitted( jobid )
    else:
        logging.warning( "Task without slurm ids" )

    for si in slurmids:
        database.addJobSlurmRef( jobid, si )

#-------------------------------------------------------------------------------
def run():
    p = multiprocessing.Process( target=pipelineLoop )
    p.start()
    return p

#-------------------------------------------------------------------------------
def checkSlurmJob( slurmids ):
    if len(slurmids) > 0:
        idsstr = ",".join(map(str,slurmids))
        command = ["ssh", remotehost, "mnq", "--job", idsstr ]
        proc = subprocess.Popen( command, stdout=subprocess.PIPE )
        output = proc.communicate()[0]

        if len(output.splitlines()) <= 1:
            return database.JOB_COMPLETED

        mm = re.search( "RUNNING", output )
        if mm is not None:
            return database.JOB_RUNNING

    return database.JOB_SUBMITTED

#-------------------------------------------------------------------------------
def sendJobCompletedEmail(jobid, uid):

    usermail = database.getUserEmail(uid)
    jdata = database.getJobInfo(jobid)

    body = ""
    with open( config.EMAIL_JOB_COMPLETE, 'r' ) as f:
        body = f.read()

    msg = MIMEText( body.format(jdata['name'], jdata['juid']))

    msg['Subject'] = config.EMAIL_JOB_COMPLETE_SUBJECT.format(jdata['name'])
    msg['From'] = config.EMAIL_SENDER
    msg['To'] = usermail

    print msg

    s = smtplib.SMTP( config.EMAIL_SMTP )
    s.sendmail( config.EMAIL_SENDER, usermail, msg.as_string())
    s.quit()

#-------------------------------------------------------------------------------
def pipelineLoop():
    try:
        logging.info( "Start pipeline loop" )
        while (1 == 1):
            time.sleep( 60 )
            # check
            zjobs = database.getJustCreatedJobs()
            if len(zjobs) > 0:
                zjids = map( lambda js: js['jid'], zjobs )
                logging.warning( "%d zombie jobs: %s", len(zjobs), str(zjids) )

            jobs = database.getActiveJobs()
            if len(jobs) > 0:
                logging.info( "Checking %d job/s", len(jobs) )

            for job in jobs:
                jobid = job['jid']
                newstate = checkSlurmJob( job['slurmids'] )
                if newstate == database.JOB_RUNNING and job['state'] != database.JOB_RUNNING:
                    logging.info( "Job %d start RUNNING", jobid )
                    database.setJobRunning( jobid )

                if newstate == database.JOB_COMPLETED:
            #         userid = job['uid']
            #         slurmid = job['slurmid']
            #         # stageout
            #         outs = ["jor_"+str(slurmid)+".out", "jor_"+str(slurmid)+".err"]
            #         for fileoutname in outs:
            #             fileout = database.createFile( userid, fileoutname )
            #             database.addJobFile( jobid, fileout, database.FILEOUT )
            #             localfile = database.getFileFullName( fileout )
            #             (localdir, localbase) = os.path.split( localfile )
            #             remotedir = os.path.join( remotehome, localdir )
            #             remotefile = os.path.join( remotehome, localfile )
            #             os.system('scp "%s:%s" "%s"' % (remotehost, remotefile, localfile) )

                    logging.info( "Job %d COMPLETED", jobid )
                    database.setJobCompleted( jobid )

                    #sendJobCompletedEmail(job['jid'], job['uid'])

    except KeyboardInterrupt:
        logging.info( "Ending pipeline loop" )

#-------------------------------------------------------------------------------
<|endoftext|>


<commit_before>import luigi

import pandas as pd

from os.path import splitext, basename, expanduser
import subprocess
import json

DECOY_ADDUCTS = ("+He,+Li,+Be,+B,+C,+N,+O,+F,+Ne,+Mg,+Al,+Si,+P,"
                 "+S,+Cl,+Ar,+Ca,+Sc,+Ti,+V,+Cr,+Mn,+Fe,+Co,+Ni,"
                 "+Cu,+Zn,+Ga,+Ge,+As,+Se,+Br,+Kr,+Rb,+Sr,+Y,+Zr,"
                 "+Nb,+Mo,+Ru,+Rh,+Pd,+Ag,+Cd,+In,+Sn,+Sb,+Te,+I,"
                 "+Xe,+Cs,+Ba,+La,+Ce,+Pr,+Nd,+Sm,+Eu,+Gd,+Tb,+Dy,"
                 "+Ho,+Ir,+Th,+Pt,+Os,+Yb,+Lu,+Bi,+Pb,+Re,+Tl,+Tm,"
                 "+U,+W,+Au,+Er,+Hf,+Hg,+Ta").split(",")

class AdductListParameter(luigi.Parameter):
    def parse(self, arguments):
        return tuple(arguments.split(','))

class CreateIsotopeDB(luigi.Task):
    instrument = luigi.DictParameter()
    molecular_db = luigi.DictParameter()

    def _output_filename(self):
        db_name = splitext(basename(self.molecular_db['sum_formulas_fn']))[0]
        fn = "{}_{}_{}.db".format(db_name,
                                  self.instrument['type'],
                                  self.instrument['res200'])
        if self.molecular_db['is_decoy']:
            fn += ".decoy"
        return fn

    def output(self):
        return luigi.LocalTarget(self._output_filename())

    def run(self):
        cmd = ['ims', 'isocalc',
               '--instrument', self.instrument['type'],
               '--resolution', str(self.instrument['res200']),
               '--adducts', ",".join(self.molecular_db['adducts']),
               self.molecular_db['sum_formulas_fn'],
               self._output_filename()]
        subprocess.check_call(cmd)

class ConvertImzMLToImzb(luigi.Task):
    imzml_fn = luigi.Parameter()

    def _output_filename(self):
        return splitext(self.imzml_fn)[0] + ".imzb"

    def output(self):
        return luigi.LocalTarget(self._output_filename())

    def run(self):
        cmd = ['ims', 'convert',
               self.imzml_fn, self._output_filename()]
        subprocess.check_call(cmd)

class RunAnnotation(luigi.Task):
    imzml_fn = luigi.Parameter()
    instrument = luigi.DictParameter()
    molecular_db = luigi.DictParameter()

    def requires(self):
        return {
            'imzb': ConvertImzMLToImzb(self.imzml_fn),
            'isotope_db': CreateIsotopeDB(self.instrument, self.molecular_db),
        }

    def _output_filename(self):
        db_id = 'decoy' if self.molecular_db['is_decoy'] else 'target'
        return splitext(basename(self.imzml_fn))[0] + ".results." + db_id

    def output(self):
        return luigi.LocalTarget(self._output_filename())

    def run(self):
        cmd = ['ims', 'detect',
               self.input()['isotope_db'].fn,
               self.input()['imzb'].fn]
        with open(self._output_filename(), "w+") as f:
            subprocess.check_call(cmd, stdout=f)

class GetAnnotationsForAdduct(luigi.Task):
    imzml_fn = luigi.Parameter()
    instrument = luigi.DictParameter()
    molecular_db = luigi.DictParameter()
    adduct = luigi.Parameter()  # empty means annotations for all adducts

    def requires(self):
        return RunAnnotation(self.imzml_fn, self.instrument, self.molecular_db)

    def _output_filename(self):
        if self.adduct:
            return self.input().fn + "_" + self.adduct
        else:
            return self.input().fn

    def output(self):
        return luigi.LocalTarget(self._output_filename())

    def run(self):
        if self.adduct:
            df = pd.read_csv(self.input().fn).fillna(0)
            df[df['adduct'] == self.adduct].to_csv(self._output_filename(), index=False)
        else:
            return self.input()

class ComputeFDR(luigi.Task):
    imzml_fn = luigi.Parameter()
    instrument = luigi.DictParameter()
    molecular_db = luigi.DictParameter()

    adduct = luigi.Parameter('')
    groundtruth_fn = luigi.Parameter('')

    def requires(self):
        return {
            'target_results': GetAnnotationsForAdduct(self.imzml_fn, self.instrument, self.molecular_db, self.adduct),
            'decoy_results': RunAnnotation(self.imzml_fn, self.instrument, self._decoy_molecular_db())
        }

    def _output_filename(self):
        prefix = splitext(basename(self.imzml_fn))[0]
        if self.adduct:
            return prefix + "_" + self.adduct + ".fdr"
        else:
            return prefix + ".fdr"

    def _decoy_adducts(self):
        result = set()
        result.update(DECOY_ADDUCTS)

        for adduct in self.molecular_db['adducts']:
            if adduct in DECOY_ADDUCTS:
                result.remove(adduct)

        return list(result)

    def _decoy_molecular_db(self):
        return {
            'sum_formulas_fn': self.molecular_db['sum_formulas_fn'],
            'adducts': self._decoy_adducts(),
            'is_decoy': True
        }

    def output(self):
        return luigi.LocalTarget(self._output_filename())

    def run(self):
        cmd = ['ims', 'fdr',
               self.input()['target_results'].fn,
               self.input()['decoy_results'].fn]
        if self.groundtruth_fn:
            cmd += ['--groundtruth', self.groundtruth_fn]
        with open(self._output_filename(), "w+") as f:
            subprocess.check_call(cmd, stdout=f, stderr=subprocess.STDOUT)

class RunFullPipeline(luigi.WrapperTask):
    config_fn = luigi.Parameter()

    def requires(self):
        with open(self.config_fn) as conf:
            self.config = json.load(conf)

        imzml = expanduser(self.config['imzml'])
        instrument = self.config['instrument']
        db = self.config['database']
        db['sum_formulas_fn'] = expanduser(db['sum_formulas'])
        db['is_decoy'] = False

        return [ComputeFDR(imzml, instrument, db, adduct)
                for adduct in db['adducts']]

if __name__ == '__main__':
    luigi.run()
<commit_msg>use hashes of db parameter to form filenames<commit_after>import luigi

import pandas as pd

from os.path import splitext, basename, expanduser
import subprocess
import json

DECOY_ADDUCTS = ("+He,+Li,+Be,+B,+C,+N,+O,+F,+Ne,+Mg,+Al,+Si,+P,"
                 "+S,+Cl,+Ar,+Ca,+Sc,+Ti,+V,+Cr,+Mn,+Fe,+Co,+Ni,"
                 "+Cu,+Zn,+Ga,+Ge,+As,+Se,+Br,+Kr,+Rb,+Sr,+Y,+Zr,"
                 "+Nb,+Mo,+Ru,+Rh,+Pd,+Ag,+Cd,+In,+Sn,+Sb,+Te,+I,"
                 "+Xe,+Cs,+Ba,+La,+Ce,+Pr,+Nd,+Sm,+Eu,+Gd,+Tb,+Dy,"
                 "+Ho,+Ir,+Th,+Pt,+Os,+Yb,+Lu,+Bi,+Pb,+Re,+Tl,+Tm,"
                 "+U,+W,+Au,+Er,+Hf,+Hg,+Ta").split(",")

class AdductListParameter(luigi.Parameter):
    def parse(self, arguments):
        return tuple(arguments.split(','))

def get_id(obj):
    return str(abs(hash(obj)))[:8]

class CreateIsotopeDB(luigi.Task):
    instrument = luigi.DictParameter()
    molecular_db = luigi.DictParameter()

    def _output_filename(self):
        db_name = splitext(basename(self.molecular_db['sum_formulas_fn']))[0]
        fn = "{}_{}_{}_{}.db".format(db_name,
                                     self.instrument['type'],
                                     self.instrument['res200'],
                                     get_id(self.molecular_db))
        if self.molecular_db['is_decoy']:
            fn += ".decoy"
        return fn

    def output(self):
        return luigi.LocalTarget(self._output_filename())

    def run(self):
        cmd = ['ims', 'isocalc',
               '--instrument', self.instrument['type'],
               '--resolution', str(self.instrument['res200']),
               '--adducts', ",".join(self.molecular_db['adducts']),
               self.molecular_db['sum_formulas_fn'],
               self._output_filename()]
        subprocess.check_call(cmd)

class ConvertImzMLToImzb(luigi.Task):
    imzml_fn = luigi.Parameter()

    def _output_filename(self):
        return splitext(self.imzml_fn)[0] + ".imzb"

    def output(self):
        return luigi.LocalTarget(self._output_filename())

    def run(self):
        cmd = ['ims', 'convert',
               self.imzml_fn, self._output_filename()]
        subprocess.check_call(cmd)

class RunAnnotation(luigi.Task):
    imzml_fn = luigi.Parameter()
    instrument = luigi.DictParameter()
    molecular_db = luigi.DictParameter()

    def requires(self):
        return {
            'imzb': ConvertImzMLToImzb(self.imzml_fn),
            'isotope_db': CreateIsotopeDB(self.instrument, self.molecular_db),
        }

    def _output_filename(self):
        db_id = get_id(self.molecular_db) + "."
        db_id += 'decoy' if self.molecular_db['is_decoy'] else 'target'
        return splitext(basename(self.imzml_fn))[0] + ".results." + db_id

    def output(self):
        return luigi.LocalTarget(self._output_filename())

    def run(self):
        cmd = ['ims', 'detect',
               self.input()['isotope_db'].fn,
               self.input()['imzb'].fn]
        with open(self._output_filename(), "w+") as f:
            subprocess.check_call(cmd, stdout=f)

class GetAnnotationsForAdduct(luigi.Task):
    imzml_fn = luigi.Parameter()
    instrument = luigi.DictParameter()
    molecular_db = luigi.DictParameter()
    adduct = luigi.Parameter()  # empty means annotations for all adducts

    def requires(self):
        return RunAnnotation(self.imzml_fn, self.instrument, self.molecular_db)

    def _output_filename(self):
        if self.adduct:
            return self.input().fn + "_" + self.adduct
        else:
            return self.input().fn

    def output(self):
        return luigi.LocalTarget(self._output_filename())

    def run(self):
        if self.adduct:
            df = pd.read_csv(self.input().fn).fillna(0)
            df[df['adduct'] == self.adduct].to_csv(self._output_filename(), index=False)
        else:
            return self.input()

class ComputeFDR(luigi.Task):
    imzml_fn = luigi.Parameter()
    instrument = luigi.DictParameter()
    molecular_db = luigi.DictParameter()

    adduct = luigi.Parameter('')
    groundtruth_fn = luigi.Parameter('')

    def requires(self):
        return {
            'target_results': GetAnnotationsForAdduct(self.imzml_fn, self.instrument, self.molecular_db, self.adduct),
            'decoy_results': RunAnnotation(self.imzml_fn, self.instrument, self._decoy_molecular_db())
        }

    def _output_filename(self):
        prefix = splitext(self.input()['decoy_results'].fn)[0] + "_" + get_id(self.molecular_db)
        if self.adduct:
            return prefix + "_" + self.adduct + ".fdr"
        else:
            return prefix + ".fdr"

    def _decoy_adducts(self):
        result = set()
        result.update(DECOY_ADDUCTS)

        for adduct in self.molecular_db['adducts']:
            if adduct in DECOY_ADDUCTS:
                result.remove(adduct)

        return tuple(list(result))

    def _decoy_molecular_db(self):
        return luigi.parameter.FrozenOrderedDict([
            ('sum_formulas_fn', self.molecular_db['sum_formulas_fn']),
            ('adducts', self._decoy_adducts()),
            ('is_decoy', True)
        ])

    def output(self):
        return luigi.LocalTarget(self._output_filename())

    def run(self):
        cmd = ['ims', 'fdr',
               self.input()['target_results'].fn,
               self.input()['decoy_results'].fn]
        if self.groundtruth_fn:
            cmd += ['--groundtruth', self.groundtruth_fn]
        with open(self._output_filename(), "w+") as f:
            subprocess.check_call(cmd, stdout=f, stderr=subprocess.STDOUT)

class RunFullPipeline(luigi.WrapperTask):
    config = luigi.DictParameter()

    def requires(self):
        imzml = self.config['imzml']
        instrument = self.config['instrument']
        db = self.config['database']

        return [ComputeFDR(imzml, instrument, db, adduct)
                for adduct in db['adducts']]

def readConfig(filename):
    with open(filename) as conf:
        config = json.load(conf, object_pairs_hook=luigi.parameter.FrozenOrderedDict)

    cfg = config.get_wrapped()
    cfg['imzml'] = expanduser(config['imzml'])
    cfg['database'].get_wrapped()['sum_formulas_fn'] = expanduser(config['database']['sum_formulas'])
    cfg['database'].get_wrapped()['adducts'] = tuple(cfg['database']['adducts'])
    cfg['database'].get_wrapped()['is_decoy'] = False
    return config

if __name__ == '__main__':
    import sys
    config = readConfig(sys.argv[1])
    luigi.build([RunFullPipeline(config)], scheduler_port=8083)
<|endoftext|>