<commit_before># -*- encoding: utf-8 -*-

from __future__ import unicode_literals
import re

rg_re = re.compile('(https?://www\.researchgate\.net/)(.*)(publication/[0-9]*)_.*/links/[0-9a-f]*.pdf')

# This section defines a priority order on the links retrieved from APIs
domain_priority = {
        'doi.org': 50,                # Links to the publisher's version in most of the cases
        'dx.doi.org': 50,                # Links to the publisher's version in most of the cases
        'ncbi.nlm.nih.gov': 40, # PubMed or PubMed Central: official version too
        'arxiv.org' : 30,        # Curated repository
        'hdl.handle.net': 20,        # Institutional repositories
        'citeseerx.ist.psu.edu': 10, # Preprints crawled on the web
}
# Academia.edu and ResearchGate are not ranked here, they are at an equal (lowest) priority
domain_blacklist = [
    'www.researchgate.net',
    # Publisher links are redundant with DOI links and often become inaccessible.
    'aacrjournals.org',
    'aanda.org',
    'aappublications.org',
    'ahajournals.org',
    'amjbot.org',
    'ams.org',
    'annals.org',
    'annualreviews.org',
    'asm.org',
    'aspetjournals.org',
    'babel.hathitrust.org',
    'biochemsoctrans.org',
    'biologists.org',
    'bioone.org',
    'bloodjournal.org',
    'cambridge.org',
    'cell.com',
    'cshlp.org',
    'degruyter.com',
    'diabetesjournals.org',
    'dl.acm.org',
    'doaj.org',
    'doi.org',
    'ersjournals.com',
    'erudit.org',
    'iop.org',
    'jamanetwork.com',
    'jbc.org',
    'jimmunol.org',
    'jneurosci.org',
    'journals.ametsoc.org',
    'journals.lww.com',
    'journals.sagepub.com',
    'journals.uchicago.edu',
    'jwildlifedis.org',
    'karger.com',
    'link.aps.org',
    'link.springer.com',
    'microbiologyresearch.org',
    'nature.com',
    'nejm.org',
    'neurology.org',
    'nrcresearchpress.com',
    'onlinelibrary.wiley.com',
    'oup.com',
    'physiology.org',
    'pubs.acs.org',
    'pubs.aeaweb.org',
    'reproduction-online.org',
    'royalsocietypublishing.org',
    'rsc.org',
    'sciencedirect.com',
    'sciencemag.org',
    'scitation.org',
    'tandfonline.com',
    'thelancet.com',
    'thieme-connect.de',
]

domain_re = re.compile(r'\s*(https?|ftp)://(([a-zA-Z0-9-_]+\.)+[a-zA-Z]+)(:[0-9]+)?/?')
def extract_domain(url):
    match = domain_re.match(url)
    if match:
        return match.group(2)

def link_rank(url):
    if 'www.ncbi.nlm.nih.gov/pubmed/' in url or 'europepmc.org/abstract/med/' in url:
        return 20
    return (- domain_priority.get(extract_domain(url), 0))

def sort_links(urls):
    return sorted(urls, key=link_rank)

def is_blacklisted(url):
    if '/accesoRestringido.pdf' in url:
        return True
    subdomain = extract_domain(url)
    for domain in domain_blacklist:
        if domain in subdomain:
            return True
    return False
<commit_msg>Add some more publishers coming up in suggestions<commit_after># -*- encoding: utf-8 -*-

from __future__ import unicode_literals
import re

rg_re = re.compile('(https?://www\.researchgate\.net/)(.*)(publication/[0-9]*)_.*/links/[0-9a-f]*.pdf')

# This section defines a priority order on the links retrieved from APIs
domain_priority = {
        'doi.org': 50,                # Links to the publisher's version in most of the cases
        'dx.doi.org': 50,                # Links to the publisher's version in most of the cases
        'ncbi.nlm.nih.gov': 40, # PubMed or PubMed Central: official version too
        'arxiv.org' : 30,        # Curated repository
        'hdl.handle.net': 20,        # Institutional repositories
        'citeseerx.ist.psu.edu': 10, # Preprints crawled on the web
}
# Academia.edu and ResearchGate are not ranked here, they are at an equal (lowest) priority
domain_blacklist = [
    'www.researchgate.net',
    # Publisher links are redundant with DOI links and often become inaccessible.
    'aacrjournals.org',
    'aanda.org',
    'aappublications.org',
    'ahajournals.org',
    'amjbot.org',
    'ams.org',
    'annals.org',
    'annualreviews.org',
    'asm.org',
    'aspetjournals.org',
    'babel.hathitrust.org',
    'biochemsoctrans.org',
    'biologists.org',
    'bioone.org',
    'bloodjournal.org',
    'cambridge.org',
    'cell.com',
    'cshlp.org',
    'degruyter.com',
    'diabetesjournals.org',
    'dl.acm.org',
    'doaj.org',
    'doi.org',
    'ersjournals.com',
    'erudit.org',
    'iop.org',
    'jamanetwork.com',
    'jbc.org',
    'jimmunol.org',
    'jneurosci.org',
    'journals.ametsoc.org',
    'journals.iucr.org',
    'journals.lww.com',
    'journals.sagepub.com',
    'journals.uchicago.edu',
    'jwildlifedis.org',
    'karger.com',
    'link.aps.org',
    'link.springer.com',
    'microbiologyresearch.org',
    'mdpi.com',
    'nature.com',
    'nejm.org',
    'neurology.org',
    'nrcresearchpress.com',
    'onlinelibrary.wiley.com',
    'oup.com',
    'parasite-journal.org',
    'physiology.org',
    'plos.org',
    'plosone.org',
    'pubs.acs.org',
    'pubs.aeaweb.org',
    'reproduction-online.org',
    'royalsocietypublishing.org',
    'rsc.org',
    'sciencedirect.com',
    'sciencemag.org',
    'scitation.org',
    'tandfonline.com',
    'thelancet.com',
    'thieme-connect.de',
]

domain_re = re.compile(r'\s*(https?|ftp)://(([a-zA-Z0-9-_]+\.)+[a-zA-Z]+)(:[0-9]+)?/?')
def extract_domain(url):
    match = domain_re.match(url)
    if match:
        return match.group(2)

def link_rank(url):
    if 'www.ncbi.nlm.nih.gov/pubmed/' in url or 'europepmc.org/abstract/med/' in url:
        return 20
    return (- domain_priority.get(extract_domain(url), 0))

def sort_links(urls):
    return sorted(urls, key=link_rank)

def is_blacklisted(url):
    if '/accesoRestringido.pdf' in url:
        return True
    subdomain = extract_domain(url)
    for domain in domain_blacklist:
        if domain in subdomain:
            return True
    return False
<|endoftext|>


<commit_before>import random

class ChooseModule:
	def __init__(self, circa):
		self.circa = circa
		self.events = {
			"cmd.roll": [self.roll],
			"message": [self.dotchoose],
			"cmd.choose": [self.choose]
		}
		self.docs = {
			"choose": "choose <option>, <option>, ... → choose a random <option>",
			"roll": "roll [[a]d[b]] → roll 'a' 'b'-sided dice (default 1d6)"
		}

	def roll(self, fr, to, msg, m):
		a, b = 1, 6
		msg = msg.strip().split(" ", 1)
		if msg:
			l, r = msg[0].split("d")
			try:
				a = int(l)
				b = int(r)
			except ValueError:
				pass
		values = [str(random.randint(1, b)) for i in range(a)]
		self.circa.say(to, fr + ": " + ", ".join(values))

	def choose(self, fr, to, msg, m):
		opts = [opt.strip() for opt in msg.split(",")]
		self.circa.say(to, fr + ": " + random.choice(opts))

	def dotchoose(self, fr, to, msg, m):
		target = fr if to == self.circa.nick else to
		if msg.startswith(".choose"):
			self.choose(fr, target, msg[len(".choose"):], m)

module = ChooseModule
<commit_msg>don't say anything if 0 options<commit_after>import random

class ChooseModule:
	def __init__(self, circa):
		self.circa = circa
		self.events = {
			"cmd.roll": [self.roll],
			"message": [self.dotchoose],
			"cmd.choose": [self.choose]
		}
		self.docs = {
			"choose": "choose <option>, <option>, ... → choose a random <option>",
			"roll": "roll [[a]d[b]] → roll 'a' 'b'-sided dice (default 1d6)"
		}

	def roll(self, fr, to, msg, m):
		a, b = 1, 6
		msg = msg.strip().split(" ", 1)
		if msg:
			l, r = msg[0].split("d")
			try:
				a = int(l)
				b = int(r)
			except ValueError:
				pass
		values = [str(random.randint(1, b)) for i in range(a)]
		self.circa.say(to, fr + ": " + ", ".join(values))

	def choose(self, fr, to, msg, m):
		opts = [opt.strip() for opt in msg.split(",")]
		if len(opts) == 0:
			return
		self.circa.say(to, fr + ": " + random.choice(opts))

	def dotchoose(self, fr, to, msg, m):
		target = fr if to == self.circa.nick else to
		if msg.startswith(".choose"):
			self.choose(fr, target, msg[len(".choose"):], m)

module = ChooseModule
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import cherrypy
import htpc
import requests
from json import dumps
import logging
from cherrypy.lib.auth2 import require, member_of
from htpc.helpers import fix_basepath, striphttp

class Deluge(object):
    session = requests.Session()

    def __init__(self):
        self.logger = logging.getLogger('modules.deluge')
        htpc.MODULES.append({
            'name': 'Deluge',
            'id': 'deluge',
            'test': htpc.WEBDIR + 'deluge/ping',
            'fields': [
                {'type': 'bool', 'label': 'Enable', 'name': 'deluge_enable'},
                {'type': 'text', 'label': 'Menu name', 'name': 'deluge_name'},
                {'type': 'text', 'label': 'IP / Host *', 'name': 'deluge_host'},
                {'type': 'text', 'label': 'Port *', 'name': 'deluge_port'},
                {'type': 'bool', 'label': 'Use SSL', 'name': 'deluge_ssl'},
                {'type': 'text', 'label': 'Basepath', 'name': 'deluge_basepath'},
                {'type': 'password', 'label': 'Password', 'name': 'deluge_password'},
                {"type": "text", "label": "Reverse proxy link", "placeholder": "", "desc": "Reverse proxy link ex: https://deluge.domain.com", "name": "deluge_reverse_proxy_link"}

            ]
        })

    @cherrypy.expose()
    @require()
    def index(self):
        return htpc.LOOKUP.get_template('deluge.html').render(scriptname='deluge', webinterface=self.webinterface())

    def webinterface(self):
        host = striphttp(htpc.settings.get('deluge_host', ''))
        port = str(htpc.settings.get('deluge_port', ''))
        deluge_basepath = fix_basepath(htpc.settings.get('deluge_basepath', ''))
        ssl = 's' if htpc.settings.get('deluge_ssl') else ''

        url = 'http%s://%s:%s%s' % (ssl, host, port, deluge_basepath)

        if htpc.settings.get('deluge_reverse_proxy_link'):
            url = htpc.settings.get('deluge_reverse_proxy_link')

        return url

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def connected(self):
        return self.fetch('web.connected')

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def connect(self, hostid):
        return self.fetch('web.connect', [hostid])

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def get_hosts(self):
        return self.fetch('web.get_hosts')

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def queue(self):
        fields = ['progress', 'is_finished', 'ratio', 'name', 'download_payload_rate',
                  'upload_payload_rate', 'eta', 'state', 'hash', 'total_size']

        return self.fetch('core.get_torrents_status', [[], fields])

    def q2(self):
        """ not in use atm, todo """
        par = ["queue", "name", "total_wanted", "state", "progress", "num_seeds",
               "total_seeds", "num_peers", "total_peers", "download_payload_rate",
               "upload_payload_rate", "eta", "ratio", "distributed_copies", "is_auto_managed",
               "time_added", "tracker_host", "save_path", "total_done", "total_uploaded",
               "max_download_speed", "max_upload_speed", "seeds_peers_ratio"]

        return self.fetch('web.update_ui', [par, {}])

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def status(self):
        ''' quick  '''
        results = self.fetch('web.update_ui', [['payload_upload_rate', 'payload_download_rate, state'], {}])
        if results['error'] is None:
            # py. 2.6..
            d = dict(tuple(results['result']['filters']['state']))
            results['result']['filters']['state'] = d

        return results

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def stats(self):
        fields = ['payload_upload_rate', 'payload_download_rate, state']
        return self.fetch('core.get_session_status', [fields])

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def start(self, torrentId):
        return self.fetch('core.resume_torrent', [[torrentId]])

    @cherrypy.expose()
    @require(member_of(htpc.role_user))
    @cherrypy.tools.json_out()
    def stop(self, torrentId=None):
        return self.fetch('core.pause_torrent', [[torrentId]])

    @cherrypy.expose()
    @require(member_of(htpc.role_user))
    @cherrypy.tools.json_out()
    def do_all(self, status):
        if status == 'resume':
            method = 'core.resume_all_torrents'
        else:
            method = 'core.pause_all_torrents'

        return self.fetch(method)

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def daemon(self, status, port):
        if status == 'start':
            action = 'web.start_daemon'
        else:
            action = 'web.stop_daemon'
        return self.fetch(action, [int(port)])

    @cherrypy.expose()
    @require(member_of(htpc.role_user))
    @cherrypy.tools.json_out()
    def set_dlspeed(self, speed):
        self.logger.debug('Set download speed to %s' % speed)
        if speed == '0':
            speed = -1
        return self.fetch('core.set_config', [{'max_download_speed': int(speed)}])

    @cherrypy.expose()
    @require(member_of(htpc.role_user))
    @cherrypy.tools.json_out()
    def set_ulspeed(self, speed):
        if speed == '0':
            speed = -1
        self.logger.debug('Set upload speed to %s' % speed)

        return self.fetch('core.set_config', [{'max_upload_speed': int(speed)}])

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def addtorrent(self, torrent, filename=''):
        result = self.fetch('core.add_torrent_file', [filename, torrent, {}])
        return result

    '''
    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def getconfig(self):
        #should be removed
        return self.fetch('core.get_config')
    '''

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def get_speed(self):
        ''' speed limit '''
        result = self.fetch('core.get_config')
        # Dunno why the f, core.get_config_values didnt work...
        d = {}
        if result:
            d['max_download_speed'] = result['result']['max_download_speed']
            d['max_upload_speed'] = result['result']['max_upload_speed']
            result['result'] = d
            return result

    @cherrypy.expose()
    @require(member_of(htpc.role_user))
    @cherrypy.tools.json_out()
    def remove(self, torrentId, removeData):
        removeDataBool = bool(int(removeData))
        return self.fetch('core.remove_torrent', [torrentId, removeDataBool])

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def to_client(self, link='', torrentname='', **kwargs):
        try:
            self.logger.info('Added %s to deluge' % torrentname)
            # Find download path
            download_path = self.fetch('core.get_config_value', ['download_location'])
            if link.startswith('magnet'):
                path = link
            else:
                # deluge doesnt like a named download...
                link = link.split('?title=')[0]
                get_url = self.fetch('web.download_torrent_from_url', [link])
                path = get_url['result']

            return self.fetch('web.add_torrents', [[{'path': path, 'options': {'download_location': download_path['result']}}]])

        except Exception as e:
            self.logger.debug('Failed adding %s to deluge %s %s' % (torrentname, link, e))

    def fetch(self, method, arguments=None):
        """ Do request to Deluge api """
        if arguments is None:
            arguments = []

        host = striphttp(htpc.settings.get('deluge_host', ''))
        port = htpc.settings.get('deluge_port', '')
        deluge_basepath = fix_basepath(htpc.settings.get('deluge_basepath', '/'))
        ssl = 's' if htpc.settings.get('deluge_ssl') else ''

        url = 'http%s://%s:%s%sjson' % (ssl, host, port, deluge_basepath)

        self.logger.debug("Request deluge method: %s arguments %s" % (method, arguments))
        try:
            # format post data
            data = {'id': 1, 'method': method, 'params': arguments}
            headers = {'Content-Type': 'application/json'}

            response = self.session.post(url, data=dumps(data), headers=headers, verify=False)
            result = response.json()
            if result and result['error']:
                self.logger.debug('Authenticating')
                self.session.post(url, data=dumps({"method": "auth.login", "params": [htpc.settings.get('deluge_password', '')], "id": 1}), verify=False)
                response = self.session.post(url, data=dumps(data), verify=False)

            return result
        except Exception as e:
            self.logger.error('Failed to fetch method %s  arguments %s %s' % (method, arguments, e))
<commit_msg>add headers to all requests in deluge.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import cherrypy
import htpc
import requests
from json import dumps
import logging
from cherrypy.lib.auth2 import require, member_of
from htpc.helpers import fix_basepath, striphttp

class Deluge(object):
    session = requests.Session()

    def __init__(self):
        self.logger = logging.getLogger('modules.deluge')
        htpc.MODULES.append({
            'name': 'Deluge',
            'id': 'deluge',
            'test': htpc.WEBDIR + 'deluge/ping',
            'fields': [
                {'type': 'bool', 'label': 'Enable', 'name': 'deluge_enable'},
                {'type': 'text', 'label': 'Menu name', 'name': 'deluge_name'},
                {'type': 'text', 'label': 'IP / Host *', 'name': 'deluge_host'},
                {'type': 'text', 'label': 'Port *', 'name': 'deluge_port'},
                {'type': 'bool', 'label': 'Use SSL', 'name': 'deluge_ssl'},
                {'type': 'text', 'label': 'Basepath', 'name': 'deluge_basepath'},
                {'type': 'password', 'label': 'Password', 'name': 'deluge_password'},
                {"type": "text", "label": "Reverse proxy link", "placeholder": "", "desc": "Reverse proxy link ex: https://deluge.domain.com", "name": "deluge_reverse_proxy_link"}

            ]
        })

    @cherrypy.expose()
    @require()
    def index(self):
        return htpc.LOOKUP.get_template('deluge.html').render(scriptname='deluge', webinterface=self.webinterface())

    def webinterface(self):
        host = striphttp(htpc.settings.get('deluge_host', ''))
        port = str(htpc.settings.get('deluge_port', ''))
        deluge_basepath = fix_basepath(htpc.settings.get('deluge_basepath', ''))
        ssl = 's' if htpc.settings.get('deluge_ssl') else ''

        url = 'http%s://%s:%s%s' % (ssl, host, port, deluge_basepath)

        if htpc.settings.get('deluge_reverse_proxy_link'):
            url = htpc.settings.get('deluge_reverse_proxy_link')

        return url

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def connected(self):
        return self.fetch('web.connected')

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def connect(self, hostid):
        return self.fetch('web.connect', [hostid])

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def get_hosts(self):
        return self.fetch('web.get_hosts')

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def queue(self):
        fields = ['progress', 'is_finished', 'ratio', 'name', 'download_payload_rate',
                  'upload_payload_rate', 'eta', 'state', 'hash', 'total_size']

        return self.fetch('core.get_torrents_status', [[], fields])

    def q2(self):
        """ not in use atm, todo """
        par = ["queue", "name", "total_wanted", "state", "progress", "num_seeds",
               "total_seeds", "num_peers", "total_peers", "download_payload_rate",
               "upload_payload_rate", "eta", "ratio", "distributed_copies", "is_auto_managed",
               "time_added", "tracker_host", "save_path", "total_done", "total_uploaded",
               "max_download_speed", "max_upload_speed", "seeds_peers_ratio"]

        return self.fetch('web.update_ui', [par, {}])

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def status(self):
        ''' quick  '''
        results = self.fetch('web.update_ui', [['payload_upload_rate', 'payload_download_rate, state'], {}])
        if results['error'] is None:
            # py. 2.6..
            d = dict(tuple(results['result']['filters']['state']))
            results['result']['filters']['state'] = d

        return results

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def stats(self):
        fields = ['payload_upload_rate', 'payload_download_rate, state']
        return self.fetch('core.get_session_status', [fields])

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def start(self, torrentId):
        return self.fetch('core.resume_torrent', [[torrentId]])

    @cherrypy.expose()
    @require(member_of(htpc.role_user))
    @cherrypy.tools.json_out()
    def stop(self, torrentId=None):
        return self.fetch('core.pause_torrent', [[torrentId]])

    @cherrypy.expose()
    @require(member_of(htpc.role_user))
    @cherrypy.tools.json_out()
    def do_all(self, status):
        if status == 'resume':
            method = 'core.resume_all_torrents'
        else:
            method = 'core.pause_all_torrents'

        return self.fetch(method)

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def daemon(self, status, port):
        if status == 'start':
            action = 'web.start_daemon'
        else:
            action = 'web.stop_daemon'
        return self.fetch(action, [int(port)])

    @cherrypy.expose()
    @require(member_of(htpc.role_user))
    @cherrypy.tools.json_out()
    def set_dlspeed(self, speed):
        self.logger.debug('Set download speed to %s' % speed)
        if speed == '0':
            speed = -1
        return self.fetch('core.set_config', [{'max_download_speed': int(speed)}])

    @cherrypy.expose()
    @require(member_of(htpc.role_user))
    @cherrypy.tools.json_out()
    def set_ulspeed(self, speed):
        if speed == '0':
            speed = -1
        self.logger.debug('Set upload speed to %s' % speed)

        return self.fetch('core.set_config', [{'max_upload_speed': int(speed)}])

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def addtorrent(self, torrent, filename=''):
        result = self.fetch('core.add_torrent_file', [filename, torrent, {}])
        return result

    '''
    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def getconfig(self):
        #should be removed
        return self.fetch('core.get_config')
    '''

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def get_speed(self):
        ''' speed limit '''
        result = self.fetch('core.get_config')
        # Dunno why the f, core.get_config_values didnt work...
        d = {}
        if result:
            d['max_download_speed'] = result['result']['max_download_speed']
            d['max_upload_speed'] = result['result']['max_upload_speed']
            result['result'] = d
            return result

    @cherrypy.expose()
    @require(member_of(htpc.role_user))
    @cherrypy.tools.json_out()
    def remove(self, torrentId, removeData):
        removeDataBool = bool(int(removeData))
        return self.fetch('core.remove_torrent', [torrentId, removeDataBool])

    @cherrypy.expose()
    @require()
    @cherrypy.tools.json_out()
    def to_client(self, link='', torrentname='', **kwargs):
        try:
            self.logger.info('Added %s to deluge' % torrentname)
            # Find download path
            download_path = self.fetch('core.get_config_value', ['download_location'])
            if link.startswith('magnet'):
                path = link
            else:
                # deluge doesnt like a named download...
                link = link.split('?title=')[0]
                get_url = self.fetch('web.download_torrent_from_url', [link])
                path = get_url['result']

            return self.fetch('web.add_torrents', [[{'path': path, 'options': {'download_location': download_path['result']}}]])

        except Exception as e:
            self.logger.debug('Failed adding %s to deluge %s %s' % (torrentname, link, e))

    def fetch(self, method, arguments=None):
        """ Do request to Deluge api """
        if arguments is None:
            arguments = []

        host = striphttp(htpc.settings.get('deluge_host', ''))
        port = htpc.settings.get('deluge_port', '')
        deluge_basepath = fix_basepath(htpc.settings.get('deluge_basepath', '/'))
        ssl = 's' if htpc.settings.get('deluge_ssl') else ''

        url = 'http%s://%s:%s%sjson' % (ssl, host, port, deluge_basepath)

        self.logger.debug("Request deluge method: %s arguments %s" % (method, arguments))
        try:
            # format post data
            data = {'id': 1, 'method': method, 'params': arguments}
            headers = {'Content-Type': 'application/json'}

            response = self.session.post(url, data=dumps(data), headers=headers, verify=False)
            result = response.json()
            if result and result['error']:
                self.logger.debug('Authenticating')
                self.session.post(url, data=dumps({"method": "auth.login", "params": [htpc.settings.get('deluge_password', '')], "id": 1}), headers=headers, verify=False)
                response = self.session.post(url, data=dumps(data), headers=headers, verify=False)

            return result
        except Exception as e:
            self.logger.error('Failed to fetch method %s  arguments %s %s' % (method, arguments, e))
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import division
import os
import sys
import time
import logging
import tempfile
import argparse
import subprocess
import distutils.spawn

REQUIRED_COMMANDS = ['mafft', 'em_cons']
OPT_DEFAULTS = {'min_reads':3, 'processes':1}
USAGE = "%(prog)s [options]"
DESCRIPTION = """Build single-strand consensus sequences from read families. Pipe sorted reads into
stdin. Prints single-strand consensus sequences to stdout. The sequence names are BARCODE.MATE, e.g.
"CTCAGATAACATACCTTATATGCA.1"."""

def main(argv):

  parser = argparse.ArgumentParser(description=DESCRIPTION)
  parser.set_defaults(**OPT_DEFAULTS)

  parser.add_argument('infile', metavar='read-families.tsv', nargs='?',
    help='The input reads, sorted into families.')
  parser.add_argument('-r', '--min-reads', type=int,
    help='The minimum number of reads required to form a family. Families with fewer reads will '
         'be skipped. Default: %(default)s.')
  parser.add_argument('-s', '--stats-file',
    help='Print statistics on the run to this file. Use "-" to print to stderr.')
  parser.add_argument('-p', '--processes', type=int,
    help='Number of processes to use. If > 1, launches this many worker subprocesses. '
         'Default: %(default)s.')
  parser.add_argument('-S', '--slurm', action='store_true',
    help='If -p > 1, prepend sub-commands with "srun -C new".')

  args = parser.parse_args(argv[1:])

  assert args.processes > 0

  # Check for required commands.
  missing_commands = []
  if args.slurm:
    REQUIRED_COMMANDS.append('srun')
  for command in REQUIRED_COMMANDS:
    if not distutils.spawn.find_executable(command):
      missing_commands.append(command)
  if missing_commands:
    fail('Error: Missing commands: "'+'", "'.join(missing_commands)+'".')

  if args.infile:
    infile = open(args.infile)
  else:
    infile = sys.stdin

  if args.stats_file:
    if args.stats_file == '-':
      logging.basicConfig(stream=sys.stderr, level=logging.INFO, format='%(message)s')
    else:
      logging.basicConfig(filename=args.stats_file, filemode='w', level=logging.INFO,
                          format='%(message)s')
  else:
    logging.disable(logging.CRITICAL)

  # Open all the worker processes, if we're using more than one.
  if args.processes > 1:
    workers = open_workers(args.processes, slurm=args.slurm, stats_file=args.stats_file)

  total_time = 0
  total_pairs = 0
  total_runs = 0
  all_pairs = 0
  all_families = 0
  family = []
  family_barcode = None
  for line in infile:
    fields = line.rstrip('\r\n').split('\t')
    if len(fields) != 7:
      continue
    (barcode, name1, seq1, qual1, name2, seq2, qual2) = fields
    # If the barcode has changed, we're in a new family.
    # Process the reads we've previously gathered as one family and start a new family.
    if barcode != family_barcode:
      if len(family) >= args.min_reads:
        all_families += 1
        if args.processes == 1:
          (elapsed, pairs) = process_family(family, family_barcode)
          if pairs > 1:
            total_time += elapsed
            total_pairs += pairs
            total_runs += 1
        else:
          i = all_families % len(workers)
          worker = workers[i]
          delegate(worker, family, family_barcode)
      family_barcode = barcode
      family = []
    family.append((name1, seq1, qual1, name2, seq2, qual2))
    all_pairs += 1
  # Process the last family.
  if len(family) >= args.min_reads:
    all_families += 1
    if args.processes == 1:
      (elapsed, pairs) = process_family(family, family_barcode)
      if pairs > 1:
        total_time += elapsed
        total_pairs += pairs
        total_runs += 1
    else:
      i = all_families % len(workers)
      worker = workers[i]
      delegate(worker, family, family_barcode)

  if args.processes > 1:
    close_workers(workers)
    compile_results(workers)
    # delete_tempfiles(workers)

  if infile is not sys.stdin:
    infile.close()

  if not args.stats_file:
    return

  # Final stats on the run.
  logging.info('Processed {} read pairs and {} multi-pair families.'.format(all_pairs, total_runs))
  per_pair = total_time / total_pairs
  per_run = total_time / total_runs
  logging.info('{:0.3f}s per pair, {:0.3f}s per run.'.format(per_pair, per_run))

def open_workers(num_workers, slurm=False, stats_file=None):
  """Open the required number of worker processes."""
  script_path = os.path.realpath(sys.argv[0])
  workers = []
  for i in range(num_workers):
    if slurm:
      command = ['srun', '-C', 'new', 'python', script_path]
    else:
      command = ['python', script_path]
    stats_subfile = None
    if stats_file:
      if stats_file == '-':
        stats_subfile = '-'
      else:
        stats_subfile = "{}.{}.log".format(stats_file, i)
      command.extend(['-s', stats_subfile])
    outfile = open("tmp-sscs-out.{}.fa".format(i), 'w')
    process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=outfile)
    worker = {'proc':process, 'outfile':outfile, 'stats':stats_subfile}
    workers.append(worker)
  return workers

def delegate(worker, family, barcode):
  """Send a family to a worker process."""
  for pair in family:
    line = barcode+'\t'+'\t'.join(pair)+'\n'
    worker['proc'].stdin.write(line)

def close_workers(workers):
  for worker in workers:
    worker['outfile'].close()
    worker['proc'].stdin.close()

def compile_results(workers):
  for worker in workers:
    with open(worker['outfile'].name, 'r') as outfile:
      for line in outfile:
        sys.stdout.write(line)

def delete_tempfiles(workers):
  for worker in workers:
    os.remove(worker['outfile'].name)
    if worker['stats']:
      os.remove(worker['stats'])

def process_family(family, barcode):
  start = time.time()
  pairs = len(family)
  if pairs == 1:
    (name1, seq1, qual1, name2, seq2, qual2) = family[0]
    print '>'+barcode+'.1'
    print seq1
    print '>'+barcode+'.2'
    print seq2
  else:
    align_path = make_msa(family, 1)
    consensus = get_consensus(align_path)
    if consensus is not None:
      print '>'+barcode+'.1'
      print consensus
    align_path = make_msa(family, 2)
    consensus = get_consensus(align_path)
    if consensus is not None:
      print '>'+barcode+'.2'
      print consensus
  end = time.time()
  elapsed = end - start
  logging.info('{} sec for {} read pairs.'.format(elapsed, pairs))
  return (elapsed, pairs)

def make_msa(family, mate):
  """Perform a multiple sequence alignment on a set of sequences.
  Uses MAFFT."""
  #TODO: Replace with tempfile.mkstemp()?
  with tempfile.NamedTemporaryFile('w', delete=False, prefix='sscs.') as family_file:
    for pair in family:
      if mate == 1:
        name = pair[0]
        seq = pair[1]
      else:
        name = pair[3]
        seq = pair[4]
      family_file.write('>'+name+'\n')
      family_file.write(seq+'\n')
  with tempfile.NamedTemporaryFile('w', delete=False, prefix='sscs.') as align_file:
    with open(os.devnull, 'w') as devnull:
      command = ['mafft', '--nuc', '--quiet', family_file.name]
      subprocess.call(command, stdout=align_file, stderr=devnull)
  os.remove(family_file.name)
  return align_file.name

def get_consensus(align_path):
  """Make a consensus from a multiple sequence alignment file and return the
  consensus sequence as a string.
  Uses the EMBOSS em_cons command."""
  # Note on em_cons output:
  # It may always be lowercase, but maybe not. It can contain "N", and possibly "-".
  with tempfile.NamedTemporaryFile('w', delete=False, prefix='sscs.') as cons_file:
    cons_path = cons_file.name
  with open(os.devnull, 'w') as devnull:
    command = ['em_cons', '-sequence', align_path, '-outseq', cons_path]
    subprocess.call(command, stderr=devnull)
  os.remove(align_path)
  if os.path.getsize(cons_path) == 0:
    os.remove(cons_path)
    return None
  else:
    consensus = read_fasta(cons_path)
    os.remove(cons_path)
    return consensus

def read_fasta(fasta_path):
  """Read a FASTA file, return the sequence.
  Uses a very narrow definition of FASTA: That returned by the "em_cons" command."""
  seq_lines = []
  at_header = True
  with open(fasta_path) as fasta_file:
    for line in fasta_file:
      if at_header and line.startswith('>'):
        at_header = False
        continue
      seq_lines.append(line.strip())
  return "".join(seq_lines)

def fail(message):
  sys.stderr.write(message+"\n")
  sys.exit(1)

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<commit_msg>sscs.py: Deduplicate code by putting all family processing into process_family().<commit_after>#!/usr/bin/env python
from __future__ import division
import os
import sys
import time
import logging
import tempfile
import argparse
import subprocess
import distutils.spawn

REQUIRED_COMMANDS = ['mafft', 'em_cons']
OPT_DEFAULTS = {'min_reads':3, 'processes':1}
USAGE = "%(prog)s [options]"
DESCRIPTION = """Build single-strand consensus sequences from read families. Pipe sorted reads into
stdin. Prints single-strand consensus sequences to stdout. The sequence names are BARCODE.MATE, e.g.
"CTCAGATAACATACCTTATATGCA.1"."""

def main(argv):

  parser = argparse.ArgumentParser(description=DESCRIPTION)
  parser.set_defaults(**OPT_DEFAULTS)

  parser.add_argument('infile', metavar='read-families.tsv', nargs='?',
    help='The input reads, sorted into families.')
  parser.add_argument('-r', '--min-reads', type=int,
    help='The minimum number of reads required to form a family. Families with fewer reads will '
         'be skipped. Default: %(default)s.')
  parser.add_argument('-s', '--stats-file',
    help='Print statistics on the run to this file. Use "-" to print to stderr.')
  parser.add_argument('-p', '--processes', type=int,
    help='Number of processes to use. If > 1, launches this many worker subprocesses. '
         'Default: %(default)s.')
  parser.add_argument('-S', '--slurm', action='store_true',
    help='If -p > 1, prepend sub-commands with "srun -C new".')

  args = parser.parse_args(argv[1:])

  assert args.processes > 0

  # Check for required commands.
  missing_commands = []
  if args.slurm:
    REQUIRED_COMMANDS.append('srun')
  for command in REQUIRED_COMMANDS:
    if not distutils.spawn.find_executable(command):
      missing_commands.append(command)
  if missing_commands:
    fail('Error: Missing commands: "'+'", "'.join(missing_commands)+'".')

  if args.infile:
    infile = open(args.infile)
  else:
    infile = sys.stdin

  if args.stats_file:
    if args.stats_file == '-':
      logging.basicConfig(stream=sys.stderr, level=logging.INFO, format='%(message)s')
    else:
      logging.basicConfig(filename=args.stats_file, filemode='w', level=logging.INFO,
                          format='%(message)s')
  else:
    logging.disable(logging.CRITICAL)

  # Open all the worker processes, if we're using more than one.
  workers = None
  if args.processes > 1:
    workers = open_workers(args.processes, slurm=args.slurm, stats_file=args.stats_file)

  stats = {'time': 0, 'pairs': 0, 'runs': 0, 'families': 0}
  all_pairs = 0
  family = []
  family_barcode = None
  for line in infile:
    fields = line.rstrip('\r\n').split('\t')
    if len(fields) != 7:
      continue
    (barcode, name1, seq1, qual1, name2, seq2, qual2) = fields
    # If the barcode has changed, we're in a new family.
    # Process the reads we've previously gathered as one family and start a new family.
    if barcode != family_barcode:
      process_family(family, family_barcode, args, workers=workers, stats=stats)
      family_barcode = barcode
      family = []
    family.append((name1, seq1, qual1, name2, seq2, qual2))
    all_pairs += 1
  # Process the last family.
  process_family(family, family_barcode, args, workers=workers, stats=stats)

  if args.processes > 1:
    close_workers(workers)
    compile_results(workers)
    # delete_tempfiles(workers)

  if infile is not sys.stdin:
    infile.close()

  if not args.stats_file:
    return

  # Final stats on the run.
  logging.info('Processed {} read pairs and {} multi-pair families.'
               .format(all_pairs, stats['runs']))
  per_pair = stats['time'] / stats['pairs']
  per_run = stats['time'] / stats['runs']
  logging.info('{:0.3f}s per pair, {:0.3f}s per run.'.format(per_pair, per_run))

def open_workers(num_workers, slurm=False, stats_file=None):
  """Open the required number of worker processes."""
  script_path = os.path.realpath(sys.argv[0])
  workers = []
  for i in range(num_workers):
    if slurm:
      command = ['srun', '-C', 'new', 'python', script_path]
    else:
      command = ['python', script_path]
    stats_subfile = None
    if stats_file:
      if stats_file == '-':
        stats_subfile = '-'
      else:
        stats_subfile = "{}.{}.log".format(stats_file, i)
      command.extend(['-s', stats_subfile])
    outfile = open("tmp-sscs-out.{}.fa".format(i), 'w')
    process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=outfile)
    worker = {'proc':process, 'outfile':outfile, 'stats':stats_subfile}
    workers.append(worker)
  return workers

def delegate(worker, family, barcode):
  """Send a family to a worker process."""
  for pair in family:
    line = barcode+'\t'+'\t'.join(pair)+'\n'
    worker['proc'].stdin.write(line)

def close_workers(workers):
  for worker in workers:
    worker['outfile'].close()
    worker['proc'].stdin.close()

def compile_results(workers):
  for worker in workers:
    with open(worker['outfile'].name, 'r') as outfile:
      for line in outfile:
        sys.stdout.write(line)

def delete_tempfiles(workers):
  for worker in workers:
    os.remove(worker['outfile'].name)
    if worker['stats']:
      os.remove(worker['stats'])

def process_family(family, barcode, args, workers=None, stats=None):
  # Pass if family doesn't contain minimum # of reads.
  if len(family) < args.min_reads:
    return
  stats['families'] += 1
  # Are we the controller process or a worker?
  if args.processes > 1:
    i = stats['families'] % len(workers)
    worker = workers[i]
    delegate(worker, family, barcode)
    return
  # We're a worker. Actually process the family.
  start = time.time()
  pairs = len(family)
  if pairs == 1:
    (name1, seq1, qual1, name2, seq2, qual2) = family[0]
    print '>'+barcode+'.1'
    print seq1
    print '>'+barcode+'.2'
    print seq2
  else:
    align_path = make_msa(family, 1)
    consensus = get_consensus(align_path)
    if consensus is not None:
      print '>'+barcode+'.1'
      print consensus
    align_path = make_msa(family, 2)
    consensus = get_consensus(align_path)
    if consensus is not None:
      print '>'+barcode+'.2'
      print consensus
  end = time.time()
  elapsed = end - start
  logging.info('{} sec for {} read pairs.'.format(elapsed, pairs))
  if stats and pairs > 1:
    stats['time'] += elapsed
    stats['pairs'] += pairs
    stats['runs'] += 1
  return

def make_msa(family, mate):
  """Perform a multiple sequence alignment on a set of sequences.
  Uses MAFFT."""
  #TODO: Replace with tempfile.mkstemp()?
  with tempfile.NamedTemporaryFile('w', delete=False, prefix='sscs.') as family_file:
    for pair in family:
      if mate == 1:
        name = pair[0]
        seq = pair[1]
      else:
        name = pair[3]
        seq = pair[4]
      family_file.write('>'+name+'\n')
      family_file.write(seq+'\n')
  with tempfile.NamedTemporaryFile('w', delete=False, prefix='sscs.') as align_file:
    with open(os.devnull, 'w') as devnull:
      command = ['mafft', '--nuc', '--quiet', family_file.name]
      subprocess.call(command, stdout=align_file, stderr=devnull)
  os.remove(family_file.name)
  return align_file.name

def get_consensus(align_path):
  """Make a consensus from a multiple sequence alignment file and return the
  consensus sequence as a string.
  Uses the EMBOSS em_cons command."""
  # Note on em_cons output:
  # It may always be lowercase, but maybe not. It can contain "N", and possibly "-".
  with tempfile.NamedTemporaryFile('w', delete=False, prefix='sscs.') as cons_file:
    cons_path = cons_file.name
  with open(os.devnull, 'w') as devnull:
    command = ['em_cons', '-sequence', align_path, '-outseq', cons_path]
    subprocess.call(command, stderr=devnull)
  os.remove(align_path)
  if os.path.getsize(cons_path) == 0:
    os.remove(cons_path)
    return None
  else:
    consensus = read_fasta(cons_path)
    os.remove(cons_path)
    return consensus

def read_fasta(fasta_path):
  """Read a FASTA file, return the sequence.
  Uses a very narrow definition of FASTA: That returned by the "em_cons" command."""
  seq_lines = []
  at_header = True
  with open(fasta_path) as fasta_file:
    for line in fasta_file:
      if at_header and line.startswith('>'):
        at_header = False
        continue
      seq_lines.append(line.strip())
  return "".join(seq_lines)

def fail(message):
  sys.stderr.write(message+"\n")
  sys.exit(1)

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<|endoftext|>


<commit_before># coding=utf8

"""
Ssdb Python Client Library.

Usage::

    >>> from ssdb import SSDBClient
    >>> c = SSDBClient(host='0.0.0.0', port=8888)
    >>> c.set('key', 'val')
    1
    >>> c.get('key')
    'val'
"""

__version__ = '0.1.0'
__author__ = 'hit9'
__license__ = 'bsd2'

import sys
import socket
import threading

if sys.version_info[0] < 3:  # compat
    rawstr = str
    nativestr = str
else:
    rawstr = lambda string: string.encode('utf8', 'replace')
    nativestr = lambda raw_string: raw_string.decode('utf8', 'replace')

# response type mappings
type_mappings = {
    'set': int,
    'setx': int,
    'expire': int,
    'ttl': int,
    'setnx': int,
    'get': str,
    'getset': str,
    'del': int,
    'incr': int,
    'exists': bool,
    'getbit': int,
    'setbit': int,
    'countbit': int,
    'substr': str,
    'strlen': int,
    'keys': list,
    'scan': list,
    'rscan': list,
    'multi_set': int,
    'multi_get': list,
    'multi_del': int,
    'hset': int,
    'hget': str,
    'hdel': int,
    'hincr': int,
    'hexists': bool,
    'hsize': int,
    'hlist': list,
    'hkeys': list,
    'hgetall': list,
    'hscan': list,
    'hrscan': list,
    'hclear': int,
    'multi_hset': int,
    'multi_hget': list,
    'multi_hdel': int,
    'zset': int,
    'zget': int,
    'zdel': int,
    'zincr': int,
    'zexists': bool,
    'zsize': int,
    'zlist': list,
    'zkeys': list,
    'zscan': list,
    'zrscan': list,
    'zrank': int,
    'zrrank': int,
    'zrange': list,
    'zrrange': list,
    'zclear': int,
    'zcount': int,
    'zsum': int,
    'zavg': int,
    'zremrangebyrank': int,
    'zremrangebyscore': int,
    'multi_zset': int,
    'multi_zget': list,
    'multi_zdel': int,
    'qsize': int,
    'qclear': int,
    'qfront': str,
    'qback': str,
    'qget': str,
    'qslice': list,
    'qpush': str,
    'qpush_front': int,
    'qpush_back': int,
    'qpop': str,
    'qpop_front': str,
    'qpop_back': str
}

py_reserved_words = {
    'delete': 'del'
}

status_ok = 'ok'
status_not_found = 'not_found'

class SSDBException(Exception):
    pass

class Connection(threading.local):
    """Ssdb connection object, usage::

        >>> conn = Connection(host='0.0.0.0', port=8888)
        >>> conn.execute(['set', 'key', 'val'])
        1
    """

    def __init__(self, host='0.0.0.0', port=8888, timeout=None):
        self.sock = None
        self.host = host
        self.port = port
        self.timeout = timeout
        self.batch_mode = False
        self.__commands = []  # commands sent cache

    def connect(self):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.sock.connect((self.host, self.port))
        self.sock.setblocking(1)  # block mode
        self.sock.settimeout(self.timeout)

    def compile(self, args):
        pattern = '{0}\n{1}\n'
        buffers = [pattern.format(len(str(arg)), arg) for arg in args]
        return ''.join(buffers) + '\n'

    def send(self, args):
        if self.sock is None:
            self.connect()
        command = self.compile(args)
        self.sock.sendall(rawstr(command))
        self.__commands.append(args[0])

    def recv(self):
        data = ''
        resps_count = len(self.__commands)
        while resps_count > 0:
            buf = nativestr(self.sock.recv(1024))
            data += buf
            resps_count -= buf.count('\n\n')
        try:
            resp = self.response(data)
        except Exception:
            raise
        finally:
            self.clear_commands()
        return resp

    def batch(self, mode=True):
        self.batch_mode = mode

    def execute(self, args):
        if self.batch_mode:
            return self.send(args)
        self.send(args)
        return self.recv()

    def clear_commands(self):
        self.__commands[:] = []

    def response(self, data):
        raw_resps = data.strip().split('\n\n')
        bodys = [raw_resp.splitlines()[1::2] for raw_resp in raw_resps]

        resps = []
        for index, lst in enumerate(bodys):
            command = self.__commands[index]
            status, body = lst[0], lst[1:]
            resps.append(self.make_response(status, body, command))

        if self.batch_mode:
            return resps
        return resps[0]

    def make_response(self, status, body, command):
        if status == status_not_found:
            return None
        elif status == status_ok:
            type = type_mappings.get(command, str)
            if type in (bool, int, str):
                return type(body[0])
            elif type is list:
                return type(body)
        else:
            error_message = '{}: {}'.format(status, body[0])
            raise SSDBException(error_message)

class SSDBClient(object):
    """Ssdb client object, usage::

        >>> c = SSDBClient(host='0.0.0.0', port=8888)
        >>> c.set('key', 'val')
        1
    """

    def __init__(self, host='0.0.0.0', port=8888, timeout=None):
        self.host = host
        self.port = port
        self.timeout = timeout
        self.conn = Connection(host=host, port=port, timeout=timeout)

    def batch(self, mode=True):
        return self.conn.batch(mode)

    def execute(self):
        return self.conn.recv()

    def __getattr__(self, name):
        name = py_reserved_words.get(name, name)

        def method(*args):
            return self.conn.execute((name,) + args)
        return method
<commit_msg>check body if empty<commit_after># coding=utf8

"""
Ssdb Python Client Library.

Usage::

    >>> from ssdb import SSDBClient
    >>> c = SSDBClient(host='0.0.0.0', port=8888)
    >>> c.set('key', 'val')
    1
    >>> c.get('key')
    'val'
"""

__version__ = '0.1.0'
__author__ = 'hit9'
__license__ = 'bsd2'

import sys
import socket
import threading

if sys.version_info[0] < 3:  # compat
    rawstr = str
    nativestr = str
else:
    rawstr = lambda string: string.encode('utf8', 'replace')
    nativestr = lambda raw_string: raw_string.decode('utf8', 'replace')

# response type mappings
type_mappings = {
    'set': int,
    'setx': int,
    'expire': int,
    'ttl': int,
    'setnx': int,
    'get': str,
    'getset': str,
    'del': int,
    'incr': int,
    'exists': bool,
    'getbit': int,
    'setbit': int,
    'countbit': int,
    'substr': str,
    'strlen': int,
    'keys': list,
    'scan': list,
    'rscan': list,
    'multi_set': int,
    'multi_get': list,
    'multi_del': int,
    'hset': int,
    'hget': str,
    'hdel': int,
    'hincr': int,
    'hexists': bool,
    'hsize': int,
    'hlist': list,
    'hkeys': list,
    'hgetall': list,
    'hscan': list,
    'hrscan': list,
    'hclear': int,
    'multi_hset': int,
    'multi_hget': list,
    'multi_hdel': int,
    'zset': int,
    'zget': int,
    'zdel': int,
    'zincr': int,
    'zexists': bool,
    'zsize': int,
    'zlist': list,
    'zkeys': list,
    'zscan': list,
    'zrscan': list,
    'zrank': int,
    'zrrank': int,
    'zrange': list,
    'zrrange': list,
    'zclear': int,
    'zcount': int,
    'zsum': int,
    'zavg': int,
    'zremrangebyrank': int,
    'zremrangebyscore': int,
    'multi_zset': int,
    'multi_zget': list,
    'multi_zdel': int,
    'qsize': int,
    'qclear': int,
    'qfront': str,
    'qback': str,
    'qget': str,
    'qslice': list,
    'qpush': str,
    'qpush_front': int,
    'qpush_back': int,
    'qpop': str,
    'qpop_front': str,
    'qpop_back': str
}

py_reserved_words = {
    'delete': 'del'
}

status_ok = 'ok'
status_not_found = 'not_found'

class SSDBException(Exception):
    pass

class Connection(threading.local):
    """Ssdb connection object, usage::

        >>> conn = Connection(host='0.0.0.0', port=8888)
        >>> conn.execute(['set', 'key', 'val'])
        1
    """

    def __init__(self, host='0.0.0.0', port=8888, timeout=None):
        self.sock = None
        self.host = host
        self.port = port
        self.timeout = timeout
        self.batch_mode = False
        self.__commands = []  # commands sent cache

    def connect(self):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.sock.connect((self.host, self.port))
        self.sock.setblocking(1)  # block mode
        self.sock.settimeout(self.timeout)

    def compile(self, args):
        pattern = '{0}\n{1}\n'
        buffers = [pattern.format(len(str(arg)), arg) for arg in args]
        return ''.join(buffers) + '\n'

    def send(self, args):
        if self.sock is None:
            self.connect()
        command = self.compile(args)
        self.sock.sendall(rawstr(command))
        self.__commands.append(args[0])

    def recv(self):
        data = ''
        resps_count = len(self.__commands)
        while resps_count > 0:
            buf = nativestr(self.sock.recv(1024))
            data += buf
            resps_count -= buf.count('\n\n')
        try:
            resp = self.response(data)
        except Exception:
            raise
        finally:
            self.clear_commands()
        return resp

    def batch(self, mode=True):
        self.batch_mode = mode

    def execute(self, args):
        if self.batch_mode:
            return self.send(args)
        self.send(args)
        return self.recv()

    def clear_commands(self):
        self.__commands[:] = []

    def response(self, data):
        raw_resps = data.strip().split('\n\n')
        bodys = [raw_resp.splitlines()[1::2] for raw_resp in raw_resps]

        resps = []
        for index, lst in enumerate(bodys):
            command = self.__commands[index]
            status, body = lst[0], lst[1:]
            resps.append(self.make_response(status, body, command))

        if self.batch_mode:
            return resps
        return resps[0]

    def make_response(self, status, body, command):
        if status == status_not_found:
            return None
        elif status == status_ok:
            type = type_mappings.get(command, str)
            if type in (bool, int, str):
                return type(body[0])
            elif type is list:
                return type(body)
        else:
            if body:
                error_message = '{}: {}'.format(status, body[0])
            else:
                error_message = status
            raise SSDBException(error_message)

class SSDBClient(object):
    """Ssdb client object, usage::

        >>> c = SSDBClient(host='0.0.0.0', port=8888)
        >>> c.set('key', 'val')
        1
    """

    def __init__(self, host='0.0.0.0', port=8888, timeout=None):
        self.host = host
        self.port = port
        self.timeout = timeout
        self.conn = Connection(host=host, port=port, timeout=timeout)

    def batch(self, mode=True):
        return self.conn.batch(mode)

    def execute(self):
        return self.conn.recv()

    def __getattr__(self, name):
        name = py_reserved_words.get(name, name)

        def method(*args):
            return self.conn.execute((name,) + args)
        return method
<|endoftext|>


<commit_before>class ModelType(object):
    """
    Enums for the differnet model types
    DO NOT CHANGE EXISTING ONES
    """

    EVENT = 0
    TEAM = 1
    MATCH = 2
    EVENT_TEAM = 3
    DISTRICT = 4
    AWARD = 5
    MEDIA = 6
<commit_msg>Update model enums to match app<commit_after>class ModelType(object):
    """
    Enums for the differnet model types
    DO NOT CHANGE EXISTING ONES
    """

    EVENT = 0
    TEAM = 1
    MATCH = 2
    EVENT_TEAM = 3
    DISTRICT = 4
    DISTRICT_TEAM = 5
    AWARD = 6
    MEDIA = 7
<|endoftext|>


<commit_before>from django.conf import settings
from django.contrib.postgres.fields import JSONField
from django.core.mail import EmailMultiAlternatives
from django.core.urlresolvers import reverse
from django.db import models
from django.utils.html import escape
import logging
import os
import re

EMAIL_RE = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
log = logging.getLogger(__name__)

def secure_random_string():
    return os.urandom(24).encode('hex')

class Party(models.Model):
    pa_id = models.CharField(max_length=100, null=False, blank=False, help_text="Peoples Assembly ID", unique=True)
    name = models.CharField(max_length=200, null=False, blank=False)
    slug = models.CharField(max_length=70, null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    @property
    def icon_url(self):
        return '/static/images/parties/%s.png' % self.slug

    def as_dict(self):
        return {
            'id': self.id,
            'pa_id': self.pa_id,
            'name': self.name,
            'slug': self.slug,
            'icon_url': self.icon_url,
        }

class ConstituencyBranch(models.Model):
    pa_id = models.CharField(max_length=100, null=False, blank=False, help_text="Peoples Assembly ID", unique=True)
    name = models.CharField(max_length=200, null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def as_dict(self):
        return {
            'id': self.id,
            'pa_id': self.pa_id,
            'name': self.name,
        }

class Person(models.Model):
    pa_id = models.CharField(max_length=100, null=False, blank=False, help_text="Peoples Assembly ID", unique=True)
    name = models.CharField(max_length=70, null=False, blank=False)
    pa_url = models.TextField(null=False, blank=False)
    in_national_assembly = models.BooleanField(default=True)
    portrait_url = models.CharField(max_length=300, null=True, blank=True)
    party = models.ForeignKey(Party, null=True, blank=False)
    constituency_branches = models.ManyToManyField(ConstituencyBranch)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def as_dict(self):
        return {
            'id': self.id,
            'pa_id': self.pa_id,
            'name': self.name,
            'contactdetails': [d.as_dict() for d in self.contactdetails.all()],
            'pa_url': self.pa_url,
            'portrait_url': self.portrait_url,
            'party': self.party.as_dict() if self.party else None,
        }

    def __unicode__(self):
        return self.name

class ContactDetail(models.Model):
    person = models.ForeignKey(Person, related_name="contactdetails")
    type = models.CharField(max_length=40, help_text="Type of contact detail")
    value = models.CharField(max_length=255, help_text="The actual detail")
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def as_dict(self):
        return {
            'id': self.id,
            'type': self.type,
            'value': self.value,
        }

    def __unicode__(self):
        return "%s (%s) %s" % (self.person.name, self.type, self.value,)

class Email(models.Model):
    to_person = models.ForeignKey(Person)
    to_addresses = models.TextField(null=True, blank=True, help_text="The actuall address(es) we sent to belonging to the recipient at the time, regardless of the addresses that we currently have for them.")
    remote_ip = models.CharField(max_length=20, help_text="User's remote IP")
    user_agent = models.CharField(max_length=200, help_text="User's user agent string")
    subject = models.CharField(max_length=200, null=False, blank=False)
    body_txt = models.TextField(null=False, blank=False)
    from_name = models.CharField(max_length=70, null=False, blank=False)
    from_email = models.EmailField(null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    # This is what we use make it viewable online, but only to anyone who knows the URL
    secure_id = models.CharField(max_length=100, blank=True, unique=True, default=secure_random_string)
    # This is what we only present to the sender so that they can answer questions
    # that we can tie to an (eventually anonymous) individual.
    # THIS SHOULD NOT BE PART OF THE STANDARD as_dict dict.
    sender_secret = models.CharField(max_length=100, blank=True, unique=True, default=secure_random_string)
    # Any data we'd like connected to the email in structured form.
    # This makes no promises about keys being present from one email to another.
    any_data = JSONField()

    @property
    def body_html(self):
        # VERY IMPORTANT: Escape any HTML someone might have entered
        html_escaped_text = escape(self.body_txt)
        return re.sub("(\r\n|\n|\r)", "
", html_escaped_text)

    def as_dict(self):
        return {
            'id': self.id,
            'to_person': self.to_person.as_dict(),
            'to_addresses': self.to_addresses,
            'subject': self.subject,
            'body_html': self.body_html,
            'body_txt': self.body_txt,
            'from_name': self.from_name,
            'from_email': self.from_email,
            'secure_id': self.secure_id,
            'any_data': self.any_data,
        }

    def send(self):
        if not EMAIL_RE.match(self.from_email):
            self.from_email = "noreply@openup.org.za"
        sender = "%s <%s>" % (self.from_name, self.from_email)
        recipients = ["%s <jd@openup.org.za>" % self.to_person.name.strip() for c in self.to_person.contactdetails.filter(type='email').all()]
        if settings.SEND_EMAILS:
            log.info("Sending email to %s from %s" % (recipients, self.from_email))

            url = settings.BASE_URL + reverse('email-detail', kwargs={'secure_id': self.secure_id})

            email = EmailMultiAlternatives(
                self.subject,
                self.body_txt,
                sender,
                recipients,
                cc=[sender],
            )
            email.attach_alternative(self.body_html, "text/html")
            email.send()
        self.to_addresses = ", ".join(recipients)
        self.save()

    def __unicode__(self):
        return u"From: %s To: %s" % (self.from_email, self.to_addresses)

class SenderQA(models.Model):
    email = models.ForeignKey(Email)
    question = models.TextField(null=False, blank=False)
    answer = models.TextField(null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def __unicode__(self):
        return u"%d Q: %s...A: %s" % (self.email.id, self.question[:100], self.answer)
<commit_msg>live emails again<commit_after>from django.conf import settings
from django.contrib.postgres.fields import JSONField
from django.core.mail import EmailMultiAlternatives
from django.core.urlresolvers import reverse
from django.db import models
from django.utils.html import escape
import logging
import os
import re

EMAIL_RE = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
log = logging.getLogger(__name__)

def secure_random_string():
    return os.urandom(24).encode('hex')

class Party(models.Model):
    pa_id = models.CharField(max_length=100, null=False, blank=False, help_text="Peoples Assembly ID", unique=True)
    name = models.CharField(max_length=200, null=False, blank=False)
    slug = models.CharField(max_length=70, null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    @property
    def icon_url(self):
        return '/static/images/parties/%s.png' % self.slug

    def as_dict(self):
        return {
            'id': self.id,
            'pa_id': self.pa_id,
            'name': self.name,
            'slug': self.slug,
            'icon_url': self.icon_url,
        }

class ConstituencyBranch(models.Model):
    pa_id = models.CharField(max_length=100, null=False, blank=False, help_text="Peoples Assembly ID", unique=True)
    name = models.CharField(max_length=200, null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def as_dict(self):
        return {
            'id': self.id,
            'pa_id': self.pa_id,
            'name': self.name,
        }

class Person(models.Model):
    pa_id = models.CharField(max_length=100, null=False, blank=False, help_text="Peoples Assembly ID", unique=True)
    name = models.CharField(max_length=70, null=False, blank=False)
    pa_url = models.TextField(null=False, blank=False)
    in_national_assembly = models.BooleanField(default=True)
    portrait_url = models.CharField(max_length=300, null=True, blank=True)
    party = models.ForeignKey(Party, null=True, blank=False)
    constituency_branches = models.ManyToManyField(ConstituencyBranch)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def as_dict(self):
        return {
            'id': self.id,
            'pa_id': self.pa_id,
            'name': self.name,
            'contactdetails': [d.as_dict() for d in self.contactdetails.all()],
            'pa_url': self.pa_url,
            'portrait_url': self.portrait_url,
            'party': self.party.as_dict() if self.party else None,
        }

    def __unicode__(self):
        return self.name

class ContactDetail(models.Model):
    person = models.ForeignKey(Person, related_name="contactdetails")
    type = models.CharField(max_length=40, help_text="Type of contact detail")
    value = models.CharField(max_length=255, help_text="The actual detail")
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def as_dict(self):
        return {
            'id': self.id,
            'type': self.type,
            'value': self.value,
        }

    def __unicode__(self):
        return "%s (%s) %s" % (self.person.name, self.type, self.value,)

class Email(models.Model):
    to_person = models.ForeignKey(Person)
    to_addresses = models.TextField(null=True, blank=True, help_text="The actuall address(es) we sent to belonging to the recipient at the time, regardless of the addresses that we currently have for them.")
    remote_ip = models.CharField(max_length=20, help_text="User's remote IP")
    user_agent = models.CharField(max_length=200, help_text="User's user agent string")
    subject = models.CharField(max_length=200, null=False, blank=False)
    body_txt = models.TextField(null=False, blank=False)
    from_name = models.CharField(max_length=70, null=False, blank=False)
    from_email = models.EmailField(null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    # This is what we use make it viewable online, but only to anyone who knows the URL
    secure_id = models.CharField(max_length=100, blank=True, unique=True, default=secure_random_string)
    # This is what we only present to the sender so that they can answer questions
    # that we can tie to an (eventually anonymous) individual.
    # THIS SHOULD NOT BE PART OF THE STANDARD as_dict dict.
    sender_secret = models.CharField(max_length=100, blank=True, unique=True, default=secure_random_string)
    # Any data we'd like connected to the email in structured form.
    # This makes no promises about keys being present from one email to another.
    any_data = JSONField()

    @property
    def body_html(self):
        # VERY IMPORTANT: Escape any HTML someone might have entered
        html_escaped_text = escape(self.body_txt)
        return re.sub("(\r\n|\n|\r)", "
", html_escaped_text)

    def as_dict(self):
        return {
            'id': self.id,
            'to_person': self.to_person.as_dict(),
            'to_addresses': self.to_addresses,
            'subject': self.subject,
            'body_html': self.body_html,
            'body_txt': self.body_txt,
            'from_name': self.from_name,
            'from_email': self.from_email,
            'secure_id': self.secure_id,
            'any_data': self.any_data,
        }

    def send(self):
        if not EMAIL_RE.match(self.from_email):
            self.from_email = "noreply@openup.org.za"
        sender = "%s <%s>" % (self.from_name, self.from_email)
        recipients = ["%s <%s>" % (self.to_person.name.strip(), c.value.strip()) for c in self.to_person.contactdetails.filter(type='email').all()]
        if settings.SEND_EMAILS:
            log.info("Sending email to %s from %s" % (recipients, self.from_email))

            url = settings.BASE_URL + reverse('email-detail', kwargs={'secure_id': self.secure_id})

            email = EmailMultiAlternatives(
                self.subject,
                self.body_txt,
                sender,
                recipients,
                cc=[sender],
            )
            email.attach_alternative(self.body_html, "text/html")
            email.send()
        self.to_addresses = ", ".join(recipients)
        self.save()

    def __unicode__(self):
        return u"From: %s To: %s" % (self.from_email, self.to_addresses)

class SenderQA(models.Model):
    email = models.ForeignKey(Email)
    question = models.TextField(null=False, blank=False)
    answer = models.TextField(null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def __unicode__(self):
        return u"%d Q: %s...A: %s" % (self.email.id, self.question[:100], self.answer)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import ssl
import subprocess
import click
from couchdbkit import Server
from couchdbkit.changes import ChangesStream
from couchdbkit.resource import CouchdbResource
from restkit import BasicAuth

def exit_with_error(msg):
    click.secho("Error: %s" % msg)
    exit(128)

def sequence_read(sequence_file):
    with open(sequence_file, 'r') as f:
        sequence = f.read()
    if sequence == '':
        return 0
    else:
        return int(sequence)

def sequence_write(sequence_file, sequence):
    with open(sequence_file, 'w') as f:
        f.write(str(sequence))

def zone_update(name, data, zone_dir):
    """ Write out zone file and reload name service """

    zone_file = os.path.join(zone_dir, "%s.zone" % name)

    if os.path.isfile(zone_file):
        created = False
    else:
        created = True

    with open("%s.tmp" % zone_file, 'w+') as f:
        f.write(data)
    os.rename("%s.tmp" % zone_file, zone_file)

    if created:
        cmd = ['/usr/sbin/nsd-control', 'addzone', name, 'zone']
        try:
            subprocess.check_call(cmd, cwd=zone_dir, shell=False, env=os.environ.copy())
        except subprocess.CalledProcessError as e:
            raise Exception("Error running '%s'. Return %s" % (cmd, e.returncode))
    else:
        cmd = ['/usr/sbin/nsd-control', 'reload', name]
        try:
            subprocess.check_call(cmd, cwd=zone_dir, shell=False, env=os.environ.copy())
        except subprocess.CalledProcessError as e:
            raise Exception("Error running '%s'. Return %s" % (cmd, e.returncode))

def zone_delete(name, zone_dir):
    zone_file = os.path.join(zone_dir, "%s.zone" % name)
    cmd = ['/usr/sbin/nsd-control', 'delzone', name]
    try:
        subprocess.check_call(cmd, cwd=zone_dir, shell=False, env=os.environ.copy())
    except subprocess.CalledProcessError as e:
        raise Exception("Error running '%s'. Return %s" % (cmd, e.returncode))
    os.remove(zone_file)

@click.command()
@click.option('--db-name', help='CouchDB Name', required=True, envvar='COUCHDB_NAME')
@click.option('--db-user', help='CouchDB User', required=True, envvar='COUCHDB_USER')
@click.option('--db-pass', help='CouchDB Password', required=True, envvar='COUCHDB_PASS')
@click.option('--db-host', help='CouchDB HOST URI', required=True, envvar='COUCHDB_URI')
@click.option('--ca_certs', help='TLS CA Cert', envvar='TLS_CACERT')
@click.option('--certfile', help='TLS Client Cert', envvar='TLS_CLIENT_CERT')
@click.option('--keyfile', help='TLS Client Key', envvar='TLS_CLIENT_KEY')
@click.option('--sequence-file', help='Sequence File', default='/var/tmp/sync.sequence')
@click.option('--zone-dir', help='Zone File Directory', default='/var/named/masters/')
def main(db_name, db_user, db_pass, db_host, sequence_file, zone_dir, **tls_args):
    # Starting Sequence for change stream
    sequence = sequence_read(sequence_file)
    click.echo('Skipping %s changes.' % sequence)
    # CouchDB Connection
    tls_args['cert_reqs'] = ssl.CERT_REQUIRED
    tls_args['ssl_version'] = ssl.PROTOCOL_TLSv1_2
    auth = CouchdbResource(filters=[BasicAuth(db_user, db_pass)], **tls_args)
    server = Server(uri=db_host, resource_instance=auth)
    db = server[db_name]

    with ChangesStream(db, since=sequence, feed='continuous', heartbeat=True) as stream:
        click.echo('Waiting for changes...')

        for change in stream:
            domain = change['id']
            seq = change['seq']
            if change.get('deleted', False) is True:
                click.echo("%s Delete for %s." % (seq, domain))
                try:
                    zone_delete(domain, zone_dir)
                except Exception, e:
                    # TODO: Add some alerting here
                    click.echo(e)
            else:
                click.echo("%s Create/Update for %s." % (seq, domain))
                try:
                    doc = db.get(docid=domain)
                    zone_update(domain, doc['data'], zone_dir)
                except Exception, e:
                    # TODO: Add some alerting here
                    click.echo(e)

            sequence_write(sequence_file, seq)   # Keep track of our sync point

if __name__ == '__main__':
    main()
<commit_msg>write out status when exiting<commit_after>#!/usr/bin/env python

import os
import ssl
import subprocess
import click
from couchdbkit import Server
from couchdbkit.changes import ChangesStream
from couchdbkit.resource import CouchdbResource
from restkit import BasicAuth

def exit_with_error(msg):
    click.secho("Error: %s" % msg)
    exit(128)

def sequence_read(sequence_file):
    with open(sequence_file, 'r') as f:
        sequence = f.read()
    if sequence == '':
        return 0
    else:
        return int(sequence)

def sequence_write(sequence_file, sequence):
    with open(sequence_file, 'w') as f:
        f.write(str(sequence))

def zone_update(name, data, zone_dir):
    """ Write out zone file and reload name service """

    zone_file = os.path.join(zone_dir, "%s.zone" % name)

    if os.path.isfile(zone_file):
        created = False
    else:
        created = True

    with open("%s.tmp" % zone_file, 'w+') as f:
        f.write(data)
    os.rename("%s.tmp" % zone_file, zone_file)

    if created:
        cmd = ['/usr/sbin/nsd-control', 'addzone', name, 'zone']
        try:
            subprocess.check_call(cmd, cwd=zone_dir, shell=False, env=os.environ.copy())
        except subprocess.CalledProcessError as e:
            raise Exception("Error running '%s'. Return %s" % (cmd, e.returncode))
    else:
        cmd = ['/usr/sbin/nsd-control', 'reload', name]
        try:
            subprocess.check_call(cmd, cwd=zone_dir, shell=False, env=os.environ.copy())
        except subprocess.CalledProcessError as e:
            raise Exception("Error running '%s'. Return %s" % (cmd, e.returncode))

def zone_delete(name, zone_dir):
    zone_file = os.path.join(zone_dir, "%s.zone" % name)
    cmd = ['/usr/sbin/nsd-control', 'delzone', name]
    try:
        subprocess.check_call(cmd, cwd=zone_dir, shell=False, env=os.environ.copy())
    except subprocess.CalledProcessError as e:
        raise Exception("Error running '%s'. Return %s" % (cmd, e.returncode))
    os.remove(zone_file)

@click.command()
@click.option('--db-name', help='CouchDB Name', required=True, envvar='COUCHDB_NAME')
@click.option('--db-user', help='CouchDB User', required=True, envvar='COUCHDB_USER')
@click.option('--db-pass', help='CouchDB Password', required=True, envvar='COUCHDB_PASS')
@click.option('--db-host', help='CouchDB HOST URI', required=True, envvar='COUCHDB_URI')
@click.option('--ca_certs', help='TLS CA Cert', envvar='TLS_CACERT')
@click.option('--certfile', help='TLS Client Cert', envvar='TLS_CLIENT_CERT')
@click.option('--keyfile', help='TLS Client Key', envvar='TLS_CLIENT_KEY')
@click.option('--sequence-file', help='Sequence File', default='/var/tmp/sync.sequence')
@click.option('--zone-dir', help='Zone File Directory', default='/var/named/masters/')
def main(db_name, db_user, db_pass, db_host, sequence_file, zone_dir, **tls_args):
    # Starting Sequence for change stream
    sequence = sequence_read(sequence_file)
    click.echo('Skipping %s changes.' % sequence)
    # CouchDB Connection
    tls_args['cert_reqs'] = ssl.CERT_REQUIRED
    tls_args['ssl_version'] = ssl.PROTOCOL_TLSv1_2
    auth = CouchdbResource(filters=[BasicAuth(db_user, db_pass)], **tls_args)
    server = Server(uri=db_host, resource_instance=auth)
    db = server[db_name]

    with ChangesStream(db, since=sequence, feed='continuous', heartbeat=True) as stream:
        click.echo('Waiting for changes...')

        for change in stream:
            domain = change['id']
            seq = change['seq']
            if change.get('deleted', False) is True:
                click.echo("%s Delete for %s." % (seq, domain))
                try:
                    zone_delete(domain, zone_dir)
                except Exception, e:
                    # TODO: Add some alerting here
                    click.echo(e)
            else:
                click.echo("%s Create/Update for %s." % (seq, domain))
                try:
                    doc = db.get(docid=domain)
                    zone_update(domain, doc['data'], zone_dir)
                except Exception, e:
                    # TODO: Add some alerting here
                    click.echo(e)

            sequence_write(sequence_file, seq)   # Keep track of our sync point

    click.echo("Stream Closed? Finished listening for changes!")

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (C) 2011 Linaro Limited
#
# Author: Michael Hudson-Doyle <michael.hudson@linaro.org>
#
# This file is part of LAVA Scheduler.
#
# LAVA Scheduler is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License version 3 as
# published by the Free Software Foundation
#
# LAVA Scheduler is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with LAVA Scheduler.  If not, see <http://www.gnu.org/licenses/>.

__version__ = (0, 4, 1, "final", 0)
<commit_msg>bump version to 0.4.2a0<commit_after># Copyright (C) 2011 Linaro Limited
#
# Author: Michael Hudson-Doyle <michael.hudson@linaro.org>
#
# This file is part of LAVA Scheduler.
#
# LAVA Scheduler is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License version 3 as
# published by the Free Software Foundation
#
# LAVA Scheduler is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with LAVA Scheduler.  If not, see <http://www.gnu.org/licenses/>.

__version__ = (0, 4, 2, "alpha", 0)
<|endoftext|>