<commit_before>## Consumer for DOE Pages for SHARE
from __future__ import unicode_literals

import time
import requests
from lxml import etree
from datetime import date, timedelta, datetime

from nameparser import HumanName

from dateutil.parser import *

from scrapi.linter import lint
from scrapi.linter.document import RawDocument, NormalizedDocument

NAME = 'doepages'
TODAY = date.today()

NAMESPACES = {'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
            'dc': 'http://purl.org/dc/elements/1.1/',
            'dcq': 'http://purl.org/dc/terms/'}

DEFAULT_ENCODING = 'UTF-8'

record_encoding = None

def copy_to_unicode(element):

    encoding = record_encoding or DEFAULT_ENCODING
    element = ''.join(element)
    if isinstance(element, unicode):
        return element
    else:
        return unicode(element, encoding=encoding)

def consume(days_back=15):
    start_date = TODAY - timedelta(days_back)
    base_url = 'http://www.osti.gov/pages/pagesxml?nrows={0}&EntryDateFrom={1}'
    url = base_url.format('1', start_date.strftime('%m/%d/%Y'))
    initial_data = requests.get(url)
    record_encoding = initial_data.encoding
    initial_doc = etree.XML(initial_data.content)

    num_results = int(initial_doc.xpath('//records/@count', namespaces=NAMESPACES)[0])

    url = base_url.format(num_results, start_date.strftime('%m/%d/%Y'))
    data = requests.get(url)
    doc = etree.XML(data.content)

    records = doc.xpath('records/record')

    xml_list = []
    for record in records:
        doc_id = record.xpath('dc:ostiId/node()', namespaces=NAMESPACES)[0]
        record = etree.tostring(record, encoding=record_encoding)
        xml_list.append(RawDocument({
                    'doc': record,
                    'source': NAME,
                    'docID': copy_to_unicode(doc_id),
                    'filetype': 'xml'
                }))

    return xml_list

def get_ids(doc, raw_doc):
    ids = {}
    ids['doi'] = (doc.xpath('//dc:doi/node()', namespaces=NAMESPACES) or [''])[0]
    ids['serviceID'] = raw_doc.get('docID')
    url = (doc.xpath('//dcq:identifier-citation/node()', namespaces=NAMESPACES) or [''])[0]
    if url == '':
        url = 'http://dx.doi.org/' + ids['doi']
    if url == '':
        raise Exception('Warning: url field is blank!')
    ids['url'] = url

    for key, value in ids.iteritems():
        ids[key] = copy_to_unicode(value)

    return ids

def get_contributors(doc):
    contributor_list = []
    full_contributors = doc.xpath('//dc:creator/node()', namespaces=NAMESPACES)[0].split(';')
    for person in full_contributors:
        name = HumanName(person)
        contributor = {
            'prefix': name.title,
            'given': name.first,
            'middle': name.middle,
            'family': name.last,
            'suffix': name.suffix,
            'email': '',
            'ORCID': ''
        }
        contributor_list.append(contributor)

    return contributor_list
def get_properties(doc):
    publisherInfo = {
        'publisher': (doc.xpath('//dcq:publisher/node()', namespaces=NAMESPACES) or [''])[0],
        'publisherSponsor': (doc.xpath('//dcq:publisherSponsor/node()', namespaces=NAMESPACES) or [''])[0],
        'publisherAvailability': (doc.xpath('//dcq:publisherAvailability/node()', namespaces=NAMESPACES) or [''])[0],
        'publisherResearch': (doc.xpath('//dcq:publisherResearch/node()', namespaces=NAMESPACES) or [''])[0],
        'publisherCountry': (doc.xpath('//dcq:publisherCountry/node()', namespaces=NAMESPACES) or [''])[0]
    }

    for key, value in publisherInfo.iteritems():
        publisherInfo[key] = copy_to_unicode(value)

    properties = {
        'language': (doc.xpath('//dc:language/node()', namespaces=NAMESPACES) or [''])[0],
        'type': (doc.xpath('//dc:type/node()', namespaces=NAMESPACES) or [''])[0],
        'typeQualifier': (doc.xpath('//dc:typeQualifier/node()', namespaces=NAMESPACES) or [''])[0],
        'relation': (doc.xpath('//dc:relation/node()', namespaces=NAMESPACES) or [''])[0],
        'coverage': (doc.xpath('//dc:coverage/node()', namespaces=NAMESPACES) or [''])[0],
        'format': (doc.xpath('//dc:format/node()', namespaces=NAMESPACES) or [''])[0],
        'rights': (doc.xpath('//dc:rights/node()', namespaces=NAMESPACES) or [''])[0],
        # TODO:  parse out some of these identifiers in the strings!
        'identifier': (doc.xpath('//dc:identifier/node()', namespaces=NAMESPACES) or [''])[0],
        'identifierReport': (doc.xpath('//dc:identifier/node()', namespaces=NAMESPACES) or [''])[0],
        'identifierDOEcontract': (doc.xpath('//dcq:identifierDOEcontract/node()', namespaces=NAMESPACES) or [''])[0],
        'identifierOther': (doc.xpath('//dc:identifierOther/node()', namespaces=NAMESPACES) or [''])[0],
        'identifier-purl': (doc.xpath('//dcq:identifier-purl/node()', namespaces=NAMESPACES) or [''])[0]
    }

    for key, value in properties.iteritems():
        properties[key] = copy_to_unicode(value)

    properties['publisherInfo'] = publisherInfo

    return properties

def get_date_created(doc):
    date_created = doc.xpath('//dc:date/node()', namespaces=NAMESPACES)[0]
    date = parse(date_created).isoformat()
    return copy_to_unicode(date)

def get_date_updated(doc):
    date_updated = doc.xpath('//dc:dateEntry/node()', namespaces=NAMESPACES)[0]
    date = parse(date_updated).isoformat()
    return copy_to_unicode(date)

def get_tags(doc):
    all_tags = doc.xpath('//dc:subject/node()', namespaces=NAMESPACES) + doc.xpath('//dc:subjectRelated/node()', namespaces=NAMESPACES)
    tags = []
    for taglist in all_tags:
        tags += taglist.split(',')
    return list(set([copy_to_unicode(tag.lower().strip()) for tag in tags]))

def normalize(raw_doc, timestamp):
    raw_doc_string = raw_doc.get('doc')
    doc = etree.XML(raw_doc_string)

    title = doc.xpath('//dc:title/node()', namespaces=NAMESPACES)[0]
    description = (doc.xpath('//dc:description/node()', namespaces=NAMESPACES) or [''])[0]
    normalized_dict = {
        'title': copy_to_unicode(title),
        'contributors': get_contributors(doc),
        'properties': get_properties(doc),
        'description': copy_to_unicode(description),
        'id': get_ids(doc, raw_doc),
        'source': NAME,
        'tags': get_tags(doc),
        'dateCreated': get_date_created(doc),
        'dateUpdated': get_date_updated(doc),
        'timestamp': timestamp
    }

    return NormalizedDocument(normalized_dict)

if __name__ == '__main__':
    print(lint(consume, normalize))
<commit_msg>removed universal time and timestamp from normalized doc<commit_after>## Consumer for DOE Pages for SHARE
from __future__ import unicode_literals

import time
import requests
from lxml import etree
from datetime import date, timedelta, datetime

from nameparser import HumanName

from dateutil.parser import *

from scrapi.linter import lint
from scrapi.linter.document import RawDocument, NormalizedDocument

NAME = 'doepages'

NAMESPACES = {'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
            'dc': 'http://purl.org/dc/elements/1.1/',
            'dcq': 'http://purl.org/dc/terms/'}

DEFAULT_ENCODING = 'UTF-8'

record_encoding = None

def copy_to_unicode(element):

    encoding = record_encoding or DEFAULT_ENCODING
    element = ''.join(element)
    if isinstance(element, unicode):
        return element
    else:
        return unicode(element, encoding=encoding)

def consume(days_back=15):
    start_date = date.today() - timedelta(days_back)
    base_url = 'http://www.osti.gov/pages/pagesxml?nrows={0}&EntryDateFrom={1}'
    url = base_url.format('1', start_date.strftime('%m/%d/%Y'))
    initial_data = requests.get(url)
    record_encoding = initial_data.encoding
    initial_doc = etree.XML(initial_data.content)

    num_results = int(initial_doc.xpath('//records/@count', namespaces=NAMESPACES)[0])

    url = base_url.format(num_results, start_date.strftime('%m/%d/%Y'))
    data = requests.get(url)
    doc = etree.XML(data.content)

    records = doc.xpath('records/record')

    xml_list = []
    for record in records:
        doc_id = record.xpath('dc:ostiId/node()', namespaces=NAMESPACES)[0]
        record = etree.tostring(record, encoding=record_encoding)
        xml_list.append(RawDocument({
                    'doc': record,
                    'source': NAME,
                    'docID': copy_to_unicode(doc_id),
                    'filetype': 'xml'
                }))

    return xml_list

def get_ids(doc, raw_doc):
    ids = {}
    ids['doi'] = (doc.xpath('//dc:doi/node()', namespaces=NAMESPACES) or [''])[0]
    ids['serviceID'] = raw_doc.get('docID')
    url = (doc.xpath('//dcq:identifier-citation/node()', namespaces=NAMESPACES) or [''])[0]
    if url == '':
        url = 'http://dx.doi.org/' + ids['doi']
    if url == '':
        raise Exception('Warning: url field is blank!')
    ids['url'] = url

    for key, value in ids.iteritems():
        ids[key] = copy_to_unicode(value)

    return ids

def get_contributors(doc):
    contributor_list = []
    full_contributors = doc.xpath('//dc:creator/node()', namespaces=NAMESPACES)[0].split(';')
    for person in full_contributors:
        name = HumanName(person)
        contributor = {
            'prefix': name.title,
            'given': name.first,
            'middle': name.middle,
            'family': name.last,
            'suffix': name.suffix,
            'email': '',
            'ORCID': ''
        }
        contributor_list.append(contributor)

    return contributor_list
def get_properties(doc):
    publisherInfo = {
        'publisher': (doc.xpath('//dcq:publisher/node()', namespaces=NAMESPACES) or [''])[0],
        'publisherSponsor': (doc.xpath('//dcq:publisherSponsor/node()', namespaces=NAMESPACES) or [''])[0],
        'publisherAvailability': (doc.xpath('//dcq:publisherAvailability/node()', namespaces=NAMESPACES) or [''])[0],
        'publisherResearch': (doc.xpath('//dcq:publisherResearch/node()', namespaces=NAMESPACES) or [''])[0],
        'publisherCountry': (doc.xpath('//dcq:publisherCountry/node()', namespaces=NAMESPACES) or [''])[0]
    }

    for key, value in publisherInfo.iteritems():
        publisherInfo[key] = copy_to_unicode(value)

    properties = {
        'language': (doc.xpath('//dc:language/node()', namespaces=NAMESPACES) or [''])[0],
        'type': (doc.xpath('//dc:type/node()', namespaces=NAMESPACES) or [''])[0],
        'typeQualifier': (doc.xpath('//dc:typeQualifier/node()', namespaces=NAMESPACES) or [''])[0],
        'relation': (doc.xpath('//dc:relation/node()', namespaces=NAMESPACES) or [''])[0],
        'coverage': (doc.xpath('//dc:coverage/node()', namespaces=NAMESPACES) or [''])[0],
        'format': (doc.xpath('//dc:format/node()', namespaces=NAMESPACES) or [''])[0],
        'rights': (doc.xpath('//dc:rights/node()', namespaces=NAMESPACES) or [''])[0],
        # TODO:  parse out some of these identifiers in the strings!
        'identifier': (doc.xpath('//dc:identifier/node()', namespaces=NAMESPACES) or [''])[0],
        'identifierReport': (doc.xpath('//dc:identifier/node()', namespaces=NAMESPACES) or [''])[0],
        'identifierDOEcontract': (doc.xpath('//dcq:identifierDOEcontract/node()', namespaces=NAMESPACES) or [''])[0],
        'identifierOther': (doc.xpath('//dc:identifierOther/node()', namespaces=NAMESPACES) or [''])[0],
        'identifier-purl': (doc.xpath('//dcq:identifier-purl/node()', namespaces=NAMESPACES) or [''])[0]
    }

    for key, value in properties.iteritems():
        properties[key] = copy_to_unicode(value)

    properties['publisherInfo'] = publisherInfo

    return properties

def get_date_created(doc):
    date_created = doc.xpath('//dc:date/node()', namespaces=NAMESPACES)[0]
    date = parse(date_created).isoformat()
    return copy_to_unicode(date)

def get_date_updated(doc):
    date_updated = doc.xpath('//dc:dateEntry/node()', namespaces=NAMESPACES)[0]
    date = parse(date_updated).isoformat()
    return copy_to_unicode(date)

def get_tags(doc):
    all_tags = doc.xpath('//dc:subject/node()', namespaces=NAMESPACES) + doc.xpath('//dc:subjectRelated/node()', namespaces=NAMESPACES)
    tags = []
    for taglist in all_tags:
        tags += taglist.split(',')
    return list(set([copy_to_unicode(tag.lower().strip()) for tag in tags]))

def normalize(raw_doc):
    raw_doc_string = raw_doc.get('doc')
    doc = etree.XML(raw_doc_string)

    title = doc.xpath('//dc:title/node()', namespaces=NAMESPACES)[0]
    description = (doc.xpath('//dc:description/node()', namespaces=NAMESPACES) or [''])[0]
    normalized_dict = {
        'title': copy_to_unicode(title),
        'contributors': get_contributors(doc),
        'properties': get_properties(doc),
        'description': copy_to_unicode(description),
        'id': get_ids(doc, raw_doc),
        'source': NAME,
        'tags': get_tags(doc),
        'dateCreated': get_date_created(doc),
        'dateUpdated': get_date_updated(doc),
    }

    return NormalizedDocument(normalized_dict)

if __name__ == '__main__':
    print(lint(consume, normalize))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# The contents of this file are subject to the Mozilla Public License
# Version 2.0 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
#    http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS"basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# OS2Webscanner was developed by Magenta in collaboration with OS2 the
# Danish community of open source municipalities (http://www.os2web.dk/).
#
# The code is currently governed by OS2 the Danish community of open
# source municipalities ( http://www.os2web.dk/ )
import os
import sys
import time
import django
from multiprocessing import Queue
from pathlib import Path

# Include the Django app
base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.append(base_dir + "/webscanner_site")
os.environ["DJANGO_SETTINGS_MODULE"] = "webscanner.settings"
django.setup()

from os2webscanner.models.scan_model import Scan
from mailscan.exchangescan.settings import NUMBER_OF_EMAIL_THREADS
from mailscan.exchangescan.exchange_server_scanner import ExchangeServerScanner

class ExchangeScanner:
    """A scanner application which can be run."""

    def __init__(self):
        """
        Initialize the scanner application.
        Takes input, argv[1], which is directly related to the scan job id in the database.
        Updates the scan status and sets the pid.
        """
        self.scan_id = sys.argv[1]
        self.scan_object = Scan.objects.get(pk=self.scan_id)
        from scanner.scanner.scanner import Scanner
        self.scanner = Scanner(self.scan_object)

    def read_users(self, user_queue, user_file):
        """ Small helper to read user-list from file
        :param user_queue: The common multiprocess queue
        :param user_file: Filename for user list
        """
        user_path = Path(user_file)
        with user_path.open('r') as f:
            users = f.read().split('\n')
        for user in users:
            if len(user.strip()) == 0:
                users.remove(user)
        for user in users:
            user_queue.put(user)

    def run(self):
        domains = self.scanner.get_domain_objects()
        for domain in domains:
            user_queue = Queue()
            self.read_users(user_queue,
                            domain.exchangedomain.get_userlist_file_path()
                            )
            print('Starting scanning for domain {}'.format(domain.url))
            scanners = {}
            for i in range(0, NUMBER_OF_EMAIL_THREADS):
                scanners[i] = ExchangeServerScanner(user_queue,
                                                    domain,
                                                    self.scanner,
                                                    None
                                                    )
                # stats.add_scanner(scanners[i])
                scanners[i].start()
                time.sleep(1)

            for key, value in scanners.items():
                value.join()

            self.scan_object.set_scan_status_done()
            print('Finished scanning.')

exchange_scanner = ExchangeScanner()
exchange_scanner.run()
<commit_msg>* Was using wrong method. Should not join master process with slave process. Instead it should wait until slave process is finished.<commit_after>#!/usr/bin/env python
# The contents of this file are subject to the Mozilla Public License
# Version 2.0 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
#    http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS"basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# OS2Webscanner was developed by Magenta in collaboration with OS2 the
# Danish community of open source municipalities (http://www.os2web.dk/).
#
# The code is currently governed by OS2 the Danish community of open
# source municipalities ( http://www.os2web.dk/ )
import os
import sys
import time
import django
from multiprocessing import Queue
from pathlib import Path

# Include the Django app
base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.append(base_dir + "/webscanner_site")
os.environ["DJANGO_SETTINGS_MODULE"] = "webscanner.settings"
django.setup()

from os2webscanner.models.scan_model import Scan
from mailscan.exchangescan.settings import NUMBER_OF_EMAIL_THREADS
from mailscan.exchangescan.exchange_server_scanner import ExchangeServerScanner

class ExchangeScanner:
    """A scanner application which can be run."""

    def __init__(self):
        """
        Initialize the scanner application.
        Takes input, argv[1], which is directly related to the scan job id in the database.
        Updates the scan status and sets the pid.
        """
        self.scan_id = sys.argv[1]
        self.scan_object = Scan.objects.get(pk=self.scan_id)
        from scanner.scanner.scanner import Scanner
        self.scanner = Scanner(self.scan_object)

    def read_users(self, user_queue, user_file):
        """ Small helper to read user-list from file
        :param user_queue: The common multiprocess queue
        :param user_file: Filename for user list
        """
        user_path = Path(user_file)
        with user_path.open('r') as f:
            users = f.read().split('\n')
        for user in users:
            if len(user.strip()) == 0:
                users.remove(user)
        for user in users:
            user_queue.put(user)

    def run(self):
        domains = self.scanner.get_domain_objects()
        for domain in domains:
            user_queue = Queue()
            self.read_users(user_queue,
                            domain.exchangedomain.get_userlist_file_path()
                            )
            print('Starting scanning for domain {}'.format(domain.url))
            scanners = {}
            for i in range(0, NUMBER_OF_EMAIL_THREADS):
                scanners[i] = ExchangeServerScanner(user_queue,
                                                    domain,
                                                    self.scanner,
                                                    None
                                                    )
                # stats.add_scanner(scanners[i])
                scanners[i].start()
                time.sleep(1)

            for key, value in scanners.items():
                while value.is_alive():
                    print('Process with pid {} is still alive'.format(value.pid))
                    time.sleep(1)

        self.scan_object.set_scan_status_done()
        print('Finished scanning.')

exchange_scanner = ExchangeScanner()
exchange_scanner.run()
<|endoftext|>


<commit_before>"""
SGMLParser-based Link extractors
"""
import re
from urlparse import urlparse, urljoin
from w3lib.url import safe_url_string
from scrapy.selector import Selector
from scrapy.link import Link
from scrapy.linkextractor import IGNORED_EXTENSIONS
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_unicode
from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension
from scrapy.utils.response import get_base_url

class BaseSgmlLinkExtractor(FixedSGMLParser):

    def __init__(self, tag="a", attr="href", unique=False, process_value=None):
        FixedSGMLParser.__init__(self)
        self.scan_tag = tag if callable(tag) else lambda t: t == tag
        self.scan_attr = attr if callable(attr) else lambda a: a == attr
        self.process_value = (lambda v: v) if process_value is None else process_value
        self.current_link = None
        self.unique = unique

    def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
        """ Do the real extraction work """
        self.reset()
        self.feed(response_text)
        self.close()

        ret = []
        if base_url is None:
            base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
        for link in self.links:
            if isinstance(link.url, unicode):
                link.url = link.url.encode(response_encoding)
            link.url = urljoin(base_url, link.url)
            link.url = safe_url_string(link.url, response_encoding)
            link.text = str_to_unicode(link.text, response_encoding, errors='replace')
            ret.append(link)

        return ret

    def _process_links(self, links):
        """ Normalize and filter extracted links

        The subclass should override it if neccessary
        """
        links = unique_list(links, key=lambda link: link.url) if self.unique else links
        return links

    def extract_links(self, response):
        # wrapper needed to allow to work directly with text
        links = self._extract_links(response.body, response.url, response.encoding)
        links = self._process_links(links)
        return links

    def reset(self):
        FixedSGMLParser.reset(self)
        self.links = []
        self.base_url = None

    def unknown_starttag(self, tag, attrs):
        if tag == 'base':
            self.base_url = dict(attrs).get('href')
        if self.scan_tag(tag):
            for attr, value in attrs:
                if self.scan_attr(attr):
                    url = self.process_value(value)
                    if url is not None:
                        link = Link(url=url, nofollow=True if dict(attrs).get('rel') == 'nofollow' else False)
                        self.links.append(link)
                        self.current_link = link

    def unknown_endtag(self, tag):
        self.current_link = None

    def handle_data(self, data):
        if self.current_link:
            self.current_link.text = self.current_link.text + data.strip()

    def matches(self, url):
        """This extractor matches with any url, since
        it doesn't contain any patterns"""
        return True

_re_type = type(re.compile("", 0))

_matches = lambda url, regexs: any((r.search(url) for r in regexs))
_is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file'])

class SgmlLinkExtractor(BaseSgmlLinkExtractor):

    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
                 tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None,
                 deny_extensions=None):
        self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)]
        self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)]
        self.allow_domains = set(arg_to_iter(allow_domains))
        self.deny_domains = set(arg_to_iter(deny_domains))
        self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
        self.canonicalize = canonicalize
        if deny_extensions is None:
            deny_extensions = IGNORED_EXTENSIONS
        self.deny_extensions = set(['.' + e for e in deny_extensions])
        tag_func = lambda x: x in tags
        attr_func = lambda x: x in attrs
        BaseSgmlLinkExtractor.__init__(self,
                                       tag=tag_func,
                                       attr=attr_func,
                                       unique=unique,
                                       process_value=process_value)

    def extract_links(self, response):
        base_url = None
        if self.restrict_xpaths:
            sel = Selector(response)
            base_url = get_base_url(response)
            body = u''.join(f
                            for x in self.restrict_xpaths
                            for f in sel.xpath(x).extract()
                            ).encode(response.encoding)
        else:
            body = response.body

        links = self._extract_links(body, response.url, response.encoding, base_url)
        links = self._process_links(links)
        return links

    def _process_links(self, links):
        links = [x for x in links if self._link_allowed(x)]
        links = BaseSgmlLinkExtractor._process_links(self, links)
        return links

    def _link_allowed(self, link):
        parsed_url = urlparse(link.url)
        allowed = _is_valid_url(link.url)
        if self.allow_res:
            allowed &= _matches(link.url, self.allow_res)
        if self.deny_res:
            allowed &= not _matches(link.url, self.deny_res)
        if self.allow_domains:
            allowed &= url_is_from_any_domain(parsed_url, self.allow_domains)
        if self.deny_domains:
            allowed &= not url_is_from_any_domain(parsed_url, self.deny_domains)
        if self.deny_extensions:
            allowed &= not url_has_any_extension(parsed_url, self.deny_extensions)
        if allowed and self.canonicalize:
            link.url = canonicalize_url(parsed_url)
        return allowed

    def matches(self, url):
        if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):
            return False
        if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):
            return False

        allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True]
        denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else []
        return any(allowed) and not any(denied)
<commit_msg>BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag<commit_after>"""
SGMLParser-based Link extractors
"""
import re
from urlparse import urlparse, urljoin
from w3lib.url import safe_url_string
from scrapy.selector import Selector
from scrapy.link import Link
from scrapy.linkextractor import IGNORED_EXTENSIONS
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_unicode
from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension
from scrapy.utils.response import get_base_url

class BaseSgmlLinkExtractor(FixedSGMLParser):

    def __init__(self, tag="a", attr="href", unique=False, process_value=None):
        FixedSGMLParser.__init__(self)
        self.scan_tag = tag if callable(tag) else lambda t: t == tag
        self.scan_attr = attr if callable(attr) else lambda a: a == attr
        self.process_value = (lambda v: v) if process_value is None else process_value
        self.current_link = None
        self.unique = unique

    def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
        """ Do the real extraction work """
        self.reset()
        self.feed(response_text)
        self.close()

        ret = []
        if base_url is None:
            base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
        for link in self.links:
            if isinstance(link.url, unicode):
                link.url = link.url.encode(response_encoding)
            link.url = urljoin(base_url, link.url)
            link.url = safe_url_string(link.url, response_encoding)
            link.text = str_to_unicode(link.text, response_encoding, errors='replace')
            ret.append(link)

        return ret

    def _process_links(self, links):
        """ Normalize and filter extracted links

        The subclass should override it if neccessary
        """
        links = unique_list(links, key=lambda link: link.url) if self.unique else links
        return links

    def extract_links(self, response):
        # wrapper needed to allow to work directly with text
        links = self._extract_links(response.body, response.url, response.encoding)
        links = self._process_links(links)
        return links

    def reset(self):
        FixedSGMLParser.reset(self)
        self.links = []
        self.base_url = None

    def unknown_starttag(self, tag, attrs):
        if tag == 'base':
            self.base_url = dict(attrs).get('href')
        if self.scan_tag(tag):
            for attr, value in attrs:
                if self.scan_attr(attr):
                    url = self.process_value(value)
                    if url is not None:
                        link = Link(url=url, nofollow=True if dict(attrs).get('rel') == 'nofollow' else False)
                        self.links.append(link)
                        self.current_link = link

    def unknown_endtag(self, tag):
        if self.scan_tag(tag):
            self.current_link = None

    def handle_data(self, data):
        if self.current_link:
            self.current_link.text = self.current_link.text + data.strip()

    def matches(self, url):
        """This extractor matches with any url, since
        it doesn't contain any patterns"""
        return True

_re_type = type(re.compile("", 0))

_matches = lambda url, regexs: any((r.search(url) for r in regexs))
_is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file'])

class SgmlLinkExtractor(BaseSgmlLinkExtractor):

    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
                 tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None,
                 deny_extensions=None):
        self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)]
        self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)]
        self.allow_domains = set(arg_to_iter(allow_domains))
        self.deny_domains = set(arg_to_iter(deny_domains))
        self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
        self.canonicalize = canonicalize
        if deny_extensions is None:
            deny_extensions = IGNORED_EXTENSIONS
        self.deny_extensions = set(['.' + e for e in deny_extensions])
        tag_func = lambda x: x in tags
        attr_func = lambda x: x in attrs
        BaseSgmlLinkExtractor.__init__(self,
                                       tag=tag_func,
                                       attr=attr_func,
                                       unique=unique,
                                       process_value=process_value)

    def extract_links(self, response):
        base_url = None
        if self.restrict_xpaths:
            sel = Selector(response)
            base_url = get_base_url(response)
            body = u''.join(f
                            for x in self.restrict_xpaths
                            for f in sel.xpath(x).extract()
                            ).encode(response.encoding)
        else:
            body = response.body

        links = self._extract_links(body, response.url, response.encoding, base_url)
        links = self._process_links(links)
        return links

    def _process_links(self, links):
        links = [x for x in links if self._link_allowed(x)]
        links = BaseSgmlLinkExtractor._process_links(self, links)
        return links

    def _link_allowed(self, link):
        parsed_url = urlparse(link.url)
        allowed = _is_valid_url(link.url)
        if self.allow_res:
            allowed &= _matches(link.url, self.allow_res)
        if self.deny_res:
            allowed &= not _matches(link.url, self.deny_res)
        if self.allow_domains:
            allowed &= url_is_from_any_domain(parsed_url, self.allow_domains)
        if self.deny_domains:
            allowed &= not url_is_from_any_domain(parsed_url, self.deny_domains)
        if self.deny_extensions:
            allowed &= not url_has_any_extension(parsed_url, self.deny_extensions)
        if allowed and self.canonicalize:
            link.url = canonicalize_url(parsed_url)
        return allowed

    def matches(self, url):
        if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):
            return False
        if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):
            return False

        allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True]
        denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else []
        return any(allowed) and not any(denied)
<|endoftext|>


<commit_before>"""
SGMLParser-based Link extractors
"""
from urlparse import urljoin
import warnings
from sgmllib import SGMLParser

from w3lib.url import safe_url_string
from scrapy.selector import Selector
from scrapy.link import Link
from scrapy.linkextractor import FilteringLinkExtractor
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import unique as unique_list, str_to_unicode
from scrapy.utils.response import get_base_url
from scrapy.exceptions import ScrapyDeprecationWarning

class BaseSgmlLinkExtractor(SGMLParser):

    def __init__(self, tag="a", attr="href", unique=False, process_value=None):
        warnings.warn(
            "BaseSgmlLinkExtractor is deprecated and will be removed in future releases!!! "
            "Please use scrapy.contrib.linkextractors.LinkExtractor",
            ScrapyDeprecationWarning
        )
        SGMLParser.__init__(self)
        self.scan_tag = tag if callable(tag) else lambda t: t == tag
        self.scan_attr = attr if callable(attr) else lambda a: a == attr
        self.process_value = (lambda v: v) if process_value is None else process_value
        self.current_link = None
        self.unique = unique

    def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
        """ Do the real extraction work """
        self.reset()
        self.feed(response_text)
        self.close()

        ret = []
        if base_url is None:
            base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
        for link in self.links:
            if isinstance(link.url, unicode):
                link.url = link.url.encode(response_encoding)
            link.url = urljoin(base_url, link.url)
            link.url = safe_url_string(link.url, response_encoding)
            link.text = str_to_unicode(link.text, response_encoding, errors='replace').strip()
            ret.append(link)

        return ret

    def _process_links(self, links):
        """ Normalize and filter extracted links

        The subclass should override it if necessary
        """
        links = unique_list(links, key=lambda link: link.url) if self.unique else links
        return links

    def extract_links(self, response):
        # wrapper needed to allow to work directly with text
        links = self._extract_links(response.body, response.url, response.encoding)
        links = self._process_links(links)
        return links

    def reset(self):
        SGMLParser.reset(self)
        self.links = []
        self.base_url = None

    def unknown_starttag(self, tag, attrs):
        if tag == 'base':
            self.base_url = dict(attrs).get('href')
        if self.scan_tag(tag):
            for attr, value in attrs:
                if self.scan_attr(attr):
                    url = self.process_value(value)
                    if url is not None:
                        link = Link(url=url, nofollow=True if dict(attrs).get('rel') == 'nofollow' else False)
                        self.links.append(link)
                        self.current_link = link

    def unknown_endtag(self, tag):
        if self.scan_tag(tag):
            self.current_link = None

    def handle_data(self, data):
        if self.current_link:
            self.current_link.text = self.current_link.text + data

    def matches(self, url):
        """This extractor matches with any url, since
        it doesn't contain any patterns"""
        return True

class SgmlLinkExtractor(FilteringLinkExtractor):

    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None,
                 deny_extensions=None):

        warnings.warn(
            "SgmlLinkExtractor is deprecated and will be removed in future releases. "
            "Please use scrapy.contrib.linkextractors.LinkExtractor",
            ScrapyDeprecationWarning
        )

        tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
        tag_func = lambda x: x in tags
        attr_func = lambda x: x in attrs

        with warnings.catch_warnings(record=True):
            lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
                unique=unique, process_value=process_value)

        super(SgmlLinkExtractor, self).__init__(lx, allow, deny,
            allow_domains, deny_domains, restrict_xpaths, canonicalize,
            deny_extensions)

        # FIXME: was added to fix a RegexLinkExtractor testcase
        self.base_url = None

    def extract_links(self, response):
        base_url = None
        if self.restrict_xpaths:
            sel = Selector(response)
            base_url = get_base_url(response)
            body = u''.join(f
                            for x in self.restrict_xpaths
                            for f in sel.xpath(x).extract()
                            ).encode(response.encoding, errors='xmlcharrefreplace')
        else:
            body = response.body

        links = self._extract_links(body, response.url, response.encoding, base_url)
        links = self._process_links(links)
        return links
<commit_msg>too much drama<commit_after>"""
SGMLParser-based Link extractors
"""
from urlparse import urljoin
import warnings
from sgmllib import SGMLParser

from w3lib.url import safe_url_string
from scrapy.selector import Selector
from scrapy.link import Link
from scrapy.linkextractor import FilteringLinkExtractor
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import unique as unique_list, str_to_unicode
from scrapy.utils.response import get_base_url
from scrapy.exceptions import ScrapyDeprecationWarning

class BaseSgmlLinkExtractor(SGMLParser):

    def __init__(self, tag="a", attr="href", unique=False, process_value=None):
        warnings.warn(
            "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
            "Please use scrapy.contrib.linkextractors.LinkExtractor",
            ScrapyDeprecationWarning
        )
        SGMLParser.__init__(self)
        self.scan_tag = tag if callable(tag) else lambda t: t == tag
        self.scan_attr = attr if callable(attr) else lambda a: a == attr
        self.process_value = (lambda v: v) if process_value is None else process_value
        self.current_link = None
        self.unique = unique

    def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
        """ Do the real extraction work """
        self.reset()
        self.feed(response_text)
        self.close()

        ret = []
        if base_url is None:
            base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
        for link in self.links:
            if isinstance(link.url, unicode):
                link.url = link.url.encode(response_encoding)
            link.url = urljoin(base_url, link.url)
            link.url = safe_url_string(link.url, response_encoding)
            link.text = str_to_unicode(link.text, response_encoding, errors='replace').strip()
            ret.append(link)

        return ret

    def _process_links(self, links):
        """ Normalize and filter extracted links

        The subclass should override it if necessary
        """
        links = unique_list(links, key=lambda link: link.url) if self.unique else links
        return links

    def extract_links(self, response):
        # wrapper needed to allow to work directly with text
        links = self._extract_links(response.body, response.url, response.encoding)
        links = self._process_links(links)
        return links

    def reset(self):
        SGMLParser.reset(self)
        self.links = []
        self.base_url = None

    def unknown_starttag(self, tag, attrs):
        if tag == 'base':
            self.base_url = dict(attrs).get('href')
        if self.scan_tag(tag):
            for attr, value in attrs:
                if self.scan_attr(attr):
                    url = self.process_value(value)
                    if url is not None:
                        link = Link(url=url, nofollow=True if dict(attrs).get('rel') == 'nofollow' else False)
                        self.links.append(link)
                        self.current_link = link

    def unknown_endtag(self, tag):
        if self.scan_tag(tag):
            self.current_link = None

    def handle_data(self, data):
        if self.current_link:
            self.current_link.text = self.current_link.text + data

    def matches(self, url):
        """This extractor matches with any url, since
        it doesn't contain any patterns"""
        return True

class SgmlLinkExtractor(FilteringLinkExtractor):

    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None,
                 deny_extensions=None):

        warnings.warn(
            "SgmlLinkExtractor is deprecated and will be removed in future releases. "
            "Please use scrapy.contrib.linkextractors.LinkExtractor",
            ScrapyDeprecationWarning
        )

        tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
        tag_func = lambda x: x in tags
        attr_func = lambda x: x in attrs

        with warnings.catch_warnings(record=True):
            lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
                unique=unique, process_value=process_value)

        super(SgmlLinkExtractor, self).__init__(lx, allow, deny,
            allow_domains, deny_domains, restrict_xpaths, canonicalize,
            deny_extensions)

        # FIXME: was added to fix a RegexLinkExtractor testcase
        self.base_url = None

    def extract_links(self, response):
        base_url = None
        if self.restrict_xpaths:
            sel = Selector(response)
            base_url = get_base_url(response)
            body = u''.join(f
                            for x in self.restrict_xpaths
                            for f in sel.xpath(x).extract()
                            ).encode(response.encoding, errors='xmlcharrefreplace')
        else:
            body = response.body

        links = self._extract_links(body, response.url, response.encoding, base_url)
        links = self._process_links(links)
        return links
<|endoftext|>


<commit_before>#!/usr/bin/env python

import re
import shutil
import subprocess
import sys
import os
import fnmatch
import io

HTML_INPUT_DIR = sys.argv[1]
BUILD_DIR = sys.argv[2]
OUTPUT_DIR = sys.argv[3]

# For some reason the handlebars executable name is passed via the
# TC_HANDLEBARS_EXE environment variable.
HANDLEBARS_EXE = os.environ['TC_HANDLEBARS_EXE'] if os.environ['TC_HANDLEBARS_EXE'] else 'handlebars'

WEB_TEMP_DIR = BUILD_DIR + "/webtemp"

def split_by_script_tags(text):
    pos = 0
    parts = []
    search_string = '<script'
    while pos < len(text):
        findres = text.find(search_string, pos + 1)
        if findres == -1:
            parts += [text[pos:]]
            pos = len(text)
        else:
            parts += [text[pos:findres]]
            pos = findres
    return parts

# We want to look through all the .html files, split them at the
# beginning of any "<script" match, and then do some business cleaning
# up the parts.
try:
    # Make a temporary directory for these chopped up template files.
    os.makedirs(WEB_TEMP_DIR)

    parts = []

    # Split all the html files by script tags.
    for root, dirnames, filenames in os.walk(HTML_INPUT_DIR):
        for filename in fnmatch.filter(filenames, "*.html"):
            path = os.path.join(root, filename)
            with io.open(path, encoding='utf-8') as f:
                text = f.read()
                tmp_parts = split_by_script_tags(text)
                parts += tmp_parts

    # Now build parts corresponding to script tags.
    named_parts = {}
    for part in parts:
        # We use the first line of the file to create the filename for
        # non-script-tag-led matches?  You could ask Michel why we do
        # that.
        if re.search('<script ((?!id=)[^>])*>', part):
            raise RuntimeError("Found script tag without id.")
        searchres = re.search('<script [^>]*id="([^"]+)"', part)
        name = searchres.group(1) if searchres else part.split('\n', 1)[0]
        part1 = re.sub('</script>|<script[^>]*>', '', part)
        part2 = re.sub('\s+', ' ', part1)
        named_parts[name] = part2

    # Write the parts to files to the temporary directory, so that we can pass them to handlebars.
    for name in named_parts:
        with io.open(WEB_TEMP_DIR + '/' + name + '.handlebars', 'w', encoding='utf-8') as f:
            f.write(named_parts[name])

    # Now, we call handlebars.
    subprocess.call([HANDLEBARS_EXE, '-m', WEB_TEMP_DIR, '-f', OUTPUT_DIR + '/template.js']);

finally:
    shutil.rmtree(WEB_TEMP_DIR)
<commit_msg>Removed build_handlebars_templates.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# Keystone monitoring script for Nagios
#
# Copyright © 2012-2014 eNovance <licensing@enovance.com>
#
# Authors:
#   Sofer Athlan-Guyot <sofer.athlan@enovance.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Requirments: python-keystoneclient, python-argparse, python

import sys
import argparse
from cinderclient.client import Client
#from novaclient.client import Client
from cinderclient import exceptions
import time
import logging
import urlparse
from datetime import datetime

DAEMON_DEFAULT_PORT=8776

STATE_OK = 0
STATE_WARNING = 1
STATE_CRITICAL = 2
STATE_UNKNOWN = 3

def script_error(msg):
    sys.stderr.write("UNKNOWN - %s" % msg)
    sys.exit(STATE_UNKNOWN)

class Novautils:
    def __init__(self, nova_client):
        self.nova_client = nova_client
        self.msgs = []
        self.volume = None
        self.start = datetime.now()

    def get_duration(self):
        return (datetime.now() - self.start).seconds

    def mangle_url(self, url):
        # need to populate management_url property
        try:
            self.nova_client.volumes.list()
        except Exception as e:
            script_error("unknown error filling object with data: %s\n" % e)

        import pdb; pdb.set_trace()

        try:
            endpoint_url = urlparse.urlparse(url)
        except Exception as e:
            script_error("you must provide an endpoint_url in the form <scheme>://<url>/ (%s)\n" % e)
        scheme = endpoint_url.scheme
        if scheme is None:
            script_error("you must provide an endpoint_url in the form <scheme>://<url>/ (%s)\n" % e)
        catalog_url = None
        try:
            catalog_url = urlparse.urlparse(self.nova_client.client.management_url)
        except Exception as e:
            script_error("unknown error parsing the catalog url : %s\n" % e)

        port = endpoint_url.port
        if port is None:
            if catalog_url.port is None:
                port = DAEMON_DEFAULT_PORT
            else:
                port = catalog_url.port

        netloc = "%s:%i" % (endpoint_url.hostname, port)
        url = urlparse.urlunparse([scheme,
                                   netloc,
                                   catalog_url.path,
                                   catalog_url.params,
                                   catalog_url.query,
                                   catalog_url.fragment])
        self.nova_client.client.management_url = url

    def check_existing_volume(self, volume_name):
        count = 0
        for s in self.nova_client.volumes.list():
            if s.display_name == volume_name:
                count += 1
        if count > 0:
            self.msgs.append("Found '%s' present %d time(s).  Won't create test volume.  Please check and delete." % (volume_name, count))

    def create_volume(self, volume_name, size):
        if not self.msgs:
            try:
                self.volume = self.nova_client.volumes.create(display_name=volume_name,
                                                             size=size)
            except Exception as e:
                self.msgs.append("Cannot create the vm %s (%s)" % (args.volume_name, e))

    def volume_ready(self, timeout):
        if not self.msgs:
            timer = 0
            while self.volume.status != "available":
                if timer >= timeout:
                    self.msgs.append("Cannot create the volume.")
                    break
                time.sleep(1)
                timer += 1
                try:
                    self.volume.get()
                except Exception as e:
                    self.msgs.append("Problem getting the status of the volume: %s" % e)
                    break

    def delete_volume(self):
        if not self.msgs or self.volume != None:
            try:
                self.volume.delete()
            except Exception as e:
                self.msgs.append("Problem deleting the volume: %s" % e)

    def volume_deleted(self, timeout):
        deleted = False
        timer = 0
        while not deleted and not self.msgs:
            time.sleep(1)
            if timer >= timeout:
                self.msgs.append("Could not delete the volume within %d seconds" % timer)
                break
            timer += 1
            try:
                self.volume.get()
            except exceptions.NotFound:
                deleted = True
            except Exception as e:
                self.msgs.append("Cannot delete the volume (%s)" % e)
                break

parser = argparse.ArgumentParser(description='Check an OpenStack Keystone server.')
parser.add_argument('--auth_url', metavar='URL', type=str,
                    required=True,
                    help='Keystone URL')

parser.add_argument('--username', metavar='username', type=str,
                    required=True,
                    help='username to use for authentication')

parser.add_argument('--password', metavar='password', type=str,
                    required=True,
                    help='password to use for authentication')

parser.add_argument('--tenant', metavar='tenant', type=str,
                    required=True,
                    help='tenant name to use for authentication')

parser.add_argument('--endpoint_url', metavar='endpoint_url', type=str,
                    help='Override the catalog endpoint.')

parser.add_argument('--endpoint_type', metavar='endpoint_type', type=str,
                    default="publicURL",
                    help='Endpoint type in the catalog request.  Public by default.')

parser.add_argument('--api_version', metavar='api_version', type=str,
                    default='1',
                    help='Version of the API to use. 2 by default.')

parser.add_argument('--timeout', metavar='timeout', type=int,
                    default=120,
                    help='Max number of second to create a vm (120 by default).')

parser.add_argument('--volume_name', metavar='volume_name', type=str,
                    default="monitoring_test",
                    help='Name of the volume to create')

parser.add_argument('--volume_size', metavar='volume_size', type=int,
                    default=1,
                    help='Size of the volume to create')

parser.add_argument('--verbose', action='count',
                    help='Print requests on stderr.')

args = parser.parse_args()
nova_client = Client(args.api_version,
                     username=args.username,
                     project_id=args.tenant,
                     api_key=args.password,
                     auth_url=args.auth_url,
                     endpoint_type=args.endpoint_type,
                     http_log_debug=args.verbose)

util = Novautils(nova_client)

if args.verbose:
    ch = logging.StreamHandler()
    nova_client.client._logger.setLevel(logging.DEBUG)
    nova_client.client._logger.addHandler(ch)

if args.endpoint_url:
    util.mangle_url(args.endpoint_url)

util.check_existing_volume(args.volume_name)
util.create_volume(args.volume_name,args.volume_size)
util.volume_ready(args.timeout)
util.delete_volume()
util.volume_deleted(args.timeout)

if util.msgs:
    print "CRITICAL - %s" % ", ".join(util.msgs)
    sys.exit(STATE_CRITICAL)

duration = util.get_duration()
print("OK - Volume spawned and deleted in %d seconds | time=%d" % (duration, duration))
sys.exit(STATE_OK)
<commit_msg>Add a check cinder using python library.<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# Keystone monitoring script for Nagios
#
# Copyright © 2012-2014 eNovance <licensing@enovance.com>
#
# Authors:
#   Sofer Athlan-Guyot <sofer.athlan@enovance.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Requirments: python-cinderclient, python-argparse, python
# ## Arguments
#
# ### Required arguments
# * `--auth_url`: Keystone URL
# * `--username`: Username to use for authentication
# * `--password`: Password to use for authentication
# * `--tenant`: Tenant name to use for authentication
#
# ### Optional arguments
#
# * `-h`: Show the help message and exit
# * `--region_name`: Region to select for authentication
# * `--endpoint_url`: Override the catalog endpoint
# * `--endpoint_type`: When not overriding, which type to use in the catalog.  Public by default.
# * `--volume_name`: Name of the volume to create (monitoring_test by default)
# * `--volume_size`: Size of the volume to create (1 GB by default)
# * `--volume_type`: With multiple backends, choose the volume type.
# * `--availability_zone`: Specify availability zone.
# * `--force_delete`: If matching volumes are found, delete them and add a notification in the message instead of getting out in critical state.
# * `--api_version`: Version of the API to use. 1 by default.
# * `--timeout`: Max number of second to create/destroy a volume (120 by default).
# * `--verbose`: Print requests on stderr
#
# ## Usage
#
# Create a test volume and delete it, only if no volume match
# `volume_name` (here `monitoring_test` by default).  If there is any
# volume it assumes that it's some leftover and exit in CRITICAL state,
# notifying the rogue volumes.  Good for testing that everything works
# properly in your environment.
#
# * `check_cinder-volume.py --auth_url $OS_AUTH_URL --username $OS_USERNAME --tenant $OS_TENANT_NAME --password $OS_PASSWORD`
#
# Here we mangle the endpoint url to override the one returned by the
# catalog.  If we assume that the url returned by the catalog is behind
# a load balancer, this enable the user to easely choose which API
# server it's going to query.
#
# * `check_cinder-volume.py --auth_url $OS_AUTH_URL --username $OS_USERNAME --tenant $OS_TENANT_NAME --password $OS_PASSWORD --endpoint_url http://localhost`
#
# Here we force de deletion of any volume found with the matching name.
# The number of found volume is returned in the output of the plugin.
#
# * `check_cinder-volume.py --auth_url $OS_AUTH_URL --username $OS_USERNAME --tenant $OS_TENANT_NAME --password $OS_PASSWORD --force_delete`
#
# For a asynchronous usage relative to a nagios check, one can use
# [cache_check.py](https://github.com/gaelL/nagios-cache-check)
#
# * `cache_check.py -c "check_cinder-volume.py --auth_url $OS_AUTH_URL --username $OS_USERNAME --tenant $OS_TENANT_NAME --password $OS_PASSWORD"`
#
import sys
import argparse
from cinderclient.client import Client
from cinderclient import exceptions
import time
import logging
import urlparse
from datetime import datetime

DAEMON_DEFAULT_PORT = 8776

STATE_OK = 0
STATE_WARNING = 1
STATE_CRITICAL = 2
STATE_UNKNOWN = 3

def script_unknown(msg):
    sys.stderr.write("UNKNOWN - %s (UTC: %s)\n" % (msg, datetime.utcnow()))
    sys.exit(STATE_UNKNOWN)

def script_critical(msg):
    sys.stderr.write("CRITICAL - %s (UTC: %s)\n" % (msg, datetime.utcnow()))
    sys.exit(STATE_CRITICAL)

# python has no "toepoch" method: http://bugs.python.org/issue2736
# now, after checking http://stackoverflow.com/a/16307378,
# and http://stackoverflow.com/a/8778548 made my mind to this approach
def totimestamp(dt=None, epoch=datetime(1970, 1, 1)):
    if not dt:
        dt = datetime.utcnow()
    td = dt - epoch
    # return td.total_seconds()
    return int((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6)
               / 1e6)

class Novautils:
    def __init__(self, nova_client):
        self.nova_client = nova_client
        self.msgs = []
        self.start = totimestamp()
        self.notifications = ["volume_creation_time=%s" % self.start]
        self.volume = None
        self.connection_done = False

    def check_connection(self, force=False):
        if not self.connection_done or force:
            try:
                # force a connection to the server
                self.connection_done = self.nova_client.limits.get()
            except Exception as e:
                script_critical("Cannot connect to cinder: %s" % e)

    def get_duration(self):
        return totimestamp() - self.start

    def mangle_url(self, url):
        # This first connection populate the structure we need inside
        # the object.  This does not cost anything if a connection has
        # already been made.
        self.check_connection()
        try:
            endpoint_url = urlparse.urlparse(url)
        except Exception as e:
            script_unknown("you must provide an endpoint_url in the form"
                           + "<scheme>://<url>/ (%s)" % e)
        scheme = endpoint_url.scheme
        if scheme is None:
            script_unknown("you must provide an endpoint_url in the form"
                           + "<scheme>://<url>/ (%s)" % e)
        catalog_url = None
        try:
            catalog_url = urlparse.urlparse(
                self.nova_client.client.management_url)
        except Exception as e:
            script_unknown("unknown error parsing the catalog url : %s" % e)

        port = endpoint_url.port
        if port is None:
            if catalog_url.port is None:
                port = DAEMON_DEFAULT_PORT
            else:
                port = catalog_url.port

        netloc = "%s:%i" % (endpoint_url.hostname, port)
        url = urlparse.urlunparse([scheme,
                                   netloc,
                                   catalog_url.path,
                                   catalog_url.params,
                                   catalog_url.query,
                                   catalog_url.fragment])
        self.nova_client.client.management_url = url

    def check_existing_volume(self, volume_name, delete):
        count = 0
        for s in self.nova_client.volumes.list():
            if s.display_name == volume_name:
                if delete:
                    # asynchronous call, we do not check that it worked
                    s.delete()
                count += 1
        if count > 0:
            if delete:
                self.notifications.append("Found '%s' present %d time(s)"
                                          % (volume_name, count))
            else:
                self.msgs.append("Found '%s' present %d time(s). "
                                 % (volume_name, count)
                                 + "Won't create test volume. "
                                 + "Please check and delete.")

    def create_volume(self, volume_name, size, availability_zone, volume_type):
        if not self.msgs:
            try:
                conf = {'display_name': volume_name,
                        'size': size}
                if volume_type:
                    conf['volume_type'] = volume_type
                if availability_zone:
                    conf['availability_zone'] = availability_zone
                self.volume = self.nova_client.volumes.create(**conf)
            except Exception as e:
                self.msgs.append("Cannot create the volume %s (%s)"
                                 % (args.volume_name, e))

    def volume_ready(self, timeout):
        if not self.msgs:
            timer = 0
            while self.volume.status != "available":
                if timer >= timeout:
                    self.msgs.append("Cannot create the volume.")
                    break
                time.sleep(1)
                timer += 1
                try:
                    self.volume.get()
                except Exception as e:
                    self.msgs.append("Problem getting the status of "
                                     + "the volume: %s" % e)
                    break

    def delete_volume(self):
        if not self.msgs or self.volume is not None:
            try:
                self.volume.delete()
            except Exception as e:
                self.msgs.append("Problem deleting the volume: %s" % e)

    def volume_deleted(self, timeout):
        deleted = False
        timer = 0
        while not deleted and not self.msgs:
            time.sleep(1)
            if timer >= timeout:
                self.msgs.append("Could not delete the volume within"
                                 + "%d seconds" % timer)
                break
            timer += 1
            try:
                self.volume.get()
            except exceptions.NotFound:
                deleted = True
            except Exception as e:
                self.msgs.append("Cannot delete the volume (%s)" % e)
                break

parser = argparse.ArgumentParser(
    description='Check an OpenStack Keystone server.')
parser.add_argument('--auth_url', metavar='URL', type=str,
                    required=True,
                    help='Keystone URL')

parser.add_argument('--username', metavar='username', type=str,
                    required=True,
                    help='username to use for authentication')

parser.add_argument('--password', metavar='password', type=str,
                    required=True,
                    help='password to use for authentication')

parser.add_argument('--tenant', metavar='tenant', type=str,
                    required=True,
                    help='tenant name to use for authentication')

parser.add_argument('--endpoint_url', metavar='endpoint_url', type=str,
                    help='Override the catalog endpoint.')

parser.add_argument('--endpoint_type', metavar='endpoint_type', type=str,
                    default="publicURL",
                    help='Endpoint type in the catalog request. '
                    + 'Public by default.')

parser.add_argument('--force_delete', action='store_true',
                    help='If matching volumes are found, delete them and add '
                    + 'a notification in the message instead of getting out '
                    + 'in critical state.')

parser.add_argument('--api_version', metavar='api_version', type=str,
                    default='1',
                    help='Version of the API to use. 1 by default.')

parser.add_argument('--timeout', metavar='timeout', type=int,
                    default=120,
                    help='Max number of second to create/delete a volume '
                    + '(120 by default).')

parser.add_argument('--volume_name', metavar='volume_name', type=str,
                    default="monitoring_test",
                    help='Name of the volume to create '
                    + '(monitoring_test by default)')

parser.add_argument('--volume_size', metavar='volume_size', type=int,
                    default=1,
                    help='Size of the volume to create (1 GB by default)')

parser.add_argument('--volume_type', metavar='volume_type', type=str,
                    default=None,
                    help='With multiple backends, choose the volume type.')

parser.add_argument('--availability_zone', metavar='availability_zone',
                    type=str,
                    default=None,
                    help='Specify availability zone.')

parser.add_argument('--verbose', action='count',
                    help='Print requests on stderr.')

args = parser.parse_args()

# this shouldn't raise any exception as no connection is done when
# creating the object.  But It may change, so I catch everything.
try:
    nova_client = Client(args.api_version,
                         username=args.username,
                         project_id=args.tenant,
                         api_key=args.password,
                         auth_url=args.auth_url,
                         endpoint_type=args.endpoint_type,
                         http_log_debug=args.verbose)
except Exception as e:
    script_critical("Error creating cinder communication object: %s" % e)

util = Novautils(nova_client)

if args.verbose:
    ch = logging.StreamHandler()
    nova_client.client._logger.setLevel(logging.DEBUG)
    nova_client.client._logger.addHandler(ch)

# Initiate the first connection and catch error.
util.check_connection()

if args.endpoint_url:
    util.mangle_url(args.endpoint_url)
    # after mangling the url, the endpoint has changed.  Check that
    # it's valid.
    util.check_connection(force=True)

util.check_existing_volume(args.volume_name, args.force_delete)
util.create_volume(args.volume_name,
                   args.volume_size,
                   args.availability_zone,
                   args.volume_type)
util.volume_ready(args.timeout)
util.delete_volume()
util.volume_deleted(args.timeout)

if util.msgs:
    script_critical(", ".join(util.msgs))

duration = util.get_duration()
notification = ""

if util.notifications:
    notification = "(" + ", ".join(util.notifications) + ")"

print("OK - Volume spawned and deleted in %d seconds %s| time=%d"
      % (duration, notification, duration))
sys.exit(STATE_OK)
<|endoftext|>


<commit_before><commit_msg>fix bug<commit_after><|endoftext|>


<commit_before>import os
from scipy_distutils.misc_util import get_path, default_config_dict

def configuration(parent_package=''):
    """
       gist only works with an X-windows server
       This will install *.gs and *.gp files to
       '%spython%s/site-packages/scipy/xplt' % (sys.prefix,sys.version[:3])
    """
    # Check for X11 libraries
    save_file = './saved_values.py'
    if not os.path.exists(save_file):
	save_file = '../saved_values.py'

    try:
        exec(open(save_file).read())
        try:
            X11 = X11
        except NameError:
            X11 = check_and_save()
    except IOError:
        X11 = check_and_save()
    if X11:
        config = default_config_dict()

        if parent_package:
            parent_package = parent_package + '.'
        local_path = get_path(__name__)

        config['packages'].append(parent_package+'xplt')

        from scipy_distutils.core import Extension
        sources = ['gistCmodule.c']
        sources = [os.path.join(local_path,x) for x in sources]

        ext = Extension(parent_package+'xplt.gistC',
                        sources,
                        include_dirs = ['/usr/include/X11'],
                        library_dirs = ['/usr/X11R6/lib'],
                        libraries = ['X11','m'])
        config['ext_modules'].append(ext)

        from glob import glob
        gist = glob(os.path.join(local_path,'gist','*.c'))
        # libraries are C static libraries
        config['libraries'].append(('gist',{'sources':gist,
                                   'macros':[('STDC_HEADERS',1)]}))

        file_ext = ['*.gs','*.gp', '*.ps', '*.help']
        xplt_files = [glob(os.path.join(local_path,x)) for x in file_ext]
	xplt_files = reduce(lambda x,y:x+y,xplt_files,[])
        xplt_path = os.path.join(local_path,'xplt')
        config['data_files'].extend( [(xplt_path,xplt_files)])

        return config

# placing the line X11=0 in saved_values will turn off xplt installation.
def check_and_save(file='saved_values.py'):
    import commands
    output = commands.getoutput('find /usr -name "libX11*" -print')
    X11 = (output != '')
    fid = open(file,'a')
    fid.write('X11 = %d\n' % X11)
    fid.close()
    return X11
<commit_msg>Introduced x11_info and fixed the inefficiency of xplt setup<commit_after>#!/usr/bin/env python

import os
from scipy_distutils.misc_util import get_path, default_config_dict, dot_join
from scipy_distutils.core import Extension
from scipy_distutils.x11_info import get_x11_info

def configuration(parent_package=''):
    """
       gist only works with an X-windows server
       This will install *.gs and *.gp files to
       '%spython%s/site-packages/scipy/xplt' % (sys.prefix,sys.version[:3])
    """
    x11_info = get_x11_info()
    if not x11_info:
        return

    config = default_config_dict('xplt',parent_package)
    local_path = get_path(__name__)

    sources = ['gistCmodule.c']
    sources = [os.path.join(local_path,x) for x in sources]

    ext = Extension(dot_join(parent_package,'xplt.gistC'),
                    sources,
                    include_dirs = x11_info.get('include_dirs',[]),
                    library_dirs = x11_info.get('library_dirs',[]),
                    libraries = x11_info.get('libraries',[]) + ['m'])
    config['ext_modules'].append(ext)

    from glob import glob
    gist = glob(os.path.join(local_path,'gist','*.c'))
    # libraries are C static libraries
    config['libraries'].append(('gist',{'sources':gist,
                                        'macros':[('STDC_HEADERS',1)]}))

    file_ext = ['*.gs','*.gp', '*.ps', '*.help']
    xplt_files = [glob(os.path.join(local_path,x)) for x in file_ext]
    xplt_files = reduce(lambda x,y:x+y,xplt_files,[])
    xplt_path = os.path.join(local_path,'xplt')
    config['data_files'].extend( [(xplt_path,xplt_files)])

    return config

if __name__ == '__main__':
    from scipy_distutils.core import setup
    setup(**configuration())
<|endoftext|>


<commit_before>"""
Author: RogueDarkJedi and Derferman
Desc: Updates a sidebar for a subreddit with the
newest Baseball data!
Usage: In the sidebar description, add the tags [](/statsstart) and
[](/statsend) where you want the table to go.
"""
import os
import logging
import reddit
import baseball
import datetime
import re
from pytz import timezone
from jinja2 import Template

logger = logging.getLogger('sfgiants')

SUBREDDIT = os.environ.get("SUBREDDIT", "SFGiants")
LEAGUE = os.environ.get("MLB_LEAGUE", "NATIONAL")
DIVISION = os.environ.get("MLB_DIVISION", "WEST")

def timestamp():
    pacific = timezone('US/Pacific')
    now = pacific.localize(datetime.datetime.utcnow())
    return now.strftime("%Y-%m-%d %I:%M %p %Z")

def all_stats():
    standings = baseball.current_standings(LEAGUE, DIVISION)
    template = Template(open('templates/all_stats.md').read())
    return template.render(standings=standings, timestamp=timestamp())

def update_standings(current_description):
    stats = all_stats()
    return re.sub(r'\[\]\(/all_statsstart\).*\[\]\(/all_statsend\)',
                  '[](/all_statsstart)\n' + stats + '\n[](/all_statsend)',
                  current_description, flags=re.S)

def update_sidebar():
    r = reddit.Client(os.environ["REDDIT_USERNAME"],
                      os.environ["REDDIT_PASSWORD"])

    about = r.about(SUBREDDIT)

    payload = {
        'description': update_standings(about['description']),
        'link_type': 'any',
        'sr': about['subreddit_id'],
        'title': about['title'],
        'type': about['subreddit_type'],
        'wikimode': about['wikimode'],
    }

    return r.update_about(SUBREDDIT, payload)

if __name__ == "__main__":
    logger.info('Starting bot')

    assert "REDDIT_USERNAME" in os.environ, "REDDIT_USERNAME required"
    assert "REDDIT_PASSWORD" in os.environ, "REDDIT_PASSWORD required"

    update_sidebar()

    logger.info('Stopping bot')
<commit_msg>Fix timezonez for real<commit_after>"""
Author: RogueDarkJedi and Derferman
Desc: Updates a sidebar for a subreddit with the
newest Baseball data!
Usage: In the sidebar description, add the tags [](/statsstart) and
[](/statsend) where you want the table to go.
"""
import os
import logging
import reddit
import baseball
import datetime
import re
from pytz import timezone
from jinja2 import Template

logger = logging.getLogger('sfgiants')

SUBREDDIT = os.environ.get("SUBREDDIT", "SFGiants")
LEAGUE = os.environ.get("MLB_LEAGUE", "NATIONAL")
DIVISION = os.environ.get("MLB_DIVISION", "WEST")

def timestamp():
    pacific = timezone('US/Pacific')
    now = datetime.datetime.now(datetime.timezone.utc)
    return now.astimezone(pacific).strftime("%Y-%m-%d %I:%M %p %Z")

def all_stats():
    standings = baseball.current_standings(LEAGUE, DIVISION)
    template = Template(open('templates/all_stats.md').read())
    return template.render(standings=standings, timestamp=timestamp())

def update_standings(current_description):
    stats = all_stats()
    return re.sub(r'\[\]\(/all_statsstart\).*\[\]\(/all_statsend\)',
                  '[](/all_statsstart)\n' + stats + '\n[](/all_statsend)',
                  current_description, flags=re.S)

def update_sidebar():
    r = reddit.Client(os.environ["REDDIT_USERNAME"],
                      os.environ["REDDIT_PASSWORD"])

    about = r.about(SUBREDDIT)

    payload = {
        'description': update_standings(about['description']),
        'link_type': 'any',
        'sr': about['subreddit_id'],
        'title': about['title'],
        'type': about['subreddit_type'],
        'wikimode': about['wikimode'],
    }

    return r.update_about(SUBREDDIT, payload)

if __name__ == "__main__":
    logger.info('Starting bot')

    assert "REDDIT_USERNAME" in os.environ, "REDDIT_USERNAME required"
    assert "REDDIT_PASSWORD" in os.environ, "REDDIT_PASSWORD required"

    update_sidebar()

    logger.info('Stopping bot')
<|endoftext|>


<commit_before>import pybedtools
import webbrowser
import requests
import os
from bs4 import BeautifulSoup
import helpers
import logging

# maintain different loggers for different functionality.
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
hgsid_logger = logging.getLogger(__name__ + '(hgsid)')
hgsid_logger.setLevel(logging.DEBUG)
_ucsc_session_instance = [None]

import settings

def change_mirror(mirror):
    settings.mirror = mirror

def UCSCSession():
    """
    Create and interact with a session in the UCSC Genome Browser.

    Use change_mirror() to change the mirror that will be used; by default
    it's http://genome.ucsc.edu.
    """
    if not _ucsc_session_instance[0]:
        _ucsc_session_instance[0] = _UCSCSession()
    return _ucsc_session_instance[0]

class _UCSCSession(object):
    def __init__(self):
        self._session = None
        self._mirror = settings.mirror
        self.cart = self.cart_info()

    def update_cart(self, **kwargs):
        """
        Update the current settings.

        For example. change the assembly with

        update_cart(db="dm3")

        """
        self.session.get(self.cart_url, data=kwargs)

    @property
    def session(self):
        """
        Creates a new session if none exists, or if the mirror has changed.
        """
        if (settings.mirror != self._mirror) or (self._session is None):
            logger.info('initilizing session')
            self._mirror = settings.mirror
            self._session = requests.session(params=dict(hgsid=self.hgsid))
        return self._session

    @property
    def hgsid(self):
        """
        Get a new hgsid if one has not been set or if the mirror has changed;
        otherwise return the existing one.
        """
        if (settings.hgsid is None) or (self._mirror != settings.mirror):
            hgsid_logger.debug('new hgsid: %s' % settings.hgsid)
            settings.hgsid = helpers.hgsid_from_response(
                requests.get(self.gateway_url))
        hgsid_logger.debug('hgsid=%s' % settings.hgsid)
        return settings.hgsid

    # -------------------------------------------------------------------------
    # Here are a bunch of getter methods that provide up-to-date URLs depending
    # on the current state of settings.mirror.
    @property
    def url(self):
        return os.path.join(settings.mirror, 'cgi-bin')

    @property
    def gateway_url(self):
        return os.path.join(self.url, 'hgGateway')

    @property
    def tracks_url(self):
        return os.path.join(self.url, 'hgTracks')

    @property
    def custom_url(self):
        return os.path.join(self.url, 'hgCustom')

    @property
    def tables_url(self):
        return os.path.join(self.url, 'hgTables')

    @property
    def cart_url(self):
        return os.path.join(self.url, 'cartDump')

    @property
    def hub_url(self):
        return os.path.join(self.url, 'hgHubConnect')

    # -------------------------------------------------------------------------

    def cart_info(self):
        """
        Return a dictionary of the current cart

        (which has all the CGI vars in it)
        """
        logger.debug('accessing cart')
        r = self.session.get(self.cart_url)
        b = BeautifulSoup(r.text)
        d = {}
        for i in b.pre.text.splitlines(False):
            items = i.split(None, 1)
            if len(items) == 1:
                k, v = items[0], None
            else:
                k, v = items
            d[k] = v
        return d

    def upload_track(self, track, trackline=None):
        """
        Uploads a track, providing an optional track line.

        The track line will automatically have a newline added if none exists.
        """
        if isinstance(track, pybedtools.BedTool):
            if trackline:
                track = track.saveas(trackline=trackline)
            track = track.fn
        else:
            if trackline:
                fout = open(pybedtools.BedTool._tmp(), 'w')
                fout.write(trackline.rstrip() + '\n')
                fout.write(open(track))
                fout.close()
                track = fout.name

        return self.session.post(
            self.custom_url,
            files={'hgt.customFile': open(track)}
        )

    def show(self, position=None):
        """
        Open a browser window at `position`.

        `position` should be either an object with chrom, start, stop
        attributes (like a pybedtools.Interval) or a string of the form
        "chrom:start-stop".

        If `position` is None, then just use the last position.
        """
        payload = {}
        payload.update({'position': self._position_string(position)})
        response = self.session.get(self.tracks_url, data=payload)
        webbrowser.open(response.url)
        return response

    def pdf(self, position=None, filename=None):
        """
        Save a PDF of the current browser view.

        If `position` is None, then use the last available position.

        If `filename` is None, then a temporary file will be created.

        Returns the created filename.
        """
        payload = {'hgt.psOutput': 'on'}
        payload.update(position=self._position_string(position))
        response = self.session.post(self.tracks_url, data=payload)
        link = helpers.pdf_link(response)
        logger.debug('PDF link: %s' % link)
        if filename is None:
            filename = pybedtools.BedTool._tmp()
        pdf_content = self.session.get(link)
        fout = open(filename, 'w')
        fout.write(pdf_content.content)
        fout.close()
        return filename

    def _position_string(self, position):
        """
        Given either a string or an object with chrom/start/stop, always return
        a position string in the form "chrom:start-stop"
        """
        if not position:
            position = self.cart_info()['position']
        if not isinstance(position, basestring):
            position = '{chrom}:{start}-{stop}'.format(position)
        return position

if __name__ == "__main__":
    u = UCSCSession()
    for fn in ['a.bed', 'b.bed']:
        x = pybedtools.example_bedtool(fn)\
            .saveas(trackline='track name=%s' % fn)
        r = u.upload_track(x)

    print "PDF file at %s" % u.pdf(position='chr1:1-2000', filename='example.pdf')
    u.show()
<commit_msg>method to update session params with [possibly a subset of] cart info<commit_after>import pybedtools
import webbrowser
import requests
import os
from bs4 import BeautifulSoup
import helpers
import logging

# maintain different loggers for different functionality.
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
hgsid_logger = logging.getLogger(__name__ + '(hgsid)')
hgsid_logger.setLevel(logging.DEBUG)
_ucsc_session_instance = [None]

import settings

def change_mirror(mirror):
    settings.mirror = mirror

def UCSCSession():
    """
    Create and interact with a session in the UCSC Genome Browser.

    Use change_mirror() to change the mirror that will be used; by default
    it's http://genome.ucsc.edu.
    """
    if not _ucsc_session_instance[0]:
        _ucsc_session_instance[0] = _UCSCSession()
    return _ucsc_session_instance[0]

class _UCSCSession(object):
    def __init__(self):
        self._session = None
        self._mirror = settings.mirror
        self.cart = self.cart_info()

    def update_cart(self, **kwargs):
        """
        Update the current settings.

        For example. change the assembly with

        update_cart(db="dm3")

        """
        self.session.get(self.cart_url, data=kwargs)

    @property
    def session(self):
        """
        Creates a new session if none exists, or if the mirror has changed.
        """
        if (settings.mirror != self._mirror) or (self._session is None):
            logger.info('initilizing session')
            self._mirror = settings.mirror
            self._session = requests.session(params=dict(hgsid=self.hgsid))
        return self._session

    @property
    def hgsid(self):
        """
        Get a new hgsid if one has not been set or if the mirror has changed;
        otherwise return the existing one.
        """
        if (settings.hgsid is None) or (self._mirror != settings.mirror):
            hgsid_logger.debug('new hgsid: %s' % settings.hgsid)
            settings.hgsid = helpers.hgsid_from_response(
                requests.get(self.gateway_url))
        hgsid_logger.debug('hgsid=%s' % settings.hgsid)
        return settings.hgsid

    # -------------------------------------------------------------------------
    # Here are a bunch of getter methods that provide up-to-date URLs depending
    # on the current state of settings.mirror.
    @property
    def url(self):
        return os.path.join(settings.mirror, 'cgi-bin')

    @property
    def gateway_url(self):
        return os.path.join(self.url, 'hgGateway')

    @property
    def tracks_url(self):
        return os.path.join(self.url, 'hgTracks')

    @property
    def custom_url(self):
        return os.path.join(self.url, 'hgCustom')

    @property
    def tables_url(self):
        return os.path.join(self.url, 'hgTables')

    @property
    def cart_url(self):
        return os.path.join(self.url, 'cartDump')

    @property
    def hub_url(self):
        return os.path.join(self.url, 'hgHubConnect')

    # -------------------------------------------------------------------------

    def cart_info(self):
        """
        Return a dictionary of the current cart

        (which has all the CGI vars in it)
        """
        logger.debug('accessing cart')
        r = self.session.get(self.cart_url)
        b = BeautifulSoup(r.text)
        d = {}
        for i in b.pre.text.splitlines(False):
            items = i.split(None, 1)
            if len(items) == 1:
                k, v = items[0], None
            else:
                k, v = items
            d[k] = v
        return d

    def update_session(self, keys=None):
        cart_info = self.cart_info()
        if keys is None:
            self.session.params.update(cart_info)
        else:
            for key in keys:
                self.session.params[key] = cart_info[key]

    def upload_track(self, track, trackline=None):
        """
        Uploads a track, providing an optional track line.

        The track line will automatically have a newline added if none exists.
        """
        if isinstance(track, pybedtools.BedTool):
            if trackline:
                track = track.saveas(trackline=trackline)
            track = track.fn
        else:
            if trackline:
                fout = open(pybedtools.BedTool._tmp(), 'w')
                fout.write(trackline.rstrip() + '\n')
                fout.write(open(track))
                fout.close()
                track = fout.name

        return self.session.post(
            self.custom_url,
            files={'hgt.customFile': open(track)}
        )

    def show(self, position=None):
        """
        Open a browser window at `position`.

        `position` should be either an object with chrom, start, stop
        attributes (like a pybedtools.Interval) or a string of the form
        "chrom:start-stop".

        If `position` is None, then just use the last position.
        """
        payload = {}
        payload.update({'position': self._position_string(position)})
        response = self.session.get(self.tracks_url, data=payload)
        webbrowser.open(response.url)
        return response

    def pdf(self, position=None, filename=None):
        """
        Save a PDF of the current browser view.

        If `position` is None, then use the last available position.

        If `filename` is None, then a temporary file will be created.

        Returns the created filename.
        """
        payload = {'hgt.psOutput': 'on'}
        payload.update(position=self._position_string(position))
        response = self.session.post(self.tracks_url, data=payload)
        link = helpers.pdf_link(response)
        logger.debug('PDF link: %s' % link)
        if filename is None:
            filename = pybedtools.BedTool._tmp()
        pdf_content = self.session.get(link)
        fout = open(filename, 'w')
        fout.write(pdf_content.content)
        fout.close()
        return filename

    def _position_string(self, position):
        """
        Given either a string or an object with chrom/start/stop, always return
        a position string in the form "chrom:start-stop"
        """
        if not position:
            position = self.cart_info()['position']
        if not isinstance(position, basestring):
            position = '{chrom}:{start}-{stop}'.format(position)
        return position

if __name__ == "__main__":
    u = UCSCSession()
    for fn in ['a.bed', 'b.bed']:
        x = pybedtools.example_bedtool(fn)\
            .saveas(trackline='track name=%s' % fn)
        r = u.upload_track(x)

    print "PDF file at %s" % u.pdf(position='chr1:1-2000', filename='example.pdf')
    u.show()
<|endoftext|>