<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import re
import datetime
from sqlalchemy import sql
from sqlalchemy import or_
from sqlalchemy.dialects.postgresql import JSONB
import random
import requests
import shortuuid

from app import db
from app import logger
from webpage import WebpageInPmhRepo
# from pmh_record import PmhRecord

from oa_local import find_normalized_license
from oa_pdf import convert_pdf_to_txt
from http_cache import http_get
from util import remove_punctuation
from util import get_sql_answer
from util import is_the_same_url
from util import normalize_title

DEBUG_BASE = False

class PageNew(db.Model):
    id = db.Column(db.Text, primary_key=True)
    url = db.Column(db.Text)
    pmh_id = db.Column(db.Text, db.ForeignKey("pmh_record.id"))
    repo_id = db.Column(db.Text)
    doi = db.Column(db.Text, db.ForeignKey("pub.id"))
    title = db.Column(db.Text)
    normalized_title = db.Column(db.Text, db.ForeignKey("pub.normalized_title"))
    authors = db.Column(JSONB)
    record_timestamp = db.Column(db.DateTime)

    scrape_updated = db.Column(db.DateTime)
    scrape_pdf_url = db.Column(db.Text)
    scrape_metadata_url = db.Column(db.Text)
    scrape_version = db.Column(db.Text)
    scrape_license = db.Column(db.Text)
    num_pub_matches = db.Column(db.Numeric)

    error = db.Column(db.Text)
    updated = db.Column(db.DateTime)

    started = db.Column(db.DateTime)
    finished = db.Column(db.DateTime)
    rand = db.Column(db.Numeric)

    match_type = db.Column(db.Text)

    __mapper_args__ = {
        "polymorphic_on": match_type,
        "polymorphic_identity": "page_new"
    }

    def __init__(self, **kwargs):
        self.id = shortuuid.uuid()[0:20]
        self.error = ""
        self.rand = random.random()
        self.updated = datetime.datetime.utcnow().isoformat()
        super(PageNew, self).__init__(**kwargs)

    @property
    def is_open(self):
        return self.scrape_metadata_url or self.scrape_pdf_url

    @property
    def is_pmc(self):
        if not self.url:
            return False
        if u"ncbi.nlm.nih.gov/pmc" in self.url:
            return True
        if u"europepmc.org/articles/" in self.url:
            return True
        return False

    @property
    def pmcid(self):
        if not self.is_pmc:
            return None
        return self.url.rsplit("/", 1)[1].lower()

    # overwritten by subclasses
    def query_for_num_pub_matches(self):
        pass

    def scrape_if_matches_pub(self):
        # if self.scrape_updated:
        #     logger.info(u"already scraped, returning: {}".format(self))
        #     return

        if self.error:
            logger.info(u"has an error! don't scrape")
            self.num_pub_matches = -1
        else:
            self.num_pub_matches = self.query_for_num_pub_matches()

        if self.num_pub_matches > 0:
            return self.scrape()

    # examples
    # https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=PMC3039489&resulttype=core&format=json&tool=oadoi
    # https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=PMC3606428&resulttype=core&format=json&tool=oadoi
    def set_info_for_pmc_page(self):
        url_template = u"https://www.ebi.ac.uk/europepmc/webservices/rest/search?query={}&resulttype=core&format=json&tool=oadoi"
        url = url_template.format(self.pmcid)

        # try:
        r = http_get(url)
        data = r.json()
        result_list = data["resultList"]["result"]
        if not result_list:
            return
        result = result_list[0]
        has_pdf = result.get("hasPDF", None)
        is_author_manuscript = result.get("authMan", None)
        is_open_access = result.get("isOpenAccess", None)
        raw_license = result.get("license", None)

        self.scrape_metadata_url = u"http://europepmc.org/articles/{}".format(self.pmcid)
        if has_pdf == u"Y":
            self.scrape_pdf_url = u"http://europepmc.org/articles/{}?pdf=render".format(self.pmcid)
        if is_author_manuscript == u"Y":
            self.scrape_version = u"acceptedVersion"
        else:
            self.scrape_version = u"publishedVersion"
        if raw_license:
            self.scrape_license = find_normalized_license(raw_license)
        elif is_open_access == "Y":
            self.scrape_license = u"implied-oa"

        # except Exception as e:
        #     self.error += u"Exception in set_info_for_pmc_page"
        #     logger.info(u"Exception in set_info_for_pmc_page")

    def scrape(self):
        self.updated = datetime.datetime.utcnow().isoformat()
        self.scrape_updated = datetime.datetime.utcnow().isoformat()
        self.scrape_pdf_url = None
        self.scrape_metadata_url = None
        self.scrape_license = None
        self.scrape_version = None
        self.error = ""

        # handle these special cases, where we compute the pdf rather than looking for it
        if "oai:arXiv.org" in self.pmh_id:
            self.scrape_metadata_url = self.url
            self.scrape_pdf_url = self.url.replace("abs", "pdf")
        if self.is_pmc:
            self.set_info_for_pmc_page()

        # delete this part at some point once we've done all the old matches we want to do
        # if not self.scrape_pdf_url:
        #     base_matches = BaseMatch.query.filter(or_(BaseMatch.url==self.url,
        #                                                BaseMatch.scrape_metadata_url==self.url,
        #                                                BaseMatch.scrape_pdf_url==self.url)).all()
        #     if base_matches:
        #         logger.info(u"** found base match version")
        #         for base_match in base_matches:
        #             self.scrape_updated = base_match.scrape_updated
        #             self.scrape_pdf_url = base_match.scrape_pdf_url
        #             self.scrape_metadata_url = base_match.scrape_metadata_url
        #             self.scrape_license = base_match.scrape_license
        #             self.scrape_version = base_match.scrape_version
        #     else:
        #         logger.info(u"did not find a base match version")

        if not self.scrape_pdf_url or not self.scrape_version:
            my_webpage = WebpageInPmhRepo(url=self.url, scraped_pdf_url=self.scrape_pdf_url)
            if not self.scrape_pdf_url:
                my_webpage.scrape_for_fulltext_link()
                if my_webpage.is_open:
                    self.scrape_updated = datetime.datetime.utcnow().isoformat()
                    self.metadata_url = self.url
                    logger.info(u"** found an open copy! {}".format(my_webpage.fulltext_url))
                    self.scrape_pdf_url = my_webpage.scraped_pdf_url
                    self.scrape_metadata_url = my_webpage.scraped_open_metadata_url
                    self.scrape_license = my_webpage.scraped_license

        if self.scrape_pdf_url and not self.scrape_version:
            have_the_pdf = False
            if my_webpage and my_webpage.r:
                history_urls = [my_webpage.r.url] + [h.url for h in my_webpage.r.history]
                if any([is_the_same_url(url, self.scrape_pdf_url) for url in history_urls]):
                    have_the_pdf = True
            if not have_the_pdf:
                logger.info(u"don't have the full pdf, so getting it to get the version")
                my_webpage.set_r_for_pdf()
            self.set_version_and_license(r=my_webpage.r)

    # use standards from https://wiki.surfnet.nl/display/DRIVERguidelines/Version+vocabulary
    # submittedVersion, acceptedVersion, publishedVersion
    def set_version_and_license(self, r=None):

        if self.is_pmc:
            self.set_info_for_pmc_page()
            return

        # set as default
        self.scrape_version = "submittedVersion"

        if not r:
            return

        try:
            # http://crossmark.dyndns.org/dialog/?doi=10.1016/j.jml.2012 at http://dspace.mit.edu/bitstream/1721.1/102417/1/Gibson_The%20syntactic.pdf
            if re.findall(u"crossmark\.[^/]*\.org/", r.text, re.IGNORECASE):
                self.scrape_version = "publishedVersion"

            text = convert_pdf_to_txt(r)
            # logger.info(text)
            if text and self.scrape_version == "submittedVersion":
                patterns = [
                    re.compile(ur"©.?\d{4}", re.UNICODE),
                    re.compile(ur"\(C\).?\d{4}", re.IGNORECASE),
                    re.compile(ur"copyright \d{4}", re.IGNORECASE),
                    re.compile(ur"all rights reserved", re.IGNORECASE),
                    re.compile(ur"This article is distributed under the terms of the Creative Commons", re.IGNORECASE),
                    re.compile(ur"this is an open access article", re.IGNORECASE)
                    ]

                for pattern in patterns:
                    matches = pattern.findall(text)
                    if matches:
                        self.scrape_version = "publishedVersion"

            logger.info(u"returning with scrape_version: {}".format(self.scrape_version))

            open_license = find_normalized_license(text)
            if open_license:
                self.scrape_license = open_license

        except Exception as e:
            self.error += u"Exception doing convert_pdf_to_txt on {}! investigate! {}".format(self.scrape_pdf_url, unicode(e.message).encode("utf-8"))
            logger.info(self.error)
            pass

    def __repr__(self):
        return u"<PageNew ( {} ) {}>".format(self.pmh_id, self.url)

class PageDoiMatch(PageNew):
    __mapper_args__ = {
        "polymorphic_identity": "doi"
    }

    def query_for_num_pub_matches(self):
        from pub import Pub
        num_pubs_with_this_doi = db.session.query(Pub.id).filter(Pub.id==self.doi).count()
        return num_pubs_with_this_doi

    def __repr__(self):
        return u"<PageDoiMatch ( {} ) {} doi:{}>".format(self.pmh_id, self.url, self.doi)

class PageTitleMatch(PageNew):
    __mapper_args__ = {
        "polymorphic_identity": "title"
    }

    def query_for_num_pub_matches(self):
        from pmh_record import title_is_too_common
        from pmh_record import title_is_too_short
        from pub import Pub

        # it takes too long to query for things like "tablecontents"
        if title_is_too_common(self.normalized_title) or title_is_too_short(self.normalized_title):
            logger.info(u"title is too common or too short, not scraping")
            return -1

        num_pubs_with_this_normalized_title = db.session.query(Pub.id).filter(Pub.normalized_title==self.normalized_title).count()
        return num_pubs_with_this_normalized_title

    def __repr__(self):
        return u"<PageTitleMatch ( {} ) {} '{}...'>".format(self.pmh_id, self.url, self.title[0:20])

class Page(db.Model):
    url = db.Column(db.Text, primary_key=True)
    id = db.Column(db.Text, db.ForeignKey("pmh_record.id"))
    source = db.Column(db.Text)
    doi = db.Column(db.Text, db.ForeignKey("pub.id"))
    title = db.Column(db.Text)
    normalized_title = db.Column(db.Text, db.ForeignKey("pub.normalized_title"))
    authors = db.Column(JSONB)

    scrape_updated = db.Column(db.DateTime)
    scrape_evidence = db.Column(db.Text)
    scrape_pdf_url = db.Column(db.Text)
    scrape_metadata_url = db.Column(db.Text)
    scrape_version = db.Column(db.Text)
    scrape_license = db.Column(db.Text)

    error = db.Column(db.Text)
    updated = db.Column(db.DateTime)

    started = db.Column(db.DateTime)
    finished = db.Column(db.DateTime)
    rand = db.Column(db.Numeric)

    def __init__(self, **kwargs):
        self.error = ""
        self.updated = datetime.datetime.utcnow().isoformat()
        super(self.__class__, self).__init__(**kwargs)

    @property
    def pmh_id(self):
        return self.id

    @property
    def is_open(self):
        return self.scrape_metadata_url or self.scrape_pdf_url

    @property
    def repo_id(self):
        if not self.pmh_id or not ":" in self.pmh_id:
            return None
        return self.pmh_id.split(":")[1]

    @property
    def pmcid(self):
        if not self.is_pmc:
            return None
        return self.url.rsplit("/", 1)[1].lower()

    @property
    def is_preprint_repo(self):
        preprint_url_fragments = [
            "precedings.nature.com",
            "10.15200/winn.",
            "/peerj.preprints",
            ".figshare.",
            "10.1101/",  #biorxiv
            "10.15363/" #thinklab
        ]
        for url_fragment in preprint_url_fragments:
            if self.url and url_fragment in self.url.lower():
                return True
        return False

    def __repr__(self):
        return u"<Page ( {} ) {} doi:{} '{}...'>".format(self.pmh_id, self.url, self.doi, self.title[0:20])

# legacy, just used for matching
class BaseMatch(db.Model):
    id = db.Column(db.Text, primary_key=True)
    base_id = db.Column(db.Text)
    doi = db.Column(db.Text, db.ForeignKey('pub.id'))
    url = db.Column(db.Text)
    scrape_updated = db.Column(db.DateTime)
    scrape_evidence = db.Column(db.Text)
    scrape_pdf_url = db.Column(db.Text)
    scrape_metadata_url = db.Column(db.Text)
    scrape_version = db.Column(db.Text)
    scrape_license = db.Column(db.Text)
    updated = db.Column(db.DateTime)

    @property
    def is_open(self):
        return self.scrape_metadata_url or self.scrape_pdf_url

<commit_msg>better pmcid finder<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import re
import datetime
from sqlalchemy import sql
from sqlalchemy import or_
from sqlalchemy.dialects.postgresql import JSONB
import random
import requests
import shortuuid

from app import db
from app import logger
from webpage import WebpageInPmhRepo
# from pmh_record import PmhRecord

from oa_local import find_normalized_license
from oa_pdf import convert_pdf_to_txt
from http_cache import http_get
from util import remove_punctuation
from util import get_sql_answer
from util import is_the_same_url
from util import normalize_title

DEBUG_BASE = False

class PageNew(db.Model):
    id = db.Column(db.Text, primary_key=True)
    url = db.Column(db.Text)
    pmh_id = db.Column(db.Text, db.ForeignKey("pmh_record.id"))
    repo_id = db.Column(db.Text)
    doi = db.Column(db.Text, db.ForeignKey("pub.id"))
    title = db.Column(db.Text)
    normalized_title = db.Column(db.Text, db.ForeignKey("pub.normalized_title"))
    authors = db.Column(JSONB)
    record_timestamp = db.Column(db.DateTime)

    scrape_updated = db.Column(db.DateTime)
    scrape_pdf_url = db.Column(db.Text)
    scrape_metadata_url = db.Column(db.Text)
    scrape_version = db.Column(db.Text)
    scrape_license = db.Column(db.Text)
    num_pub_matches = db.Column(db.Numeric)

    error = db.Column(db.Text)
    updated = db.Column(db.DateTime)

    started = db.Column(db.DateTime)
    finished = db.Column(db.DateTime)
    rand = db.Column(db.Numeric)

    match_type = db.Column(db.Text)

    __mapper_args__ = {
        "polymorphic_on": match_type,
        "polymorphic_identity": "page_new"
    }

    def __init__(self, **kwargs):
        self.id = shortuuid.uuid()[0:20]
        self.error = ""
        self.rand = random.random()
        self.updated = datetime.datetime.utcnow().isoformat()
        super(PageNew, self).__init__(**kwargs)

    @property
    def is_open(self):
        return self.scrape_metadata_url or self.scrape_pdf_url

    @property
    def is_pmc(self):
        if not self.url:
            return False
        if u"ncbi.nlm.nih.gov/pmc" in self.url:
            return True
        if u"europepmc.org/articles/" in self.url:
            return True
        return False

    @property
    def pmcid(self):
        if not self.is_pmc:
            return None
        matches = re.findall(u"(pmc\d+)", self.url, re.IGNORECASE)
        if not matches:
            return None
        return matches[0].lower()

    # overwritten by subclasses
    def query_for_num_pub_matches(self):
        pass

    def scrape_if_matches_pub(self):
        # if self.scrape_updated:
        #     logger.info(u"already scraped, returning: {}".format(self))
        #     return

        if self.error:
            logger.info(u"has an error! don't scrape")
            self.num_pub_matches = -1
        else:
            self.num_pub_matches = self.query_for_num_pub_matches()

        if self.num_pub_matches > 0:
            return self.scrape()

    # examples
    # https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=PMC3039489&resulttype=core&format=json&tool=oadoi
    # https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=PMC3606428&resulttype=core&format=json&tool=oadoi
    def set_info_for_pmc_page(self):
        if not self.pmcid:
            return

        url_template = u"https://www.ebi.ac.uk/europepmc/webservices/rest/search?query={}&resulttype=core&format=json&tool=oadoi"
        url = url_template.format(self.pmcid)

        # try:
        r = http_get(url)
        data = r.json()
        result_list = data["resultList"]["result"]
        if not result_list:
            return
        result = result_list[0]
        has_pdf = result.get("hasPDF", None)
        is_author_manuscript = result.get("authMan", None)
        is_open_access = result.get("isOpenAccess", None)
        raw_license = result.get("license", None)

        self.scrape_metadata_url = u"http://europepmc.org/articles/{}".format(self.pmcid)
        if has_pdf == u"Y":
            self.scrape_pdf_url = u"http://europepmc.org/articles/{}?pdf=render".format(self.pmcid)
        if is_author_manuscript == u"Y":
            self.scrape_version = u"acceptedVersion"
        else:
            self.scrape_version = u"publishedVersion"
        if raw_license:
            self.scrape_license = find_normalized_license(raw_license)
        elif is_open_access == "Y":
            self.scrape_license = u"implied-oa"

        # except Exception as e:
        #     self.error += u"Exception in set_info_for_pmc_page"
        #     logger.info(u"Exception in set_info_for_pmc_page")

    def scrape(self):
        self.updated = datetime.datetime.utcnow().isoformat()
        self.scrape_updated = datetime.datetime.utcnow().isoformat()
        self.scrape_pdf_url = None
        self.scrape_metadata_url = None
        self.scrape_license = None
        self.scrape_version = None
        self.error = ""

        # handle these special cases, where we compute the pdf rather than looking for it
        if "oai:arXiv.org" in self.pmh_id:
            self.scrape_metadata_url = self.url
            self.scrape_pdf_url = self.url.replace("abs", "pdf")
        if self.is_pmc:
            self.set_info_for_pmc_page()

        # delete this part at some point once we've done all the old matches we want to do
        # if not self.scrape_pdf_url:
        #     base_matches = BaseMatch.query.filter(or_(BaseMatch.url==self.url,
        #                                                BaseMatch.scrape_metadata_url==self.url,
        #                                                BaseMatch.scrape_pdf_url==self.url)).all()
        #     if base_matches:
        #         logger.info(u"** found base match version")
        #         for base_match in base_matches:
        #             self.scrape_updated = base_match.scrape_updated
        #             self.scrape_pdf_url = base_match.scrape_pdf_url
        #             self.scrape_metadata_url = base_match.scrape_metadata_url
        #             self.scrape_license = base_match.scrape_license
        #             self.scrape_version = base_match.scrape_version
        #     else:
        #         logger.info(u"did not find a base match version")

        if not self.scrape_pdf_url or not self.scrape_version:
            my_webpage = WebpageInPmhRepo(url=self.url, scraped_pdf_url=self.scrape_pdf_url)
            if not self.scrape_pdf_url:
                my_webpage.scrape_for_fulltext_link()
                if my_webpage.is_open:
                    self.scrape_updated = datetime.datetime.utcnow().isoformat()
                    self.metadata_url = self.url
                    logger.info(u"** found an open copy! {}".format(my_webpage.fulltext_url))
                    self.scrape_pdf_url = my_webpage.scraped_pdf_url
                    self.scrape_metadata_url = my_webpage.scraped_open_metadata_url
                    self.scrape_license = my_webpage.scraped_license

        if self.scrape_pdf_url and not self.scrape_version:
            have_the_pdf = False
            if my_webpage and my_webpage.r:
                history_urls = [my_webpage.r.url] + [h.url for h in my_webpage.r.history]
                if any([is_the_same_url(url, self.scrape_pdf_url) for url in history_urls]):
                    have_the_pdf = True
            if not have_the_pdf:
                logger.info(u"don't have the full pdf, so getting it to get the version")
                my_webpage.set_r_for_pdf()
            self.set_version_and_license(r=my_webpage.r)

    # use standards from https://wiki.surfnet.nl/display/DRIVERguidelines/Version+vocabulary
    # submittedVersion, acceptedVersion, publishedVersion
    def set_version_and_license(self, r=None):

        if self.is_pmc:
            self.set_info_for_pmc_page()
            return

        # set as default
        self.scrape_version = "submittedVersion"

        if not r:
            return

        try:
            # http://crossmark.dyndns.org/dialog/?doi=10.1016/j.jml.2012 at http://dspace.mit.edu/bitstream/1721.1/102417/1/Gibson_The%20syntactic.pdf
            if re.findall(u"crossmark\.[^/]*\.org/", r.text, re.IGNORECASE):
                self.scrape_version = "publishedVersion"

            text = convert_pdf_to_txt(r)
            # logger.info(text)
            if text and self.scrape_version == "submittedVersion":
                patterns = [
                    re.compile(ur"©.?\d{4}", re.UNICODE),
                    re.compile(ur"\(C\).?\d{4}", re.IGNORECASE),
                    re.compile(ur"copyright \d{4}", re.IGNORECASE),
                    re.compile(ur"all rights reserved", re.IGNORECASE),
                    re.compile(ur"This article is distributed under the terms of the Creative Commons", re.IGNORECASE),
                    re.compile(ur"this is an open access article", re.IGNORECASE)
                    ]

                for pattern in patterns:
                    matches = pattern.findall(text)
                    if matches:
                        self.scrape_version = "publishedVersion"

            logger.info(u"returning with scrape_version: {}".format(self.scrape_version))

            open_license = find_normalized_license(text)
            if open_license:
                self.scrape_license = open_license

        except Exception as e:
            self.error += u"Exception doing convert_pdf_to_txt on {}! investigate! {}".format(self.scrape_pdf_url, unicode(e.message).encode("utf-8"))
            logger.info(self.error)
            pass

    def __repr__(self):
        return u"<PageNew ( {} ) {}>".format(self.pmh_id, self.url)

class PageDoiMatch(PageNew):
    __mapper_args__ = {
        "polymorphic_identity": "doi"
    }

    def query_for_num_pub_matches(self):
        from pub import Pub
        num_pubs_with_this_doi = db.session.query(Pub.id).filter(Pub.id==self.doi).count()
        return num_pubs_with_this_doi

    def __repr__(self):
        return u"<PageDoiMatch ( {} ) {} doi:{}>".format(self.pmh_id, self.url, self.doi)

class PageTitleMatch(PageNew):
    __mapper_args__ = {
        "polymorphic_identity": "title"
    }

    def query_for_num_pub_matches(self):
        from pmh_record import title_is_too_common
        from pmh_record import title_is_too_short
        from pub import Pub

        # it takes too long to query for things like "tablecontents"
        if title_is_too_common(self.normalized_title) or title_is_too_short(self.normalized_title):
            logger.info(u"title is too common or too short, not scraping")
            return -1

        num_pubs_with_this_normalized_title = db.session.query(Pub.id).filter(Pub.normalized_title==self.normalized_title).count()
        return num_pubs_with_this_normalized_title

    def __repr__(self):
        return u"<PageTitleMatch ( {} ) {} '{}...'>".format(self.pmh_id, self.url, self.title[0:20])

class Page(db.Model):
    url = db.Column(db.Text, primary_key=True)
    id = db.Column(db.Text, db.ForeignKey("pmh_record.id"))
    source = db.Column(db.Text)
    doi = db.Column(db.Text, db.ForeignKey("pub.id"))
    title = db.Column(db.Text)
    normalized_title = db.Column(db.Text, db.ForeignKey("pub.normalized_title"))
    authors = db.Column(JSONB)

    scrape_updated = db.Column(db.DateTime)
    scrape_evidence = db.Column(db.Text)
    scrape_pdf_url = db.Column(db.Text)
    scrape_metadata_url = db.Column(db.Text)
    scrape_version = db.Column(db.Text)
    scrape_license = db.Column(db.Text)

    error = db.Column(db.Text)
    updated = db.Column(db.DateTime)

    started = db.Column(db.DateTime)
    finished = db.Column(db.DateTime)
    rand = db.Column(db.Numeric)

    def __init__(self, **kwargs):
        self.error = ""
        self.updated = datetime.datetime.utcnow().isoformat()
        super(self.__class__, self).__init__(**kwargs)

    @property
    def pmh_id(self):
        return self.id

    @property
    def is_open(self):
        return self.scrape_metadata_url or self.scrape_pdf_url

    @property
    def repo_id(self):
        if not self.pmh_id or not ":" in self.pmh_id:
            return None
        return self.pmh_id.split(":")[1]

    @property
    def pmcid(self):
        if not self.is_pmc:
            return None
        return self.url.rsplit("/", 1)[1].lower()

    @property
    def is_preprint_repo(self):
        preprint_url_fragments = [
            "precedings.nature.com",
            "10.15200/winn.",
            "/peerj.preprints",
            ".figshare.",
            "10.1101/",  #biorxiv
            "10.15363/" #thinklab
        ]
        for url_fragment in preprint_url_fragments:
            if self.url and url_fragment in self.url.lower():
                return True
        return False

    def __repr__(self):
        return u"<Page ( {} ) {} doi:{} '{}...'>".format(self.pmh_id, self.url, self.doi, self.title[0:20])

# legacy, just used for matching
class BaseMatch(db.Model):
    id = db.Column(db.Text, primary_key=True)
    base_id = db.Column(db.Text)
    doi = db.Column(db.Text, db.ForeignKey('pub.id'))
    url = db.Column(db.Text)
    scrape_updated = db.Column(db.DateTime)
    scrape_evidence = db.Column(db.Text)
    scrape_pdf_url = db.Column(db.Text)
    scrape_metadata_url = db.Column(db.Text)
    scrape_version = db.Column(db.Text)
    scrape_license = db.Column(db.Text)
    updated = db.Column(db.DateTime)

    @property
    def is_open(self):
        return self.scrape_metadata_url or self.scrape_pdf_url

<|endoftext|>


<commit_before>from .base import *             # NOQA
import sys
import logging.config

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

# Turn off debug while imported by Celery with a workaround
# See http://stackoverflow.com/a/4806384
if "celery" in sys.argv[0]:
    DEBUG = False

# Django Debug Toolbar
INSTALLED_APPS += ('debug_toolbar.apps.DebugToolbarConfig',)

# Postgres-specifi migrations
if 'postgres' in DATABASES['default']['ENGINE']:
    INSTALLED_APPS += ('postgres',)

# Show emails to console in DEBUG mode
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

# Log everything to the logs directory at the top
LOGFILE_ROOT = join(dirname(BASE_DIR), 'logs')

# Reset logging
# (see http://www.caktusgroup.com/blog/2015/01/27/Django-Logging-Configuration-logging_config-default-settings-logger/)

LOGGING_CONFIG = None
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': (
                '[%(asctime)s] %(levelname)s '
                '[%(pathname)s:%(lineno)s] %(message)s'
            ),
            'datefmt': "%d/%b/%Y %H:%M:%S"
        },
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
    },
    'handlers': {
        'django_log_file': {
            'level': 'DEBUG',
            'class': 'logging.FileHandler',
            'filename': join(LOGFILE_ROOT, 'django.log'),
            'formatter': 'verbose'
        },
        'proj_log_file': {
            'level': 'DEBUG',
            'class': 'logging.FileHandler',
            'filename': join(LOGFILE_ROOT, 'project.log'),
            'formatter': 'verbose'
        },
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        }
    },
    'loggers': {
        'django': {
            'handlers': ['django_log_file', 'console',],
            'propagate': True,
            'level': 'DEBUG',
        },
        'project': {
            'handlers': ['proj_log_file', 'console', ],
            'level': 'DEBUG',
        },
    }
}

logging.config.dictConfig(LOGGING)
<commit_msg>Turn off console debug message<commit_after>from .base import *             # NOQA
import sys
import logging.config

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

# Turn off debug while imported by Celery with a workaround
# See http://stackoverflow.com/a/4806384
if "celery" in sys.argv[0]:
    DEBUG = False

# Django Debug Toolbar
INSTALLED_APPS += ('debug_toolbar.apps.DebugToolbarConfig',)

# Postgres-specifi migrations
if 'postgres' in DATABASES['default']['ENGINE']:
    INSTALLED_APPS += ('postgres',)

# Show emails to console in DEBUG mode
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

# Log everything to the logs directory at the top
LOGFILE_ROOT = join(dirname(BASE_DIR), 'logs')

# Reset logging
# (see http://www.caktusgroup.com/blog/2015/01/27/Django-Logging-Configuration-logging_config-default-settings-logger/)

LOGGING_CONFIG = None
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': (
                '[%(asctime)s] %(levelname)s '
                '[%(pathname)s:%(lineno)s] %(message)s'
            ),
            'datefmt': "%d/%b/%Y %H:%M:%S"
        },
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
    },
    'handlers': {
        'django_log_file': {
            'level': 'DEBUG',
            'class': 'logging.FileHandler',
            'filename': join(LOGFILE_ROOT, 'django.log'),
            'formatter': 'verbose'
        },
        'proj_log_file': {
            'level': 'DEBUG',
            'class': 'logging.FileHandler',
            'filename': join(LOGFILE_ROOT, 'project.log'),
            'formatter': 'verbose'
        },
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        }
    },
    'loggers': {
        'django': {
            'handlers': ['django_log_file', ],
            'propagate': True,
            'level': 'DEBUG',
        },
        'project': {
            'handlers': ['proj_log_file', 'console', ],
            'level': 'DEBUG',
        },
    }
}

logging.config.dictConfig(LOGGING)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Jayesh Kariya <jayeshk@saltstack.com>`
'''
# Import Python libs
from __future__ import absolute_import

# Import Salt Testing Libs
from salttesting import skipIf, TestCase
from salttesting.mock import (
    NO_MOCK,
    NO_MOCK_REASON,
    MagicMock,
    mock_open,
    patch)

from salttesting.helpers import ensure_in_syspath

ensure_in_syspath('../../')

# Import Salt Libs
import salt.utils
from salt.modules import timezone
import os
from salt.exceptions import CommandExecutionError
from salt.exceptions import SaltInvocationError

# Globals
timezone.__grains__ = {}
timezone.__salt__ = {}
timezone.__opts__ = {}

@skipIf(NO_MOCK, NO_MOCK_REASON)
class TimezoneTestCase(TestCase):
    '''
    Test cases for salt.modules.timezone
    '''
    def test_get_zone(self):
        '''
        Test to get current timezone (i.e. America/Denver)
        '''
        mock = MagicMock(side_effect=['Time zone: A', 'A'])
        with patch.object(salt.utils, 'which', return_value=True):
            with patch.dict(timezone.__salt__, {'cmd.run': mock}):
                self.assertEqual(timezone.get_zone(), 'A')

                self.assertRaises(CommandExecutionError, timezone.get_zone)

        with patch.object(salt.utils, 'which', return_value=False):
            file_data = '\n'.join(['#', 'A'])
            with patch('salt.utils.fopen', mock_open(read_data=file_data),
                       create=True) as mfile:
                mfile.return_value.__iter__.return_value = file_data.splitlines()
                with patch.dict(timezone.__grains__, {'os_family': 'Debian'}):
                    self.assertEqual(timezone.get_zone(), '#\nA')

                with patch.dict(timezone.__grains__, {'os_family': 'Gentoo'}):
                    self.assertEqual(timezone.get_zone(), '#\nA')

            with patch.dict(timezone.__grains__, {'os_family': 'FreeBSD'}):
                with patch.object(os, 'readlink',
                                  return_value='/usr/share/zoneinfo/'):
                    self.assertEqual(timezone.get_zone(), '')

            with patch.dict(timezone.__grains__, {'os_family': 'Solaris'}):
                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A=B')}):
                    self.assertEqual(timezone.get_zone(), 'B')

    def test_get_zonecode(self):
        '''
        Test to get current timezone (i.e. PST, MDT, etc)
        '''
        with patch.dict(timezone.__salt__, {'cmd.run':
                                            MagicMock(return_value='A')}):
            self.assertEqual(timezone.get_zonecode(), 'A')

    def test_get_offset(self):
        '''
        Test to get current numeric timezone offset from UCT (i.e. -0700)
        '''
        with patch.dict(timezone.__salt__, {'cmd.run':
                                            MagicMock(return_value='A')}):
            self.assertEqual(timezone.get_offset(), 'A')

    def test_set_zone(self):
        '''
        Test to unlinks, then symlinks /etc/localtime to the set timezone.
        '''
        ret = ('Zone does not exist: /usr/share/lib/zoneinfo/timezone')
        mock = MagicMock(side_effect=[False, True, True])
        with patch.dict(timezone.__grains__, {'os_family': 'Solaris'}):
            with patch.object(os.path, 'exists', mock):
                self.assertEqual(timezone.set_zone('timezone'), ret)

                with patch.object(os, 'unlink', return_value=None):
                    with patch.dict(timezone.__salt__,
                                    {'file.sed':
                                     MagicMock(return_value=None)}):
                        self.assertTrue(timezone.set_zone('timezone'))

    def test_zone_compare(self):
        '''
        Test to checks the hash sum between the given timezone, and the
        one set in /etc/localtime.
        '''
        with patch.dict(timezone.__grains__, {'os_family': 'Solaris'}):
            self.assertEqual(timezone.zone_compare('timezone'),
                             'Not implemented for Solaris family')

        with patch.dict(timezone.__grains__, {'os_family': 'Sola'}):
            with patch.object(os.path, 'exists', return_value=False):
                self.assertEqual(timezone.zone_compare('timezone'),
                                 'Error: /etc/localtime does not exist.')

        with patch.object(os.path, 'exists', return_value=True):
            with patch.dict(timezone.__grains__, {'os_family': 'Sola'}):
                with patch.dict(timezone.__opts__, {'hash_type': 'md5'}):
                    with patch.object(salt.utils, 'get_hash',
                                      side_effect=IOError('foo')):
                        self.assertRaises(SaltInvocationError,
                                          timezone.zone_compare, 't')

                    with patch.object(salt.utils, 'get_hash',
                                      side_effect=['A', IOError('foo')]):
                        self.assertRaises(CommandExecutionError,
                                          timezone.zone_compare, 't')

                    with patch.object(salt.utils, 'get_hash',
                                      side_effect=['A', 'A']):
                        self.assertTrue(timezone.zone_compare('timezone'))

                    with patch.object(salt.utils, 'get_hash',
                                      side_effect=['A', 'B']):
                        self.assertFalse(timezone.zone_compare('timezone'))

    def test_get_hwclock(self):
        '''
        Test to get current hardware clock setting (UTC or localtime)
        '''
        with patch.object(salt.utils, 'which', return_value=True):
            with patch.dict(timezone.__salt__,
                            {'cmd.run':
                             MagicMock(return_value='rtc in local tz:yes\n')}):
                self.assertEqual(timezone.get_hwclock(), 'localtime')

            with patch.dict(timezone.__salt__,
                            {'cmd.run':
                             MagicMock(return_value='rtc in local tz:No\n')}):
                self.assertEqual(timezone.get_hwclock(), 'UTC')

            with patch.dict(timezone.__salt__,
                            {'cmd.run':
                             MagicMock(return_value='rtc')}):
                self.assertRaises(CommandExecutionError, timezone.get_hwclock)

        with patch.object(salt.utils, 'which', return_value=False):
            with patch.dict(timezone.__grains__, {'os_family': 'RedHat'}):
                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A')}):
                    self.assertEqual(timezone.get_hwclock(), 'A')

            with patch.dict(timezone.__grains__, {'os_family': 'Suse'}):
                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A')}):
                    self.assertEqual(timezone.get_hwclock(), 'A')

            with patch.dict(timezone.__grains__, {'os_family': 'Debian'}):
                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A=yes')}):
                    self.assertEqual(timezone.get_hwclock(), 'UTC')

                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A=no')}):
                    self.assertEqual(timezone.get_hwclock(), 'localtime')

                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A')}):
                    self.assertEqual(timezone.get_hwclock(), 'A')

            with patch.dict(timezone.__grains__, {'os_family': 'Gentoo'}):
                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A=B')}):
                    self.assertEqual(timezone.get_hwclock(), 'B')

        mock = MagicMock(return_value=True)
        with patch.object(os.path, 'isfile', mock):
            fl_data = 'zone_info=GMT'
            with patch('salt.utils.fopen',
                       mock_open(read_data=fl_data),
                       create=True) as mfile:
                mfile.return_value.__iter__.return_value = fl_data.splitlines()
                with patch.object(salt.utils, 'which', return_value=False):
                    with patch.dict(timezone.__grains__,
                                    {'os_family': 'Solaris'}):
                        self.assertEqual(timezone.get_hwclock(), 'UTC')

        mock = MagicMock(return_value=True)
        with patch.object(os.path, 'isfile', mock):
            fl_data = 'A=GMT'
            with patch('salt.utils.fopen',
                       mock_open(read_data=fl_data),
                       create=True) as mfile:
                mfile.return_value.__iter__.return_value = fl_data.splitlines()
                with patch.object(salt.utils, 'which', return_value=False):
                    with patch.dict(timezone.__grains__,
                                    {'os_family': 'Solaris'}):
                        self.assertEqual(timezone.get_hwclock(), 'localtime')

        with patch.object(salt.utils, 'which', return_value=False):
            with patch.dict(timezone.__grains__, {'os_family': 'Solaris'}):
                mock = MagicMock(return_value=False)
                with patch.object(os.path, 'isfile', mock):
                    self.assertEqual(timezone.get_hwclock(), 'UTC')

    def test_set_hwclock(self):
        '''
        Test to sets the hardware clock to be either UTC or localtime
        '''
        ret = ('UTC is the only choice for SPARC architecture')
        ret1 = ('Zone does not exist: /usr/share/zoneinfo/America/Denver')
        with patch.object(timezone, 'get_zone',
                          return_value='America/Denver'):

            with patch.dict(timezone.__grains__, {'os_family': 'Solaris',
                                                  'cpuarch': 'sparc'}):
                self.assertEqual(timezone.set_hwclock('clock'), ret)

                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value=None)}):
                    self.assertTrue(timezone.set_hwclock('localtime'))

                    self.assertTrue(timezone.set_hwclock('UTC'))

            with patch.dict(timezone.__grains__, {'os_family': 'Sola'}):
                with patch.object(os.path, 'exists', return_value=False):
                    self.assertEqual(timezone.set_hwclock('clock'), ret1)

                with patch.object(os.path, 'exists', return_value=True):
                    with patch.object(os, 'unlink', return_value=None):
                        with patch.object(os, 'symlink', return_value=None):
                            self.assertTrue(timezone.set_hwclock('clock'))

if __name__ == '__main__':
    from integration import run_tests
    run_tests(TimezoneTestCase, needs_daemon=False)
<commit_msg>removed assertion error<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Jayesh Kariya <jayeshk@saltstack.com>`
'''
# Import Python libs
from __future__ import absolute_import

# Import Salt Testing Libs
from salttesting import skipIf, TestCase
from salttesting.mock import (
    NO_MOCK,
    NO_MOCK_REASON,
    MagicMock,
    mock_open,
    patch)

from salttesting.helpers import ensure_in_syspath

ensure_in_syspath('../../')

# Import Salt Libs
import salt.utils
from salt.modules import timezone
import os
from salt.exceptions import CommandExecutionError
from salt.exceptions import SaltInvocationError

# Globals
timezone.__grains__ = {}
timezone.__salt__ = {}
timezone.__opts__ = {}

@skipIf(NO_MOCK, NO_MOCK_REASON)
class TimezoneTestCase(TestCase):
    '''
    Test cases for salt.modules.timezone
    '''
    def test_get_zone(self):
        '''
        Test to get current timezone (i.e. America/Denver)
        '''
        mock = MagicMock(side_effect=['Time zone: A', 'A'])
        with patch.object(salt.utils, 'which', return_value=True):
            with patch.dict(timezone.__salt__, {'cmd.run': mock}):
                self.assertEqual(timezone.get_zone(), 'A')

                self.assertRaises(CommandExecutionError, timezone.get_zone)

        with patch.object(salt.utils, 'which', return_value=False):
            file_data = '\n'.join(['#', 'A'])
            with patch('salt.utils.fopen', mock_open(read_data=file_data),
                       create=True) as mfile:
                mfile.return_value.__iter__.return_value = file_data.splitlines()
                with patch.dict(timezone.__grains__, {'os_family': 'Debian'}):
                    self.assertEqual(timezone.get_zone(), '#\nA')

                with patch.dict(timezone.__grains__, {'os_family': 'Gentoo'}):
                    self.assertEqual(timezone.get_zone(), '')

            with patch.dict(timezone.__grains__, {'os_family': 'FreeBSD'}):
                with patch.object(os, 'readlink',
                                  return_value='/usr/share/zoneinfo/'):
                    self.assertEqual(timezone.get_zone(), '')

            with patch.dict(timezone.__grains__, {'os_family': 'Solaris'}):
                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A=B')}):
                    self.assertEqual(timezone.get_zone(), 'B')

    def test_get_zonecode(self):
        '''
        Test to get current timezone (i.e. PST, MDT, etc)
        '''
        with patch.dict(timezone.__salt__, {'cmd.run':
                                            MagicMock(return_value='A')}):
            self.assertEqual(timezone.get_zonecode(), 'A')

    def test_get_offset(self):
        '''
        Test to get current numeric timezone offset from UCT (i.e. -0700)
        '''
        with patch.dict(timezone.__salt__, {'cmd.run':
                                            MagicMock(return_value='A')}):
            self.assertEqual(timezone.get_offset(), 'A')

    def test_set_zone(self):
        '''
        Test to unlinks, then symlinks /etc/localtime to the set timezone.
        '''
        ret = ('Zone does not exist: /usr/share/lib/zoneinfo/timezone')
        mock = MagicMock(side_effect=[False, True, True])
        with patch.dict(timezone.__grains__, {'os_family': 'Solaris'}):
            with patch.object(os.path, 'exists', mock):
                self.assertEqual(timezone.set_zone('timezone'), ret)

                with patch.object(os, 'unlink', return_value=None):
                    with patch.dict(timezone.__salt__,
                                    {'file.sed':
                                     MagicMock(return_value=None)}):
                        self.assertTrue(timezone.set_zone('timezone'))

    def test_zone_compare(self):
        '''
        Test to checks the hash sum between the given timezone, and the
        one set in /etc/localtime.
        '''
        with patch.dict(timezone.__grains__, {'os_family': 'Solaris'}):
            self.assertEqual(timezone.zone_compare('timezone'),
                             'Not implemented for Solaris family')

        with patch.dict(timezone.__grains__, {'os_family': 'Sola'}):
            with patch.object(os.path, 'exists', return_value=False):
                self.assertEqual(timezone.zone_compare('timezone'),
                                 'Error: /etc/localtime does not exist.')

        with patch.object(os.path, 'exists', return_value=True):
            with patch.dict(timezone.__grains__, {'os_family': 'Sola'}):
                with patch.dict(timezone.__opts__, {'hash_type': 'md5'}):
                    with patch.object(salt.utils, 'get_hash',
                                      side_effect=IOError('foo')):
                        self.assertRaises(SaltInvocationError,
                                          timezone.zone_compare, 't')

                    with patch.object(salt.utils, 'get_hash',
                                      side_effect=['A', IOError('foo')]):
                        self.assertRaises(CommandExecutionError,
                                          timezone.zone_compare, 't')

                    with patch.object(salt.utils, 'get_hash',
                                      side_effect=['A', 'A']):
                        self.assertTrue(timezone.zone_compare('timezone'))

                    with patch.object(salt.utils, 'get_hash',
                                      side_effect=['A', 'B']):
                        self.assertFalse(timezone.zone_compare('timezone'))

    def test_get_hwclock(self):
        '''
        Test to get current hardware clock setting (UTC or localtime)
        '''
        with patch.object(salt.utils, 'which', return_value=True):
            with patch.dict(timezone.__salt__,
                            {'cmd.run':
                             MagicMock(return_value='rtc in local tz:yes\n')}):
                self.assertEqual(timezone.get_hwclock(), 'localtime')

            with patch.dict(timezone.__salt__,
                            {'cmd.run':
                             MagicMock(return_value='rtc in local tz:No\n')}):
                self.assertEqual(timezone.get_hwclock(), 'UTC')

            with patch.dict(timezone.__salt__,
                            {'cmd.run':
                             MagicMock(return_value='rtc')}):
                self.assertRaises(CommandExecutionError, timezone.get_hwclock)

        with patch.object(salt.utils, 'which', return_value=False):
            with patch.dict(timezone.__grains__, {'os_family': 'RedHat'}):
                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A')}):
                    self.assertEqual(timezone.get_hwclock(), 'A')

            with patch.dict(timezone.__grains__, {'os_family': 'Suse'}):
                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A')}):
                    self.assertEqual(timezone.get_hwclock(), 'A')

            with patch.dict(timezone.__grains__, {'os_family': 'Debian'}):
                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A=yes')}):
                    self.assertEqual(timezone.get_hwclock(), 'UTC')

                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A=no')}):
                    self.assertEqual(timezone.get_hwclock(), 'localtime')

                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A')}):
                    self.assertEqual(timezone.get_hwclock(), 'A')

            with patch.dict(timezone.__grains__, {'os_family': 'Gentoo'}):
                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value='A=B')}):
                    self.assertEqual(timezone.get_hwclock(), 'B')

        mock = MagicMock(return_value=True)
        with patch.object(os.path, 'isfile', mock):
            fl_data = 'zone_info=GMT'
            with patch('salt.utils.fopen',
                       mock_open(read_data=fl_data),
                       create=True) as mfile:
                mfile.return_value.__iter__.return_value = fl_data.splitlines()
                with patch.object(salt.utils, 'which', return_value=False):
                    with patch.dict(timezone.__grains__,
                                    {'os_family': 'Solaris'}):
                        self.assertEqual(timezone.get_hwclock(), 'UTC')

        mock = MagicMock(return_value=True)
        with patch.object(os.path, 'isfile', mock):
            fl_data = 'A=GMT'
            with patch('salt.utils.fopen',
                       mock_open(read_data=fl_data),
                       create=True) as mfile:
                mfile.return_value.__iter__.return_value = fl_data.splitlines()
                with patch.object(salt.utils, 'which', return_value=False):
                    with patch.dict(timezone.__grains__,
                                    {'os_family': 'Solaris'}):
                        self.assertEqual(timezone.get_hwclock(), 'localtime')

        with patch.object(salt.utils, 'which', return_value=False):
            with patch.dict(timezone.__grains__, {'os_family': 'Solaris'}):
                mock = MagicMock(return_value=False)
                with patch.object(os.path, 'isfile', mock):
                    self.assertEqual(timezone.get_hwclock(), 'UTC')

    def test_set_hwclock(self):
        '''
        Test to sets the hardware clock to be either UTC or localtime
        '''
        ret = ('UTC is the only choice for SPARC architecture')
        ret1 = ('Zone does not exist: /usr/share/zoneinfo/America/Denver')
        with patch.object(timezone, 'get_zone',
                          return_value='America/Denver'):

            with patch.dict(timezone.__grains__, {'os_family': 'Solaris',
                                                  'cpuarch': 'sparc'}):
                self.assertEqual(timezone.set_hwclock('clock'), ret)

                with patch.dict(timezone.__salt__,
                                {'cmd.run':
                                 MagicMock(return_value=None)}):
                    self.assertTrue(timezone.set_hwclock('localtime'))

                    self.assertTrue(timezone.set_hwclock('UTC'))

            with patch.dict(timezone.__grains__, {'os_family': 'Sola'}):
                with patch.object(os.path, 'exists', return_value=False):
                    self.assertEqual(timezone.set_hwclock('clock'), ret1)

                with patch.object(os.path, 'exists', return_value=True):
                    with patch.object(os, 'unlink', return_value=None):
                        with patch.object(os, 'symlink', return_value=None):
                            self.assertTrue(timezone.set_hwclock('clock'))

if __name__ == '__main__':
    from integration import run_tests
    run_tests(TimezoneTestCase, needs_daemon=False)
<|endoftext|>


<commit_before>from microbit import *

class Apa:
    def __init__(self, num_leds):
        self.buffer = bytearray(4)
        self.leds = [bytearray([0xE0,0,0x0,0]) for i in range(num_leds)]

    def write(self):
        spi.write(b'\x00\x00\x00\x00')
        for led in self.leds:
            for i in range(4):
                self.buffer[i] = led[i]
            spi.write(self.buffer)
        spi.write(b'\x00\x00\x00\x00')

    def limit(self, intensity):
        # intensity ranges from 0 (off) to 31 (very bright)
        # but leading three bits must be set to 1, so add 0xE0
        offset = 0xE0
        if intensity < 0: return offset
        if intensity > 31: return offset + 31
        return offset + intensity

    def set_led(self, n, intensity, r, b, g):
        self.leds[n][0] = 0xE0 + self.limit(intensity)
        self.leds[n][1] = r
        self.leds[n][2] = b
        self.leds[n][3] = g

# bar has 8 leds
apa = Apa(8)

# initialise the SPI bus using defaults.
# Pin 13 of the microbit is therefore used as CI (SCLCK) - the clock
# Pin 15 of the ,icorbit is used as DI (MOSI) - tha data

spi.init()

# turn each LED blue

for i in range(8):
    apa.set_led(i, 10, 0x30, 0x0, 0x0)
    apa.write()
    sleep(100)
# turn each LED off

for i in range(8):
    apa.set_led(i, 0, 0x00, 0x0, 0x0)
    apa.write()
    sleep(100)
 <commit_msg>Corrected typos in comments.<commit_after>from microbit import *

class Apa:
    def __init__(self, num_leds):
        self.buffer = bytearray(4)
        self.leds = [bytearray([0xE0,0,0x0,0]) for i in range(num_leds)]

    def write(self):
        spi.write(b'\x00\x00\x00\x00')
        for led in self.leds:
            for i in range(4):
                self.buffer[i] = led[i]
            spi.write(self.buffer)
        spi.write(b'\x00\x00\x00\x00')

    def limit(self, intensity):
        # intensity ranges from 0 (off) to 31 (very bright)
        # but leading three bits must be set to 1, so add 0xE0
        offset = 0xE0
        if intensity < 0: return offset
        if intensity > 31: return offset + 31
        return offset + intensity

    def set_led(self, n, intensity, r, b, g):
        self.leds[n][0] = 0xE0 + self.limit(intensity)
        self.leds[n][1] = r
        self.leds[n][2] = b
        self.leds[n][3] = g

# bar has 8 leds
apa = Apa(8)

# initialise the SPI bus using defaults.
# Pin 13 of the microbit is therefore used as CI (SCLCK) - the clock
# Pin 15 of the microbit is used as DI (MOSI) - the data

spi.init()

# turn each LED blue

for i in range(8):
    apa.set_led(i, 10, 0x30, 0x0, 0x0)
    apa.write()
    sleep(100)
# turn each LED off

for i in range(8):
    apa.set_led(i, 0, 0x00, 0x0, 0x0)
    apa.write()
    sleep(100)
 <|endoftext|>


<commit_before>from api import app
from api.database import db
from api.models import User, Group, App, Log
from flask_restless import APIManager
#from flask_login import current_user

'''''''''
def auth_func(*args, **kw):
    if not current_user.is_authenticated():
        raise ProcessingException(description='Not authenticated!', code=401)
'''''''''
manager = APIManager(app, flask_sqlalchemy_db=db)

# /api/user , /api/user/<int>
manager.create_api(User,  exclude_columns=['password_hash'], methods=['GET', 'POST', 'DELETE','PUT'])

# /api/group , /api/group/<int>
manager.create_api(Group, exclude_columns=['users.password_hash', 'users.group_id'], methods=['GET', 'POST', 'DELETE','PUT'])

# /api/app , /api/app/<int>
manager.create_api(App, methods=['GET', 'POST', 'DELETE','PUT'])

# /api/log , /api/log/<int>
manager.create_api(Log, methods=['GET', 'POST'])<commit_msg>Setup api auth function, change api for fetching app from outside<commit_after>from api import app
from api.database import db
from api.models import User, Group, App, Log
from flask_restless import APIManager, ProcessingException
from flask_login import current_user

def auth_func(*args, **kw):
    if not hasattr(current_user,'username'):
        raise ProcessingException(description='Not authenticated!', code=401)

manager = APIManager(app, flask_sqlalchemy_db=db, preprocessors=dict(POST=[auth_func], DELETE=[auth_func], PUT=[auth_func]))

# /api/user , /api/user/<int>
manager.create_api(User,  exclude_columns=['password_hash'], methods=['GET', 'POST', 'DELETE','PUT'], preprocessors=dict(GET_SINGLE=[auth_func], GET_MANY=[auth_func]))

# /api/group , /api/group/<int>
manager.create_api(Group, exclude_columns=['users.password_hash', 'users.group_id'], methods=['GET', 'POST', 'DELETE','PUT'], preprocessors=dict(GET_SINGLE=[auth_func], GET_MANY=[auth_func]))

# /api/app , /api/app/<int>
manager.create_api(App, exclude_columns=['creator.password_hash'], methods=['GET', 'POST', 'DELETE','PUT'], preprocessors=dict(GET_SINGLE=[auth_func], GET_MANY=[auth_func]))

# /api/app , /api/app/<int>
manager.create_api(App, include_columns=['name','link'], url_prefix='/api/v2', methods=['GET'])

# /api/log , /api/log/<int>
manager.create_api(Log, exclude_columns=['author.password_hash'], methods=['GET', 'POST'], preprocessors=dict(GET_SINGLE=[auth_func], GET_MANY=[auth_func]))

<|endoftext|>


<commit_before>"""
    :codeauthor: Gareth J. Greenaway <gareth@saltstack.com>
"""

import logging

import salt.config
import salt.proxy.netmiko_px as netmiko_proxy
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.mock import MagicMock, patch
from tests.support.runtests import RUNTIME_VARS
from tests.support.unit import TestCase

log = logging.getLogger(__name__)

class MockNetmikoConnection:
    def is_alive(self):
        return False

    def send_config_set(self, *args, **kwargs):
        return args, kwargs

class NetmikoProxyTestCase(TestCase, LoaderModuleMockMixin):
    def setup_loader_modules(self):
        return {netmiko_proxy: {}}

    def test_check_virtual(self):

        """ check netmiko_proxy virtual method - return value """

        with patch.object(netmiko_proxy, "HAS_NETMIKO", True):
            result = netmiko_proxy.__virtual__()
            self.assertIn("netmiko", result)

        expected = (
            False,
            "The netmiko proxy module requires netmiko library to be installed.",
        )
        with patch.object(netmiko_proxy, "HAS_NETMIKO", False):
            result = netmiko_proxy.__virtual__()
            self.assertEqual(expected, result)

    def test_init_skip_connect_on_init_true(self):

        """ check netmiko_proxy init method """

        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = True

        result = netmiko_proxy.init(opts)
        self.assertIn("netmiko_device", netmiko_proxy.__context__)
        self.assertIn("args", netmiko_proxy.__context__["netmiko_device"])

        self.assertIn("initialized", netmiko_proxy.__context__["netmiko_device"])
        self.assertFalse(netmiko_proxy.__context__["netmiko_device"]["initialized"])

        self.assertIn("up", netmiko_proxy.__context__["netmiko_device"])
        self.assertTrue(netmiko_proxy.__context__["netmiko_device"]["up"])

        self.assertIn("always_alive", netmiko_proxy.__context__["netmiko_device"])

        self.assertNotIn("conneciton", netmiko_proxy.__context__["netmiko_device"])

    def test_init_skip_connect_on_init_false(self):

        """ check netmiko_proxy init method """

        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = False

        mock_make_con = MagicMock()
        with patch.object(netmiko_proxy, "make_con", mock_make_con):
            result = netmiko_proxy.init(opts)

        self.assertIn("netmiko_device", netmiko_proxy.__context__)
        self.assertIn("args", netmiko_proxy.__context__["netmiko_device"])

        self.assertIn("initialized", netmiko_proxy.__context__["netmiko_device"])
        self.assertTrue(netmiko_proxy.__context__["netmiko_device"]["initialized"])

        self.assertIn("up", netmiko_proxy.__context__["netmiko_device"])
        self.assertTrue(netmiko_proxy.__context__["netmiko_device"]["up"])

        self.assertIn("always_alive", netmiko_proxy.__context__["netmiko_device"])

        self.assertNotIn("conneciton", netmiko_proxy.__context__["netmiko_device"])

    def test_make_con(self):

        """ check netmiko_proxy make_con method """

        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = True

        netmiko_proxy.init(opts)

        mock_connection = MockNetmikoConnection

        with patch.object(netmiko_proxy, "ConnectHandler", mock_connection):
            result = netmiko_proxy.make_con()
            self.assertNotEqual(result, None)

    def test_make_con_raise_exception(self):

        """ check netmiko_proxy make_con method """

        def raise_exception(*arg, **kwarg):
            raise Exception("expected")

        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = True

        netmiko_proxy.init(opts)

        mock_connection = MockNetmikoConnection

        with patch.object(netmiko_proxy, "DEFAULT_CONNECTION_TIMEOUT", 0), patch.object(
            netmiko_proxy, "ConnectHandler", raise_exception
        ):
            result = None
            try:
                result = netmiko_proxy.make_con(0)
            except Exception:  # pylint: disable=broad-except
                self.assertEqual(result, None)

    def test_ping(self):

        """ check netmiko_proxy ping method """

        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = True

        netmiko_proxy.init(opts)

        result = netmiko_proxy.ping()
        self.assertEqual(result, True)

    def test_alive(self):

        """ check netmiko_proxy alive method """

        # Always alive False with skip_connect_on_init on True
        # should return alive as True
        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = True
        opts["proxy_always_alive"] = False

        netmiko_proxy.init(opts)

        result = netmiko_proxy.alive(opts)
        self.assertTrue(result)

        # Always alive True with skip_connect_on_init on True
        # should return alive as False
        opts["proxy_always_alive"] = True

        netmiko_proxy.init(opts)

        result = netmiko_proxy.alive(opts)
        self.assertFalse(result)

        # Always alive True with skip_connect_on_init on False
        # should return alive as True
        opts["skip_connect_on_init"] = False
        opts["proxy_always_alive"] = True

        mock_make_con = MagicMock()
        with patch.object(netmiko_proxy, "make_con", mock_make_con):
            netmiko_proxy.init(opts)
            result = netmiko_proxy.alive(opts)
            self.assertTrue(result)

    def test_initialized(self):

        """ check netmiko_proxy alive method """

        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = True

        netmiko_proxy.init(opts)

        result = netmiko_proxy.initialized()
        self.assertFalse(result)

        # Always alive True with skip_connect_on_init on False
        # should return alive as True
        opts["skip_connect_on_init"] = False

        mock_make_con = MagicMock()
        with patch.object(netmiko_proxy, "make_con", mock_make_con):
            netmiko_proxy.init(opts)
            result = netmiko_proxy.initialized()
            self.assertTrue(result)
<commit_msg>Create the ConnectHandler mock when running on a system that does not have netmiko installed.<commit_after>"""
    :codeauthor: Gareth J. Greenaway <gareth@saltstack.com>
"""

import logging

import salt.config
import salt.proxy.netmiko_px as netmiko_proxy
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.mock import MagicMock, patch
from tests.support.runtests import RUNTIME_VARS
from tests.support.unit import TestCase

log = logging.getLogger(__name__)

class MockNetmikoConnection:
    def is_alive(self):
        return False

    def send_config_set(self, *args, **kwargs):
        return args, kwargs

class NetmikoProxyTestCase(TestCase, LoaderModuleMockMixin):
    def setup_loader_modules(self):
        return {netmiko_proxy: {}}

    def test_check_virtual(self):

        """ check netmiko_proxy virtual method - return value """

        with patch.object(netmiko_proxy, "HAS_NETMIKO", True):
            result = netmiko_proxy.__virtual__()
            self.assertIn("netmiko", result)

        expected = (
            False,
            "The netmiko proxy module requires netmiko library to be installed.",
        )
        with patch.object(netmiko_proxy, "HAS_NETMIKO", False):
            result = netmiko_proxy.__virtual__()
            self.assertEqual(expected, result)

    def test_init_skip_connect_on_init_true(self):

        """ check netmiko_proxy init method """

        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = True

        result = netmiko_proxy.init(opts)
        self.assertIn("netmiko_device", netmiko_proxy.__context__)
        self.assertIn("args", netmiko_proxy.__context__["netmiko_device"])

        self.assertIn("initialized", netmiko_proxy.__context__["netmiko_device"])
        self.assertFalse(netmiko_proxy.__context__["netmiko_device"]["initialized"])

        self.assertIn("up", netmiko_proxy.__context__["netmiko_device"])
        self.assertTrue(netmiko_proxy.__context__["netmiko_device"]["up"])

        self.assertIn("always_alive", netmiko_proxy.__context__["netmiko_device"])

        self.assertNotIn("conneciton", netmiko_proxy.__context__["netmiko_device"])

    def test_init_skip_connect_on_init_false(self):

        """ check netmiko_proxy init method """

        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = False

        mock_make_con = MagicMock()
        with patch.object(netmiko_proxy, "make_con", mock_make_con):
            result = netmiko_proxy.init(opts)

        self.assertIn("netmiko_device", netmiko_proxy.__context__)
        self.assertIn("args", netmiko_proxy.__context__["netmiko_device"])

        self.assertIn("initialized", netmiko_proxy.__context__["netmiko_device"])
        self.assertTrue(netmiko_proxy.__context__["netmiko_device"]["initialized"])

        self.assertIn("up", netmiko_proxy.__context__["netmiko_device"])
        self.assertTrue(netmiko_proxy.__context__["netmiko_device"]["up"])

        self.assertIn("always_alive", netmiko_proxy.__context__["netmiko_device"])

        self.assertNotIn("conneciton", netmiko_proxy.__context__["netmiko_device"])

    def test_make_con(self):

        """ check netmiko_proxy make_con method """

        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = True

        netmiko_proxy.init(opts)

        mock_connection = MockNetmikoConnection

        with patch.object(
            netmiko_proxy, "ConnectHandler", mock_connection, create=True
        ):
            result = netmiko_proxy.make_con()
            self.assertNotEqual(result, None)

    def test_make_con_raise_exception(self):

        """ check netmiko_proxy make_con method """

        def raise_exception(*arg, **kwarg):
            raise Exception("expected")

        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = True

        netmiko_proxy.init(opts)

        mock_connection = MockNetmikoConnection

        with patch.object(netmiko_proxy, "DEFAULT_CONNECTION_TIMEOUT", 0), patch.object(
            netmiko_proxy, "ConnectHandler", raise_exception, create=True
        ):
            result = None
            try:
                result = netmiko_proxy.make_con(0)
            except Exception:  # pylint: disable=broad-except
                self.assertEqual(result, None)

    def test_ping(self):

        """ check netmiko_proxy ping method """

        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = True

        netmiko_proxy.init(opts)

        result = netmiko_proxy.ping()
        self.assertEqual(result, True)

    def test_alive(self):

        """ check netmiko_proxy alive method """

        # Always alive False with skip_connect_on_init on True
        # should return alive as True
        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = True
        opts["proxy_always_alive"] = False

        netmiko_proxy.init(opts)

        result = netmiko_proxy.alive(opts)
        self.assertTrue(result)

        # Always alive True with skip_connect_on_init on True
        # should return alive as False
        opts["proxy_always_alive"] = True

        netmiko_proxy.init(opts)

        result = netmiko_proxy.alive(opts)
        self.assertFalse(result)

        # Always alive True with skip_connect_on_init on False
        # should return alive as True
        opts["skip_connect_on_init"] = False
        opts["proxy_always_alive"] = True

        mock_make_con = MagicMock()
        with patch.object(netmiko_proxy, "make_con", mock_make_con):
            netmiko_proxy.init(opts)
            result = netmiko_proxy.alive(opts)
            self.assertTrue(result)

    def test_initialized(self):

        """ check netmiko_proxy alive method """

        opts = salt.config.proxy_config(
            RUNTIME_VARS.TMP_PROXY_CONF_DIR + "/proxy", minion_id="proxytest"
        )
        opts["skip_connect_on_init"] = True

        netmiko_proxy.init(opts)

        result = netmiko_proxy.initialized()
        self.assertFalse(result)

        # Always alive True with skip_connect_on_init on False
        # should return alive as True
        opts["skip_connect_on_init"] = False

        mock_make_con = MagicMock()
        with patch.object(netmiko_proxy, "make_con", mock_make_con):
            netmiko_proxy.init(opts)
            result = netmiko_proxy.initialized()
            self.assertTrue(result)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Thomas Jackson <jacksontj.89@gmail.com>`
'''

# Import python libs
from __future__ import absolute_import
import os
import threading
import time

import zmq.eventloop.ioloop
# support pyzmq 13.0.x, TODO: remove once we force people to 14.0.x
if not hasattr(zmq.eventloop.ioloop, 'ZMQIOLoop'):
    zmq.eventloop.ioloop.ZMQIOLoop = zmq.eventloop.ioloop.IOLoop
from tornado.testing import AsyncTestCase

import tornado.gen

import salt.config
import salt.utils
import salt.transport.server
import salt.transport.client
import salt.exceptions

# Import Salt Testing libs
from salttesting import TestCase
from salttesting.helpers import ensure_in_syspath
ensure_in_syspath('../')

import integration

# Import Salt libs
from unit.transport.req_test import ReqChannelMixin
from unit.transport.pub_test import PubChannelMixin

# TODO: move to a library?
def get_config_file_path(filename):
    return os.path.join(integration.TMP, 'config', filename)

class BaseZMQReqCase(TestCase):
    '''
    Test the req server/client pair
    '''
    @classmethod
    def setUpClass(cls):
        cls.master_opts = salt.config.master_config(get_config_file_path('master'))
        cls.master_opts.update({
            'transport': 'zeromq',
            'auto_accept': True,
        })

        cls.minion_opts = salt.config.minion_config(get_config_file_path('minion'))
        cls.minion_opts.update({
            'transport': 'zeromq',
            'auth_timeout': 5,
            'auth_tries': 1,
            'master_uri': 'tcp://127.0.0.1:{0}'.format(cls.minion_opts['master_port']),
        })

        cls.process_manager = salt.utils.process.ProcessManager(name='ReqServer_ProcessManager')

        cls.server_channel = salt.transport.server.ReqServerChannel.factory(cls.master_opts)
        cls.server_channel.pre_fork(cls.process_manager)

        cls.io_loop = zmq.eventloop.ioloop.ZMQIOLoop()
        cls.server_channel.post_fork(cls._handle_payload, io_loop=cls.io_loop)

        cls.server_thread = threading.Thread(target=cls.io_loop.start)
        cls.server_thread.daemon = True
        cls.server_thread.start()

    @classmethod
    def tearDownClass(cls):
        cls.process_manager.kill_children()
        time.sleep(2)  # Give the procs a chance to fully close before we stop the io_loop
        cls.io_loop.stop()
        cls.server_channel.close()

class ClearReqTestCases(BaseZMQReqCase, ReqChannelMixin):
    '''
    Test all of the clear msg stuff
    '''
    def setUp(self):
        self.channel = salt.transport.client.ReqChannel.factory(self.minion_opts, crypt='clear')

    @classmethod
    @tornado.gen.coroutine
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        raise tornado.gen.Return((payload, {'fun': 'send_clear'}))

class AESReqTestCases(BaseZMQReqCase, ReqChannelMixin):
    def setUp(self):
        self.channel = salt.transport.client.ReqChannel.factory(self.minion_opts)

    @classmethod
    @tornado.gen.coroutine
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        raise tornado.gen.Return((payload, {'fun': 'send'}))

    # TODO: make failed returns have a specific framing so we can raise the same exception
    # on encrypted channels
    def test_badload(self):
        '''
        Test a variety of bad requests, make sure that we get some sort of error
        '''
        msgs = ['', [], tuple()]
        for msg in msgs:
            with self.assertRaises(salt.exceptions.AuthenticationError):
                ret = self.channel.send(msg, timeout=1)

class BaseZMQPubCase(AsyncTestCase):
    '''
    Test the req server/client pair
    '''
    @classmethod
    def setUpClass(cls):
        cls.master_opts = salt.config.master_config(get_config_file_path('master'))
        cls.master_opts.update({
            'transport': 'zeromq',
            'auto_accept': True,
        })

        cls.minion_opts = salt.config.minion_config(get_config_file_path('minion'))
        cls.minion_opts.update({
            'transport': 'zeromq',
            'master_ip': '127.0.0.1',
            'master_uri': 'tcp://127.0.0.1:{0}'.format(cls.minion_opts['master_port']),
        })

        cls.process_manager = salt.utils.process.ProcessManager(name='ReqServer_ProcessManager')

        cls.server_channel = salt.transport.server.PubServerChannel.factory(cls.master_opts)
        cls.server_channel.pre_fork(cls.process_manager)

        # we also require req server for auth
        cls.req_server_channel = salt.transport.server.ReqServerChannel.factory(cls.master_opts)
        cls.req_server_channel.pre_fork(cls.process_manager)

        cls._server_io_loop = zmq.eventloop.ioloop.ZMQIOLoop()
        cls.req_server_channel.post_fork(cls._handle_payload, io_loop=cls._server_io_loop)

        cls.server_thread = threading.Thread(target=cls._server_io_loop.start)
        cls.server_thread.daemon = True
        cls.server_thread.start()

    @classmethod
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        return payload, {'fun': 'send_clear'}

    @classmethod
    def tearDownClass(cls):
        cls.process_manager.kill_children()
        cls._server_io_loop.stop()
        cls.req_server_channel.close()

    def setUp(self):
        super(BaseZMQPubCase, self).setUp()
        self._start_handlers = dict(self.io_loop._handlers)

    def tearDown(self):
        super(BaseZMQPubCase, self).tearDown()
        failures = []
        for k, v in self.io_loop._handlers.iteritems():
            if self._start_handlers.get(k) != v:
                failures.append((k, v))
        if len(failures) > 0:
            raise Exception('FDs still attached to the IOLoop: {0}'.format(failures))

class AsyncPubChannelTest(BaseZMQPubCase, PubChannelMixin):
    '''
    Tests around the publish system
    '''
    def get_new_ioloop(self):
        return zmq.eventloop.ioloop.ZMQIOLoop()

if __name__ == '__main__':
    from integration import run_tests
    run_tests(ClearReqTestCases, needs_daemon=False)
    run_tests(AESReqTestCases, needs_daemon=False)
<commit_msg>Change pub test to return global ioloop instance<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Thomas Jackson <jacksontj.89@gmail.com>`
'''

# Import python libs
from __future__ import absolute_import
import os
import threading
import time

import zmq.eventloop.ioloop
# support pyzmq 13.0.x, TODO: remove once we force people to 14.0.x
if not hasattr(zmq.eventloop.ioloop, 'ZMQIOLoop'):
    zmq.eventloop.ioloop.ZMQIOLoop = zmq.eventloop.ioloop.IOLoop
from tornado.testing import AsyncTestCase

import tornado.gen

import salt.config
import salt.utils
import salt.transport.server
import salt.transport.client
import salt.exceptions

# Import Salt Testing libs
from salttesting import TestCase
from salttesting.helpers import ensure_in_syspath
ensure_in_syspath('../')

import integration

# Import Salt libs
from unit.transport.req_test import ReqChannelMixin
from unit.transport.pub_test import PubChannelMixin

# TODO: move to a library?
def get_config_file_path(filename):
    return os.path.join(integration.TMP, 'config', filename)

class BaseZMQReqCase(TestCase):
    '''
    Test the req server/client pair
    '''
    @classmethod
    def setUpClass(cls):
        cls.master_opts = salt.config.master_config(get_config_file_path('master'))
        cls.master_opts.update({
            'transport': 'zeromq',
            'auto_accept': True,
        })

        cls.minion_opts = salt.config.minion_config(get_config_file_path('minion'))
        cls.minion_opts.update({
            'transport': 'zeromq',
            'auth_timeout': 5,
            'auth_tries': 1,
            'master_uri': 'tcp://127.0.0.1:{0}'.format(cls.minion_opts['master_port']),
        })

        cls.process_manager = salt.utils.process.ProcessManager(name='ReqServer_ProcessManager')

        cls.server_channel = salt.transport.server.ReqServerChannel.factory(cls.master_opts)
        cls.server_channel.pre_fork(cls.process_manager)

        cls.io_loop = zmq.eventloop.ioloop.ZMQIOLoop()
        cls.server_channel.post_fork(cls._handle_payload, io_loop=cls.io_loop)

        cls.server_thread = threading.Thread(target=cls.io_loop.start)
        cls.server_thread.daemon = True
        cls.server_thread.start()

    @classmethod
    def tearDownClass(cls):
        cls.process_manager.kill_children()
        time.sleep(2)  # Give the procs a chance to fully close before we stop the io_loop
        cls.io_loop.stop()
        cls.server_channel.close()

class ClearReqTestCases(BaseZMQReqCase, ReqChannelMixin):
    '''
    Test all of the clear msg stuff
    '''
    def setUp(self):
        self.channel = salt.transport.client.ReqChannel.factory(self.minion_opts, crypt='clear')

    @classmethod
    @tornado.gen.coroutine
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        raise tornado.gen.Return((payload, {'fun': 'send_clear'}))

class AESReqTestCases(BaseZMQReqCase, ReqChannelMixin):
    def setUp(self):
        self.channel = salt.transport.client.ReqChannel.factory(self.minion_opts)

    @classmethod
    @tornado.gen.coroutine
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        raise tornado.gen.Return((payload, {'fun': 'send'}))

    # TODO: make failed returns have a specific framing so we can raise the same exception
    # on encrypted channels
    def test_badload(self):
        '''
        Test a variety of bad requests, make sure that we get some sort of error
        '''
        msgs = ['', [], tuple()]
        for msg in msgs:
            with self.assertRaises(salt.exceptions.AuthenticationError):
                ret = self.channel.send(msg, timeout=1)

class BaseZMQPubCase(AsyncTestCase):
    '''
    Test the req server/client pair
    '''
    @classmethod
    def setUpClass(cls):
        cls.master_opts = salt.config.master_config(get_config_file_path('master'))
        cls.master_opts.update({
            'transport': 'zeromq',
            'auto_accept': True,
        })

        cls.minion_opts = salt.config.minion_config(get_config_file_path('minion'))
        cls.minion_opts.update({
            'transport': 'zeromq',
            'master_ip': '127.0.0.1',
            'master_uri': 'tcp://127.0.0.1:{0}'.format(cls.minion_opts['master_port']),
        })

        cls.process_manager = salt.utils.process.ProcessManager(name='ReqServer_ProcessManager')

        cls.server_channel = salt.transport.server.PubServerChannel.factory(cls.master_opts)
        cls.server_channel.pre_fork(cls.process_manager)

        # we also require req server for auth
        cls.req_server_channel = salt.transport.server.ReqServerChannel.factory(cls.master_opts)
        cls.req_server_channel.pre_fork(cls.process_manager)

        cls._server_io_loop = zmq.eventloop.ioloop.ZMQIOLoop()
        cls.req_server_channel.post_fork(cls._handle_payload, io_loop=cls._server_io_loop)

        cls.server_thread = threading.Thread(target=cls._server_io_loop.start)
        cls.server_thread.daemon = True
        cls.server_thread.start()

    @classmethod
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        return payload, {'fun': 'send_clear'}

    @classmethod
    def tearDownClass(cls):
        cls.process_manager.kill_children()
        cls._server_io_loop.stop()
        cls.req_server_channel.close()

    def setUp(self):
        super(BaseZMQPubCase, self).setUp()
        self._start_handlers = dict(self.io_loop._handlers)

    def tearDown(self):
        super(BaseZMQPubCase, self).tearDown()
        failures = []
        for k, v in self.io_loop._handlers.iteritems():
            if self._start_handlers.get(k) != v:
                failures.append((k, v))
        if len(failures) > 0:
            raise Exception('FDs still attached to the IOLoop: {0}'.format(failures))

class AsyncPubChannelTest(BaseZMQPubCase, PubChannelMixin):
    '''
    Tests around the publish system
    '''
    def get_new_ioloop(self):
        return zmq.eventloop.ioloop.ZMQIOLoop().instance()

if __name__ == '__main__':
    from integration import run_tests
    run_tests(ClearReqTestCases, needs_daemon=False)
    run_tests(AESReqTestCases, needs_daemon=False)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Thomas Jackson <jacksontj.89@gmail.com>`
'''

# Import python libs
from __future__ import absolute_import
import os
import time
import threading
import platform

import zmq.eventloop.ioloop
# support pyzmq 13.0.x, TODO: remove once we force people to 14.0.x
if not hasattr(zmq.eventloop.ioloop, 'ZMQIOLoop'):
    zmq.eventloop.ioloop.ZMQIOLoop = zmq.eventloop.ioloop.IOLoop
from tornado.testing import AsyncTestCase

import tornado.gen

import salt.config
import salt.ext.six as six
import salt.utils
import salt.transport.server
import salt.transport.client
import salt.exceptions

# Import Salt Testing libs
from salttesting import TestCase, skipIf
from salttesting.helpers import ensure_in_syspath
ensure_in_syspath('../')

import integration

# Import Salt libs
from unit.transport.req_test import ReqChannelMixin
from unit.transport.pub_test import PubChannelMixin

ON_SUSE = False
if 'SuSE' in platform.dist():
    ON_SUSE = True

# TODO: move to a library?
def get_config_file_path(filename):
    return os.path.join(integration.TMP, 'config', filename)

class BaseZMQReqCase(TestCase):
    '''
    Test the req server/client pair
    '''
    @classmethod
    def setUpClass(cls):
        if not hasattr(cls, '_handle_payload'):
            return
        cls.master_opts = salt.config.master_config(get_config_file_path('master'))
        cls.master_opts.update({
            'transport': 'zeromq',
            'auto_accept': True,
        })

        cls.minion_opts = salt.config.minion_config(get_config_file_path('minion'))
        cls.minion_opts.update({
            'transport': 'zeromq',
            'auth_timeout': 5,
            'auth_tries': 1,
            'master_uri': 'tcp://127.0.0.1:{0}'.format(cls.minion_opts['master_port']),
        })

        cls.process_manager = salt.utils.process.ProcessManager(name='ReqServer_ProcessManager')

        cls.server_channel = salt.transport.server.ReqServerChannel.factory(cls.master_opts)
        cls.server_channel.pre_fork(cls.process_manager)

        cls.io_loop = zmq.eventloop.ioloop.ZMQIOLoop()
        cls.io_loop.make_current()
        cls.server_channel.post_fork(cls._handle_payload, io_loop=cls.io_loop)

        cls.server_thread = threading.Thread(target=cls.io_loop.start)
        cls.server_thread.daemon = True
        cls.server_thread.start()

    @classmethod
    def tearDownClass(cls):
        if not hasattr(cls, '_handle_payload'):
            return
        # Attempting to kill the children hangs the test suite.
        # Let the test suite handle this instead.
        cls.process_manager.kill_children()
        time.sleep(2)  # Give the procs a chance to fully close before we stop the io_loop
        cls.io_loop.stop()
        cls.server_channel.close()
        del cls.server_channel

    @classmethod
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        return payload, {'fun': 'send_clear'}

class ClearReqTestCases(BaseZMQReqCase, ReqChannelMixin):
    '''
    Test all of the clear msg stuff
    '''
    def setUp(self):
        self.channel = salt.transport.client.ReqChannel.factory(self.minion_opts, crypt='clear')

    @classmethod
    @tornado.gen.coroutine
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        raise tornado.gen.Return((payload, {'fun': 'send_clear'}))

@skipIf(ON_SUSE, 'Skipping until https://github.com/saltstack/salt/issues/32902 gets fixed')
class AESReqTestCases(BaseZMQReqCase, ReqChannelMixin):
    def setUp(self):
        self.channel = salt.transport.client.ReqChannel.factory(self.minion_opts)

    @classmethod
    @tornado.gen.coroutine
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        raise tornado.gen.Return((payload, {'fun': 'send'}))

    # TODO: make failed returns have a specific framing so we can raise the same exception
    # on encrypted channels
    def test_badload(self):
        '''
        Test a variety of bad requests, make sure that we get some sort of error
        '''
        msgs = ['', [], tuple()]
        for msg in msgs:
            with self.assertRaises(salt.exceptions.AuthenticationError):
                ret = self.channel.send(msg, timeout=1)

class BaseZMQPubCase(AsyncTestCase):
    '''
    Test the req server/client pair
    '''
    @classmethod
    def setUpClass(cls):
        cls.master_opts = salt.config.master_config(get_config_file_path('master'))
        cls.master_opts.update({
            'transport': 'zeromq',
            'auto_accept': True,
        })

        cls.minion_opts = salt.config.minion_config(get_config_file_path('minion'))
        cls.minion_opts.update({
            'transport': 'zeromq',
            'master_ip': '127.0.0.1',
            'master_uri': 'tcp://127.0.0.1:{0}'.format(cls.minion_opts['master_port']),
        })

        cls.process_manager = salt.utils.process.ProcessManager(name='ReqServer_ProcessManager')

        cls.server_channel = salt.transport.server.PubServerChannel.factory(cls.master_opts)
        cls.server_channel.pre_fork(cls.process_manager)

        # we also require req server for auth
        cls.req_server_channel = salt.transport.server.ReqServerChannel.factory(cls.master_opts)
        cls.req_server_channel.pre_fork(cls.process_manager)

        cls._server_io_loop = zmq.eventloop.ioloop.ZMQIOLoop()
        cls.req_server_channel.post_fork(cls._handle_payload, io_loop=cls._server_io_loop)

        cls.server_thread = threading.Thread(target=cls._server_io_loop.start)
        cls.server_thread.daemon = True
        cls.server_thread.start()

    @classmethod
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        return payload, {'fun': 'send_clear'}

    @classmethod
    def tearDownClass(cls):
        cls.process_manager.kill_children()
        cls._server_io_loop.stop()
        cls.req_server_channel.close()

    def setUp(self):
        super(BaseZMQPubCase, self).setUp()
        self._start_handlers = dict(self.io_loop._handlers)

    def tearDown(self):
        super(BaseZMQPubCase, self).tearDown()
        failures = []
        for k, v in six.iteritems(self.io_loop._handlers):
            if self._start_handlers.get(k) != v:
                failures.append((k, v))
        if len(failures) > 0:
            raise Exception('FDs still attached to the IOLoop: {0}'.format(failures))

@skipIf(True, 'Skip until we can devote time to fix this test')
class AsyncPubChannelTest(BaseZMQPubCase, PubChannelMixin):
    '''
    Tests around the publish system
    '''
    def get_new_ioloop(self):
        return zmq.eventloop.ioloop.ZMQIOLoop()

if __name__ == '__main__':
    from integration import run_tests
    run_tests(ClearReqTestCases, needs_daemon=False)
    run_tests(AESReqTestCases, needs_daemon=False)
<commit_msg>Increased timeout to test_badload zeromq test<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Thomas Jackson <jacksontj.89@gmail.com>`
'''

# Import python libs
from __future__ import absolute_import
import os
import time
import threading
import platform

import zmq.eventloop.ioloop
# support pyzmq 13.0.x, TODO: remove once we force people to 14.0.x
if not hasattr(zmq.eventloop.ioloop, 'ZMQIOLoop'):
    zmq.eventloop.ioloop.ZMQIOLoop = zmq.eventloop.ioloop.IOLoop
from tornado.testing import AsyncTestCase

import tornado.gen

import salt.config
import salt.ext.six as six
import salt.utils
import salt.transport.server
import salt.transport.client
import salt.exceptions

# Import Salt Testing libs
from salttesting import TestCase, skipIf
from salttesting.helpers import ensure_in_syspath
ensure_in_syspath('../')

import integration

# Import Salt libs
from unit.transport.req_test import ReqChannelMixin
from unit.transport.pub_test import PubChannelMixin

ON_SUSE = False
if 'SuSE' in platform.dist():
    ON_SUSE = True

# TODO: move to a library?
def get_config_file_path(filename):
    return os.path.join(integration.TMP, 'config', filename)

class BaseZMQReqCase(TestCase):
    '''
    Test the req server/client pair
    '''
    @classmethod
    def setUpClass(cls):
        if not hasattr(cls, '_handle_payload'):
            return
        cls.master_opts = salt.config.master_config(get_config_file_path('master'))
        cls.master_opts.update({
            'transport': 'zeromq',
            'auto_accept': True,
        })

        cls.minion_opts = salt.config.minion_config(get_config_file_path('minion'))
        cls.minion_opts.update({
            'transport': 'zeromq',
            'auth_timeout': 5,
            'auth_tries': 1,
            'master_uri': 'tcp://127.0.0.1:{0}'.format(cls.minion_opts['master_port']),
        })

        cls.process_manager = salt.utils.process.ProcessManager(name='ReqServer_ProcessManager')

        cls.server_channel = salt.transport.server.ReqServerChannel.factory(cls.master_opts)
        cls.server_channel.pre_fork(cls.process_manager)

        cls.io_loop = zmq.eventloop.ioloop.ZMQIOLoop()
        cls.io_loop.make_current()
        cls.server_channel.post_fork(cls._handle_payload, io_loop=cls.io_loop)

        cls.server_thread = threading.Thread(target=cls.io_loop.start)
        cls.server_thread.daemon = True
        cls.server_thread.start()

    @classmethod
    def tearDownClass(cls):
        if not hasattr(cls, '_handle_payload'):
            return
        # Attempting to kill the children hangs the test suite.
        # Let the test suite handle this instead.
        cls.process_manager.kill_children()
        time.sleep(2)  # Give the procs a chance to fully close before we stop the io_loop
        cls.io_loop.stop()
        cls.server_channel.close()
        del cls.server_channel

    @classmethod
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        return payload, {'fun': 'send_clear'}

class ClearReqTestCases(BaseZMQReqCase, ReqChannelMixin):
    '''
    Test all of the clear msg stuff
    '''
    def setUp(self):
        self.channel = salt.transport.client.ReqChannel.factory(self.minion_opts, crypt='clear')

    @classmethod
    @tornado.gen.coroutine
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        raise tornado.gen.Return((payload, {'fun': 'send_clear'}))

@skipIf(ON_SUSE, 'Skipping until https://github.com/saltstack/salt/issues/32902 gets fixed')
class AESReqTestCases(BaseZMQReqCase, ReqChannelMixin):
    def setUp(self):
        self.channel = salt.transport.client.ReqChannel.factory(self.minion_opts)

    @classmethod
    @tornado.gen.coroutine
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        raise tornado.gen.Return((payload, {'fun': 'send'}))

    # TODO: make failed returns have a specific framing so we can raise the same exception
    # on encrypted channels
    def test_badload(self):
        '''
        Test a variety of bad requests, make sure that we get some sort of error
        '''
        msgs = ['', [], tuple()]
        for msg in msgs:
            with self.assertRaises(salt.exceptions.AuthenticationError):
                ret = self.channel.send(msg, timeout=5)

class BaseZMQPubCase(AsyncTestCase):
    '''
    Test the req server/client pair
    '''
    @classmethod
    def setUpClass(cls):
        cls.master_opts = salt.config.master_config(get_config_file_path('master'))
        cls.master_opts.update({
            'transport': 'zeromq',
            'auto_accept': True,
        })

        cls.minion_opts = salt.config.minion_config(get_config_file_path('minion'))
        cls.minion_opts.update({
            'transport': 'zeromq',
            'master_ip': '127.0.0.1',
            'master_uri': 'tcp://127.0.0.1:{0}'.format(cls.minion_opts['master_port']),
        })

        cls.process_manager = salt.utils.process.ProcessManager(name='ReqServer_ProcessManager')

        cls.server_channel = salt.transport.server.PubServerChannel.factory(cls.master_opts)
        cls.server_channel.pre_fork(cls.process_manager)

        # we also require req server for auth
        cls.req_server_channel = salt.transport.server.ReqServerChannel.factory(cls.master_opts)
        cls.req_server_channel.pre_fork(cls.process_manager)

        cls._server_io_loop = zmq.eventloop.ioloop.ZMQIOLoop()
        cls.req_server_channel.post_fork(cls._handle_payload, io_loop=cls._server_io_loop)

        cls.server_thread = threading.Thread(target=cls._server_io_loop.start)
        cls.server_thread.daemon = True
        cls.server_thread.start()

    @classmethod
    def _handle_payload(cls, payload):
        '''
        TODO: something besides echo
        '''
        return payload, {'fun': 'send_clear'}

    @classmethod
    def tearDownClass(cls):
        cls.process_manager.kill_children()
        cls._server_io_loop.stop()
        cls.req_server_channel.close()

    def setUp(self):
        super(BaseZMQPubCase, self).setUp()
        self._start_handlers = dict(self.io_loop._handlers)

    def tearDown(self):
        super(BaseZMQPubCase, self).tearDown()
        failures = []
        for k, v in six.iteritems(self.io_loop._handlers):
            if self._start_handlers.get(k) != v:
                failures.append((k, v))
        if len(failures) > 0:
            raise Exception('FDs still attached to the IOLoop: {0}'.format(failures))

@skipIf(True, 'Skip until we can devote time to fix this test')
class AsyncPubChannelTest(BaseZMQPubCase, PubChannelMixin):
    '''
    Tests around the publish system
    '''
    def get_new_ioloop(self):
        return zmq.eventloop.ioloop.ZMQIOLoop()

if __name__ == '__main__':
    from integration import run_tests
    run_tests(ClearReqTestCases, needs_daemon=False)
    run_tests(AESReqTestCases, needs_daemon=False)
<|endoftext|>