<commit_before>#!/usr/bin/env python
import gevent
import os
import re
import urllib2
import urlparse

from gevent import monkey; monkey.patch_all()

from flask import Flask, render_template, send_from_directory, request
from lxml.html import parse

# initialization
app = Flask(__name__)

app.config.update(
    DEBUG = True,
)

# controllers
@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static'), 'ico/favicon.ico')

@app.route('/css/<filename>')
def css(filename):
    return send_from_directory(os.path.join(app.root_path, 'static'), "css/%s" % filename)

@app.route('/js/<filename>')
def js(filename):
    return send_from_directory(os.path.join(app.root_path, 'static'), "js/%s" % filename)

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

@app.route("/")
def index():
    value = request.args.get('slug')
    if value == None:
        value = ""
    return render_template('index.html', value=value)

@app.route("/about")
def about():
    return render_template('about.html')

@app.route("/slug/<path:slug>")
def check(slug):
    return service_check("/%s" % slug)

# Helper functions
def extract_service_domain_from_link(link):
    domain = urlparse.urlparse(link).netloc
    if re.match('^.*\.service\.gov\.uk$',domain):
        return True, domain
    else:
        return False, "The link is not to something on the service.gov.uk domain"

def find_link_from_slug(govuk_slug):
    try:
        service_link = None
        html = urllib2.urlopen("https://www.gov.uk%s" % govuk_slug)
        doc = parse(html).getroot()
        for link in doc.cssselect('a'):
            if link.text_content() == 'Start now':
                service_link = link.get('href')
        if service_link != None:
            return True, service_link
        else:
            return False, "Could not find 'Start now' link on https://www.gov.uk%s" % govuk_slug
    except IOError:
        return False, "https://www.gov.uk%s" % govuk_slug

def header_dict(headers):
    dict = {}
    for header in headers:
        key, value = header.split(': ', 1)
        dict[key.lower()] = value.rstrip()
    return dict

def format_output(status, title, description):
        return render_template('check.html', status=status, title=title, description=description)

# Service checks
def check_bare_ssl_domain_redirects_to_slug(domain, slug):
    correct_location = "https://www.gov.uk%s" % slug
    bare_domain = "https://%s/" % domain
    url = urllib2.urlopen(bare_domain)
    location = url.geturl()
    correct_location = "https://www.gov.uk%s" % slug
    check_title = "The bare service domain should redirect back to the GOV.UK start page"
    check_description = """
    In order to make sure that all transactions begin and end on GOV.UK, it is important that
    the bare domain (%s) redirects back to the GOV.UK start page (%s), so that if users are
    typing the URL from memory, they get a consistent user experience and their browser does
    not cache the wrong entry page.
    """ % (bare_domain, bare_domain, correct_location, correct_location)
    if location == correct_location:
        return True, check_title, check_description
    else:
        return False, check_title, check_description

def check_listening_on_http(domain):
    check_title = "The service should enforce SSL"
    check_description = """
    Users must have confidence that any information they are submitting to a service, including
    pages they visit, is not available to a 3rd-party. In order to enforce this, the service should
    either reject non-SSL connections, or should immediately redirect them to secured connection via SSL.
    """
    try:
        url = urllib2.urlopen("http://%s/" % domain, timeout=1)
        location = url.geturl()
        ssl_location = "https://%s/" % domain
        if location == ssl_location:
            return True, "%s (Service redirects HTTP to HTTPS)" % check_title, check_description
        else:
            return False, check_title, check_description
    except IOError:
        return True, "%s (Service does not listen on HTTP)" % check_title, check_description

def check_for_HSTS_header(link):

    check_title = "The service should set a Strict-Transport-Security (HSTS) header"
    check_description = """
    To reduce the chance that traffic for a user can be intercepted, the service
    should notify the browser that in future it should only use secure connections.
    It can do this by setting an HTTP Header called 'Strict-Transport-Security'.
    """
    try:
        url = urllib2.urlopen(link)
        headers = header_dict(url.info().headers)
        if 'strict-transport-security' in headers.keys():
            return True, check_title, check_description
        else:
            return False, check_title, check_description
    except urllib2.HTTPError as e:
        return False, check_title, "Error: %s" % e

def check_for_www(domain):
    check_title = "The service domain format should be www.{service}.service.gov.uk"
    check_description = """
    The Service Manual states that Users must interact with a single domain and that it
    will be www.{service}.service.gov.uk. It is permissible to create extra domains for
    example for Content Delivery Networks, Assets or Administration, however the user-facing
    domain should be prefixed by www.
    """
    if re.match('^www\.[^.]+\.service\.gov\.uk$', domain):
        return True, check_title, check_description
    else:
        return False, check_title, check_description

def check_for_robots_txt(domain):
    check_title = "The service should have a robots.txt file"
    check_description = """
    Every service hosted on a service.gov.uk domain must have a robots.txt file asking search engines
    not to index any part of the site. More details can be found on the Web Robots pages
    """
    try:
        url = urllib2.urlopen("https://%s/robots.txt" % domain)
        headers = header_dict(url.info().headers)
        if headers['content-type'] == "text/plain":
            return True, check_title, check_description
        else:
            return False, check_title, "The robots.txt file exists, but is %s rather than text/plain." % headers['content-type']
    except urllib2.HTTPError as e:
        return False, check_title, "Could not find robots.txt (Error: %s)" % e

def check_cookies(link):
    failed = False
    check_title = "Cookies should be Secure, HttpOnly and scoped to the service domain"
    check_description = """
    Cookies used on www.{service}.service.gov.uk must be scoped to the originating domain only.
    Cookies must not be scoped to the domain servicename.service.gov.uk. Cookies must be sent with
    the Secure attribute and should, where appropriate, be sent with the HttpOnly
    attribute. These flags provide additional assurances about how cookies will be handled by browsers.
    """
    result, domain = extract_service_domain_from_link(link)
    cookie_domain = "domain=" + domain
    url = urllib2.urlopen(link)
    headers = url.info().headers
    for header in headers:
        key, value = header.rstrip().split(': ', 1)
        if key.lower() == 'set-cookie':
            cookie_settings = value.lower().split('; ')
            if 'httponly' not in cookie_settings:
                check_description += "

HttpOnly is not set
<"
                check_description += "  Set-Cookie: %s
" % value
                failed = True
            if 'secure' not in cookie_settings:
                check_description += "

Secure is not set
"
                check_description += "  Set-Cookie: %s
" % value
                failed = True
            if cookie_domain not in cookie_settings:
                check_description += "

Cookie not scoped to domain=%s
" % domain
                check_description += "  Set-Cookie: %s
" % value
                failed = True
    if failed:
        return False, check_title, check_description
    else:
        return True, check_title, check_description

# Main logic process
def service_check(slug):

    output = ""
    result, link = find_link_from_slug(slug)
    if result:
        output += format_output(result,
                                "The GOV.UK start page should link to the service",
                                """All transactions should start on GOV.UK with a transaction start page.
                                You supplied the start page of https://www.gov.uk%s
                                which appears to link to a service: %s
                                """ % (slug, slug, link, link))
        result, domain = extract_service_domain_from_link(link)
        if result:
            check1 = gevent.spawn(check_bare_ssl_domain_redirects_to_slug, domain, slug)
            check2 = gevent.spawn(check_listening_on_http, domain)
            check3 = gevent.spawn(check_for_www, domain)
            check4 = gevent.spawn(check_for_HSTS_header,link)
            check5 = gevent.spawn(check_for_robots_txt,domain)
            check6 = gevent.spawn(check_cookies,link)
            checks = [check1, check2, check3, check4, check5, check6]
            gevent.joinall(checks)
            for check in checks:
                status, message, description = check.value
                output += "%s\n" % format_output(status, message, description)
        else:
            output += format_output(result,
                                    "The GOV.UK start page should link to the service",
                                    """All transactions should start on GOV.UK with a transaction start page.
                                    You supplied the start page of https://www.gov.uk%s
                                    which appears to have a 'Start now' button, but it does not link to something on the
                                    service.gov.uk domain as it points to %s.""" % (slug, slug, link, link))
    else:
        output += format_output(result,
                                "The GOV.UK start page should link to the service",
                                """All transactions should start on GOV.UK with a transaction start page.
                                You supplied the start page of https://www.gov.uk%s,
                                but either the page does not exist, or I cannot find a 'Start now' link on this
                                page pointing to a service.""" % (slug, slug))
    return render_template('service_check.html', output=output, link=link)

# launch
if __name__ == "__main__":
    port = int(os.environ.get("PORT", 5000))
    app.run(host='0.0.0.0', port=port, debug=True)
<commit_msg>Clarify error for non-service.gov.uk domains<commit_after>#!/usr/bin/env python
import gevent
import os
import re
import urllib2
import urlparse

from gevent import monkey; monkey.patch_all()

from flask import Flask, render_template, send_from_directory, request
from lxml.html import parse

# initialization
app = Flask(__name__)

app.config.update(
    DEBUG = True,
)

# controllers
@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static'), 'ico/favicon.ico')

@app.route('/css/<filename>')
def css(filename):
    return send_from_directory(os.path.join(app.root_path, 'static'), "css/%s" % filename)

@app.route('/js/<filename>')
def js(filename):
    return send_from_directory(os.path.join(app.root_path, 'static'), "js/%s" % filename)

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

@app.route("/")
def index():
    value = request.args.get('slug')
    if value == None:
        value = ""
    return render_template('index.html', value=value)

@app.route("/about")
def about():
    return render_template('about.html')

@app.route("/slug/<path:slug>")
def check(slug):
    return service_check("/%s" % slug)

# Helper functions
def extract_service_domain_from_link(link):
    domain = urlparse.urlparse(link).netloc
    if re.match('^.*\.service\.gov\.uk$',domain):
        return True, domain
    else:
        return False, "The link is not to something on the service.gov.uk domain"

def find_link_from_slug(govuk_slug):
    try:
        service_link = None
        html = urllib2.urlopen("https://www.gov.uk%s" % govuk_slug)
        doc = parse(html).getroot()
        for link in doc.cssselect('a'):
            if link.text_content() == 'Start now':
                service_link = link.get('href')
        if service_link != None:
            return True, service_link
        else:
            return False, "Could not find 'Start now' link on https://www.gov.uk%s" % govuk_slug
    except IOError:
        return False, "https://www.gov.uk%s" % govuk_slug

def header_dict(headers):
    dict = {}
    for header in headers:
        key, value = header.split(': ', 1)
        dict[key.lower()] = value.rstrip()
    return dict

def format_output(status, title, description):
        return render_template('check.html', status=status, title=title, description=description)

# Service checks
def check_bare_ssl_domain_redirects_to_slug(domain, slug):
    correct_location = "https://www.gov.uk%s" % slug
    bare_domain = "https://%s/" % domain
    url = urllib2.urlopen(bare_domain)
    location = url.geturl()
    correct_location = "https://www.gov.uk%s" % slug
    check_title = "The bare service domain should redirect back to the GOV.UK start page"
    check_description = """
    In order to make sure that all transactions begin and end on GOV.UK, it is important that
    the bare domain (%s) redirects back to the GOV.UK start page (%s), so that if users are
    typing the URL from memory, they get a consistent user experience and their browser does
    not cache the wrong entry page.
    """ % (bare_domain, bare_domain, correct_location, correct_location)
    if location == correct_location:
        return True, check_title, check_description
    else:
        return False, check_title, check_description

def check_listening_on_http(domain):
    check_title = "The service should enforce SSL"
    check_description = """
    Users must have confidence that any information they are submitting to a service, including
    pages they visit, is not available to a 3rd-party. In order to enforce this, the service should
    either reject non-SSL connections, or should immediately redirect them to secured connection via SSL.
    """
    try:
        url = urllib2.urlopen("http://%s/" % domain, timeout=1)
        location = url.geturl()
        ssl_location = "https://%s/" % domain
        if location == ssl_location:
            return True, "%s (Service redirects HTTP to HTTPS)" % check_title, check_description
        else:
            return False, check_title, check_description
    except IOError:
        return True, "%s (Service does not listen on HTTP)" % check_title, check_description

def check_for_HSTS_header(link):

    check_title = "The service should set a Strict-Transport-Security (HSTS) header"
    check_description = """
    To reduce the chance that traffic for a user can be intercepted, the service
    should notify the browser that in future it should only use secure connections.
    It can do this by setting an HTTP Header called 'Strict-Transport-Security'.
    """
    try:
        url = urllib2.urlopen(link)
        headers = header_dict(url.info().headers)
        if 'strict-transport-security' in headers.keys():
            return True, check_title, check_description
        else:
            return False, check_title, check_description
    except urllib2.HTTPError as e:
        return False, check_title, "Error: %s" % e

def check_for_www(domain):
    check_title = "The service domain format should be www.{service}.service.gov.uk"
    check_description = """
    The Service Manual states that Users must interact with a single domain and that it
    will be www.{service}.service.gov.uk. It is permissible to create extra domains for
    example for Content Delivery Networks, Assets or Administration, however the user-facing
    domain should be prefixed by www.
    """
    if re.match('^www\.[^.]+\.service\.gov\.uk$', domain):
        return True, check_title, check_description
    else:
        return False, check_title, check_description

def check_for_robots_txt(domain):
    check_title = "The service should have a robots.txt file"
    check_description = """
    Every service hosted on a service.gov.uk domain must have a robots.txt file asking search engines
    not to index any part of the site. More details can be found on the Web Robots pages
    """
    try:
        url = urllib2.urlopen("https://%s/robots.txt" % domain)
        headers = header_dict(url.info().headers)
        if headers['content-type'] == "text/plain":
            return True, check_title, check_description
        else:
            return False, check_title, "The robots.txt file exists, but is %s rather than text/plain." % headers['content-type']
    except urllib2.HTTPError as e:
        return False, check_title, "Could not find robots.txt (Error: %s)" % e

def check_cookies(link):
    failed = False
    check_title = "Cookies should be Secure, HttpOnly and scoped to the service domain"
    check_description = """
    Cookies used on www.{service}.service.gov.uk must be scoped to the originating domain only.
    Cookies must not be scoped to the domain servicename.service.gov.uk. Cookies must be sent with
    the Secure attribute and should, where appropriate, be sent with the HttpOnly
    attribute. These flags provide additional assurances about how cookies will be handled by browsers.
    """
    result, domain = extract_service_domain_from_link(link)
    cookie_domain = "domain=" + domain
    url = urllib2.urlopen(link)
    headers = url.info().headers
    for header in headers:
        key, value = header.rstrip().split(': ', 1)
        if key.lower() == 'set-cookie':
            cookie_settings = value.lower().split('; ')
            if 'httponly' not in cookie_settings:
                check_description += "

HttpOnly is not set
<"
                check_description += "  Set-Cookie: %s
" % value
                failed = True
            if 'secure' not in cookie_settings:
                check_description += "

Secure is not set
"
                check_description += "  Set-Cookie: %s
" % value
                failed = True
            if cookie_domain not in cookie_settings:
                check_description += "

Cookie not scoped to domain=%s
" % domain
                check_description += "  Set-Cookie: %s
" % value
                failed = True
    if failed:
        return False, check_title, check_description
    else:
        return True, check_title, check_description

# Main logic process
def service_check(slug):

    output = ""
    result, link = find_link_from_slug(slug)
    if result:
        output += format_output(result,
                                "The GOV.UK start page should link to the service",
                                """All transactions should start on GOV.UK with a transaction start page.
                                You supplied the start page of https://www.gov.uk%s
                                which appears to link to a service: %s
                                """ % (slug, slug, link, link))
        result, domain = extract_service_domain_from_link(link)
        if result:
            check1 = gevent.spawn(check_bare_ssl_domain_redirects_to_slug, domain, slug)
            check2 = gevent.spawn(check_listening_on_http, domain)
            check3 = gevent.spawn(check_for_www, domain)
            check4 = gevent.spawn(check_for_HSTS_header,link)
            check5 = gevent.spawn(check_for_robots_txt,domain)
            check6 = gevent.spawn(check_cookies,link)
            checks = [check1, check2, check3, check4, check5, check6]
            gevent.joinall(checks)
            for check in checks:
                status, message, description = check.value
                output += "%s\n" % format_output(status, message, description)
        else:
            output += format_output(result,
                                    "The GOV.UK start page should link to service on a service.gov.uk domain",
                                    """You supplied the start page of https://www.gov.uk%s
                                    which appears to have a 'Start now' button, but it does not link to something on the
                                    service.gov.uk domain as it points to %s.""" % (slug, slug, link, link))
    else:
        output += format_output(result,
                                "The GOV.UK start page should link to the service",
                                """All transactions should start on GOV.UK with a transaction start page.
                                You supplied the start page of https://www.gov.uk%s,
                                but either the page does not exist, or I cannot find a 'Start now' link on this
                                page pointing to a service.""" % (slug, slug))
    return render_template('service_check.html', output=output, link=link)

# launch
if __name__ == "__main__":
    port = int(os.environ.get("PORT", 5000))
    app.run(host='0.0.0.0', port=port, debug=True)
<|endoftext|>


<commit_before>import os
import shutil
import codecs
from glob import glob
import logging
import zipfile

from django.template import Template, Context, loader as template_loader
from django.contrib.auth.models import SiteProfileNotAvailable
from django.core.exceptions import ObjectDoesNotExist
from django.conf import settings

from builds import utils as version_utils
from core.utils import copy_to_app_servers, copy_file_to_app_servers
from doc_builder.base import BaseBuilder, restoring_chdir
from projects.utils import run
from tastyapi import apiv2

log = logging.getLogger(__name__)

TEMPLATE_DIR = '%s/readthedocs/templates/sphinx' % settings.SITE_ROOT
STATIC_DIR = '%s/_static' % TEMPLATE_DIR

class BaseSphinx(BaseBuilder):
    """
    The parent for most sphinx builders.
    """

    def __init__(self, *args, **kwargs):
        super(BaseSphinx, self).__init__(*args, **kwargs)
        self.old_artifact_path = os.path.join(self.version.project.conf_dir(self.version.slug), self.sphinx_build_dir)

    @restoring_chdir
    def build(self, **kwargs):
        project = self.version.project
        os.chdir(project.conf_dir(self.version.slug))
        force_str = " -E " if self._force else ""
        if project.use_virtualenv:
            build_command = "%s %s -b %s -D language=%s . %s " % (
                project.venv_bin(version=self.version.slug,
                                 bin='sphinx-build'),
                force_str,
                self.sphinx_builder,
                project.language,
                self.sphinx_build_dir,
                )
        else:
            build_command = ("sphinx-build %s -b %s -D language=%s . %s"
                             % (
                                force_str,
                                self.sphinx_builder,
                                project.language,
                                self.sphinx_build_dir,
                                )
                             )
        results = run(build_command, shell=True)
        return results

    def append_conf(self, **kwargs):
        """Modify the given ``conf.py`` file from a whitelisted user's project.
        """
        project = self.version.project
        #Open file for appending.
        outfile = codecs.open(project.conf_file(self.version.slug),
                              encoding='utf-8', mode='a')
        outfile.write("\n")
        conf_py_path = version_utils.get_conf_py_path(self.version)
        remote_version = version_utils.get_vcs_version_slug(self.version)
        github_info = version_utils.get_github_username_repo(self.version)
        bitbucket_info = version_utils.get_bitbucket_username_repo(self.version)
        if github_info[0] is None:
            display_github = False
        else:
            display_github = True
        if bitbucket_info[0] is None:
            display_bitbucket = False
        else:
            display_bitbucket = True

        rtd_ctx = Context({
            'versions': project.api_versions(),
            'downloads': self.version.get_downloads(pretty=True),
            'current_version': self.version.slug,
            'project': project,
            'settings': settings,
            'static_path': STATIC_DIR,
            'template_path': TEMPLATE_DIR,
            'conf_py_path': conf_py_path,
            'downloads': apiv2.version(self.version.pk).downloads.get()['downloads'],
            'api_host': getattr(settings, 'SLUMBER_API_HOST', 'https://readthedocs.org'),
            # GitHub
            'github_user': github_info[0],
            'github_repo': github_info[1],
            'github_version':  remote_version,
            'display_github': display_github,
            # BitBucket
            'bitbucket_user': bitbucket_info[0],
            'bitbucket_repo': bitbucket_info[1],
            'bitbucket_version':  remote_version,
            'display_bitbucket': display_bitbucket,
        })
        rtd_string = template_loader.get_template('doc_builder/conf.py.tmpl').render(rtd_ctx)
        outfile.write(rtd_string)

class HtmlBuilder(BaseSphinx):
    type = 'sphinx'
    sphinx_builder = 'readthedocs'
    sphinx_build_dir = '_build/html'

class HtmlDirBuilder(HtmlBuilder):
    type = 'sphinx_htmldir'
    sphinx_builder = 'readthedocsdirhtml'

class SingleHtmlBuilder(HtmlBuilder):
    type = 'sphinx_singlehtml'
    sphinx_builder = 'readthedocssinglehtml'

class SearchBuilder(BaseSphinx):
    type = 'sphinx_search'
    sphinx_builder = 'json'
    sphinx_build_dir = '_build/json'

class LocalMediaBuilder(BaseSphinx):
    type = 'sphinx_localmedia'
    sphinx_builder = 'readthedocssinglehtmllocalmedia'
    sphinx_build_dir = '_build/localmedia'

    @restoring_chdir
    def move(self, **kwargs):
        log.info("Creating zip file from %s" % self.old_artifact_path)
        target_file = os.path.join(self.target, '%s.zip' % self.version.project.slug)
        if not os.path.exists(self.target):
            os.makedirs(self.target)
        if os.path.exists(target_file):
            os.remove(target_file)

        # Create a <slug>.zip file
        os.chdir(self.old_artifact_path)
        archive = zipfile.ZipFile(target_file, 'w')
        for root, subfolders, files in os.walk('.'):
            for file in files:
                to_write = os.path.join(root, file)
                archive.write(
                    filename=to_write,
                    arcname=os.path.join("%s-%s" % (self.version.project.slug,
                                                    self.version.slug),
                                         to_write)
                )
        archive.close()

class EpubBuilder(BaseSphinx):
    type = 'sphinx_epub'
    sphinx_builder = 'epub'
    sphinx_build_dir = '_build/epub'

    def move(self, **kwargs):
        from_globs = glob(os.path.join(self.old_artifact_path, "*.epub"))
        if not os.path.exists(self.target):
            os.makedirs(self.target)
        if from_globs:
            from_file = from_globs[0]
            to_file = os.path.join(self.target, "%s.epub" % self.version.project.slug)
            run('mv -f %s %s' % (from_file, to_file))

class PdfBuilder(BaseSphinx):
    type = 'sphinx_pdf'
    sphinx_build_dir = '_build/latex'

    @restoring_chdir
    def build(self, **kwargs):
        project = self.version.project
        os.chdir(project.conf_dir(self.version.slug))
        #Default to this so we can return it always.
        results = {}
        if project.use_virtualenv:
            latex_results = run('%s -b latex -D language=%s -d _build/doctrees . _build/latex'
                                % (project.venv_bin(version=self.version.slug,
                                                   bin='sphinx-build'), project.language))
        else:
            latex_results = run('sphinx-build -b latex -D language=%s -d _build/doctrees '
                                '. _build/latex' % project.language)

        if latex_results[0] == 0:
            os.chdir('_build/latex')
            tex_files = glob('*.tex')

            if tex_files:
                # Run LaTeX -> PDF conversions
                pdflatex_cmds = [('pdflatex -interaction=nonstopmode %s'
                                 % tex_file) for tex_file in tex_files]
                pdf_results = run(*pdflatex_cmds)
            else:
                pdf_results = (0, "No tex files found", "No tex files found")

            results = [
                latex_results[0] + pdf_results[0],
                latex_results[1] + pdf_results[1],
                latex_results[2] + pdf_results[2],
            ]
        else:
            results = latex_results
        return results

    def move(self, **kwargs):
        if not os.path.exists(self.target):
            os.makedirs(self.target)
        exact = os.path.join(self.old_artifact_path, "%s.pdf" % self.version.project.slug)
        if os.path.exists(exact):
            from_file = exact
        else:
            from_globs = glob(os.path.join(self.old_artifact_path, "*.pdf"))
            if from_globs:
                from_file = from_globs[0]
        to_file = os.path.join(self.target, "%s.pdf" % self.version.project.slug)
        run('mv -f %s %s' % (from_file, to_file))

<commit_msg>Defend against no pdf build<commit_after>import os
import shutil
import codecs
from glob import glob
import logging
import zipfile

from django.template import Template, Context, loader as template_loader
from django.contrib.auth.models import SiteProfileNotAvailable
from django.core.exceptions import ObjectDoesNotExist
from django.conf import settings

from builds import utils as version_utils
from core.utils import copy_to_app_servers, copy_file_to_app_servers
from doc_builder.base import BaseBuilder, restoring_chdir
from projects.utils import run
from tastyapi import apiv2

log = logging.getLogger(__name__)

TEMPLATE_DIR = '%s/readthedocs/templates/sphinx' % settings.SITE_ROOT
STATIC_DIR = '%s/_static' % TEMPLATE_DIR

class BaseSphinx(BaseBuilder):
    """
    The parent for most sphinx builders.
    """

    def __init__(self, *args, **kwargs):
        super(BaseSphinx, self).__init__(*args, **kwargs)
        self.old_artifact_path = os.path.join(self.version.project.conf_dir(self.version.slug), self.sphinx_build_dir)

    @restoring_chdir
    def build(self, **kwargs):
        project = self.version.project
        os.chdir(project.conf_dir(self.version.slug))
        force_str = " -E " if self._force else ""
        if project.use_virtualenv:
            build_command = "%s %s -b %s -D language=%s . %s " % (
                project.venv_bin(version=self.version.slug,
                                 bin='sphinx-build'),
                force_str,
                self.sphinx_builder,
                project.language,
                self.sphinx_build_dir,
                )
        else:
            build_command = ("sphinx-build %s -b %s -D language=%s . %s"
                             % (
                                force_str,
                                self.sphinx_builder,
                                project.language,
                                self.sphinx_build_dir,
                                )
                             )
        results = run(build_command, shell=True)
        return results

    def append_conf(self, **kwargs):
        """Modify the given ``conf.py`` file from a whitelisted user's project.
        """
        project = self.version.project
        #Open file for appending.
        outfile = codecs.open(project.conf_file(self.version.slug),
                              encoding='utf-8', mode='a')
        outfile.write("\n")
        conf_py_path = version_utils.get_conf_py_path(self.version)
        remote_version = version_utils.get_vcs_version_slug(self.version)
        github_info = version_utils.get_github_username_repo(self.version)
        bitbucket_info = version_utils.get_bitbucket_username_repo(self.version)
        if github_info[0] is None:
            display_github = False
        else:
            display_github = True
        if bitbucket_info[0] is None:
            display_bitbucket = False
        else:
            display_bitbucket = True

        rtd_ctx = Context({
            'versions': project.api_versions(),
            'downloads': self.version.get_downloads(pretty=True),
            'current_version': self.version.slug,
            'project': project,
            'settings': settings,
            'static_path': STATIC_DIR,
            'template_path': TEMPLATE_DIR,
            'conf_py_path': conf_py_path,
            'downloads': apiv2.version(self.version.pk).downloads.get()['downloads'],
            'api_host': getattr(settings, 'SLUMBER_API_HOST', 'https://readthedocs.org'),
            # GitHub
            'github_user': github_info[0],
            'github_repo': github_info[1],
            'github_version':  remote_version,
            'display_github': display_github,
            # BitBucket
            'bitbucket_user': bitbucket_info[0],
            'bitbucket_repo': bitbucket_info[1],
            'bitbucket_version':  remote_version,
            'display_bitbucket': display_bitbucket,
        })
        rtd_string = template_loader.get_template('doc_builder/conf.py.tmpl').render(rtd_ctx)
        outfile.write(rtd_string)

class HtmlBuilder(BaseSphinx):
    type = 'sphinx'
    sphinx_builder = 'readthedocs'
    sphinx_build_dir = '_build/html'

class HtmlDirBuilder(HtmlBuilder):
    type = 'sphinx_htmldir'
    sphinx_builder = 'readthedocsdirhtml'

class SingleHtmlBuilder(HtmlBuilder):
    type = 'sphinx_singlehtml'
    sphinx_builder = 'readthedocssinglehtml'

class SearchBuilder(BaseSphinx):
    type = 'sphinx_search'
    sphinx_builder = 'json'
    sphinx_build_dir = '_build/json'

class LocalMediaBuilder(BaseSphinx):
    type = 'sphinx_localmedia'
    sphinx_builder = 'readthedocssinglehtmllocalmedia'
    sphinx_build_dir = '_build/localmedia'

    @restoring_chdir
    def move(self, **kwargs):
        log.info("Creating zip file from %s" % self.old_artifact_path)
        target_file = os.path.join(self.target, '%s.zip' % self.version.project.slug)
        if not os.path.exists(self.target):
            os.makedirs(self.target)
        if os.path.exists(target_file):
            os.remove(target_file)

        # Create a <slug>.zip file
        os.chdir(self.old_artifact_path)
        archive = zipfile.ZipFile(target_file, 'w')
        for root, subfolders, files in os.walk('.'):
            for file in files:
                to_write = os.path.join(root, file)
                archive.write(
                    filename=to_write,
                    arcname=os.path.join("%s-%s" % (self.version.project.slug,
                                                    self.version.slug),
                                         to_write)
                )
        archive.close()

class EpubBuilder(BaseSphinx):
    type = 'sphinx_epub'
    sphinx_builder = 'epub'
    sphinx_build_dir = '_build/epub'

    def move(self, **kwargs):
        from_globs = glob(os.path.join(self.old_artifact_path, "*.epub"))
        if not os.path.exists(self.target):
            os.makedirs(self.target)
        if from_globs:
            from_file = from_globs[0]
            to_file = os.path.join(self.target, "%s.epub" % self.version.project.slug)
            run('mv -f %s %s' % (from_file, to_file))

class PdfBuilder(BaseSphinx):
    type = 'sphinx_pdf'
    sphinx_build_dir = '_build/latex'

    @restoring_chdir
    def build(self, **kwargs):
        project = self.version.project
        os.chdir(project.conf_dir(self.version.slug))
        #Default to this so we can return it always.
        results = {}
        if project.use_virtualenv:
            latex_results = run('%s -b latex -D language=%s -d _build/doctrees . _build/latex'
                                % (project.venv_bin(version=self.version.slug,
                                                   bin='sphinx-build'), project.language))
        else:
            latex_results = run('sphinx-build -b latex -D language=%s -d _build/doctrees '
                                '. _build/latex' % project.language)

        if latex_results[0] == 0:
            os.chdir('_build/latex')
            tex_files = glob('*.tex')

            if tex_files:
                # Run LaTeX -> PDF conversions
                pdflatex_cmds = [('pdflatex -interaction=nonstopmode %s'
                                 % tex_file) for tex_file in tex_files]
                pdf_results = run(*pdflatex_cmds)
            else:
                pdf_results = (0, "No tex files found", "No tex files found")

            results = [
                latex_results[0] + pdf_results[0],
                latex_results[1] + pdf_results[1],
                latex_results[2] + pdf_results[2],
            ]
        else:
            results = latex_results
        return results

    def move(self, **kwargs):
        if not os.path.exists(self.target):
            os.makedirs(self.target)
        exact = os.path.join(self.old_artifact_path, "%s.pdf" % self.version.project.slug)
        if os.path.exists(exact):
            from_file = exact
        else:
            from_globs = glob(os.path.join(self.old_artifact_path, "*.pdf"))
            if from_globs:
                from_file = from_globs[0]
        if from_file:
            to_file = os.path.join(self.target, "%s.pdf" % self.version.project.slug)
            run('mv -f %s %s' % (from_file, to_file))

<|endoftext|>


<commit_before>#!/usr/bin/env python
import os, re
import web
from utils import zip2rep, simplegraphs, apipublish
import blog

web.config.debug = True
web.template.Template.globals['commify'] = web.commify
web.template.Template.globals['int'] = int
render = web.template.render('templates/', base='base')
db = web.database(dbn=os.environ.get('DATABASE_ENGINE', 'postgres'),
                  db='watchdog_dev')

options = r'(?:\.(html|xml|rdf|n3|json))'
urls = (
  r'/', 'index',
  r'/us/(?:index%s)?' % options, 'find',
  r'/us/([A-Z][A-Z])', 'redistrict',
  r'/us/([a-z][a-z])%s?' % options, 'state',
  r'/us/([A-Z][A-Z]-\d+)', 'redistrict',
  r'/us/([a-z][a-z]-\d+)%s?' % options, 'district',
  r'/(us|p)/by/(.*)/distribution.png', 'sparkdist',
  r'/(us|p)/by/(.*)', 'dproperty',
  r'/p/(.*?)%s?' % options, 'politician',
  r'/about(/?)', 'about',
  r'/about/api', 'aboutapi',
  r'/about/feedback', 'feedback',
  r'/blog', 'reblog',
  r'/blog(/.*)', blog.app,
  r'/data/(.*)', 'staticdata'
)

class index:
    def GET(self):
        return render.index()

class about:
    def GET(self, endslash=None):
        if not endslash: raise web.seeother('/about/')
        return render.about()

class aboutapi:
    def GET(self):
        return render.about_api()

class feedback:
    def GET(self):
        raise web.seeother('/about')

    def POST(self):
        i = web.input(email='info@watchdog.net')
        web.sendmail('Feedback <%s>' % i.email, 'Watchdog <info@watchdog.net>',
          'watchdog.net feedback',
          i.content +'\n\n' + web.ctx.ip)

        return render.feedback_thanks()

class reblog:
    def GET(self):
        raise web.seeother('/blog/')

class find:
    def GET(self, format=None):
        i = web.input(address=None)
        join = ['district' + ' LEFT OUTER JOIN politician '
                             'ON (politician.district = district.name)']
        if i.get('zip'):
            try:
                dists = zip2rep.zip2dist(i.zip, i.address)
            except zip2rep.BadAddress:
                return render.find_badaddr(i.zip, i.address)
            if len(dists) == 1:
                raise web.seeother('/us/%s' % dists[0].lower())
            elif len(dists) == 0:
                return render.find_none(i.zip)
            else:
                dists = db.select(join, where=web.sqlors('name=', dists))
                return render.find_multi(i.zip, dists)
        else:
            out = apipublish.publish({
              'uri': apipublish.generic(lambda x: 'http://watchdog.net/us/' +
                                        x.name.lower()),
              'type': 'District',
              'name state district voting': apipublish.identity,
              'wikipedia': apipublish.URI,
             }, db.select('district'), format)
            if out is not False:
                return out

            dists = db.select(join, order='name asc')
            return render.districtlist(dists)

class state:
    def GET(self, state, format=None):
        state = state.upper()
        try:
            state = db.select('state', where='code=$state', vars=locals())[0]
        except IndexError:
            raise web.notfound

        out = apipublish.publish({
          'uri': 'http://watchdog.net/us/' + state.code.lower(),
          'type': 'State',
          'wikipedia': apipublish.URI,
          'code fipscode name status': apipublish.identity,
        }, [state], format)
        if out is not False:
            return out

        districts = db.select('district',
                              where='state=$state.code',
                              order='district asc',
                              vars=locals())

        return render.state(state, districts.list())

class redistrict:
    def GET(self, district):
        return web.seeother('/us/' + district.lower())

class district:
    def GET(self, district, format=None):
        try:
            district = district.upper()
            d = db.select(['district', 'state', 'politician'],
                          what=('district.*, '
                                'state.name as state_name, '
                                'politician.firstname as pol_firstname, '
                                'politician.lastname as pol_lastname, '
                                'politician.id as pol_id, '
                                'politician.photo_path as pol_photo_path'),
                          where=('district.name = $district AND '
                                 'district.state = state.code AND '
                                 'politician.district = district.name'),
                          vars=locals())[0]
        except IndexError:
            raise web.notfound

        out = apipublish.publish({
          'uri': 'http://watchdog.net/us/' + district.lower(),
          'type': 'District',
          'state': apipublish.URI('http://watchdog.net/us/' + d.state.lower()),
          'wikipedia almanac': apipublish.URI,
          'name voting area_sqmi cook_index poverty_pct median_income '
          'est_population est_population_year outline center_lat '
          'center_lng zoom_level': apipublish.identity,
        }, [d], format)
        if out is not False:
            return out

        if d.district == 0:
            d.districtth = 'at-large'
        else:
            d.districtth = web.nthstr(d.district)

        return render.district(d)

class politician:
    def GET(self, polid, format=None):
        if polid != polid.lower():
            raise web.seeother('/p/' + polid.lower())

        if polid == "" or polid == "index":
            p = db.select(['politician'], order='district asc').list()

            out = apipublish.publish({
              'uri': apipublish.generic(lambda x: 'http://watchdog.net/p/' +
                                        x.id),
              'type': 'Politician',
              'district': lambda x: apipublish.URI('http://watchdog.net/us/' +
                                                   x.lower()),
              'wikipedia': apipublish.URI,
             }, p, format)
            if out is not False:
                return out

            return render.pollist(p)

        try:
            p = db.select(['politician', 'district'],
                          what=("politician.*, "
                                "district.center_lat as d0, "
                                "district.center_lng as d1, "
                                "district.zoom_level as d2"),
                          where=('id=$polid AND '
                                 'district.name = politician.district'),
                          vars=locals())[0]
        except IndexError:
            raise web.notfound

        out = apipublish.publish({
          'uri': 'http://watchdog.net/p/' + polid,
          'type': 'Politician',
          'district': apipublish.URI('http://watchdog.net/us/' + p.district.lower()),
          'wikipedia': apipublish.URI,
          'bioguideid opensecretsid govtrackid gender birthday firstname '
          'middlename lastname officeurl party religion photo_path '
          'photo_credit_url photo_credit_text': apipublish.identity,
         }, [p], format)
        if out is not False:
            return out

        return render.politician(p)

r_safeproperty = re.compile('^[a-z0-9_]+$')
table_map = {'us': 'district', 'p': 'politician'}

class dproperty:
    def GET(self, table, what):
        try:
            table = table_map[table]
        except KeyError:
            raise web.notfound
        if not r_safeproperty.match(what): raise web.notfound

        maxnum = float(db.select(table,
                                 what='max(%s) as m' % what,
                                 vars=locals())[0].m)
        items = db.select(table,
                          what="*, 100*(%s/$maxnum) as pct" % what,
                          order='%s desc' % what,
                          where='%s is not null' % what,
                          vars=locals()).list()
        for item in items:
            if table == 'district':
                item.id = 'd' + item.name
                item.path = '/us/' + item.name.lower()
            elif table == 'politician':
                item.name = item.firstname + ' ' + item.lastname
                item.path = '/p/' + item.id
        return render.dproperty(items, what)

class sparkdist:
    def GET(self, table, what):
        try:
            table = table_map[table]
        except KeyError:
            raise web.notfound
        if not r_safeproperty.match(what): raise web.notfound

        inp = web.input(point=None)
        points = db.select(table, what=what, order=what+' desc', where=what+' is not null')
        points = [x[what] for x in points.list()]

        web.header('Content-Type', 'image/png')
        return simplegraphs.sparkline(points, inp.point)

class staticdata:
    def GET(self, path):
        if not web.config.debug:
            raise web.notfound

        assert '..' not in path, 'security'
        return file('data/' + path).read()

app = web.application(urls, globals())
if __name__ == "__main__": app.run()
<commit_msg>oops, forgot to add to the api time to think about refactoring<commit_after>#!/usr/bin/env python
import os, re
import web
from utils import zip2rep, simplegraphs, apipublish
import blog

web.config.debug = True
web.template.Template.globals['commify'] = web.commify
web.template.Template.globals['int'] = int
render = web.template.render('templates/', base='base')
db = web.database(dbn=os.environ.get('DATABASE_ENGINE', 'postgres'),
                  db='watchdog_dev')

options = r'(?:\.(html|xml|rdf|n3|json))'
urls = (
  r'/', 'index',
  r'/us/(?:index%s)?' % options, 'find',
  r'/us/([A-Z][A-Z])', 'redistrict',
  r'/us/([a-z][a-z])%s?' % options, 'state',
  r'/us/([A-Z][A-Z]-\d+)', 'redistrict',
  r'/us/([a-z][a-z]-\d+)%s?' % options, 'district',
  r'/(us|p)/by/(.*)/distribution.png', 'sparkdist',
  r'/(us|p)/by/(.*)', 'dproperty',
  r'/p/(.*?)%s?' % options, 'politician',
  r'/about(/?)', 'about',
  r'/about/api', 'aboutapi',
  r'/about/feedback', 'feedback',
  r'/blog', 'reblog',
  r'/blog(/.*)', blog.app,
  r'/data/(.*)', 'staticdata'
)

class index:
    def GET(self):
        return render.index()

class about:
    def GET(self, endslash=None):
        if not endslash: raise web.seeother('/about/')
        return render.about()

class aboutapi:
    def GET(self):
        return render.about_api()

class feedback:
    def GET(self):
        raise web.seeother('/about')

    def POST(self):
        i = web.input(email='info@watchdog.net')
        web.sendmail('Feedback <%s>' % i.email, 'Watchdog <info@watchdog.net>',
          'watchdog.net feedback',
          i.content +'\n\n' + web.ctx.ip)

        return render.feedback_thanks()

class reblog:
    def GET(self):
        raise web.seeother('/blog/')

class find:
    def GET(self, format=None):
        i = web.input(address=None)
        join = ['district' + ' LEFT OUTER JOIN politician '
                             'ON (politician.district = district.name)']
        if i.get('zip'):
            try:
                dists = zip2rep.zip2dist(i.zip, i.address)
            except zip2rep.BadAddress:
                return render.find_badaddr(i.zip, i.address)
            if len(dists) == 1:
                raise web.seeother('/us/%s' % dists[0].lower())
            elif len(dists) == 0:
                return render.find_none(i.zip)
            else:
                dists = db.select(join, where=web.sqlors('name=', dists))
                return render.find_multi(i.zip, dists)
        else:
            out = apipublish.publish({
              'uri': apipublish.generic(lambda x: 'http://watchdog.net/us/' +
                                        x.name.lower()),
              'type': 'District',
              'name state district voting': apipublish.identity,
              'wikipedia': apipublish.URI,
             }, db.select('district'), format)
            if out is not False:
                return out

            dists = db.select(join, order='name asc')
            return render.districtlist(dists)

class state:
    def GET(self, state, format=None):
        state = state.upper()
        try:
            state = db.select('state', where='code=$state', vars=locals())[0]
        except IndexError:
            raise web.notfound

        out = apipublish.publish({
          'uri': 'http://watchdog.net/us/' + state.code.lower(),
          'type': 'State',
          'wikipedia': apipublish.URI,
          'code fipscode name status': apipublish.identity,
        }, [state], format)
        if out is not False:
            return out

        districts = db.select('district',
                              where='state=$state.code',
                              order='district asc',
                              vars=locals())

        return render.state(state, districts.list())

class redistrict:
    def GET(self, district):
        return web.seeother('/us/' + district.lower())

class district:
    def GET(self, district, format=None):
        try:
            district = district.upper()
            d = db.select(['district', 'state', 'politician'],
                          what=('district.*, '
                                'state.name as state_name, '
                                'politician.firstname as pol_firstname, '
                                'politician.lastname as pol_lastname, '
                                'politician.id as pol_id, '
                                'politician.photo_path as pol_photo_path'),
                          where=('district.name = $district AND '
                                 'district.state = state.code AND '
                                 'politician.district = district.name'),
                          vars=locals())[0]
        except IndexError:
            raise web.notfound

        out = apipublish.publish({
          'uri': 'http://watchdog.net/us/' + district.lower(),
          'type': 'District',
          'state': apipublish.URI('http://watchdog.net/us/' + d.state.lower()),
          'wikipedia almanac': apipublish.URI,
          'name voting area_sqmi cook_index poverty_pct median_income '
          'est_population est_population_year outline center_lat '
          'center_lng zoom_level': apipublish.identity,
        }, [d], format)
        if out is not False:
            return out

        if d.district == 0:
            d.districtth = 'at-large'
        else:
            d.districtth = web.nthstr(d.district)

        return render.district(d)

class politician:
    def GET(self, polid, format=None):
        if polid != polid.lower():
            raise web.seeother('/p/' + polid.lower())

        if polid == "" or polid == "index":
            p = db.select(['politician'], order='district asc').list()

            out = apipublish.publish({
              'uri': apipublish.generic(lambda x: 'http://watchdog.net/p/' +
                                        x.id),
              'type': 'Politician',
              'district': lambda x: apipublish.URI('http://watchdog.net/us/' +
                                                   x.lower()),
              'wikipedia': apipublish.URI,
             }, p, format)
            if out is not False:
                return out

            return render.pollist(p)

        try:
            p = db.select(['politician', 'district'],
                          what=("politician.*, "
                                "district.center_lat as d0, "
                                "district.center_lng as d1, "
                                "district.zoom_level as d2"),
                          where=('id=$polid AND '
                                 'district.name = politician.district'),
                          vars=locals())[0]
        except IndexError:
            raise web.notfound

        out = apipublish.publish({
          'uri': 'http://watchdog.net/p/' + polid,
          'type': 'Politician',
          'district': apipublish.URI('http://watchdog.net/us/' + p.district.lower()),
          'wikipedia': apipublish.URI,
          'bioguideid opensecretsid govtrackid gender birthday firstname '
          'middlename lastname officeurl party religion photo_path '
          'photo_credit_url photo_credit_text '
          'n_speeches words_per_speech': apipublish.identity,
         }, [p], format)
        if out is not False:
            return out

        return render.politician(p)

r_safeproperty = re.compile('^[a-z0-9_]+$')
table_map = {'us': 'district', 'p': 'politician'}

class dproperty:
    def GET(self, table, what):
        try:
            table = table_map[table]
        except KeyError:
            raise web.notfound
        if not r_safeproperty.match(what): raise web.notfound

        maxnum = float(db.select(table,
                                 what='max(%s) as m' % what,
                                 vars=locals())[0].m)
        items = db.select(table,
                          what="*, 100*(%s/$maxnum) as pct" % what,
                          order='%s desc' % what,
                          where='%s is not null' % what,
                          vars=locals()).list()
        for item in items:
            if table == 'district':
                item.id = 'd' + item.name
                item.path = '/us/' + item.name.lower()
            elif table == 'politician':
                item.name = item.firstname + ' ' + item.lastname
                item.path = '/p/' + item.id
        return render.dproperty(items, what)

class sparkdist:
    def GET(self, table, what):
        try:
            table = table_map[table]
        except KeyError:
            raise web.notfound
        if not r_safeproperty.match(what): raise web.notfound

        inp = web.input(point=None)
        points = db.select(table, what=what, order=what+' desc', where=what+' is not null')
        points = [x[what] for x in points.list()]

        web.header('Content-Type', 'image/png')
        return simplegraphs.sparkline(points, inp.point)

class staticdata:
    def GET(self, path):
        if not web.config.debug:
            raise web.notfound

        assert '..' not in path, 'security'
        return file('data/' + path).read()

app = web.application(urls, globals())
if __name__ == "__main__": app.run()
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : ml-playground
# Timestamp : 13-09-2017 19:28
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
Document your module here
"""<commit_msg>script comments<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : ml-playground
# Timestamp : 13-09-2017 19:28
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This is a script for quick and dirty tests on machine learning concepts
"""
<|endoftext|>


<commit_before>import fcntl
import select
import numpy as np
import cv2
import v4l2
import v4l2capture # USE THIS FORK: https://github.com/gebart/python-v4l2capture

class WebCam(object):

    def __init__(self, dev_path, num_buffers, ctlval_exposure):
        self.dev_path        = dev_path
        self.num_buffers     = num_buffers
        self.ctlval_exposure = ctlval_exposure

        self.dev = open(self.dev_path, 'r')

        # set the JPEG compression quality to the maximum level possible
        try:
            jpegcomp = v4l2.v4l2_jpegcompression()
            fcntl.ioctl(self.dev, v4l2.VIDIOC_G_JPEGCOMP, jpegcomp)
            jpegcomp.quality = 100
            fcntl.ioctl(self.dev, v4l2.VIDIOC_S_JPEGCOMP, jpegcomp)
        except (IOError, OSError):
            print('WebCam: failed to set control: JPEG compression quality')

        # disable automatic gain control
        try:
            ctrl = v4l2.v4l2_control()
            ctrl.id    = v4l2.V4L2_CID_AUTOGAIN
            ctrl.value = 0
            fcntl.ioctl(self.dev, v4l2.VIDIOC_S_CTRL, ctrl)
        except (IOError, OSError):
            print('WebCam: failed to set control: automatic gain')

        # set exposure to the desired level
        try:
            ctrl = v4l2.v4l2_control()
            ctrl.id    = v4l2.V4L2_CID_EXPOSURE
            ctrl.value = int(self.ctlval_exposure)
            fcntl.ioctl(self.dev, v4l2.VIDIOC_S_CTRL, ctrl)
        except (IOError, OSError):
            print('WebCam: failed to set control: exposure')

        # ensure that the device supports 'JFIF JPEG' format video capture
        fmt = v4l2.v4l2_format()
        fmt.type = v4l2.V4L2_BUF_TYPE_VIDEO_CAPTURE
        fcntl.ioctl(self.dev, v4l2.VIDIOC_G_FMT, fmt)
        assert (fmt.fmt.pix.pixelformat == v4l2.V4L2_PIX_FMT_JPEG)

        # sanity-check the allegedly supported camera width and height
        assert (fmt.fmt.win.w.left > 0 and fmt.fmt.win.w.left <= 10240)
        assert (fmt.fmt.win.w.top  > 0 and fmt.fmt.win.w.top  <= 10240)
        self.res_wanted = (fmt.fmt.win.w.left, fmt.fmt.win.w.top)

        self.camera = v4l2capture.Video_device(self.dev_path)

        self.res_actual = self.camera.set_format(self.res_wanted[0], self.res_wanted[1], yuv420=0, fourcc='JPEG')

        self.camera.create_buffers(self.num_buffers)
        self.camera.queue_all_buffers()
        self.camera.start()

    def __del__(self):
        if hasattr(self, 'camera'):
            self.camera.stop()
            self.camera.close()
        if hasattr(self, 'dev'):
            self.dev.close()

    # get the ACTUAL webcam frame width
    def get_res_x(self):
        return self.res_actual[0]

    # get the ACTUAL webcam frame height
    def get_res_y(self):
        return self.res_actual[1]

    # get the most recent frame from the webcam, waiting if necessary, and throwing away stale frames if any
    # (the frame is a numpy array in BGR format)
    def get_fresh_frame(self):
        self.block_until_frame_ready()

        frames = []
        while self.has_frames_available():
            frames += [self.get_one_frame()]

        # decode the JPEG from the webcam into BGR for OpenCV's use
        return cv2.imdecode(np.fromstring(frames[-1], dtype=np.uint8), cv2.IMREAD_COLOR)

    # get one frame from the webcam buffer; the frame is not guaranteed to be the most recent frame available!
    # (the frame is a JPEG byte string)
    def get_one_frame(self):
        return self.camera.read_and_queue()

    # block until the webcam has at least one frame ready
    def block_until_frame_ready(self):
        select.select((self.camera,), (), ())

    # query whether the webcam has at least one frame ready for us to read (non-blocking)
    def has_frames_available(self):
        readable, writable, exceptional = select.select((self.camera,), (), (), 0.0)
        return (len(readable) != 0)

    def is_ctrl_supported(self, id):
        query = v4l2.v4l2_queryctrl()
        query.id = id
        try:
            fcntl.ioctl(self.dev, v4l2.VIDIOC_QUERYCTRL, query)
        except (IOError, OSError):
            return False
        if ((query.flags & v4l2.V4L2_CTRL_FLAG_DISABLED) != 0):
            return False
        return True
<commit_msg>Reimplement webcam code using multiprocessing library for concurrency<commit_after>import os
import fcntl
import select
import datetime
import multiprocessing
import numpy as np
import cv2
import v4l2
import v4l2capture # USE THIS FORK: https://github.com/gebart/python-v4l2capture

class WebCam(object):

    def __init__(self, dev_path, num_buffers, ctlval_exposure):
        self.dev_path        = dev_path
        self.num_buffers     = num_buffers
        self.ctlval_exposure = ctlval_exposure

        self.dev = open(self.dev_path, 'r')

        # set the JPEG compression quality to the maximum level possible
        try:
            jpegcomp = v4l2.v4l2_jpegcompression()
            fcntl.ioctl(self.dev, v4l2.VIDIOC_G_JPEGCOMP, jpegcomp)
            jpegcomp.quality = 100
            fcntl.ioctl(self.dev, v4l2.VIDIOC_S_JPEGCOMP, jpegcomp)
        except (IOError, OSError):
            print('WebCam: failed to set control: JPEG compression quality')

        # disable automatic gain control
        try:
            ctrl = v4l2.v4l2_control()
            ctrl.id    = v4l2.V4L2_CID_AUTOGAIN
            ctrl.value = 0
            fcntl.ioctl(self.dev, v4l2.VIDIOC_S_CTRL, ctrl)
        except (IOError, OSError):
            print('WebCam: failed to set control: automatic gain')

        # set exposure to the desired level
        try:
            ctrl = v4l2.v4l2_control()
            ctrl.id    = v4l2.V4L2_CID_EXPOSURE
            ctrl.value = int(self.ctlval_exposure)
            fcntl.ioctl(self.dev, v4l2.VIDIOC_S_CTRL, ctrl)
        except (IOError, OSError):
            print('WebCam: failed to set control: exposure')

        # ensure that the device supports 'JFIF JPEG' format video capture
        fmt = v4l2.v4l2_format()
        fmt.type = v4l2.V4L2_BUF_TYPE_VIDEO_CAPTURE
        fcntl.ioctl(self.dev, v4l2.VIDIOC_G_FMT, fmt)
        assert (fmt.fmt.pix.pixelformat == v4l2.V4L2_PIX_FMT_JPEG)

        # sanity-check the allegedly supported camera width and height
        assert (fmt.fmt.win.w.left > 0 and fmt.fmt.win.w.left <= 10240)
        assert (fmt.fmt.win.w.top  > 0 and fmt.fmt.win.w.top  <= 10240)
        self.res_wanted = (fmt.fmt.win.w.left, fmt.fmt.win.w.top)

        self.camera = v4l2capture.Video_device(self.dev_path)

        self.res_actual = self.camera.set_format(self.res_wanted[0], self.res_wanted[1], yuv420=0, fourcc='JPEG')

        self.camera.create_buffers(self.num_buffers)
        self.camera.queue_all_buffers()
        self.camera.start()

        self.proc_exit_r, self.proc_exit_w = os.pipe()

        self.frames_out, self.frames_in = multiprocessing.Pipe(duplex=False)

        self.proc = multiprocessing.Process(target=self.monitor, name='WebCam Process')
        self.proc.start()

    def __del__(self):
        if hasattr(self, 'proc') and self.proc.is_alive():
            # first attempt to use the pipe/select/join method to cleanly end the monitor process
            os.write(self.proc_exit_w, '!')
            self.proc.join(1.0)
            # failing that, kill it the brutal way
            if self.proc.is_alive():
                print('WebCam: monitor process didn\'t end; terminating it forcefully')
                self.proc.terminate()
        if hasattr(self, 'proc_exit_r'):
            os.close(self.proc_exit_r)
        if hasattr(self, 'proc_exit_w'):
            os.close(self.proc_exit_w)
        if hasattr(self, 'camera'):
            self.camera.stop()
            self.camera.close()
        if hasattr(self, 'dev'):
            self.dev.close()

    # get the ACTUAL webcam frame width
    def get_res_x(self):
        return self.res_actual[0]

    # get the ACTUAL webcam frame height
    def get_res_y(self):
        return self.res_actual[1]

    # get the most recent frame from the webcam, waiting if necessary, and throwing away stale frames if any
    # (the frame is a numpy array in BGR format)
    def get_fresh_frame(self):
        # block until there's at least one frame in the pipe
        self.frames_in.poll(None)

        frames = []
        while self.frames_in.poll():
            frames += [self.frames_in.read()]

        # return only the most recently acquired frame
        return frames[-1]

    def is_ctrl_supported(self, id):
        query = v4l2.v4l2_queryctrl()
        query.id = id
        try:
            fcntl.ioctl(self.dev, v4l2.VIDIOC_QUERYCTRL, query)
        except (IOError, OSError):
            return False
        if ((query.flags & v4l2.V4L2_CTRL_FLAG_DISABLED) != 0):
            return False
        return True

    # [WEBCAM PROCESS] webcam monitor loop
    def monitor(self):
        while True:
            readable, writable, exceptional = select.select((self.camera, self.proc_exit_r), (), (), 0.5)

            # the main thread is asking us to terminate
            if self.proc_exit_r in readable:
                return

            # grab as many frames as are available to us right now
            while self.has_frames_available():
                now = datetime.datetime.utcnow() # TODO
                frames_jpeg += [self.camera.read_and_queue()]

            # decode the compressed JPEG image data into BGR format for OpenCV's use
            for frame_jpeg in frames_jpeg:
                frames_bgr += [cv2.imdecode(np.fromstring(frame_jpeg, dtype=np.uint8), cv2.IMREAD_COLOR)]

            # send the decoded frames across the pipe to the main process
            for frame_bgr in frames_bgr:
                self.frames_in.send(frame_bgr)

    # [WEBCAM PROCESS] does the webcam have at least one frame ready for us to read? (non-blocking)
    def has_frames_available(self):
        readable, writable, exceptional = select.select((self.camera,), (), (), 0.0)
        return (self.camera in readable)
<|endoftext|>


<commit_before># cv compare: https://stackoverflow.com/questions/11094481/capturing-a-single-image-from-my-webcam-in-java-or-python
# cv settings: https://stackoverflow.com/questions/32943227/python-opencv-capture-images-from-webcam
# another compareg: https://softwarerecs.stackexchange.com/questions/18134/python-library-for-taking-camera-images
# Timestamp: http://startgrid.blogspot.com/2012/08/tutorial-creating-timestamp-on.html
# fswebcam: https://www.raspberrypi.org/documentation/usage/webcams/
# fswebcam doc: https://www.systutorials.com/docs/linux/man/1-fswebcam/
import time
import datetime
import wget
import os
from subprocess import call
#from my_globals import settings  #import settings from my_globals
import my_globals

def remove_image(filename):
    # remove current picture
    try:
        os.remove(filename)
    except OSError:
        pass

def get_cat_picture(filename):
    #url = "http://lorempixel.com/1024/768/cats/"   # cats
    url = "http://207.251.86.238/cctv448.jpg"       # NY trafic cam
    remove_image(filename)
    cat_pic = wget.download(url, out=filename, bar=None)
    #print ("filename: ", cat_pic)

def get_Picture(FAKEWEBCAM):
    print ("\n--- Getting picture ------------------")
    filename = my_globals.settings["storage_dir"] + my_globals.settings["img_name"]      # full path to image
    #print ("Img Filename: ", filename)
    remove_image(filename)
    if FAKEWEBCAM == 1:     # get fake picture
        get_cat_picture(filename)
        #return
    else:                   # get picture from webcam
        global filename
        # setup some metadata for fswebcam
        compression = "45"
        device      = "/dev/video0"
        resolution  = "1280x720"
        #resolution  = "1024x600
        #resolution = "640x480"     # Brandon's res choice
        textcolor   = "#0000cc00"
        font        = "luxisr:14"
        title       = str(my_globals.settings["name"])
        subtitle    = "cpu null"  #"cpu: {} C".format(cpu_temp())
        info        =  str(my_globals.settings["uuid"])

        # call fswebcam to take the picture
        #call(["fswebcam", "-S 3", "--jpeg", compression, "-d", device, "-r", resolution, "--scale", "960x540",
         #"--top-banner", "--text-colour", textcolor, "--font", font,
         # "--title", title, "--subtitle", subtitle, "--info", info, filename])

        # call v4lctl to take the picture
        call(["v4lctl", "-c", device, "snap", "jpeg", resolution, filename])
        print ("Img size: ", call(["du", "-h", filename]))    # prints size of picture

    # overlay reference
    # overlay_text = "/usr/bin/convert "+ filename + "  -pointsize 36 -fill white -annotate +40+728 '" + "hello" + "' "
    # overlay_text += " -pointsize 36 -fill white -annotate +40+630 'Your Text annotation here ' " + filename

    # The [:-4] truncates the last 4 characters of the string which is used cut out some microsecond digits
    text = '"Smartsettia - %s UTC"' % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    overlay_text = "convert " + filename + " -gravity North   -background YellowGreen  -splice 0x18 \
          -annotate +0+2 " + text + " " + filename

    # print ("convert command: %s" % overlay_text)      # debugger to see command executed
    call ([overlay_text], shell=True)
<commit_msg>Logging update: webcam<commit_after># cv compare: https://stackoverflow.com/questions/11094481/capturing-a-single-image-from-my-webcam-in-java-or-python
# cv settings: https://stackoverflow.com/questions/32943227/python-opencv-capture-images-from-webcam
# another compareg: https://softwarerecs.stackexchange.com/questions/18134/python-library-for-taking-camera-images
# Timestamp: http://startgrid.blogspot.com/2012/08/tutorial-creating-timestamp-on.html
# fswebcam: https://www.raspberrypi.org/documentation/usage/webcams/
# fswebcam doc: https://www.systutorials.com/docs/linux/man/1-fswebcam/
import time
import datetime
import wget
import os
from subprocess import call
#from my_globals import settings  #import settings from my_globals
import my_globals
import logging

def remove_image(filename):
    # remove current picture
    try:
        os.remove(filename)
    except OSError:
        pass

def get_cat_picture(filename):
    #url = "http://lorempixel.com/1024/768/cats/"   # cats
    url = "http://207.251.86.238/cctv448.jpg"       # NY trafic cam
    remove_image(filename)
    cat_pic = wget.download(url, out=filename, bar=None)

def get_Picture(FAKEWEBCAM):
    logging.info ("Webcam: --- Getting picture ------------------")
    filename = my_globals.settings["storage_dir"] + my_globals.settings["img_name"]      # full path to image
    logging.debug("Webcam: Img Filename: "+ filename)
    remove_image(filename)
    if FAKEWEBCAM == 1:     # get fake picture
        get_cat_picture(filename)
    else:                   # get picture from webcam
        # setup some metadata for fswebcam
        compression = "45"
        device      = "/dev/video0"
        resolution  = "1280x720"
        #resolution  = "1024x600
        #resolution = "640x480"     # Brandon's res choice
        textcolor   = "#0000cc00"
        font        = "luxisr:14"
        title       = str(my_globals.settings["name"])
        subtitle    = "cpu null"  #"cpu: {} C".format(cpu_temp())
        info        =  str(my_globals.settings["uuid"])

        # call fswebcam to take the picture
        #call(["fswebcam", "-S 3", "--jpeg", compression, "-d", device, "-r", resolution, "--scale", "960x540",
         #"--top-banner", "--text-colour", textcolor, "--font", font,
         # "--title", title, "--subtitle", subtitle, "--info", info, filename])

        # call v4lctl to take the picture
        call(["v4lctl", "-c", device, "snap", "jpeg", resolution, filename])
        logging.info ("Webcam: Img size: " + call(["du", "-h", filename]))    # prints size of picture
        #check_output(["/opt/vc/bin/vcgencmd measure_temp | cut -c6-9"], shell=True)[:-1].decode('utf-8')

    # overlay reference
    # overlay_text = "/usr/bin/convert "+ filename + "  -pointsize 36 -fill white -annotate +40+728 '" + "hello" + "' "
    # overlay_text += " -pointsize 36 -fill white -annotate +40+630 'Your Text annotation here ' " + filename

    # The [:-4] truncates the last 4 characters of the string which is used cut out some microsecond digits
    text = '"Smartsettia - %s UTC"' % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    overlay_text = "convert " + filename + " -gravity North   -background YellowGreen  -splice 0x18 \
          -annotate +0+2 " + text + " " + filename

    #logging.debug ("Webcam: convert command: %s" % overlay_text)      # debugger to see command executed
    call ([overlay_text], shell=True)<|endoftext|>


<commit_before>from urllib.request import urlopen
from urllib.error import HTTPError
from urllib.error import URLError
from urllib.parse import urlparse
from urllib.parse import quote_plus as q
from bs4 import BeautifulSoup
import re
import datetime
import random
import socks
import socket
import requests
import time
import sys

class Crawler:
    def __init__(self, page):
        self.internalLinks = set()
        self.externalLinks = set()
        self.pages = set()
        self.current_page = page
        random.seed(datetime.datetime.now())

    def get_internal_links(self, bsObj, includeUrl):
        scm = urlparse(includeUrl).scheme
        loc = urlparse(includeUrl).netloc
        includeUrl = scm + "://" + loc

        reurl = re.compile("^(\/|.*" + includeUrl + ")")

        for link in bsObj.findAll("a", href=reurl):
            url = q(link.attrs['href'], safe="/:")
            if url is not None:
                if url.startswith("/"):
                    the_link = includeUrl + url
                else:
                    the_link = url
                if link not in self.pages:
                    self.internalLinks.add(the_link)

    def get_external_links(self, bsObj, excludeUrl):
        reurl = re.compile("^(http|www)((?!" + excludeUrl + ").)*$")

        for link in bsObj.findAll("a", href=reurl):
            url = q(link.attrs['href'], safe="/:")
            if url is not None:
                if url not in self.pages:
                    self.externalLinks.add(url)

    def get_links(self):
        # html = urlopen(self.current_page).read()
        session = requests.Session()
        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; rv:38.0)"
                   "Gecko/20100101 Firefox/38.0",
                   "Accept": "text/html,application/xhtml+xml,application/xml;"
                   "q=0.9,image/webp,*/*;q=0.8"}
        html = session.get(self.current_page, headers=headers)
        bsObj = BeautifulSoup(html.text, "html.parser")

        links = self.get_external_links(bsObj,
                                        urlparse(self.current_page).netloc)
        if links is not None:
            for link in links:
                if link not in self.pages:
                    self.externalLinks.add(link)

        links = self.get_internal_links(bsObj, self.current_page)
        if links is not None:
            for link in links:
                if link not in self.pages:
                    self.internalLinks.add(link)

    def next_page(self):
        if len(self.externalLinks) != 0:
            self.current_page = self.externalLinks.pop()
            self.pages.add(self.current_page)
        elif len(self.internalLinks) != 0:
            self.current_page = self.internalLinks.pop()
            self.pages.add(self.current_page)
        return self.current_page

    def run(self):
        self.externalLinks.add(self.current_page)
        self.internalLinks.add(self.current_page)
        while (self.next_page() is not None):
            try:
                print(self.current_page)
                self.get_links()
            except HTTPError:
                pass
            except URLError:
                pass
            except Exception:
                pass
# connect TOR
socks.set_default_proxy(socks.SOCKS5, "localhost", 9150)
socket.socket = socks.socksocket

def getaddrinfo(*args):
    return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
socket.getaddrinfo = getaddrinfo

# TEST CODES
# session = requests.Session()
# headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; rv:38.0)"
#           "Gecko/20100101 Firefox/38.0",
#           "Accept": "text/html,application/xhtml+xml,application/xml;"
#           "q=0.9,image/webp,*/*;q=0.8"}

# html = session.get("http://skunksworkedp2cg.onion:80", headers=headers)
# print(html.text)
# bsObj = BeautifulSoup(html.text, "html.parser")

# test crawler
#crawler = Crawler("http://kcg.edu")
crawler = Crawler("http://skunksworkedp2cg.onion:80")
crawler.run()

<commit_msg>update: db management<commit_after>from urllib.error import HTTPError
from urllib.error import URLError
from urllib.parse import urlparse
from urllib.parse import quote_plus as q
from bs4 import BeautifulSoup
import re
import datetime
import random
import socks
import socket
import requests
import sqlite3

class Crawler:
    def __init__(self, page):
        self.internalLinks = set()
        self.externalLinks = set()
        self.pages = set()
        self.bsObj = None
        self.current_page = page
        self.conn = sqlite3.connect('web.db')
        random.seed(datetime.datetime.now())

    def get_internal_links(self, includeUrl):
        scm = urlparse(includeUrl).scheme
        loc = urlparse(includeUrl).netloc
        includeUrl = scm + "://" + loc

        reurl = re.compile("^(\/|.*" + includeUrl + ")")

        for link in self.bsObj.findAll("a", href=reurl):
            url = q(link.attrs['href'], safe="/:")
            if url is not None:
                if url.startswith("/"):
                    the_link = includeUrl + url
                else:
                    the_link = url
                if link not in self.pages:
                    self.internalLinks.add(the_link)

    def get_external_links(self, excludeUrl):
        reurl = re.compile("^(http|www)((?!" + excludeUrl + ").)*$")

        for link in self.bsObj.findAll("a", href=reurl):
            url = q(link.attrs['href'], safe="/:")
            if url is not None:
                if url not in self.pages:
                    self.externalLinks.add(url)

    def get_links(self):
        # html = urlopen(self.current_page).read()
        session = requests.Session()
        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; rv:38.0)"
                                 "Gecko/20100101 Firefox/38.0",
                   "Accept": "text/html,application/xhtml+xml,application/xml;"
                             "q=0.9,image/webp,*/*;q=0.8"}
        html = session.get(self.current_page, headers=headers)
        self.bsObj = BeautifulSoup(html.text, "html.parser")

        links = self.get_external_links(urlparse(self.current_page).netloc)
        if links is not None:
            for link in links:
                if link not in self.pages:
                    self.externalLinks.add(link)

        links = self.get_internal_links(self.current_page)
        if links is not None:
            for link in links:
                if link not in self.pages:
                    self.internalLinks.add(link)

    def next_page(self):
        if len(self.externalLinks) != 0:
            self.current_page = self.externalLinks.pop()
            self.pages.add(self.current_page)
        elif len(self.internalLinks) != 0:
            self.current_page = self.internalLinks.pop()
            self.pages.add(self.current_page)
        return self.current_page

    def run(self):
        self.externalLinks.add(self.current_page)
        self.internalLinks.add(self.current_page)
        while (self.next_page() is not None):
            try:
                print(self.current_page)
                self.get_links()
                c = self.conn.cursor()
                try:
                    data = (self.current_page,
                            self.bsObj.head.title.get_text(),
                            str(self.bsObj.body))
                    c.execute(
                        'INSERT INTO web(url, title, body) VALUES (?,?,?)',
                        data)
                    self.conn.commit()
                except Exception as e:
                    c.close()
                    self.conn = sqlite3.connect('web.db')
                    print(e)
            except HTTPError:
                pass
            except URLError:
                pass
            except Exception:
                pass
# connect TOR
socks.set_default_proxy(socks.SOCKS5, "localhost", 9150)
socket.socket = socks.socksocket

def getaddrinfo(*args):
    return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]

socket.getaddrinfo = getaddrinfo

# TEST CODES
# session = requests.Session()
# headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; rv:38.0)"
#           "Gecko/20100101 Firefox/38.0",
#           "Accept": "text/html,application/xhtml+xml,application/xml;"
#           "q=0.9,image/webp,*/*;q=0.8"}

# html = session.get("http://skunksworkedp2cg.onion:80", headers=headers)
# print(html.text)
# bsObj = BeautifulSoup(html.text, "html.parser")

# test crawler
#crawler = Crawler("http://kcg.edu")
crawler = Crawler("http://skunksworkedp2cg.onion:80")
crawler.run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function

from owmeta_core.dataobject import DatatypeProperty, ObjectProperty, Alias

from .muscle import Muscle
from .cell import Cell
from .biology import BiologyType
from .network import Network
from .worm_common import WORM_RDF_TYPE

class Worm(BiologyType):
    """ A representation of the whole worm """

    class_context = BiologyType.class_context

    rdf_type = WORM_RDF_TYPE

    scientific_name = DatatypeProperty()
    ''' Scientific name for the organism '''

    name = Alias(scientific_name)
    ''' Alias to `scientific_name` '''

    muscle = ObjectProperty(value_type=Muscle, multiple=True)
    ''' A type of muscle which is in the worm '''

    cell = ObjectProperty(value_type=Cell)
    ''' A type of cell in the worm '''

    neuron_network = ObjectProperty(value_type=Network, inverse_of=(Network, 'worm'))
    ''' The neuron network of the worm '''

    def __init__(self, scientific_name=False, **kwargs):
        super(Worm, self).__init__(**kwargs)

        if scientific_name:
            self.scientific_name(scientific_name)
        else:
            self.scientific_name("C. elegans")

    def get_neuron_network(self):
        """
        Return the neuron network of the worm.

        Example::

            # Grabs the representation of the neuronal network
            >>> net = P.Worm().get_neuron_network()

            # Grab a specific neuron
            >>> aval = net.aneuron('AVAL')

            >>> aval.type()
            set([u'interneuron'])

            #show how many connections go out of AVAL
            >>> aval.connection.count('pre')
            77

        :returns: An object to work with the network of the worm
        :rtype: owmeta.Network
        """
        return self.neuron_network()

    def muscles(self):
        """
        Get all Muscle objects attached to the Worm.

        Example::

            >>> muscles = P.Worm().muscles()
            >>> len(muscles)
            96

        :returns: A set of all muscles
        :rtype: :py:class:`set`
         """
        return set(x for x in self._muscles_helper())

    def _muscles_helper(self):
        for x in self.muscle.get():
            yield x

    def get_semantic_net(self):
        """
         Get the underlying semantic network as an RDFLib Graph

        :returns: A semantic network containing information about the worm
        :rtype: rdflib.ConjunctiveGraph
         """

        return self.rdf

    def defined_augment(self):
        ''' True if the name is defined '''
        return self.name.has_defined_value()

    def identifier_augment(self, *args, **kwargs):
        ''' Result is derived from the name property '''
        return self.make_identifier(self.name.defined_values[0])
<commit_msg>Using more conventional default for `scientific_name` `Worm` param<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function

from owmeta_core.dataobject import DatatypeProperty, ObjectProperty, Alias

from .muscle import Muscle
from .cell import Cell
from .biology import BiologyType
from .network import Network
from .worm_common import WORM_RDF_TYPE

class Worm(BiologyType):
    """ A representation of the whole worm """

    class_context = BiologyType.class_context

    rdf_type = WORM_RDF_TYPE

    scientific_name = DatatypeProperty()
    ''' Scientific name for the organism '''

    name = Alias(scientific_name)
    ''' Alias to `scientific_name` '''

    muscle = ObjectProperty(value_type=Muscle, multiple=True)
    ''' A type of muscle which is in the worm '''

    cell = ObjectProperty(value_type=Cell)
    ''' A type of cell in the worm '''

    neuron_network = ObjectProperty(value_type=Network, inverse_of=(Network, 'worm'))
    ''' The neuron network of the worm '''

    def __init__(self, scientific_name=None, **kwargs):
        super(Worm, self).__init__(**kwargs)

        if scientific_name:
            self.scientific_name(scientific_name)
        else:
            self.scientific_name("C. elegans")

    def get_neuron_network(self):
        """
        Return the neuron network of the worm.

        Example::

            # Grabs the representation of the neuronal network
            >>> net = P.Worm().get_neuron_network()

            # Grab a specific neuron
            >>> aval = net.aneuron('AVAL')

            >>> aval.type()
            set([u'interneuron'])

            #show how many connections go out of AVAL
            >>> aval.connection.count('pre')
            77

        :returns: An object to work with the network of the worm
        :rtype: owmeta.Network
        """
        return self.neuron_network()

    def muscles(self):
        """
        Get all Muscle objects attached to the Worm.

        Example::

            >>> muscles = P.Worm().muscles()
            >>> len(muscles)
            96

        :returns: A set of all muscles
        :rtype: :py:class:`set`
         """
        return set(x for x in self._muscles_helper())

    def _muscles_helper(self):
        for x in self.muscle.get():
            yield x

    def get_semantic_net(self):
        """
         Get the underlying semantic network as an RDFLib Graph

        :returns: A semantic network containing information about the worm
        :rtype: rdflib.ConjunctiveGraph
         """

        return self.rdf

    def defined_augment(self):
        ''' True if the name is defined '''
        return self.name.has_defined_value()

    def identifier_augment(self, *args, **kwargs):
        ''' Result is derived from the name property '''
        return self.make_identifier(self.name.defined_values[0])
<|endoftext|>