<commit_before>import cookielib
import urllib
import urllib2
import os
import pickle
from bs4 import BeautifulSoup
from flask import Flask, session, render_template

app = Flask(__name__)

def getCases(html, name, names):
    for row in html.find(class_="nameList").find_all('tr'):
        cols = row.find_all('td')
        if len(cols) > 4:
            if name not in cols[1].string:
                return True
            names.append({
                'caseNumber': cols[0].span.a.string.strip(),
                'name': cols[1].string.strip(),
                'charge': cols[2].string.strip(),
                'date': cols[3].string.strip(),
                'status': cols[4].string.strip()
            })
        elif len(cols) > 3:
            if name not in cols[1].get_text() and name not in \
                    cols[2].get_text():
                return True
            names.append({
                'caseNumber': cols[0].span.a.string.strip(),
                'name': cols[1].get_text(),
                'otherName': cols[2].get_text(),
                'status': cols[3].string.strip()
            })
    return False

def lookupCases(opener, name, court, division):
    cases = []

    data = urllib.urlencode({
        'category': division,
        'lastName': name,
        'courtId': court,
        'submitValue': 'N'})
    cases_url = u"http://ewsocis1.courts.state.va.us/CJISWeb/Search.do"
    searchResults = opener.open(cases_url, data)
    html = searchResults.read()
    done = getCases(BeautifulSoup(html), name, cases)

    data = urllib.urlencode({
        'courtId': court,
        'pagelink': 'Next',
        'lastCaseProcessed': '',
        'firstCaseProcessed': '',
        'lastNameProcessed': '',
        'firstNameProcessed': '',
        'category': division,
        'firstCaseSerialNumber': 0,
        'lastCaseSerialNumber': 0,
        'searchType': '',
        'emptyList': ''})

    count = 1
    search_url = u"http://ewsocis1.courts.state.va.us/CJISWeb/Search.do"
    while(not done and count < 6):
        search_results = opener.open(search_url, data)
        html = search_results.read()
        done = getCases(BeautifulSoup(html), name, cases)
        count += 1
    return cases

def getCasesInVirginiaBeach(html, name, names):
    resultsTable = html.find(class_="tablesorter")
    if resultsTable is None:
        return True

    for row in resultsTable.find('tbody').find_all('tr'):
        cols = row.find_all('td')
        if len(cols) > 5:
            names.append({
                'caseNumber': cols[0].a.string or '',
                'link': 'https://vbcircuitcourt.com' + cols[0].a['href'],
                'otherName': cols[1].string or '',
                'caseStyle': ''.join(cols[2].findAll(text=True))
                .replace('\r\n', ' ') or '',
                'name': ''.join(cols[3].findAll(text=True))
                .replace('\r\n', ' ') or '',
                'partyType': cols[4].string.capitalize() + ':',
                'status': cols[5].string or ''
            })
    return False

def lookupCasesInVirginiaBeach(name, division):
    cases = []

    url = 'https://vbcircuitcourt.com/public/search.do?searchType=1&indexName=publiccasesearch&q=' + name.replace(' ', '+')
    url = url + '%20FilterByCourtType:"' + division + '"'

    searchResults = urllib2.urlopen(url)
    html = searchResults.read()
    done = getCasesInVirginiaBeach(BeautifulSoup(html), name, cases)

    count = 1
    while(not done and count < 6):
        searchResults = urllib2.urlopen(url + '&start=' + str(count * 30))
        html = searchResults.read()
        done = getCasesInVirginiaBeach(BeautifulSoup(html), name, cases)
        count += 1
    return cases

@app.route("/search/<name>/court/<path:court>")
def searchCourt(name, court):
    if 'cookies' not in session:
        return "Error. Please reload the page."

    cookieJar = cookielib.CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJar))
    opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.11) Gecko/20101012 Firefox/3.6.11')]
    for cookie in pickle.loads(session['cookies']):
        cookieJar.set_cookie(cookie)

    courtId = court[:3]
    data = urllib.urlencode({
        'courtId': courtId,
        'courtType': 'C',
        'caseType': 'ALL',
        'testdos': False,
        'sessionCreate': 'NEW',
        'whichsystem': court})
    place = opener.open('http://ewsocis1.courts.state.va.us/CJISWeb/MainMenu.do', data)

    courtSearch = {'name': court[5:], 'id': courtId}
    courtSearch['criminalCases'] = lookupCases(opener, name.upper(),
                                               courtId, 'R')
    courtSearch['civilCases'] = lookupCases(opener, name.upper(),
                                            courtId, 'CIVIL')

    if 'Virginia Beach' in court:
        courtSearch['criminalCases'].extend(lookupCasesInVirginiaBeach(name, 'CRIMINAL'))
        courtSearch['civilCases'].extend(lookupCasesInVirginiaBeach(name, 'CIVIL'))

    return render_template('court.html', court=courtSearch)

@app.route("/search/<name>")
def search(name):
    cookieJar = cookielib.CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJar))
    opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.11) Gecko/20101012 Firefox/3.6.11')]
    home = opener.open('http://ewsocis1.courts.state.va.us/CJISWeb/circuit.jsp')

    session['cookies'] = pickle.dumps(list(cookieJar))

    courts = []
    html = BeautifulSoup(home.read())
    for option in html.find_all('option'):
        courts.append({
            'fullName': option['value'],
            'id': option['value'][:3],
            'name': option['value'][5:]
        })

    return render_template('search.html', data={'name': name.upper(), 'courts': courts})

@app.route("/")
def index():
    return render_template('index.html')

if __name__ == "__main__":
    # Bind to PORT if defined, otherwise default to 5000.
    port = int(os.environ.get('PORT', 5000))
    app.secret_key = 'doesnt-need-to-be-secret'
    app.run(host='0.0.0.0', port=port, debug=True)
<commit_msg>pep8 - line too long > 80 characters<commit_after>import cookielib
import urllib
import urllib2
import os
import pickle
from bs4 import BeautifulSoup
from flask import Flask, session, render_template

app = Flask(__name__)

def getCases(html, name, names):
    for row in html.find(class_="nameList").find_all('tr'):
        cols = row.find_all('td')
        if len(cols) > 4:
            if name not in cols[1].string:
                return True
            names.append({
                'caseNumber': cols[0].span.a.string.strip(),
                'name': cols[1].string.strip(),
                'charge': cols[2].string.strip(),
                'date': cols[3].string.strip(),
                'status': cols[4].string.strip()
            })
        elif len(cols) > 3:
            if name not in cols[1].get_text() and name not in \
                    cols[2].get_text():
                return True
            names.append({
                'caseNumber': cols[0].span.a.string.strip(),
                'name': cols[1].get_text(),
                'otherName': cols[2].get_text(),
                'status': cols[3].string.strip()
            })
    return False

def lookupCases(opener, name, court, division):
    cases = []

    data = urllib.urlencode({
        'category': division,
        'lastName': name,
        'courtId': court,
        'submitValue': 'N'})
    cases_url = u"http://ewsocis1.courts.state.va.us/CJISWeb/Search.do"
    searchResults = opener.open(cases_url, data)
    html = searchResults.read()
    done = getCases(BeautifulSoup(html), name, cases)

    data = urllib.urlencode({
        'courtId': court,
        'pagelink': 'Next',
        'lastCaseProcessed': '',
        'firstCaseProcessed': '',
        'lastNameProcessed': '',
        'firstNameProcessed': '',
        'category': division,
        'firstCaseSerialNumber': 0,
        'lastCaseSerialNumber': 0,
        'searchType': '',
        'emptyList': ''})

    count = 1
    search_url = u"http://ewsocis1.courts.state.va.us/CJISWeb/Search.do"
    while(not done and count < 6):
        search_results = opener.open(search_url, data)
        html = search_results.read()
        done = getCases(BeautifulSoup(html), name, cases)
        count += 1
    return cases

def getCasesInVirginiaBeach(html, name, names):
    resultsTable = html.find(class_="tablesorter")
    if resultsTable is None:
        return True

    for row in resultsTable.find('tbody').find_all('tr'):
        cols = row.find_all('td')
        if len(cols) > 5:
            names.append({
                'caseNumber': cols[0].a.string or '',
                'link': 'https://vbcircuitcourt.com' + cols[0].a['href'],
                'otherName': cols[1].string or '',
                'caseStyle': ''.join(cols[2].findAll(text=True))
                .replace('\r\n', ' ') or '',
                'name': ''.join(cols[3].findAll(text=True))
                .replace('\r\n', ' ') or '',
                'partyType': cols[4].string.capitalize() + ':',
                'status': cols[5].string or ''
            })
    return False

def lookupCasesInVirginiaBeach(name, division):
    cases = []

    url = 'https://vbcircuitcourt.com/public/search.do?searchType=1&indexName=publiccasesearch&q=' + name.replace(' ', '+')
    url = url + '%20FilterByCourtType:"' + division + '"'

    searchResults = urllib2.urlopen(url)
    html = searchResults.read()
    done = getCasesInVirginiaBeach(BeautifulSoup(html), name, cases)

    count = 1
    while(not done and count < 6):
        searchResults = urllib2.urlopen(url + '&start=' + str(count * 30))
        html = searchResults.read()
        done = getCasesInVirginiaBeach(BeautifulSoup(html), name, cases)
        count += 1
    return cases

@app.route("/search/<name>/court/<path:court>")
def searchCourt(name, court):
    if 'cookies' not in session:
        return "Error. Please reload the page."

    cookieJar = cookielib.CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJar))
    opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.11) Gecko/20101012 Firefox/3.6.11')]
    for cookie in pickle.loads(session['cookies']):
        cookieJar.set_cookie(cookie)

    courtId = court[:3]
    data = urllib.urlencode({
        'courtId': courtId,
        'courtType': 'C',
        'caseType': 'ALL',
        'testdos': False,
        'sessionCreate': 'NEW',
        'whichsystem': court})
    place = opener.open('http://ewsocis1.courts.state.va.us/CJISWeb/MainMenu.do', data)

    courtSearch = {'name': court[5:], 'id': courtId}
    courtSearch['criminalCases'] = lookupCases(opener, name.upper(),
                                               courtId, 'R')
    courtSearch['civilCases'] = lookupCases(opener, name.upper(),
                                            courtId, 'CIVIL')

    if 'Virginia Beach' in court:
        courtSearch['criminalCases'].extend(lookupCasesInVirginiaBeach(name, 'CRIMINAL'))
        courtSearch['civilCases'].extend(lookupCasesInVirginiaBeach(name, 'CIVIL'))

    return render_template('court.html', court=courtSearch)

@app.route("/search/<name>")
def search(name):
    cookieJar = cookielib.CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJar))
    opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.11) Gecko/20101012 Firefox/3.6.11')]
    home = opener.open('http://ewsocis1.courts.state.va.us/CJISWeb/circuit.jsp')

    session['cookies'] = pickle.dumps(list(cookieJar))

    courts = []
    html = BeautifulSoup(home.read())
    for option in html.find_all('option'):
        courts.append({
            'fullName': option['value'],
            'id': option['value'][:3],
            'name': option['value'][5:]
        })
    data = {'name': name.upper(), 'courts': courts}
    return render_template('search.html', data=data)

@app.route("/")
def index():
    return render_template('index.html')

if __name__ == "__main__":
    # Bind to PORT if defined, otherwise default to 5000.
    port = int(os.environ.get('PORT', 5000))
    app.secret_key = 'doesnt-need-to-be-secret'
    app.run(host='0.0.0.0', port=port, debug=True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Virtual machine image management tools
'''

from __future__ import absolute_import

# Import python libs
import os
import shutil
import logging
import tempfile

# Import salt libs
import salt.crypt
import salt.utils
import salt.utils.cloud
import salt.config
import salt.syspaths
import uuid

# Set up logging
log = logging.getLogger(__name__)

# Don't shadow built-in's.
__func_alias__ = {
    'apply_': 'apply'
}

def _file_or_content(file_):
    if os.path.exists(file_):
        with salt.utils.fopen(file_) as fic:
            return fic.read()
    return file_

def prep_bootstrap(mpt):
    '''
    Update and get the random script to a random place

    CLI Example:

    .. code-block:: bash

        salt '*' seed.prep_bootstrap /tmp

    '''
    # Verify that the boostrap script is downloaded
    bs_ = __salt__['config.gather_bootstrap_script']()
    fpd_ = os.path.join(mpt, 'tmp', "{0}".format(
        uuid.uuid4()))
    if not os.path.exists(fpd_):
        os.makedirs(fpd_)
    os.chmod(fpd_, 0o700)
    fp_ = os.path.join(fpd_, os.path.basename(bs_))
    # Copy script into tmp
    shutil.copy(bs_, fp_)
    return fp_

def _mount(path, ftype):
    mpt = None
    if ftype == 'block':
        mpt = tempfile.mkdtemp()
        if not __salt__['mount.mount'](mpt, path):
            os.rmdir(mpt)
            return None
    elif ftype == 'dir':
        return path
    elif ftype == 'file':
        mpt = __salt__['img.mount_image'](path)
        if not mpt:
            return None
    return mpt

def _umount(mpt, ftype):
    if ftype == 'block':
        __salt__['mount.umount'](mpt)
        os.rmdir(mpt)
    elif ftype == 'file':
        __salt__['img.umount_image'](mpt)

def apply_(path, id_=None, config=None, approve_key=True, install=True,
           prep_install=False):
    '''
    Seed a location (disk image, directory, or block device) with the
    minion config, approve the minion's key, and/or install salt-minion.

    CLI Example:

    .. code-block:: bash

        salt 'minion' seed.apply path id [config=config_data] \\
                [gen_key=(true|false)] [approve_key=(true|false)] \\
                [install=(true|false)]

    path
        Full path to the directory, device, or disk image  on the target
        minion's file system.

    id
        Minion id with which to seed the path.

    config
        Minion configuration options. By default, the 'master' option is set to
        the target host's 'master'.

    approve_key
        Request a pre-approval of the generated minion key. Requires
        that the salt-master be configured to either auto-accept all keys or
        expect a signing request from the target host. Default: true.

    install
        Install salt-minion, if absent. Default: true.

    prep_install
        Prepare the bootstrap script, but don't run it. Default: false
    '''
    stats = __salt__['file.stats'](path, follow_symlinks=True)
    if not stats:
        return '{0} does not exist'.format(path)
    ftype = stats['type']
    path = stats['target']
    mpt = _mount(path, ftype)

    if not mpt:
        return '{0} could not be mounted'.format(path)

    tmp = os.path.join(mpt, 'tmp')
    try:
        os.makedirs(tmp)
    except OSError:
        if not os.path.isdir(tmp):
            raise
    cfg_files = mkconfig(config, tmp=tmp, id_=id_, approve_key=approve_key)

    if _check_install(mpt):
        # salt-minion is already installed, just move the config and keys
        # into place
        log.info('salt-minion pre-installed on image, '
                 'configuring as {0}'.format(id_))
        minion_config = salt.config.minion_config(cfg_files['config'])
        pki_dir = minion_config['pki_dir']
        if not os.path.isdir():
            __salt__['file.makedirs'](
                os.path.join(mpt, pki_dir)
            )
        os.rename(cfg_files['privkey'], os.path.join(
            mpt, pki_dir.lstrip('/'), 'minion.pem'))
        os.rename(cfg_files['pubkey'], os.path.join(
            mpt, pki_dir.lstrip('/'), 'minion.pub'))
        os.rename(cfg_files['config'], os.path.join(mpt, 'etc/salt/minion'))
        res = True
    elif install:
        log.info('attempting to install salt-minion to '
                 '{0}'.format(mpt))
        res = _install(mpt)
    elif prep_install:
        log.error('The prep_install option is no longer supported. Please use '
                  'the bootstrap script installed with Salt, located at {0}.'
                  .format(salt.syspaths.BOOTSTRAP))
        res = False
    else:
        log.warning('No useful action performed on {0}'.format(mpt))
        res = False

    _umount(mpt, ftype)
    return res

def mkconfig(config=None,
             tmp=None,
             id_=None,
             approve_key=True,
             pub_key=None,
             priv_key=None):
    '''
    Generate keys and config and put them in a tmp directory.

    pub_key
        absolute path or file content of an optional preseeded salt key

    priv_key
        absolute path or file content of an optional preseeded salt key

    CLI Example:

    .. code-block:: bash

        salt 'minion' seed.mkconfig [config=config_data] [tmp=tmp_dir] \\
                [id_=minion_id] [approve_key=(true|false)]
    '''
    if tmp is None:
        tmp = tempfile.mkdtemp()
    if config is None:
        config = {}
    if 'master' not in config and __opts__['master'] != 'salt':
        config['master'] = __opts__['master']
    if id_:
        config['id'] = id_

    # Write the new minion's config to a tmp file
    tmp_config = os.path.join(tmp, 'minion')
    with salt.utils.fopen(tmp_config, 'w+') as fp_:
        fp_.write(salt.utils.cloud.salt_config_to_yaml(config))

    # Generate keys for the minion
    pubkeyfn = os.path.join(tmp, 'minion.pub')
    privkeyfn = os.path.join(tmp, 'minion.pem')
    preseeded = pub_key and priv_key
    if preseeded:
        with salt.utils.fopen(pubkeyfn, 'w') as fic:
            fic.write(_file_or_content(pub_key))
        with salt.utils.fopen(privkeyfn, 'w') as fic:
            fic.write(_file_or_content(priv_key))
        os.chmod(pubkeyfn, 0o600)
        os.chmod(privkeyfn, 0o600)
    else:
        salt.crypt.gen_keys(tmp, 'minion', 2048)
    if approve_key and not preseeded:
        with salt.utils.fopen(pubkeyfn) as fp_:
            pubkey = fp_.read()
            __salt__['pillar.ext']({'virtkey': [id_, pubkey]})

    return {'config': tmp_config, 'pubkey': pubkeyfn, 'privkey': privkeyfn}

def _install(mpt):
    '''
    Determine whether salt-minion is installed and, if not,
    install it.
    Return True if install is successful or already installed.
    '''

    _check_resolv(mpt)
    boot_ = (prep_bootstrap(mpt)
             or salt.syspaths.BOOTSTRAP)
    # Exec the chroot command
    cmd = 'if type salt-minion; then exit 0; '
    cmd += 'else sh {0} -c /tmp; fi'.format(salt.syspaths.BOOTSTRAP)
    return not __salt__['cmd.run_chroot'](mpt, cmd, python_shell=True)['retcode']

def _check_resolv(mpt):
    '''
    Check that the resolv.conf is present and populated
    '''
    resolv = os.path.join(mpt, 'etc/resolv.conf')
    replace = False
    if os.path.islink(resolv):
        resolv = os.path.realpath(resolv)
        if not os.path.isdir(os.path.dirname(resolv)):
            os.makedirs(os.path.dirname(resolv))
    if not os.path.isfile(resolv):
        replace = True
    if not replace:
        with salt.utils.fopen(resolv, 'rb') as fp_:
            conts = fp_.read()
            if 'nameserver' not in conts:
                replace = True
    if replace:
        shutil.copy('/etc/resolv.conf', resolv)

def _check_install(root):
    sh_ = '/bin/sh'
    if os.path.isfile(os.path.join(root, 'bin/bash')):
        sh_ = '/bin/bash'

    cmd = ('if ! type salt-minion; then exit 1; fi')
    cmd = 'chroot \'{0}\' {1} -c {2!r}'.format(
        root,
        sh_,
        cmd)

    return not __salt__['cmd.retcode'](cmd,
                                       output_loglevel='quiet',
                                       python_shell=True)
<commit_msg>Add argument to isdir()<commit_after># -*- coding: utf-8 -*-
'''
Virtual machine image management tools
'''

from __future__ import absolute_import

# Import python libs
import os
import shutil
import logging
import tempfile

# Import salt libs
import salt.crypt
import salt.utils
import salt.utils.cloud
import salt.config
import salt.syspaths
import uuid

# Set up logging
log = logging.getLogger(__name__)

# Don't shadow built-in's.
__func_alias__ = {
    'apply_': 'apply'
}

def _file_or_content(file_):
    if os.path.exists(file_):
        with salt.utils.fopen(file_) as fic:
            return fic.read()
    return file_

def prep_bootstrap(mpt):
    '''
    Update and get the random script to a random place

    CLI Example:

    .. code-block:: bash

        salt '*' seed.prep_bootstrap /tmp

    '''
    # Verify that the boostrap script is downloaded
    bs_ = __salt__['config.gather_bootstrap_script']()
    fpd_ = os.path.join(mpt, 'tmp', "{0}".format(
        uuid.uuid4()))
    if not os.path.exists(fpd_):
        os.makedirs(fpd_)
    os.chmod(fpd_, 0o700)
    fp_ = os.path.join(fpd_, os.path.basename(bs_))
    # Copy script into tmp
    shutil.copy(bs_, fp_)
    return fp_

def _mount(path, ftype):
    mpt = None
    if ftype == 'block':
        mpt = tempfile.mkdtemp()
        if not __salt__['mount.mount'](mpt, path):
            os.rmdir(mpt)
            return None
    elif ftype == 'dir':
        return path
    elif ftype == 'file':
        mpt = __salt__['img.mount_image'](path)
        if not mpt:
            return None
    return mpt

def _umount(mpt, ftype):
    if ftype == 'block':
        __salt__['mount.umount'](mpt)
        os.rmdir(mpt)
    elif ftype == 'file':
        __salt__['img.umount_image'](mpt)

def apply_(path, id_=None, config=None, approve_key=True, install=True,
           prep_install=False):
    '''
    Seed a location (disk image, directory, or block device) with the
    minion config, approve the minion's key, and/or install salt-minion.

    CLI Example:

    .. code-block:: bash

        salt 'minion' seed.apply path id [config=config_data] \\
                [gen_key=(true|false)] [approve_key=(true|false)] \\
                [install=(true|false)]

    path
        Full path to the directory, device, or disk image  on the target
        minion's file system.

    id
        Minion id with which to seed the path.

    config
        Minion configuration options. By default, the 'master' option is set to
        the target host's 'master'.

    approve_key
        Request a pre-approval of the generated minion key. Requires
        that the salt-master be configured to either auto-accept all keys or
        expect a signing request from the target host. Default: true.

    install
        Install salt-minion, if absent. Default: true.

    prep_install
        Prepare the bootstrap script, but don't run it. Default: false
    '''
    stats = __salt__['file.stats'](path, follow_symlinks=True)
    if not stats:
        return '{0} does not exist'.format(path)
    ftype = stats['type']
    path = stats['target']
    mpt = _mount(path, ftype)

    if not mpt:
        return '{0} could not be mounted'.format(path)

    tmp = os.path.join(mpt, 'tmp')
    try:
        os.makedirs(tmp)
    except OSError:
        if not os.path.isdir(tmp):
            raise
    cfg_files = mkconfig(config, tmp=tmp, id_=id_, approve_key=approve_key)

    if _check_install(mpt):
        # salt-minion is already installed, just move the config and keys
        # into place
        log.info('salt-minion pre-installed on image, '
                 'configuring as {0}'.format(id_))
        minion_config = salt.config.minion_config(cfg_files['config'])
        pki_dir = minion_config['pki_dir']
        if not os.path.isdir(os.path.join(mpt, pki_dir)):
            __salt__['file.makedirs'](
                os.path.join(mpt, pki_dir)
            )
        os.rename(cfg_files['privkey'], os.path.join(
            mpt, pki_dir.lstrip('/'), 'minion.pem'))
        os.rename(cfg_files['pubkey'], os.path.join(
            mpt, pki_dir.lstrip('/'), 'minion.pub'))
        os.rename(cfg_files['config'], os.path.join(mpt, 'etc/salt/minion'))
        res = True
    elif install:
        log.info('attempting to install salt-minion to '
                 '{0}'.format(mpt))
        res = _install(mpt)
    elif prep_install:
        log.error('The prep_install option is no longer supported. Please use '
                  'the bootstrap script installed with Salt, located at {0}.'
                  .format(salt.syspaths.BOOTSTRAP))
        res = False
    else:
        log.warning('No useful action performed on {0}'.format(mpt))
        res = False

    _umount(mpt, ftype)
    return res

def mkconfig(config=None,
             tmp=None,
             id_=None,
             approve_key=True,
             pub_key=None,
             priv_key=None):
    '''
    Generate keys and config and put them in a tmp directory.

    pub_key
        absolute path or file content of an optional preseeded salt key

    priv_key
        absolute path or file content of an optional preseeded salt key

    CLI Example:

    .. code-block:: bash

        salt 'minion' seed.mkconfig [config=config_data] [tmp=tmp_dir] \\
                [id_=minion_id] [approve_key=(true|false)]
    '''
    if tmp is None:
        tmp = tempfile.mkdtemp()
    if config is None:
        config = {}
    if 'master' not in config and __opts__['master'] != 'salt':
        config['master'] = __opts__['master']
    if id_:
        config['id'] = id_

    # Write the new minion's config to a tmp file
    tmp_config = os.path.join(tmp, 'minion')
    with salt.utils.fopen(tmp_config, 'w+') as fp_:
        fp_.write(salt.utils.cloud.salt_config_to_yaml(config))

    # Generate keys for the minion
    pubkeyfn = os.path.join(tmp, 'minion.pub')
    privkeyfn = os.path.join(tmp, 'minion.pem')
    preseeded = pub_key and priv_key
    if preseeded:
        with salt.utils.fopen(pubkeyfn, 'w') as fic:
            fic.write(_file_or_content(pub_key))
        with salt.utils.fopen(privkeyfn, 'w') as fic:
            fic.write(_file_or_content(priv_key))
        os.chmod(pubkeyfn, 0o600)
        os.chmod(privkeyfn, 0o600)
    else:
        salt.crypt.gen_keys(tmp, 'minion', 2048)
    if approve_key and not preseeded:
        with salt.utils.fopen(pubkeyfn) as fp_:
            pubkey = fp_.read()
            __salt__['pillar.ext']({'virtkey': [id_, pubkey]})

    return {'config': tmp_config, 'pubkey': pubkeyfn, 'privkey': privkeyfn}

def _install(mpt):
    '''
    Determine whether salt-minion is installed and, if not,
    install it.
    Return True if install is successful or already installed.
    '''

    _check_resolv(mpt)
    boot_ = (prep_bootstrap(mpt)
             or salt.syspaths.BOOTSTRAP)
    # Exec the chroot command
    cmd = 'if type salt-minion; then exit 0; '
    cmd += 'else sh {0} -c /tmp; fi'.format(salt.syspaths.BOOTSTRAP)
    return not __salt__['cmd.run_chroot'](mpt, cmd, python_shell=True)['retcode']

def _check_resolv(mpt):
    '''
    Check that the resolv.conf is present and populated
    '''
    resolv = os.path.join(mpt, 'etc/resolv.conf')
    replace = False
    if os.path.islink(resolv):
        resolv = os.path.realpath(resolv)
        if not os.path.isdir(os.path.dirname(resolv)):
            os.makedirs(os.path.dirname(resolv))
    if not os.path.isfile(resolv):
        replace = True
    if not replace:
        with salt.utils.fopen(resolv, 'rb') as fp_:
            conts = fp_.read()
            if 'nameserver' not in conts:
                replace = True
    if replace:
        shutil.copy('/etc/resolv.conf', resolv)

def _check_install(root):
    sh_ = '/bin/sh'
    if os.path.isfile(os.path.join(root, 'bin/bash')):
        sh_ = '/bin/bash'

    cmd = ('if ! type salt-minion; then exit 1; fi')
    cmd = 'chroot \'{0}\' {1} -c {2!r}'.format(
        root,
        sh_,
        cmd)

    return not __salt__['cmd.retcode'](cmd,
                                       output_loglevel='quiet',
                                       python_shell=True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import itertools

import scipy

from base import Minimizer, repeat_or_iter
from lbfgs import Lbfgs
from bfgs import Bfgs
from sbfgs import SBfgs
from util import optimize_some, optimize_while

class KrylovSubspaceDescent(Minimizer):
    """Minimize a function using Krylov subspace descent as described in

      Vinyals, Povey (2011)
      http://opt.kyb.tuebingen.mpg.de/papers/opt2011_vinyals.pdf
    """

    def __init__(
        self, wrt, f, fprime, f_Hp, n_bases,
        args, hessian_args, krylov_args,
        floor_fisher=False, precond_hessian=False, floor_hessian=False,
        stop=1, verbose=False):

        super(KrylovSubspaceDescent, self).__init__(
            wrt, args=args, stop=stop, verbose=verbose)
        self.f = f
        self.fprime = fprime
        self.f_Hp = f_Hp

        self.n_bases = n_bases
        self.basis = scipy.zeros((n_bases, self.wrt.shape[0]))
        self.coefficients = scipy.zeros(n_bases)

        self.hessian_args = hessian_args
        self.krylov_args = krylov_args
        self.floor_fisher = floor_fisher
        self.precond_hessian = precond_hessian
        self.floor_hessian = floor_hessian
        self.floor_eps = 1E-4

    def _f_krylov(self, x, *args, **kwargs):
        wrt = self.wrt + scipy.dot(x, self.basis)
        return self.f(wrt, *args, **kwargs)

    def _f_krylov_prime(self, x, *args, **kwargs):
        wrt = self.wrt + scipy.dot(x, self.basis)
        df_dwrt = self.fprime(wrt, *args, **kwargs)
        return scipy.dot(self.basis, df_dwrt)

    def _calc_krylov_basis(self, grad, step):
        args, kwargs = self.hessian_args.next()
        n_bases = self.n_bases

        diag_fisher = grad**2

        if self.floor_fisher:
            diag_fisher_max = diag_fisher.max()
            df_floor = diag_fisher_max * self.floor_eps
            diag_fisher = scipy.clip(diag_fisher, df_floor, diag_fisher_max)

        inv_diag_fisher = 1 / diag_fisher

        H = scipy.empty((n_bases, n_bases))
        W = scipy.empty((n_bases, grad.shape[0]))
        V = self.basis

        V[0] = grad / diag_fisher
        V[0] = V[0] / scipy.sqrt(scipy.inner(V[0], V[0]))

        for i in range(0, n_bases):
            w = self.f_Hp(self.wrt, V[i], *args, **kwargs)
            if i < n_bases - 1:
                u = w / diag_fisher
            elif i == n_bases - 1:
                u = step

            for j in range(i + 1):
                H[i, j] = H[j, i] = scipy.inner(w, V[j])
                u -= scipy.inner(u, V[j]) * V[j]
            if i < n_bases - 1:
                V[i + 1] = u / scipy.sqrt(scipy.inner(u, u))

        if self.precond_hessian:
            if self.floor_hessian:
                w, v = scipy.linalg.eigh(H)
                w_max = w.max()
                w_floor = w_max * self.floor_eps
                w = scipy.clip(w, w_floor, w_max)
                H = scipy.dot(v, scipy.dot(scipy.diag(w), v.T))

            C = scipy.linalg.cholesky(H, lower=True)
            Cinv = scipy.linalg.inv(C)
            V[:] = scipy.dot(Cinv, V)

        self.hessian = H

    def __iter__(self):
        step = scipy.ones(self.wrt.shape)
        while True:
            _args, _kwargs = self.args.next()
            self.coefficients *= 0
            grad = self.fprime(self.wrt, *_args, **_kwargs)
            self._calc_krylov_basis(grad, step)

            # Minimize subobjective.
            subargs, subkwargs = self.krylov_args.next()

            subopt = SBfgs(
                self.coefficients, self._f_krylov, self._f_krylov_prime,
                args=itertools.repeat((subargs, subkwargs)))

            def log(info):
                print 'inner loop loss', info['loss']
                print '=' * 20

            info = optimize_while(subopt, 1E-4, log=log)
            loss = info['loss']

            # Take search step.
            step[:] = scipy.dot(self.coefficients, self.basis)
            self.wrt += step
            yield dict(
                loss=loss, step=step, grad=grad, basis=self.basis,
                coefficients=self.coefficients)

            print 'parameter hash', self.wrt.sum(), (self.wrt**2).sum()
            print '-' * 20
            print '=' * 20
            print '-' * 20
<commit_msg>specify hessian as calculated by basis calculation<commit_after># -*- coding: utf-8 -*-

import itertools

import scipy

from base import Minimizer, repeat_or_iter
from lbfgs import Lbfgs
from bfgs import Bfgs
from sbfgs import SBfgs
from util import optimize_some, optimize_while

class KrylovSubspaceDescent(Minimizer):
    """Minimize a function using Krylov subspace descent as described in

      Vinyals, Povey (2011)
      http://opt.kyb.tuebingen.mpg.de/papers/opt2011_vinyals.pdf
    """

    def __init__(
        self, wrt, f, fprime, f_Hp, n_bases,
        args, hessian_args, krylov_args,
        floor_fisher=False, precond_hessian=False, floor_hessian=False,
        stop=1, verbose=False):

        super(KrylovSubspaceDescent, self).__init__(
            wrt, args=args, stop=stop, verbose=verbose)
        self.f = f
        self.fprime = fprime
        self.f_Hp = f_Hp

        self.n_bases = n_bases
        self.basis = scipy.zeros((n_bases, self.wrt.shape[0]))
        self.coefficients = scipy.zeros(n_bases)

        self.hessian_args = hessian_args
        self.krylov_args = krylov_args
        self.floor_fisher = floor_fisher
        self.precond_hessian = precond_hessian
        self.floor_hessian = floor_hessian
        self.floor_eps = 1E-4

    def _f_krylov(self, x, *args, **kwargs):
        wrt = self.wrt + scipy.dot(x, self.basis)
        return self.f(wrt, *args, **kwargs)

    def _f_krylov_prime(self, x, *args, **kwargs):
        wrt = self.wrt + scipy.dot(x, self.basis)
        df_dwrt = self.fprime(wrt, *args, **kwargs)
        return scipy.dot(self.basis, df_dwrt)

    def _calc_krylov_basis(self, grad, step):
        args, kwargs = self.hessian_args.next()
        n_bases = self.n_bases

        diag_fisher = grad**2

        if self.floor_fisher:
            diag_fisher_max = diag_fisher.max()
            df_floor = diag_fisher_max * self.floor_eps
            diag_fisher = scipy.clip(diag_fisher, df_floor, diag_fisher_max)

        inv_diag_fisher = 1 / diag_fisher

        H = scipy.empty((n_bases, n_bases))
        W = scipy.empty((n_bases, grad.shape[0]))
        V = self.basis

        V[0] = grad / diag_fisher
        V[0] = V[0] / scipy.sqrt(scipy.inner(V[0], V[0]))

        for i in range(0, n_bases):
            w = self.f_Hp(self.wrt, V[i], *args, **kwargs)
            if i < n_bases - 1:
                u = w / diag_fisher
            elif i == n_bases - 1:
                u = step

            for j in range(i + 1):
                H[i, j] = H[j, i] = scipy.inner(w, V[j])
                u -= scipy.inner(u, V[j]) * V[j]
            if i < n_bases - 1:
                V[i + 1] = u / scipy.sqrt(scipy.inner(u, u))

        if self.precond_hessian:
            if self.floor_hessian:
                w, v = scipy.linalg.eigh(H)
                w_max = w.max()
                w_floor = w_max * self.floor_eps
                w = scipy.clip(w, w_floor, w_max)
                H = scipy.dot(v, scipy.dot(scipy.diag(w), v.T))

            C = scipy.linalg.cholesky(H, lower=True)
            Cinv = scipy.linalg.inv(C)
            V[:] = scipy.dot(Cinv, V)

        self.hessian = H

    def __iter__(self):
        step = scipy.ones(self.wrt.shape)
        while True:
            _args, _kwargs = self.args.next()
            self.coefficients *= 0
            grad = self.fprime(self.wrt, *_args, **_kwargs)
            self._calc_krylov_basis(grad, step)

            # Minimize subobjective.
            subargs, subkwargs = self.krylov_args.next()

            subopt = SBfgs(
                self.coefficients, self._f_krylov, self._f_krylov_prime,
                initial_inv_hessian=self.hessian,
                args=itertools.repeat((subargs, subkwargs)))

            def log(info):
                print 'inner loop loss', info['loss']
                print '=' * 20

            info = optimize_while(subopt, 1E-4, log=log)
            loss = info['loss']

            # Take search step.
            step[:] = scipy.dot(self.coefficients, self.basis)
            self.wrt += step
            yield dict(
                loss=loss, step=step, grad=grad, basis=self.basis,
                coefficients=self.coefficients)

            print 'parameter hash', self.wrt.sum(), (self.wrt**2).sum()
            print '-' * 20
            print '=' * 20
            print '-' * 20
<|endoftext|>


<commit_before>import os
from gittip import wireup
from gittip.testing import Harness
from gittip.testing.client import TestClient

class TestCanonizer(Harness):

    def setUp(self):
        self.client = TestClient()
        Harness.setUp(self)

        self._blech = ( os.environ['CANONICAL_SCHEME']
                      , os.environ['CANONICAL_HOST']
                       )
        os.environ['CANONICAL_SCHEME'] = 'https'
        os.environ['CANONICAL_HOST'] = 'www.gittip.com'
        wireup.canonical()

    def tearDown(self):
        os.environ['CANONICAL_SCHEME'] = self._blech[0]
        os.environ['CANONICAL_HOST'] = self._blech[1]
        wireup.canonical()

    def test_canonize_canonizes(self):
        response = self.client.get("/", HTTP_HOST='www.gittip.com', HTTP_X_FORWARDED_PROTO='http')
        assert response.code == 302
        assert response.headers['Location'] == 'https://www.gittip.com/'
<commit_msg>Here's a failing test for #940<commit_after>from __future__ import absolute_import, division, print_function, unicode_literals

import os
from gittip import wireup
from gittip.testing import Harness
from gittip.testing.client import TestClient
from gittip.models.participant import Participant

class TestCanonizer(Harness):

    def setUp(self):
        self.client = TestClient()
        Harness.setUp(self)

        self._blech = ( os.environ['CANONICAL_SCHEME']
                      , os.environ['CANONICAL_HOST']
                       )
        os.environ['CANONICAL_SCHEME'] = 'https'
        os.environ['CANONICAL_HOST'] = 'www.gittip.com'
        wireup.canonical()

    def tearDown(self):
        os.environ['CANONICAL_SCHEME'] = self._blech[0]
        os.environ['CANONICAL_HOST'] = self._blech[1]
        wireup.canonical()

    def test_canonize_canonizes(self):
        response = self.client.get("/", HTTP_HOST='www.gittip.com', HTTP_X_FORWARDED_PROTO='http')
        assert response.code == 302
        assert response.headers['Location'] == 'https://www.gittip.com/'

    def test_canonize_doesnt_mess_up_auth(self):
        # https://github.com/gittip/www.gittip.com/issues/940

        self.make_participant('alice')

        # Make a normal authenticated request.
        normal = self.client.get( "/"
                                , user='alice'
                                , HTTP_X_FORWARDED_PROTO='https'
                                , HTTP_HOST='www.gittip.com'
                                 )
        alice = Participant.from_username('alice')
        assert normal.headers.cookie['session'].value == alice.session_token

        # Now make a request that canonizer will redirect.
        redirect = self.client.get( "/"
                                  , user='alice'
                                  , HTTP_X_FORWARDED_PROTO='http'
                                  , HOST='www.gittip.com'
                                   )
        assert 'session' not in redirect.headers.cookie
<|endoftext|>


<commit_before>##############################################################################
#
# Copyright (c) 2004 TINY SPRL. (http://tiny.be) All Rights Reserved.
#                    Fabien Pinckaers <fp@tiny.Be>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

from report.interface import report_int
import pooler
import tools

from report import render

from xml.dom import minidom
import libxml2
import libxslt

import time, os

class report_printscreen_list(report_int):
	def __init__(self, name):
		report_int.__init__(self, name)

	def _parse_node(self, root_node):
		result = []
		for node in root_node.childNodes:
			if node.localName == 'field':
				attrsa = node.attributes
				attrs = {}
				if not attrsa is None:
					for i in range(attrsa.length):
						attrs[attrsa.item(i).localName] = attrsa.item(i).nodeValue
				result.append(attrs['name'])
			else:
				result.extend(self._parse_node(node))
		return result

	def _parse_string(self, view):
		dom = minidom.parseString(view)
		return self._parse_node(dom)

	def create(self, cr, uid, ids, datas, context={}):
		datas['ids'] = ids
		model = pooler.get_pool(cr.dbname).get(datas['model'])
		result = model.fields_view_get(cr, uid, view_type='tree', context=context)

		fields_order = self._parse_string(result['arch'])
		rows = model.read(cr, uid, datas['ids'], result['fields'].keys() )
		res = self._create_table(uid, datas['ids'], result['fields'], fields_order, rows, context, model._description)
		return (self.obj.get(), 'pdf')

	def _create_table(self, uid, ids, fields, fields_order, results, context, title=''):
		pageSize=[297.0,210.0]

		impl = minidom.getDOMImplementation()
		new_doc = impl.createDocument(None, "report", None)

		# build header
		config = new_doc.createElement("config")

		def _append_node(name, text):
			n = new_doc.createElement(name)
			t = new_doc.createTextNode(text)
			n.appendChild(t)
			config.appendChild(n)

		_append_node('date', time.strftime('%d/%m/%Y'))
		_append_node('PageSize', '%.2fmm,%.2fmm' % tuple(pageSize))
		_append_node('PageWidth', '%.2f' % (pageSize[0] * 2.8346,))
		_append_node('PageHeight', '%.2f' %(pageSize[1] * 2.8346,))

		_append_node('report-header', title)

		l = []
		t = 0
		strmax = (pageSize[0]-40) * 2.8346
		for f in fields_order:
			s = 0
			if fields[f]['type'] in ('date','time','float','integer'):
				s = 60
				strmax -= s
			else:
				t += fields[f].get('size', 56) / 28 + 1
			l.append(s)
		for pos in range(len(l)):
			if not l[pos]:
				s = fields[fields_order[pos]].get('size', 56) / 28 + 1
				l[pos] = strmax * s / t
		_append_node('tableSize', ','.join(map(str,l)) )
		new_doc.childNodes[0].appendChild(config)
		header = new_doc.createElement("header")

		for f in fields_order:
			field = new_doc.createElement("field")
			field_txt = new_doc.createTextNode('%s' % (fields[f]['string'],))
			field.appendChild(field_txt)
			header.appendChild(field)

		new_doc.childNodes[0].appendChild(header)

		lines = new_doc.createElement("lines")
		for line in results:
			node_line = new_doc.createElement("row")
			for f in fields_order:
				if fields[f]['type']=='many2one' and line[f]:
					line[f] = line[f][1]
				if fields[f]['type'] in ('one2many','many2many') and line[f]:
					line[f] = '( '+str(len(line[f])) + ' )'
				col = new_doc.createElement("col")
				col.setAttribute('tree','no')
				if line[f] != None:
					txt = new_doc.createTextNode(str(line[f] or ''))
				else:
					txt = new_doc.createTextNode('/')
				col.appendChild(txt)
				node_line.appendChild(col)
			lines.appendChild(node_line)
		new_doc.childNodes[0].appendChild(lines)

		styledoc = libxml2.parseFile(os.path.join(tools.config['root_path'],'addons/base/report/custom_new.xsl'))
		style = libxslt.parseStylesheetDoc(styledoc)
		doc = libxml2.parseDoc(new_doc.toxml())
		rml_obj = style.applyStylesheet(doc, None)
		rml = style.saveResultToString(rml_obj)

		self.obj = render.rml(rml)
		self.obj.render()
		return True
report_printscreen_list('report.printscreen.list')

<commit_msg>report total<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

# Replaces the default formatter by one which understands GCC output and colorizes it.

__author__ = __maintainer__ = "Jérôme Carretero <cJ-waf@zougloub.eu>"
__copyright__ = "Jérôme Carretero, 2012"

import sys
from waflib import Logs

class ColorGCCFormatter(Logs.formatter):
	def __init__(self, colors):
		self.colors = colors
		Logs.formatter.__init__(self)
	def format(self, rec):
		frame = sys._getframe()
		while frame:
			func = frame.f_code.co_name
			if func == 'exec_command':
				cmd = frame.f_locals['cmd']
				if isinstance(cmd, list) and ('gcc' in cmd[0] or 'g++' in cmd[0]):
					lines = []
					for line in rec.msg.splitlines():
						if 'warning: ' in line:
							lines.append(self.colors.YELLOW + line)
						elif 'error: ' in line:
							lines.append(self.colors.RED + line)
						elif 'note: ' in line:
							lines.append(self.colors.CYAN + line)
						else:
							lines.append(line)
					rec.msg = "\n".join(lines)
			frame = frame.f_back
		return Logs.formatter.format(self, rec)

def options(opt):
	Logs.log.handlers[0].setFormatter(ColorGCCFormatter(Logs.colors))

<commit_msg>Colorgcc for other compilers? Issue 1732<commit_after>#!/usr/bin/env python
# encoding: utf-8

# Replaces the default formatter by one which understands GCC output and colorizes it.

__author__ = __maintainer__ = "Jérôme Carretero <cJ-waf@zougloub.eu>"
__copyright__ = "Jérôme Carretero, 2012"

import sys
from waflib import Logs

class ColorGCCFormatter(Logs.formatter):
	def __init__(self, colors):
		self.colors = colors
		Logs.formatter.__init__(self)
	def format(self, rec):
		frame = sys._getframe()
		while frame:
			func = frame.f_code.co_name
			if func == 'exec_command':
				cmd = frame.f_locals.get('cmd')
				if isinstance(cmd, list) and ('gcc' in cmd[0] or 'g++' in cmd[0]):
					lines = []
					for line in rec.msg.splitlines():
						if 'warning: ' in line:
							lines.append(self.colors.YELLOW + line)
						elif 'error: ' in line:
							lines.append(self.colors.RED + line)
						elif 'note: ' in line:
							lines.append(self.colors.CYAN + line)
						else:
							lines.append(line)
					rec.msg = "\n".join(lines)
			frame = frame.f_back
		return Logs.formatter.format(self, rec)

def options(opt):
	Logs.log.handlers[0].setFormatter(ColorGCCFormatter(Logs.colors))

<|endoftext|>


<commit_before>import sys
import pysolr
import requests
import logging
import json

from django.conf import settings

from hypermap.aggregator.utils import layer2dict

SEARCH_URL = settings.REGISTRY_SEARCH_URL.split('+')[1]

LOGGER = logging.getLogger(__name__)

class SolrHypermap(object):

    def layers_to_solr(self, layers):
        """
        Sync n layers in Solr.
        """

        layers_dict_list = []
        layers_success_ids = []
        layers_errors_ids = []

        for layer in layers:
            layer_dict, message = layer2dict(layer)
            if not layer_dict:
                layers_errors_ids.append([layer.id, message])
                LOGGER.error(message)
            else:
                layers_dict_list.append(layer_dict)
                layers_success_ids.append(layer.id)

        layers_json = json.dumps(layers_dict_list)
        try:
            url_solr_update = '%s/solr/hypermap/update/json/docs' % SEARCH_URL
            headers = {"content-type": "application/json"}
            params = {"commitWithin": 1500}
            requests.post(url_solr_update, data=layers_json, params=params, headers=headers)
            LOGGER.info('Solr synced for the given layers')
        except Exception:
            message = "Error saving solr records: %s" % sys.exc_info()[1]
            layers_errors_ids.append([-1, message])
            LOGGER.error(message)
            return False, layers_errors_ids

        return True, layers_errors_ids

    def layer_to_solr(self, layer):
        """
        Sync a layer in Solr.
        """
        success = True
        message = 'Synced layer id %s to Solr' % layer.id

        layer_dict, message = layer2dict(layer)
        if not layer_dict:
            success = False
        else:
            layer_json = json.dumps(layer_dict)
            try:
                url_solr_update = '%s/solr/hypermap/update/json/docs' % SEARCH_URL
                headers = {"content-type": "application/json"}
                params = {"commitWithin": 1500}
                res = requests.post(url_solr_update, data=layer_json, params=params,  headers=headers)
                res = res.json()
                if 'error' in res:
                    success = False
                    message = "Error syncing layer id %s to Solr: %s" % (layer.id, res["error"].get("msg"))
            except Exception, e:
                success = False
                message = "Error syncing layer id %s to Solr: %s" % (layer.id, sys.exc_info()[1])
                LOGGER.error(e, exc_info=True)
        if success:
            LOGGER.info(message)
        else:
            LOGGER.error(message)
        return success, message

    def clear_solr(self, catalog="hypermap"):
        """Clear all indexes in the solr core"""
        solr_url = "{0}/solr/{1}".format(SEARCH_URL, catalog)
        solr = pysolr.Solr(solr_url, timeout=60)
        solr.delete(q='*:*')
        LOGGER.debug('Solr core cleared')

    def remove_layer(self, layer_uiid, catalog="hypermap"):
        """
        Remove a layer from Solr.
        """
        solr_url = "{0}/solr/{1}".format(SEARCH_URL, catalog)
        solr = pysolr.Solr(solr_url, timeout=60)
        solr.delete(q='uuid:%s' % layer_uiid)
        LOGGER.debug('Layer %s removed from Solr' % layer_uiid)

    def update_schema(self, catalog="hypermap"):
        """
        set the mapping in solr.
        :param catalog: core
        :return:
        """
        schema_url = "{0}/solr/{1}/schema".format(SEARCH_URL, catalog)

        # create a special type to draw better heatmaps.
        location_rpt_quad_5m_payload = {
            "add-field-type": {
                "name": "location_rpt_quad_5m",
                "class": "solr.SpatialRecursivePrefixTreeFieldType",
                "geo": False,
                "worldBounds": "ENVELOPE(-180, 180, 180, -180)",
                "prefixTree": "packedQuad",
                "distErrPct": "0.025",
                "maxDistErr": "0.001",
                "distanceUnits": "degrees"
            }
        }
        requests.post(schema_url, json=location_rpt_quad_5m_payload)

        # now the other fields
        fields = [
            {"name": "abstract", "type": "string"},
            {"name": "area", "type": "tdouble"},
            {"name": "availability", "type": "string"},
            {"name": "bbox", "type": "location_rpt_quad_5m"},
            {"name": "domain_name", "type": "string"},
            {"name": "id", "type": "tlong", "required": True},
            {"name": "is_public", "type": "boolean"},
            {"name": "is_valid", "type": "boolean"},
            {"name": "last_status", "type": "boolean"},
            {"name": "layer_category", "type": "string"},
            {"name": "layer_date", "type": "tdate", "docValues": True},
            {"name": "layer_datetype", "type": "string"},
            {"name": "layer_id", "type": "tlong"},
            {"name": "layer_originator", "type": "string"},
            {"name": "layer_username", "type": "string"},
            {"name": "location", "type": "string"},
            {"name": "max_x", "type": "tdouble"},
            {"name": "max_y", "type": "tdouble"},
            {"name": "min_x", "type": "tdouble"},
            {"name": "min_y", "type": "tdouble"},
            {"name": "name", "type": "string"},
            {"name": "recent_reliability", "type": "tdouble"},
            {"name": "reliability", "type": "tdouble"},
            {"name": "service_id", "type": "tlong"},
            {"name": "service_type", "type": "string"},
            {"name": "srs", "type": "string", "multiValued": True},
            {"name": "tile_url", "type": "string"},
            {"name": "title", "type": "string"},
            {"name": "type", "type": "string"},
            {"name": "url", "type": "string"},
            {"name": "uuid", "type": "string", "required": True},
            {"name": "centroid_y", "type": "tdouble"},
            {"name": "centroid_x", "type": "tdouble"},
        ]

        headers = {
            "Content-type": "application/json"
        }

        for field in fields:
            data = {
                "add-field": field
            }
            requests.post(schema_url, json=data, headers=headers)
<commit_msg>Fixes path to solr<commit_after>import sys
import pysolr
import requests
import logging
import json

from django.conf import settings

from hypermap.aggregator.utils import layer2dict

SEARCH_URL = settings.REGISTRY_SEARCH_URL.split('+')[1]

LOGGER = logging.getLogger(__name__)

class SolrHypermap(object):

    def layers_to_solr(self, layers):
        """
        Sync n layers in Solr.
        """

        layers_dict_list = []
        layers_success_ids = []
        layers_errors_ids = []

        for layer in layers:
            layer_dict, message = layer2dict(layer)
            if not layer_dict:
                layers_errors_ids.append([layer.id, message])
                LOGGER.error(message)
            else:
                layers_dict_list.append(layer_dict)
                layers_success_ids.append(layer.id)

        layers_json = json.dumps(layers_dict_list)
        try:
            url_solr_update = '%s/solr/hypermap/update/json/docs' % SEARCH_URL
            headers = {"content-type": "application/json"}
            params = {"commitWithin": 1500}
            requests.post(url_solr_update, data=layers_json, params=params, headers=headers)
            LOGGER.info('Solr synced for the given layers')
        except Exception:
            message = "Error saving solr records: %s" % sys.exc_info()[1]
            layers_errors_ids.append([-1, message])
            LOGGER.error(message)
            return False, layers_errors_ids

        return True, layers_errors_ids

    def layer_to_solr(self, layer):
        """
        Sync a layer in Solr.
        """
        success = True
        message = 'Synced layer id %s to Solr' % layer.id

        layer_dict, message = layer2dict(layer)
        if not layer_dict:
            success = False
        else:
            layer_json = json.dumps(layer_dict)
            try:
                url_solr_update = '%s/solr/hypermap/update/json/docs' % SEARCH_URL
                headers = {"content-type": "application/json"}
                params = {"commitWithin": 1500}
                res = requests.post(url_solr_update, data=layer_json, params=params,  headers=headers)
                res = res.json()
                if 'error' in res:
                    success = False
                    message = "Error syncing layer id %s to Solr: %s" % (layer.id, res["error"].get("msg"))
            except Exception, e:
                success = False
                message = "Error syncing layer id %s to Solr: %s" % (layer.id, sys.exc_info()[1])
                LOGGER.error(e, exc_info=True)
        if success:
            LOGGER.info(message)
        else:
            LOGGER.error(message)
        return success, message

    def clear_solr(self, catalog="hypermap"):
        """Clear all indexes in the solr core"""
        solr_url = "{0}/solr/{1}".format(SEARCH_URL, catalog)
        solr = pysolr.Solr(solr_url, timeout=60)
        solr.delete(q='*:*')
        LOGGER.debug('Solr core cleared')

    def remove_layer(self, layer_uiid, catalog="hypermap"):
        """
        Remove a layer from Solr.
        """
        solr_url = "{0}/solr/{1}".format(SEARCH_URL, catalog)
        solr = pysolr.Solr(solr_url, timeout=60)
        solr.delete(q='uuid:%s' % layer_uiid)
        LOGGER.debug('Layer %s removed from Solr' % layer_uiid)

    def update_schema(self, catalog="hypermap"):
        """
        set the mapping in solr.
        :param catalog: core
        :return:
        """
        schema_url = "{0}/solr/{1}/schema".format(SEARCH_URL, catalog)
        print schema_url

        # create a special type to draw better heatmaps.
        location_rpt_quad_5m_payload = {
            "add-field-type": {
                "name": "location_rpt_quad_5m",
                "class": "solr.SpatialRecursivePrefixTreeFieldType",
                "geo": False,
                "worldBounds": "ENVELOPE(-180, 180, 180, -180)",
                "prefixTree": "packedQuad",
                "distErrPct": "0.025",
                "maxDistErr": "0.001",
                "distanceUnits": "degrees"
            }
        }
        requests.post(schema_url, json=location_rpt_quad_5m_payload)

        # now the other fields
        fields = [
            {"name": "abstract", "type": "string"},
            {"name": "area", "type": "tdouble"},
            {"name": "availability", "type": "string"},
            {"name": "bbox", "type": "location_rpt_quad_5m"},
            {"name": "domain_name", "type": "string"},
            {"name": "id", "type": "tlong", "required": True},
            {"name": "is_public", "type": "boolean"},
            {"name": "is_valid", "type": "boolean"},
            {"name": "last_status", "type": "boolean"},
            {"name": "layer_category", "type": "string"},
            {"name": "layer_date", "type": "tdate", "docValues": True},
            {"name": "layer_datetype", "type": "string"},
            {"name": "layer_id", "type": "tlong"},
            {"name": "layer_originator", "type": "string"},
            {"name": "layer_username", "type": "string"},
            {"name": "location", "type": "string"},
            {"name": "max_x", "type": "tdouble"},
            {"name": "max_y", "type": "tdouble"},
            {"name": "min_x", "type": "tdouble"},
            {"name": "min_y", "type": "tdouble"},
            {"name": "name", "type": "string"},
            {"name": "recent_reliability", "type": "tdouble"},
            {"name": "reliability", "type": "tdouble"},
            {"name": "service_id", "type": "tlong"},
            {"name": "service_type", "type": "string"},
            {"name": "srs", "type": "string", "multiValued": True},
            {"name": "tile_url", "type": "string"},
            {"name": "title", "type": "string"},
            {"name": "type", "type": "string"},
            {"name": "url", "type": "string"},
            {"name": "uuid", "type": "string", "required": True},
            {"name": "centroid_y", "type": "tdouble"},
            {"name": "centroid_x", "type": "tdouble"},
        ]

        headers = {
            "Content-type": "application/json"
        }

        for field in fields:
            data = {
                "add-field": field
            }
            requests.post(schema_url, json=data, headers=headers)
<|endoftext|>


<commit_before># stdlib
from typing import Optional

# third party
from nacl.signing import SigningKey

# syft absolute
from syft import Domain  # type: ignore
from syft import Network  # type: ignore
from syft import logger  # noqa F401 # type: ignore
from syft.core.node.common.client import Client
from syft.core.node.common.node_table import Base
from syft.core.node.common.node_table.utils import seed_db

# grid absolute
from grid.core.config import settings
from grid.db.session import get_db_engine
from grid.db.session import get_db_session

if settings.NODE_TYPE.lower() == "domain":
    node = Domain("Domain", db_engine=get_db_engine())
elif settings.NODE_TYPE.lower() == "network":
    node = Network("Network", db_engine=get_db_engine())
else:
    raise Exception(
        "Don't know NODE_TYPE "
        + str(settings.NODE_TYPE)
        + ". Please set "
        + "NODE_TYPE to either 'Domain' or 'Network'."
    )

node.loud_print()

Base.metadata.create_all(get_db_engine())

if len(node.setup):  # Check if setup was defined previously
    node.name = node.setup.node_name

# Moving this to get called WITHIN Domain and Network so that they can operate in standalone mode
if not len(node.roles):  # Check if roles were registered previously
    seed_db(get_db_session())

def get_client(signing_key: Optional[SigningKey] = None) -> Client:
    return node.get_client(signing_key=signing_key)
<commit_msg>fix: remove db init from grid/core/node<commit_after># stdlib
from typing import Optional

# third party
from nacl.signing import SigningKey

# syft absolute
from syft import Domain  # type: ignore
from syft import Network  # type: ignore
from syft.core.node.common.client import Client
from syft.core.node.common.node_table.utils import seed_db

# grid absolute
from grid.core.config import settings
from grid.db.session import get_db_engine
from grid.db.session import get_db_session

if settings.NODE_TYPE.lower() == "domain":
    node = Domain("Domain", db_engine=get_db_engine())
elif settings.NODE_TYPE.lower() == "network":
    node = Network("Network", db_engine=get_db_engine())
else:
    raise Exception(
        "Don't know NODE_TYPE "
        + str(settings.NODE_TYPE)
        + ". Please set "
        + "NODE_TYPE to either 'Domain' or 'Network'."
    )

node.loud_print()

if len(node.setup):  # Check if setup was defined previously
    node.name = node.setup.node_name

# Moving this to get called WITHIN Domain and Network so that they can operate in standalone mode
if not len(node.roles):  # Check if roles were registered previously
    seed_db(get_db_session())

def get_client(signing_key: Optional[SigningKey] = None) -> Client:
    return node.get_client(signing_key=signing_key)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Classes to assist with GFF parsing. It tries to keep track of the partent/
child relationships and allows for fetching of parent or children features.

Contains the following classes:
    GFFError:   An error when a queried ID is not found in the GFF data.
    GFFFeature: Reads a line from a GFF file and parses out the feature info
    GFFHandler: Handles the parsing of the whole GFF file and allows access
                to parents, children, or "siblings" of individual features.
"""

class GFFError(LookupError):
    """Raised when a user asks for a feature ID that is not present in the GFF
    data."""

class GFFFeature(object):
    """This reads the data out of a single line of a GFF. It is a *very* ugly
    class, but GFF is a *very* ugly file format, and I don't know of a better
    way to handle this data."""
    __slots__ = [
        'seqid',
        'source',
        'type',
        'start',
        'end',
        'score',
        'strand',
        'phase',
        'ID',
        'Name',
        'Alias',
        'Parent',
        'Target',
        'Gap',
        'Derives_from',
        'Note',
        'Dbxref',
        'Ontology_term',
        'Is_circular'
        ]

    def __init__(self, line):
        self.seqid = None
        self.source = None
        self.type = None
        self.start = None
        self.end = None
        self.score = None
        self.strand = None
        self.phase = None
        self.ID = None
        self.Name = None
        self.Alias = None
        self.Parent = None
        self.Target = None
        self.Gap = None
        self.Derives_from = None
        self.Note = None
        self.Dbxref = None
        self.Ontology_term = None
        self.Is_circular = None
        self.read_gff_line(line)

    def read_gff_line(self, line):
        fields = line.strip().split('\t')
        #   Fixed fields are the first eight
        self.seqid = fields[0]
        self.source = fields[1]
        self.type = fields[2]
        self.start = fields[3]
        self.end = fields[4]
        self.score = fields[5]
        self.strand = fields[6]
        self.phase = fields[7]
        #   For the next field, we split it and look for keywords
        attrib = fields[8].split(';')
        for a in attrib:
            if a.startswith('ID'):
                self.ID = a.split('=')[1]
            elif a.startswith('Name'):
                self.Name = a.split('=')[1]
            elif a.startswith('Alias'):
                alias = a.split('=')[1]
                self.Alias = alias.split(',')
            elif a.startswith('Parent'):
                parent = a.split('=')[1]
                self.Parent = parent.split(',')
            elif a.startswith('Target'):
                self.Target = a.split('=')[1]
            elif a.startswith('Gap'):
                self.Gap = a.split('=')[1]
            elif a.startswith('Derives_from'):
                self.Derives_from = a.split('=')[1]
            elif a.startswith('Note'):
                note = a.split('=')[1]
                self.Note = note.split(',')
            elif a.startsiwth('Dbxref'):
                dbxref = a.split('=')[1]
                self.Dbxref = dbxref.split(',')
            elif a.startswith('Ontology_term'):
                ont = a.split('=')[1]
                self.Ontology_term = ont.split(',')
            elif a.startswith('Is_circular'):
                self.Is_circular = a.split('=')[1]

class GFFHandler(object):
    """Handles the parsing of the entire GFF file, line by line. Defines
    methods that allow for retrieval of specific features, or groups of
    features.

    Contains the following methods:
        get_feature(self, feat_id)
            Retuns a single feature, or raises an exception when the ID is not
            found

        get_parents(self, feat_id, type=None)
            Returns the parents of a feature, or raises an exception when the
            ID is not found. If the type keyword is specified, then only
            features of a certain type are returned. If there are no features
            of the supplied type, then NoneType is returned. Defaults to
            returning all parents. This will only traverse up one level.

        get_children(self, feat_id, type=None)
            Returns the children of a feature, or raises an exception when the
            ID is not found. If the type keyword is specified, then only child
            features of the specified type are returned. If there are no child
            features of that type, then NoneType is returned. Defaults to
            returning all children. This will only traverse down one level.

        get_siblings(self, feat_id, type=None)
            Returns the "siblings" (features that share parents), or raises an
            exception when the ID is not found. If the type keyword is
            specified, then only features of that type are returned. If there
            are no sibling features of that type, then NoneType is returned.
            Defaults to returning all siblings. This does not check parents
            of parents or children of children.
    """
    gff_data = []
    gff_ids = []

    def __init__(self):
        pass

    def gff_parse(self, fname):
        with open(fname, 'r') as f:
            for line in f:
                #   Skip lines starting with #; they are directives or comments
                if line.startswith('#'):
                    continue
                #   Skip blank lines. I guess we should support CRLF lines too
                elif line in ('\n', '\r\n'):
                    continue
                else:
                    g = GFFFeature(line)
                    self.gff_data.append(g)
                    self.gff_ids.append(g.ID)

    def get_feature(self, feat_id):
        if feat_id not in self.gff_ids:
            raise GFFError('ID {fid} not found in GFF!'.format(fid=feat_id))
        else:
            for f in self.gff_data:
                if f.ID == feat_id:
                    return(f)

    def get_parents(self, feat_id, feat_type=None):
        if feat_id not in self.gff_ids:
            raise GFFError('ID {fid} not found in GFF!'.format(fid=feat_id))
        else:
            parents = []
            targeted_features = []
            #   We search through the list and save features that match the
            #   query ID
            for f in self.gff_data:
                if f.Parent and feat_id == f.ID:
                    if feat_type:
                        if f.type == feat_type:
                            parents += f.Parent
                    else:
                        parents += f.Parent
            #   We then search AGAIN to get the features that match the parent
            #   feature ID. This is not very efficient, but given the structure
            #   of the data, it works.
            for f in self.gff_data:
                if f.ID in parents:
                    targeted_features.append(f)
            return(targeted_features)

    def get_children(self, feat_id, feat_type=None):
        if feat_id not in self.gff_ids:
            raise GFFError('ID {fid} not found in GFF!'.format(fid=feat_id))
        else:
            targeted_features = []
            #   Searching for child features is easy, since we just have to see
            #   if our query ID is in the Parent GFF attributes.
            for f in self.gff_data:
                if f.Parent and feat_id in f.Parent:
                    if feat_type:
                        if f.type == feat_type:
                            targeted_features.append(f)
                    else:
                        targeted_features.append(f)
            return(targeted_features)

    def get_siblings(self, feat_id, feat_type=None):
        if feat_id not in self.gff_ids:
            raise GFFError('ID {fid} not found in GFF!'.format(fid=feat_id))
        #   This is pretty easy, too. We just take the query ID, find its
        #   parents, and then find all children of those parents. These are the
        #   "siblings"
        parents = self.get_parents(feat_id, feat_type=feat_type)
        sibs = []
        for p in parents:
            sibs += self.get_children(p.ID, feat_type=feat_type)
        return(sibs)
<commit_msg>Update gff_parse.py to get all features in a given interval<commit_after>#!/usr/bin/env python
"""Classes to assist with GFF parsing. It tries to keep track of the partent/
child relationships and allows for fetching of parent or children features.

Contains the following classes:
    GFFError:   An error when a queried ID is not found in the GFF data.
    GFFFeature: Reads a line from a GFF file and parses out the feature info
    GFFHandler: Handles the parsing of the whole GFF file and allows access
                to parents, children, or "siblings" of individual features.
"""

class GFFError(LookupError):
    """Raised when a user asks for a feature ID that is not present in the GFF
    data."""

class GFFFeature(object):
    """This reads the data out of a single line of a GFF. It is a *very* ugly
    class, but GFF is a *very* ugly file format, and I don't know of a better
    way to handle this data."""
    __slots__ = [
        'seqid',
        'source',
        'type',
        'start',
        'end',
        'score',
        'strand',
        'phase',
        'ID',
        'Name',
        'Alias',
        'Parent',
        'Target',
        'Gap',
        'Derives_from',
        'Note',
        'Dbxref',
        'Ontology_term',
        'Is_circular'
        ]

    def __init__(self, line):
        self.seqid = None
        self.source = None
        self.type = None
        self.start = None
        self.end = None
        self.score = None
        self.strand = None
        self.phase = None
        self.ID = None
        self.Name = None
        self.Alias = None
        self.Parent = None
        self.Target = None
        self.Gap = None
        self.Derives_from = None
        self.Note = None
        self.Dbxref = None
        self.Ontology_term = None
        self.Is_circular = None
        self.read_gff_line(line)

    def read_gff_line(self, line):
        fields = line.strip().split('\t')
        #   Fixed fields are the first eight
        self.seqid = fields[0]
        self.source = fields[1]
        self.type = fields[2]
        self.start = int(fields[3])
        self.end = int(fields[4])
        self.score = fields[5]
        self.strand = fields[6]
        self.phase = fields[7]
        #   For the next field, we split it and look for keywords
        attrib = fields[8].split(';')
        for a in attrib:
            if a.startswith('ID'):
                self.ID = a.split('=')[1]
            elif a.startswith('Name'):
                self.Name = a.split('=')[1]
            elif a.startswith('Alias'):
                alias = a.split('=')[1]
                self.Alias = alias.split(',')
            elif a.startswith('Parent'):
                parent = a.split('=')[1]
                self.Parent = parent.split(',')
            elif a.startswith('Target'):
                self.Target = a.split('=')[1]
            elif a.startswith('Gap'):
                self.Gap = a.split('=')[1]
            elif a.startswith('Derives_from'):
                self.Derives_from = a.split('=')[1]
            elif a.startswith('Note'):
                note = a.split('=')[1]
                self.Note = note.split(',')
            elif a.startsiwth('Dbxref'):
                dbxref = a.split('=')[1]
                self.Dbxref = dbxref.split(',')
            elif a.startswith('Ontology_term'):
                ont = a.split('=')[1]
                self.Ontology_term = ont.split(',')
            elif a.startswith('Is_circular'):
                self.Is_circular = a.split('=')[1]

class GFFHandler(object):
    """Handles the parsing of the entire GFF file, line by line. Defines
    methods that allow for retrieval of specific features, or groups of
    features.

    Contains the following methods:
        get_feature(self, feat_id)
            Retuns a single feature, or raises an exception when the ID is not
            found

        get_parents(self, feat_id, type=None)
            Returns the parents of a feature, or raises an exception when the
            ID is not found. If the type keyword is specified, then only
            features of a certain type are returned. If there are no features
            of the supplied type, then empty list is returned. Defaults to
            returning all parents. This will only traverse up one level.

        get_children(self, feat_id, type=None)
            Returns the children of a feature, or raises an exception when the
            ID is not found. If the type keyword is specified, then only child
            features of the specified type are returned. If there are no child
            features of that type, then empty list is returned. Defaults to
            returning all children. This will only traverse down one level.

        get_siblings(self, feat_id, type=None)
            Returns the "siblings" (features that share parents), or raises an
            exception when the ID is not found. If the type keyword is
            specified, then only features of that type are returned. If there
            are no sibling features of that type, then empty list is returned.
            Defaults to returning all siblings. This does not check parents
            of parents or children of children.

        chrom_features(self, chrom, left, right, type=None)
            Returns all features on the specified chromosome, or raises an
            exception when that chromosome is not found. If the type keyword is
            specified, then only features of that type are returned. If the
            left keyowrd is specified, then features that have bases to the
            _right_ of the specified base are returned. If the right keyword is
            specified, then features with bases to the _left_ of the offset
            are returned. These effctively limit the search to a certain
            region. If there are no features of the specified type in the given
            region, then empty list is returned. Defaults to all types and the
            entire chromosome.
    """
    gff_data = []
    gff_ids = []

    def __init__(self):
        pass

    def gff_parse(self, fname):
        with open(fname, 'r') as f:
            for line in f:
                #   Skip lines starting with #; they are directives or comments
                if line.startswith('#'):
                    continue
                #   Skip blank lines. I guess we should support CRLF lines too
                elif line in ('\n', '\r\n'):
                    continue
                else:
                    g = GFFFeature(line)
                    self.gff_data.append(g)
                    self.gff_ids.append(g.ID)

    def get_feature(self, feat_id):
        if feat_id not in self.gff_ids:
            raise GFFError('ID {fid} not found in GFF!'.format(fid=feat_id))
        else:
            for f in self.gff_data:
                if f.ID == feat_id:
                    return(f)

    def get_parents(self, feat_id, feat_type=None):
        if feat_id not in self.gff_ids:
            raise GFFError('ID {fid} not found in GFF!'.format(fid=feat_id))
        else:
            parents = []
            targeted_features = []
            #   We search through the list and save features that match the
            #   query ID
            for f in self.gff_data:
                if f.Parent and feat_id == f.ID:
                    if feat_type:
                        if f.type == feat_type:
                            parents += f.Parent
                    else:
                        parents += f.Parent
            #   We then search AGAIN to get the features that match the parent
            #   feature ID. This is not very efficient, but given the structure
            #   of the data, it works.
            for f in self.gff_data:
                if f.ID in parents:
                    targeted_features.append(f)
            return(targeted_features)

    def get_children(self, feat_id, feat_type=None):
        if feat_id not in self.gff_ids:
            raise GFFError('ID {fid} not found in GFF!'.format(fid=feat_id))
        else:
            targeted_features = []
            #   Searching for child features is easy, since we just have to see
            #   if our query ID is in the Parent GFF attributes.
            for f in self.gff_data:
                if f.Parent and feat_id in f.Parent:
                    if feat_type:
                        if f.type == feat_type:
                            targeted_features.append(f)
                    else:
                        targeted_features.append(f)
            return(targeted_features)

    def get_siblings(self, feat_id, feat_type=None):
        if feat_id not in self.gff_ids:
            raise GFFError('ID {fid} not found in GFF!'.format(fid=feat_id))
        #   This is pretty easy, too. We just take the query ID, find its
        #   parents, and then find all children of those parents. These are the
        #   "siblings"
        parents = self.get_parents(feat_id, feat_type=feat_type)
        sibs = []
        for p in parents:
            sibs += self.get_children(p.ID, feat_type=feat_type)
        return(sibs)

    def chrom_features(self, chrom, left=None, right=None, feat_type=None):
        targeted_features = []
        filtered_features = []
        for f in self.gff_data:
            if f.seqid == chrom:
                if feat_type:
                    if f.type == feat_type:
                        targeted_features.append(f)
                else:
                    targeted_features.append(f)
        #   Now we go through the list of targeted features and filter based
        #   on position
        if left or right:
            #   if left is not specified, set it to 0
            if not left:
                left = 0
            #   Same with right, except make it huge
            if not right:
                right = 1e99
            for t in targeted_features:
                if t.end >= left or t.start <= right:
                    filtered_features.append(t)
        else:
            filtered_features = targeted_features
        return(filtered_features)
<|endoftext|>


<commit_before>from bibliopixel import animation
from bibliopixel.colors import COLORS

"""
This is an example Animation class which has a single field -
`color` with the default value `COLORS.green`.

You can edit this as you like, add and remove fields and
write Python code in general.
"""

class Example26(animation.Animation):
    def __init__(
            self, *args,
            # The fields for your class go here!
            color=COLORS.green,

            # End of the fields for your class
            **kwds):
        super().__init__(*args, **kwds)
        self.color = color
        # Your initialization code goes here.

    def step(self, amt=1):
        this_pixel = self.cur_step % len(self.color_list)

        # Set the previous pixel to black.
        self.color_list[this_pixel - 1] = COLORS.black

        # Set this pixel to the color
        self.color_list[this_pixel] = self.color

    #
    # Everything below here is optional, and you can delete it if you aren't
    # using it.
    #

    # pre_run is called right before the animation starts running.
    def pre_run(self):
        super().pre_run()
        # Your code goes here.

    def cleanup(self, clean_layout=True):
        super().cleanup(clean_layout)
        # Your code goes here.

class Example28(Example26):
    def step(self, amt=1):
        this_pixel = self.cur_step % len(self.color_list)

        # Set the previous pixel to black.
        self.color_list[this_pixel - 1] = COLORS.black

        # Set this pixel to the color
        self.color_list[this_pixel] = self.color
        self.color_list[this_pixel - 2] = COLORS.yellow
        self.color_list[this_pixel - 3] = COLORS.black
<commit_msg>Fix import error in test class<commit_after>from bibliopixel.animation import animation
from bibliopixel.colors import COLORS

"""
This is an example Animation class which has a single field -
`color` with the default value `COLORS.green`.

You can edit this as you like, add and remove fields and
write Python code in general.
"""

class Example26(animation.Animation):
    def __init__(
            self, *args,
            # The fields for your class go here!
            color=COLORS.green,

            # End of the fields for your class
            **kwds):
        super().__init__(*args, **kwds)
        self.color = color
        # Your initialization code goes here.

    def step(self, amt=1):
        this_pixel = self.cur_step % len(self.color_list)

        # Set the previous pixel to black.
        self.color_list[this_pixel - 1] = COLORS.black

        # Set this pixel to the color
        self.color_list[this_pixel] = self.color

    #
    # Everything below here is optional, and you can delete it if you aren't
    # using it.
    #

    # pre_run is called right before the animation starts running.
    def pre_run(self):
        super().pre_run()
        # Your code goes here.

    def cleanup(self, clean_layout=True):
        super().cleanup(clean_layout)
        # Your code goes here.

class Example28(Example26):
    def step(self, amt=1):
        this_pixel = self.cur_step % len(self.color_list)

        # Set the previous pixel to black.
        self.color_list[this_pixel - 1] = COLORS.black

        # Set this pixel to the color
        self.color_list[this_pixel] = self.color
        self.color_list[this_pixel - 2] = COLORS.yellow
        self.color_list[this_pixel - 3] = COLORS.black
<|endoftext|>


<commit_before>from __future__ import print_function, absolute_import, division

from boto import ses

class AwsSesPlugin(object):

    def __init__(self, resources, region=None, sender=None, subject=None,
                 recipients=None, body=None):
        self.region = region
        self.mail_sender = sender
        self.subject = subject
        self.mail_recipients = recipients or []
        self.mail_body = body
        self.resources = resources

    @property
    def sender(self):
        return self.mail_sender

    @property
    def recipients(self):
        return self.mail_recipients

    @property
    def body(self):
        return self.mail_body

    def send_email(self):
        conn = ses.connect_to_region(region_name=self.region)

        conn.send_email(
            source=self.sender,
            subject=self.subject,
            body=self.body,
            to_addresses=self.recipients)

    def run(self):
        self.send_email()
<commit_msg>PIO-124: add logging<commit_after>from __future__ import print_function, absolute_import, division

from boto import ses
import logging

class AwsSesPlugin(object):

    def __init__(self, resources, region=None, sender=None, subject=None,
                 recipients=None, body=None):
        self.region = region
        self.mail_sender = sender
        self.subject = subject
        self.mail_recipients = recipients or []
        self.mail_body = body
        self.resources = resources
        self.logger = logging.getLogger(__name__)

    @property
    def sender(self):
        return self.mail_sender

    @property
    def recipients(self):
        return self.mail_recipients

    @property
    def body(self):
        return self.mail_body

    def send_email(self):
        conn = ses.connect_to_region(region_name=self.region)

        self.logger.info("Sending Email to %s", ", ".join(self.recipients))

        conn.send_email(
            source=self.sender,
            subject=self.subject,
            body=self.body,
            to_addresses=self.recipients)

    def run(self):
        self.send_email()
<|endoftext|>