<commit_before>from models.user import User
from flask import Blueprint, jsonify, request, make_response, abort
from flask.ext.login import login_user, current_user, logout_user
from modules.content import get as _

user = Blueprint('user', __name__, url_prefix='/api/users')

def _login(user):
    """
    Login a given user, and return an appropriate response.
    Used by signup, login, and reset password.
    """
    login_user(user, remember=True)
    resp = make_response(jsonify(user=user.deliver(access='private')))
    resp.set_cookie('logged_in', '1')
    return resp

@user.route('/<user_id>/', methods=['GET'])
def get_user(user_id):
    """Get the user by their ID."""
    user = User.get(id=user_id)
    if user:
        return jsonify(
            user=user.deliver(
                access='private' if user.is_current_user() else None
            )
        )
    return abort(404)

@user.route('/current/', methods=['GET'])
def get_current_user():
    """Get current user's information."""
    if current_user.is_authenticated():
        return jsonify(user=current_user.deliver(access='private'))
    return abort(401)

@user.route('/', methods=['POST'])
def create_user():
    """Create user."""
    user, errors = User.insert(request.json)
    if len(errors):
        return jsonify(errors=errors), 400
    return _login(user)

@user.route('/login/', methods=['POST'])
def login():
    """Login user."""
    user = User.get(name=request.json.get('name'))
    if not user:
        return jsonify(errors=[{
            'name': 'name',
            'message': _('user', 'no_user'),
        }]), 404
    if not user.is_password_valid(request.json.get('password')):
        return jsonify(errors=[{
            'name': 'password',
            'message': _('user', 'no_match'),
        }]), 400
    return _login(user)

@user.route('/logout/', methods=['POST'])
def logout():
    """Logout user."""
    logout_user()
    resp = make_response('')
    resp.set_cookie('logged_in', '0')
    return resp, 204

@user.route('/<user_id>/', methods=['PUT'])
def update_user(user_id):
    """Update the user. Must be the current user."""
    user = User.get(id=user_id)
    if not user:
        return abort(404)
    if not user.is_current_user():
        return abort(401)
    user, errors = user.update(request.json)
    if len(errors):
        return jsonify(errors=errors), 400
    return jsonify(user=user.deliver(access='private'))

@user.route('/token/', methods=['POST'])
def create_token():
    """Create an email token for the user."""
    user = User.get(email=request.json.get('email'))
    if not user:
        return abort(404)
    user.get_email_token()
    return '', 204

@user.route('/password/', methods=['POST'])
def create_password():
    """Update a user's password if the token is valid."""
    user = User.get(id=request.json.get('id'))
    if not user:
        return abort(404)
    valid = user.is_valid_token(request.json.get('token'))
    if not valid:
        return abort(403)
    user.update_password(request.json.get('password'))
    return _login(user)

@user.route('/<user_id>/menu/', methods=['GET'])
def get_menu(user_id):
    """
    Get the list of menu items given the user's context.
    """
    pass

@user.route('/<user_id>/sets/', methods=['GET'])
def get_user_sets(user_id):
    """
    Get the list of sets the user has added.
    """
    pass

@user.route('/<user_id>/sets/', methods=['POST'])
def add_set(user_id):
    """
    Adds a set to the learner's list of sets.
    """
    pass

@user.route('/<user_id>/sets/<set_id>/', method=['DELETE'])
def remove_set(user_id, set_id):
    """
    Removes a set from the learner's list of sets.
    """
    pass
<commit_msg>Fix broken unit tests<commit_after>from models.user import User
from flask import Blueprint, jsonify, request, make_response, abort
from flask.ext.login import login_user, current_user, logout_user
from modules.content import get as _

user = Blueprint('user', __name__, url_prefix='/api/users')

def _login(user):
    """
    Login a given user, and return an appropriate response.
    Used by signup, login, and reset password.
    """
    login_user(user, remember=True)
    resp = make_response(jsonify(user=user.deliver(access='private')))
    resp.set_cookie('logged_in', '1')
    return resp

@user.route('/<user_id>/', methods=['GET'])
def get_user(user_id):
    """Get the user by their ID."""
    user = User.get(id=user_id)
    if user:
        return jsonify(
            user=user.deliver(
                access='private' if user.is_current_user() else None
            )
        )
    return abort(404)

@user.route('/current/', methods=['GET'])
def get_current_user():
    """Get current user's information."""
    if current_user.is_authenticated():
        return jsonify(user=current_user.deliver(access='private'))
    return abort(401)

@user.route('/', methods=['POST'])
def create_user():
    """Create user."""
    user, errors = User.insert(request.json)
    if len(errors):
        return jsonify(errors=errors), 400
    return _login(user)

@user.route('/login/', methods=['POST'])
def login():
    """Login user."""
    user = User.get(name=request.json.get('name'))
    if not user:
        return jsonify(errors=[{
            'name': 'name',
            'message': _('user', 'no_user'),
        }]), 404
    if not user.is_password_valid(request.json.get('password')):
        return jsonify(errors=[{
            'name': 'password',
            'message': _('user', 'no_match'),
        }]), 400
    return _login(user)

@user.route('/logout/', methods=['POST'])
def logout():
    """Logout user."""
    logout_user()
    resp = make_response('')
    resp.set_cookie('logged_in', '0')
    return resp, 204

@user.route('/<user_id>/', methods=['PUT'])
def update_user(user_id):
    """Update the user. Must be the current user."""
    user = User.get(id=user_id)
    if not user:
        return abort(404)
    if not user.is_current_user():
        return abort(401)
    user, errors = user.update(request.json)
    if len(errors):
        return jsonify(errors=errors), 400
    return jsonify(user=user.deliver(access='private'))

@user.route('/token/', methods=['POST'])
def create_token():
    """Create an email token for the user."""
    user = User.get(email=request.json.get('email'))
    if not user:
        return abort(404)
    user.get_email_token()
    return '', 204

@user.route('/password/', methods=['POST'])
def create_password():
    """Update a user's password if the token is valid."""
    user = User.get(id=request.json.get('id'))
    if not user:
        return abort(404)
    valid = user.is_valid_token(request.json.get('token'))
    if not valid:
        return abort(403)
    user.update_password(request.json.get('password'))
    return _login(user)

@user.route('/<user_id>/menu/', methods=['GET'])
def get_menu(user_id):
    """
    Get the list of menu items given the user's context.
    """
    pass

@user.route('/<user_id>/sets/', methods=['GET'])
def get_user_sets(user_id):
    """
    Get the list of sets the user has added.
    """
    pass

@user.route('/<user_id>/sets/', methods=['POST'])
def add_set(user_id):
    """
    Adds a set to the learner's list of sets.
    """
    pass

@user.route('/<user_id>/sets/<set_id>/', methods=['DELETE'])
def remove_set(user_id, set_id):
    """
    Removes a set from the learner's list of sets.
    """
    pass
<|endoftext|>


<commit_before>
#  FoXPath, tools for processing FoXPath test suites,
#  by Martin Keegan, Mark Brough and Ben Webb

#  Copyright (C) 2013  Ben Webb
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

import re
import datetime
from functools import partial

def generate_mappings():
    mappings = []

    def add(regex):
        def append_to_mappings(fn):
            mappings.append((re.compile(regex),fn))
            return fn
        return append_to_mappings

    def add_partial(regex):
        def append_to_mappings(fn):
            def partial_fn(groups):
                return partial(fn, groups=groups)
            mappings.append((re.compile(regex), partial_fn))
            return fn
        return append_to_mappings

    def add_partial_with_list(regex):
        def append_to_mappings(fn):
            def partial_fn(groups):
                return partial(fn, groups=groups)
            mappings.append((re.compile(regex), partial_fn))
            return fn
        return append_to_mappings

    @add('(\S*) is an? (.*)\?')
    def is_an(groups):
        if groups[1] == 'iso date':
            return None
        elif groups[1] == 'integer':
            def int_check(x):
                try:
                    int(x)
                    return True
                except ValueError:
                    return False
            def is_an_integer(activity):
                return reduce(lambda x,y: x and y,
                            map(lambda x: int_check(x),
                                    activity.xpath(groups[0])),
                            False)
            return is_an_integer

    @add_partial('(\S*) has more than (\S*) characters\?')
    def text_chars(activity, groups):
        return bool(reduce(lambda x,y: x or y,
                        map(lambda x: len(x)>int(groups[1]),
                            activity.xpath(groups[0])),
                        False))

    def rm_blank(alist):
        return filter(lambda x: x!='', alist)

    @add_partial('(\S*) sum to (\S*)\?')
    def sum(activity, groups):
        return (reduce(lambda x,y: float(x)+float(y),
                           rm_blank(activity.xpath(groups[0])),
                           0)
                   == float(groups[1]))

    @add_partial('(\S*) exists (\S*) times?\?')
    def exist_times(activity, groups):
        return len(rm_blank(activity.xpath(groups[0]))) == int(groups[1])

    @add_partial('(\S*) exists more than (\S*) times?\?')
    def exist_times(activity, groups):
        return len(rm_blank(activity.xpath(groups[0]))) > int(groups[1])

    def exist_check(activity, xpath):
        return bool(rm_blank(activity.xpath(xpath)))

    def exist_check_list(activity, xpath, codelist):
        outcome = False
        try:
            if bool(str(activity.xpath(xpath)[0]) in codelist):
                outcome = True
            elif bool(str(activity.xpath(xpath)[0]).lowercase in codelist):
                outcome = True
            elif bool(str(activity.xpath(xpath)[0]).uppercase in codelist):
                outcome = True
            return outcome
        except Exception:
            return False

    def x_months_ago_check(activity, xpath, months, many=False):
        outcome = False
        months = int(months)
        current_date = datetime.datetime.utcnow()
        if many:
            for check in activity.findall(many):
                try:
                    if ((current_date-datetime.datetime.strptime(check.xpath(xpath)[0], "%Y-%m-%d"))
                        < (datetime.timedelta(days=(30*months)))):
                        outcome = True
                except IndexError:
                    pass
        else:
            try:
                if ((current_date-(datetime.datetime.strptime(activity.xpath(xpath)[0], "%Y-%m-%d")))
                        < (datetime.timedelta(days=(30*months)))):
                    outcome = True
            except IndexError:
                pass
        return outcome

    @add_partial('only one of (\S*) or (\S*) exists\?')
    def exist_xor(activity, groups):
        return (exist_check(activity, groups[0]) !=
                exist_check(activity, groups[1]))

    @add_partial('(\S*) or (\S*) exists\?')
    def exist_or(activity, groups):
        return (exist_check(activity, groups[0]) or
                exist_check(activity, groups[1]))

    @add_partial('(\S*) exists\?')
    def exist(activity, groups):
        return exist_check(activity, groups[0])

    @add_partial_with_list('(\S*) is on list (\S*)\?')
    def exist_list(data, groups):
        return exist_check_list(data['activity'], groups[0], data['lists'][groups[1]])

    @add_partial('(\S*) or (\S*) \(for each (\S*)\) is less than (\S*) months? ago\?')
    def less_than_x_months_ago(activity, groups):
        return (x_months_ago_check(activity, groups[0], groups[3]) or
                x_months_ago_check(activity, groups[1], groups[3], groups[2]))

    return mappings
<commit_msg>Make compatible with new current data test<commit_after>
#  FoXPath, tools for processing FoXPath test suites,
#  by Martin Keegan, Mark Brough and Ben Webb

#  Copyright (C) 2013  Ben Webb
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

import re
import datetime
from functools import partial

def generate_mappings():
    mappings = []

    def add(regex):
        def append_to_mappings(fn):
            mappings.append((re.compile(regex),fn))
            return fn
        return append_to_mappings

    def add_partial(regex):
        def append_to_mappings(fn):
            def partial_fn(groups):
                return partial(fn, groups=groups)
            mappings.append((re.compile(regex), partial_fn))
            return fn
        return append_to_mappings

    def add_partial_with_list(regex):
        def append_to_mappings(fn):
            def partial_fn(groups):
                return partial(fn, groups=groups)
            mappings.append((re.compile(regex), partial_fn))
            return fn
        return append_to_mappings

    @add('(\S*) is an? (.*)\?')
    def is_an(groups):
        if groups[1] == 'iso date':
            return None
        elif groups[1] == 'integer':
            def int_check(x):
                try:
                    int(x)
                    return True
                except ValueError:
                    return False
            def is_an_integer(activity):
                return reduce(lambda x,y: x and y,
                            map(lambda x: int_check(x),
                                    activity.xpath(groups[0])),
                            False)
            return is_an_integer

    @add_partial('(\S*) has more than (\S*) characters\?')
    def text_chars(activity, groups):
        return bool(reduce(lambda x,y: x or y,
                        map(lambda x: len(x)>int(groups[1]),
                            activity.xpath(groups[0])),
                        False))

    def rm_blank(alist):
        return filter(lambda x: x!='', alist)

    @add_partial('(\S*) sum to (\S*)\?')
    def sum(activity, groups):
        return (reduce(lambda x,y: float(x)+float(y),
                           rm_blank(activity.xpath(groups[0])),
                           0)
                   == float(groups[1]))

    @add_partial('(\S*) exists (\S*) times?\?')
    def exist_times(activity, groups):
        return len(rm_blank(activity.xpath(groups[0]))) == int(groups[1])

    @add_partial('(\S*) exists more than (\S*) times?\?')
    def exist_times(activity, groups):
        return len(rm_blank(activity.xpath(groups[0]))) > int(groups[1])

    def exist_check(activity, xpath):
        return bool(rm_blank(activity.xpath(xpath)))

    def exist_check_list(activity, xpath, codelist):
        outcome = False
        try:
            if bool(str(activity.xpath(xpath)[0]) in codelist):
                outcome = True
            elif bool(str(activity.xpath(xpath)[0]).lowercase in codelist):
                outcome = True
            elif bool(str(activity.xpath(xpath)[0]).uppercase in codelist):
                outcome = True
            return outcome
        except Exception:
            return False

    def x_months_ago_check(activity, xpath, months, many=False):
        outcome = False
        months = int(months)
        current_date = datetime.datetime.utcnow()
        if many:
            for check in activity.findall(many):
                try:
                    if ((current_date-datetime.datetime.strptime(check.xpath(xpath)[0], "%Y-%m-%d"))
                        < (datetime.timedelta(days=(30*months)))):
                        outcome = True
                except IndexError:
                    pass
        else:
            try:
                if ((current_date-(datetime.datetime.strptime(activity.xpath(xpath)[0], "%Y-%m-%d")))
                        < (datetime.timedelta(days=(30*months)))):
                    outcome = True
            except IndexError:
                pass
        return outcome

    @add_partial('only one of (\S*) or (\S*) exists\?')
    def exist_xor(activity, groups):
        return (exist_check(activity, groups[0]) !=
                exist_check(activity, groups[1]))

    @add_partial('(\S*) or (\S*) exists\?')
    def exist_or(activity, groups):
        return (exist_check(activity, groups[0]) or
                exist_check(activity, groups[1]))

    @add_partial('(\S*) exists\?')
    def exist(activity, groups):
        return exist_check(activity, groups[0])

    @add_partial_with_list('(\S*) is on list (\S*)\?')
    def exist_list(data, groups):
        return exist_check_list(data['activity'], groups[0], data['lists'][groups[1]])

    @add_partial('(\S*) or (\S*) \(for each (\S*)\) is less than (\S*) months? ago\?')
    def less_than_x_months_ago(activity, groups):
        return (x_months_ago_check(activity, groups[0], groups[3]) or
                x_months_ago_check(activity, groups[1], groups[3], groups[2]))

    @add_partial('(\S*) or (\S*) is less than (\S*) months? ago\?')
    def less_than_x_months_ago(activity, groups):
        return (x_months_ago_check(activity, groups[0], groups[2]) or
                x_months_ago_check(activity, groups[1], groups[2]))

    return mappings
<|endoftext|>


<commit_before>"""
I/O functions: reading and writing molecules.
"""

__author__ = "Steven Kearnes"
__copyright__ = "Copyright 2014, Stanford University"
__license__ = "3-clause BSD"

import gzip
import warnings

from rdkit import Chem
from rdkit.Chem.SaltRemover import SaltRemover

def guess_mol_format(filename):
    """
    Guess molecule file format from filename. Currently supports SDF and
    SMILES.

    Parameters
    ----------
    filename : str
        Filename.
    """
    if filename.endswith(('.sdf', '.sdf.gz')):
        mol_format = 'sdf'
    elif filename.endswith(('.smi', '.smi.gz', '.can', '.can.gz',
                            '.ism', '.ism.gz')):
        mol_format = 'smi'
    else:
        raise NotImplementedError('Unrecognized file format.')
    return mol_format

class MolReader(object):
    """
    Read molecules from files and file-like objects. Supports SDF or SMILES
    format.

    Parameters
    ----------
    remove_salts : bool, optional (default True)
        Whether to remove salts from molecules.
    """
    def __init__(self, remove_salts=True):
        self.remove_salts = remove_salts
        self.salt_remover = SaltRemover()

    def read_mols_from_file(self, filename, mol_format=None):
        """
        Read molecules from a file.

        Parameters
        ----------
        filename : str
            Filename.
        mol_format : str, optional
            Molecule file format. Currently supports 'sdf' and 'smi'. If
            not provided, this method will attempt to infer it from the
            filename.

        Returns
        -------
        A generator yielding multi-conformer Mol objects.
        """
        if mol_format is None:
            mol_format = guess_mol_format(filename)
        if filename.endswith('.gz'):
            f = gzip.open(filename)
        else:
            f = open(filename)
        for mol in self.read_mols(f, mol_format=mol_format):
            yield mol
        f.close()

    def read_mols(self, f, mol_format):
        """
        Read molecules from a file-like object.

        Molecule conformers are grouped into a single molecule. Two
        molecules are considered conformers of the same molecule if they:
        * Are contiguous in the file
        * Have identical (canonical isomeric) SMILES strings
        * Have identical compound names (a warning is issued if compounds
            lack names)

        Parameters
        ----------
        f : file
            File-like object.
        mol_format : str
            Molecule file format. Currently supports 'sdf' and 'smi'.

        Returns
        -------
        A generator yielding multi-conformer Mol objects.
        """
        source = self._read_mols(f, mol_format)
        mol = source.next()
        if mol.HasProp("_Name"):
            mol_name = mol.GetProp("_Name")
        else:
            mol_name = None
        mol_smiles = Chem.MolToSmiles(mol, isomericSmiles=True, canonical=True)
        while True:
            try:
                new = source.next()

                # on error, skip and move to the next multiconformer mol
                if new is None:
                    mol_smiles = None
                    continue
            except StopIteration:
                break
            if new.HasProp("_Name"):
                new_name = new.GetProp("_Name")
            else:
                new_name = None
            new_smiles = Chem.MolToSmiles(new, isomericSmiles=True,
                                          canonical=True)
            assert new_smiles
            if new_smiles == mol_smiles and new_name == mol_name:
                if not new_name:
                    warnings.warn("Grouping conformers of an unnamed " +
                                  "molecule.")
                assert new.GetNumConformers() == 1
                for conf in new.GetConformers():
                    mol.AddConformer(conf)
            else:
                if self.remove_salts:
                    mol = self.salt_remover.StripMol(mol)
                yield mol
                mol = new
                if mol.HasProp("_Name"):
                    mol_name = mol.GetProp("_Name")
                else:
                    mol_name = None
                mol_smiles = Chem.MolToSmiles(mol, isomericSmiles=True,
                                              canonical=True)
        if self.remove_salts:
            mol = self.salt_remover.StripMol(mol)
        yield mol

    def _read_mols(self, f, mol_format):
        """
        Read molecules from a file-like object.

        This method returns individual conformers from a file and does not
        attempt to combine them into multiconformer Mol objects.

        Parameters
        ----------
        f : file
            File-like object.
        mol_format : str
            Molecule file format. Currently supports 'sdf' and 'smi'.

        Returns
        -------
        A generator yielding single-conformer Mol objects.
        """
        if mol_format == 'sdf':
            for mol in Chem.ForwardSDMolSupplier(f):
                yield mol
        elif mol_format == 'smi':
            for line in f.readlines():
                line = line.strip().split()
                if len(line) > 1:
                    smiles, name = line
                else:
                    smiles = line
                    name = None
                mol = Chem.MolFromSmiles(smiles)
                if name is not None:
                    mol.SetProp('_Name', name)
                yield mol
        else:
            raise NotImplementedError('Unrecognized mol_format "{}"'.format(
                mol_format))

def read_mols_from_file(filename, mol_format=None, remove_salts=True):
    """
    Read molecules from a file.

    Parameters
    ----------
    filename : str
        Filename.
    mol_format : str, optional
        Molecule file format. Currently supports 'sdf' and 'smi'. If
        not provided, this method will attempt to infer it from the
        filename.
    remove_salts : bool, optional (default True)
        Whether to remove salts from molecules.
    """
    reader = MolReader(remove_salts=remove_salts)
    for mol in reader.read_mols_from_file(filename, mol_format):
        yield mol

def read_mols(f, mol_format, remove_salts=True):
    """
    Read molecules from a file-like object.

    Parameters
    ----------
    f : file
        File-like object.
    mol_format : str
        Molecule file format. Currently supports 'sdf' and 'smi'.
    remove_salts : bool, optional (default True)
        Whether to remove salts from molecules.
    """
    reader = MolReader(remove_salts=remove_salts)
    for mol in reader.read_mols(f, mol_format):
        yield mol

class MolWriter(object):
    """
    Write molecules to files or file-like objects. Supports SDF or SMILES
    format.

    Parameters
    ----------
    f : file, optional
        File-like object.
    mol_format : str, optional
        Molecule file format. Currently supports 'sdf' and 'smi'.
    """
    def __init__(self, f=None, mol_format=None):
        self.f = f
        self.mol_format = mol_format

    def __del__(self):
        self.close()

    def open(self, filename, mol_format=None):
        """
        Open output file.

        Parameters
        ----------
        filename : str
            Filename.
        mol_format : str, optional
            Molecule file format. Currently supports 'sdf' and 'smi'.
        """
        if filename.endswith('.gz'):
            self.f = gzip.open(filename, 'wb')
        else:
            self.f = open(filename, 'wb')
        if mol_format is None:
            self.mol_format = guess_mol_format(filename)
        else:
            self.mol_format = mol_format

    def close(self):
        """
        Close output file.
        """
        if self.f is not None:
            self.f.close()

    def write(self, mols):
        """
        Write molecules to a file.

        Parameters
        ----------
        mols : iterable
            Molecules to write.
        """
        if self.mol_format == 'sdf':
            w = Chem.SDWriter(self.f)
            for mol in mols:
                if mol.GetNumConformers():
                    for conf in mol.GetConformers():
                        w.write(mol, confId=conf.GetId())
                else:
                    w.write(mol)
            w.close()
        elif self.mol_format == 'smi':
            w = Chem.SmilesWriter(self.f)
            for mol in mols:
                w.write(mol)
            w.close()

def write_mols_to_file(mols, filename, mol_format=None):
    """
    Write molecules to a file.

    Parameters
    ----------
    mols : iterable
        Molecules to write.
    filename : str
        Output filename.
    mol_format : str, optional
        Molecule file format. Currently supports 'sdf' and 'smi'. If
        not provided, this method will attempt to infer it from the
        filename.
    """
    writer = MolWriter()
    writer.open(filename, mol_format)
    writer.write(mols)
    writer.close()

def write_mols(mols, f, mol_format):
    """
    Write molecules to a file-like object.

    Parameters
    ----------
    mols : iterable
        Molecules to write.
    f : file
        File-like object.
    mol_format : str
        Molecule file format. Currently supports 'sdf' and 'smi'.
    """
    writer = MolWriter(f, mol_format)
    writer.write(mols)
<commit_msg>Cleanup MolReader<commit_after>"""
I/O functions: reading and writing molecules.
"""

__author__ = "Steven Kearnes"
__copyright__ = "Copyright 2014, Stanford University"
__license__ = "3-clause BSD"

import gzip

from rdkit import Chem
from rdkit.Chem.SaltRemover import SaltRemover

def guess_mol_format(filename):
    """
    Guess molecule file format from filename. Currently supports SDF and
    SMILES.

    Parameters
    ----------
    filename : str
        Filename.
    """
    if filename.endswith(('.sdf', '.sdf.gz')):
        mol_format = 'sdf'
    elif filename.endswith(('.smi', '.smi.gz', '.can', '.can.gz',
                            '.ism', '.ism.gz')):
        mol_format = 'smi'
    else:
        raise NotImplementedError('Unrecognized file format.')
    return mol_format

class MolReader(object):
    """
    Read molecules from files and file-like objects. Supports SDF or SMILES
    format.

    Parameters
    ----------
    remove_salts : bool, optional (default True)
        Whether to remove salts from molecules.
    """
    def __init__(self, remove_salts=True):
        self.remove_salts = remove_salts
        self.salt_remover = SaltRemover()

    def clean_mol(self, mol):
        """
        Clean a molecule.

        Parameters
        ----------
        mol : RDKit Mol
            Molecule.
        """
        if self.remove_salts:
            mol = self.salt_remover.StripMol(mol)
        return mol

    def read_mols_from_file(self, filename, mol_format=None):
        """
        Read molecules from a file.

        Parameters
        ----------
        filename : str
            Filename.
        mol_format : str, optional
            Molecule file format. Currently supports 'sdf' and 'smi'. If
            not provided, this method will attempt to infer it from the
            filename.

        Returns
        -------
        A generator yielding multi-conformer Mol objects.
        """
        if mol_format is None:
            mol_format = guess_mol_format(filename)
        if filename.endswith('.gz'):
            f = gzip.open(filename)
        else:
            f = open(filename)
        for mol in self.read_mols(f, mol_format=mol_format):
            yield mol
        f.close()

    def read_mols(self, f, mol_format):
        """
        Read molecules from a file-like object.

        Molecule conformers are grouped into a single molecule. Two
        molecules are considered conformers of the same molecule if they:
        * Are contiguous in the file
        * Have identical (canonical isomeric) SMILES strings
        * Have identical compound names (if set)

        Parameters
        ----------
        f : file
            File-like object.
        mol_format : str
            Molecule file format. Currently supports 'sdf' and 'smi'.

        Returns
        -------
        A generator yielding multi-conformer Mol objects.
        """
        source = self._read_mols(f, mol_format)
        mol = source.next()
        for new in source:
            if self.is_same_molecule(mol, new):
                assert new.GetNumConformers() == 1
                for conf in new.GetConformers():
                    mol.AddConformer(conf)
            else:
                mol = self.clean_mol(mol)
                yield mol
                mol = new
        mol = self.clean_mol(mol)
        yield mol

    def is_same_molecule(self, a, b):
        """
        Test whether two molecules are conformers of the same molecule.

        Test for:
        * Identical (canonical isomeric) SMILES strings
        * Identical compound names (if set)

        Parameters
        ----------
        a, b : RDKit Mol
            Molecules to compare.
        """
        a_name = None
        if a.HasProp('_Name'):
            a_name = a.GetProp('_Name')
        b_name = None
        if b.HasProp('_Name'):
            b_name = b.GetProp('_Name')
        a_smiles = Chem.MolToSmiles(a, isomericSmiles=True, canonical=True)
        b_smiles = Chem.MolToSmiles(b, isomericSmiles=True, canonical=True)
        assert a_smiles and b_smiles
        return a_smiles == b_smiles and a_name == b_name

    def _read_mols(self, f, mol_format):
        """
        Read molecules from a file-like object.

        This method returns individual conformers from a file and does not
        attempt to combine them into multiconformer Mol objects.

        Parameters
        ----------
        f : file
            File-like object.
        mol_format : str
            Molecule file format. Currently supports 'sdf' and 'smi'.

        Returns
        -------
        A generator yielding single-conformer Mol objects.
        """
        if mol_format == 'sdf':
            for mol in Chem.ForwardSDMolSupplier(f):
                yield mol
        elif mol_format == 'smi':
            for line in f.readlines():
                line = line.strip().split()
                if len(line) > 1:
                    smiles, name = line
                else:
                    smiles = line
                    name = None
                mol = Chem.MolFromSmiles(smiles)
                if name is not None:
                    mol.SetProp('_Name', name)
                yield mol
        else:
            raise NotImplementedError('Unrecognized mol_format "{}"'.format(
                mol_format))

class MolWriter(object):
    """
    Write molecules to files or file-like objects. Supports SDF or SMILES
    format.

    Parameters
    ----------
    f : file, optional
        File-like object.
    mol_format : str, optional
        Molecule file format. Currently supports 'sdf' and 'smi'.
    """
    def __init__(self, f=None, mol_format=None):
        self.f = f
        self.mol_format = mol_format

    def __del__(self):
        self.close()

    def open(self, filename, mol_format=None):
        """
        Open output file.

        Parameters
        ----------
        filename : str
            Filename.
        mol_format : str, optional
            Molecule file format. Currently supports 'sdf' and 'smi'.
        """
        if filename.endswith('.gz'):
            self.f = gzip.open(filename, 'wb')
        else:
            self.f = open(filename, 'wb')
        if mol_format is None:
            self.mol_format = guess_mol_format(filename)
        else:
            self.mol_format = mol_format

    def close(self):
        """
        Close output file.
        """
        if self.f is not None:
            self.f.close()

    def write(self, mols):
        """
        Write molecules to a file.

        Parameters
        ----------
        mols : iterable
            Molecules to write.
        """
        if self.mol_format == 'sdf':
            w = Chem.SDWriter(self.f)
            for mol in mols:
                if mol.GetNumConformers():
                    for conf in mol.GetConformers():
                        w.write(mol, confId=conf.GetId())
                else:
                    w.write(mol)
            w.close()
        elif self.mol_format == 'smi':
            w = Chem.SmilesWriter(self.f)
            for mol in mols:
                w.write(mol)
            w.close()

def read_mols_from_file(filename, mol_format=None, remove_salts=True):
    """
    Read molecules from a file.

    Parameters
    ----------
    filename : str
        Filename.
    mol_format : str, optional
        Molecule file format. Currently supports 'sdf' and 'smi'. If
        not provided, this method will attempt to infer it from the
        filename.
    remove_salts : bool, optional (default True)
        Whether to remove salts from molecules.
    """
    reader = MolReader(remove_salts=remove_salts)
    for mol in reader.read_mols_from_file(filename, mol_format):
        yield mol

def read_mols(f, mol_format, remove_salts=True):
    """
    Read molecules from a file-like object.

    Parameters
    ----------
    f : file
        File-like object.
    mol_format : str
        Molecule file format. Currently supports 'sdf' and 'smi'.
    remove_salts : bool, optional (default True)
        Whether to remove salts from molecules.
    """
    reader = MolReader(remove_salts=remove_salts)
    for mol in reader.read_mols(f, mol_format):
        yield mol

def write_mols_to_file(mols, filename, mol_format=None):
    """
    Write molecules to a file.

    Parameters
    ----------
    mols : iterable
        Molecules to write.
    filename : str
        Output filename.
    mol_format : str, optional
        Molecule file format. Currently supports 'sdf' and 'smi'. If
        not provided, this method will attempt to infer it from the
        filename.
    """
    writer = MolWriter()
    writer.open(filename, mol_format)
    writer.write(mols)
    writer.close()

def write_mols(mols, f, mol_format):
    """
    Write molecules to a file-like object.

    Parameters
    ----------
    mols : iterable
        Molecules to write.
    f : file
        File-like object.
    mol_format : str
        Molecule file format. Currently supports 'sdf' and 'smi'.
    """
    writer = MolWriter(f, mol_format)
    writer.write(mols)
<|endoftext|>


<commit_before>from flask import Flask, Markup, render_template, request
import os
import json
import httplib
import urllib
import re

import indexer, config, async

app = Flask('webapp', template_folder=os.getcwd() + "/templates")

@app.route("/")
def index():
    return render_template("index.html", peers=config.neighbors, port=config.webapp)

@app.route("/query", methods=["GET"])
def query():
    return indexer.search(re.match("search=(.*)", request.data).group(1))

@app.route("/neighbors", methods=['GET'])
def neighbors():
    print config.neighbors
    print request.remote_addr
    if not request.remote_addr in config.neighbors:
        async.event.asynchronous_callback(
            client.SandwichGetter.bootstrap_into_network,
            (request.remote_addr))
    return json.dumps(config.neighbors)

@app.route("/search", methods=["GET"])
def search():
    x = ""
    if not request.args.get("host"):
        for n in config.neighbors:
            conn = httplib.HTTPConnection("%s:%d" % (n, config.webapp),
                                          timeout=config.timeout)
            conn.request("GET", "/query",
                         urllib.urlencode({'search': request.args.get("search")}))
            x = conn.getresponse().read()
            conn.close()
    else:
        conn = httplib.HTTPConnection("%s:%d" % (request.args.get("host"),
                                                 config.webapp), timeout=config.timeout)

        conn.request("GET", "/query", urllib.urlencode({'search': ""}))
        x = conn.getresponse().read()
        conn.close()
    return x

def run():

    app.debug = config.debug
    # flask is dumb and tries to restart infinitely if we fork it into a subprocess.
    # this is bad. So we disable it, and hate on flask a bit.
    app.run(port=config.webapp, use_reloader=False, host='0.0.0.0')

if __name__ == '__main__':
    app.debug = True
    app.run()
<commit_msg>imports client<commit_after>from flask import Flask, Markup, render_template, request
import os
import json
import httplib
import urllib
import re

import indexer, config, async, client

app = Flask('webapp', template_folder=os.getcwd() + "/templates")

@app.route("/")
def index():
    return render_template("index.html", peers=config.neighbors, port=config.webapp)

@app.route("/query", methods=["GET"])
def query():
    return indexer.search(re.match("search=(.*)", request.data).group(1))

@app.route("/neighbors", methods=['GET'])
def neighbors():
    print config.neighbors
    print request.remote_addr
    if not request.remote_addr in config.neighbors:
        async.event.asynchronous_callback(
            client.SandwichGetter.bootstrap_into_network,
            (request.remote_addr))
    return json.dumps(config.neighbors)

@app.route("/search", methods=["GET"])
def search():
    x = ""
    if not request.args.get("host"):
        for n in config.neighbors:
            conn = httplib.HTTPConnection("%s:%d" % (n, config.webapp),
                                          timeout=config.timeout)
            conn.request("GET", "/query",
                         urllib.urlencode({'search': request.args.get("search")}))
            x = conn.getresponse().read()
            conn.close()
    else:
        conn = httplib.HTTPConnection("%s:%d" % (request.args.get("host"),
                                                 config.webapp), timeout=config.timeout)

        conn.request("GET", "/query", urllib.urlencode({'search': ""}))
        x = conn.getresponse().read()
        conn.close()
    return x

def run():

    app.debug = config.debug
    # flask is dumb and tries to restart infinitely if we fork it into a subprocess.
    # this is bad. So we disable it, and hate on flask a bit.
    app.run(port=config.webapp, use_reloader=False, host='0.0.0.0')

if __name__ == '__main__':
    app.debug = True
    app.run()
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import codecs
import ConfigParser as configparser
import logging
import optparse
import os
import signal
import StringIO
import sys

import gobject
gobject.threads_init()

import pkg_resources
import pykka.debug

# Extract any command line arguments. This needs to be done before GStreamer is
# imported, so that GStreamer doesn't hijack e.g. ``--help``.
mopidy_args = sys.argv[1:]
sys.argv[1:] = []

# Add ../ to the path so we can run Mopidy from a Git checkout without
# installing it on the system.
sys.path.insert(
    0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))

from mopidy import exceptions, settings
from mopidy.audio import Audio
from mopidy.config import default_config, config_schemas
from mopidy.core import Core
from mopidy.utils import (
    config as config_utils, deps, log, path, process, versioning)

logger = logging.getLogger('mopidy.main')

def main():
    signal.signal(signal.SIGTERM, process.exit_handler)
    signal.signal(signal.SIGUSR1, pykka.debug.log_thread_tracebacks)

    loop = gobject.MainLoop()
    options = parse_options()
    config_files = options.config.split(':')
    config_overrides = options.overrides

    try:
        logging_config = load_config(config_files, config_overrides)
        log.setup_logging(
            logging_config, options.verbosity_level, options.save_debug_log)
        extensions = load_extensions()
        raw_config = load_config(config_files, config_overrides, extensions)
        extensions = filter_enabled_extensions(raw_config, extensions)
        config = validate_config(raw_config, config_schemas, extensions)
        log.setup_log_levels(config)
        check_old_folders()
        setup_settings()
        # Anything that wants to exit after this point must use
        # process.exit_proces as actors have been started.
        audio = setup_audio(config)
        backends = setup_backends(config, extensions, audio)
        core = setup_core(audio, backends)
        setup_frontends(config, extensions, core)
        loop.run()
    except exceptions.SettingsError as ex:
        logger.error(ex.message)
    except KeyboardInterrupt:
        logger.info('Interrupted. Exiting...')
    except Exception as ex:
        logger.exception(ex)
    finally:
        loop.quit()
        stop_frontends(extensions)
        stop_core()
        stop_backends(extensions)
        stop_audio()
        process.stop_remaining_actors()

def check_config_override(option, opt, override):
    try:
        section, remainder = override.split('/', 1)
        key, value = remainder.split('=', 1)
        return (section, key, value)
    except ValueError:
        raise optparse.OptionValueError(
            'option %s: must have the format section/key=value' % opt)

def parse_options():
    parser = optparse.OptionParser(
        version='Mopidy %s' % versioning.get_version())

    # Ugly extension of optparse type checking magic :/
    optparse.Option.TYPES += ('config_override',)
    optparse.Option.TYPE_CHECKER['config_override'] = check_config_override

    # NOTE First argument to add_option must be bytestrings on Python < 2.6.2
    # See https://github.com/mopidy/mopidy/issues/302 for details
    parser.add_option(
        b'-q', '--quiet',
        action='store_const', const=0, dest='verbosity_level',
        help='less output (warning level)')
    parser.add_option(
        b'-v', '--verbose',
        action='count', default=1, dest='verbosity_level',
        help='more output (debug level)')
    parser.add_option(
        b'--save-debug-log',
        action='store_true', dest='save_debug_log',
        help='save debug log to "./mopidy.log"')
    parser.add_option(
        b'--show-config',
        action='callback', callback=show_config_callback,
        help='show current config')
    parser.add_option(
        b'--list-deps',
        action='callback', callback=deps.list_deps_optparse_callback,
        help='list dependencies and their versions')
    parser.add_option(
        b'--config',
        action='store', dest='config',
        default='/etc/mopidy/mopidy.conf:$XDG_CONFIG_DIR/mopidy/mopidy.conf',
        help='config files to use, colon seperated, later files override')
    parser.add_option(
        b'-o', b'--option',
        action='append', dest='overrides', type='config_override',
        help='`section/key=value` values to override config options')
    return parser.parse_args(args=mopidy_args)[0]

def show_config_callback(option, opt, value, parser):
    # TODO: don't use callback for this as --config or -o set after
    # --show-config will be ignored.
    files = getattr(parser.values, 'config', '').split(':')
    overrides = getattr(parser.values, 'overrides', [])

    extensions = load_extensions()
    raw_config = load_config(files, overrides, extensions)
    enabled_extensions = filter_enabled_extensions(raw_config, extensions)
    config = validate_config(raw_config, config_schemas, enabled_extensions)

    output = []
    for section_name, schema in config_schemas.items():
        options = config.get(section_name, {})
        if not options:
            continue
        output.append(schema.format(section_name, options))

    for extension in extensions:
        if extension in enabled_extensions:
            schema = extension.get_config_schema()
            options = config.get(extension.ext_name, {})
            output.append(schema.format(extension.ext_name, options))
        else:
            lines = ['[%s]' % extension.ext_name, 'enabled = false',
                     '# Config hidden as extension is disabled']
            output.append('\n'.join(lines))

    print '\n\n'.join(output)
    sys.exit(0)

def check_old_folders():
    # TODO: add old settings and pre extension storage locations?
    old_settings_folder = os.path.expanduser('~/.mopidy')

    if not os.path.isdir(old_settings_folder):
        return

    logger.warning(
        'Old settings folder found at %s, settings.py should be moved '
        'to %s, any cache data should be deleted. See release notes for '
        'further instructions.', old_settings_folder, path.SETTINGS_PATH)

def load_extensions():
    extensions = []
    for entry_point in pkg_resources.iter_entry_points('mopidy.ext'):
        logger.debug('Loading entry point: %s', entry_point)

        try:
            extension_class = entry_point.load()
        except pkg_resources.DistributionNotFound as ex:
            logger.info(
                'Disabled extension %s: Dependency %s not found',
                entry_point.name, ex)
            continue

        extension = extension_class()

        logger.debug(
            'Loaded extension: %s %s', extension.dist_name, extension.version)

        if entry_point.name != extension.ext_name:
            logger.warning(
                'Disabled extension %(ep)s: entry point name (%(ep)s) '
                'does not match extension name (%(ext)s)',
                {'ep': entry_point.name, 'ext': extension.ext_name})
            continue

        try:
            extension.validate_environment()
        except exceptions.ExtensionError as ex:
            logger.info(
                'Disabled extension %s: %s', entry_point.name, ex.message)
            continue

        extensions.append(extension)

    names = (e.ext_name for e in extensions)
    logging.debug('Discovered extensions: %s', ', '.join(names))
    return extensions

def filter_enabled_extensions(raw_config, extensions):
    boolean = config_utils.Boolean()
    enabled_extensions = []
    enabled_names = []
    disabled_names = []

    for extension in extensions:
        # TODO: handle key and value errors.
        enabled = raw_config[extension.ext_name]['enabled']
        if boolean.deserialize(enabled):
            enabled_extensions.append(extension)
            enabled_names.append(extension.ext_name)
        else:
            disabled_names.append(extension.ext_name)

    logging.info('Enabled extensions: %s', ', '.join(enabled_names))
    logging.info('Disabled extensions: %s', ', '.join(disabled_names))
    return enabled_extensions

def load_config(files, overrides, extensions=None):
    parser = configparser.RawConfigParser()

    files = [path.expand_path(f) for f in files]
    sources = ['builtin-defaults'] + files + ['command-line']
    logger.info('Loading config from: %s', ', '.join(sources))

    # Read default core config
    parser.readfp(StringIO.StringIO(default_config))

    # Read default extension config
    for extension in extensions or []:
        parser.readfp(StringIO.StringIO(extension.get_default_config()))

    # Load config from a series of config files
    for filename in files:
        # TODO: if this is the initial load of logging config we might not have
        # a logger at this point, we might want to handle this better.
        try:
            filehandle = codecs.open(filename, encoding='utf-8')
            parser.readfp(filehandle)
        except IOError:
            logger.debug('Config file %s not found; skipping', filename)
            continue
        except UnicodeDecodeError:
            logger.error('Config file %s is not UTF-8 encoded', filename)
            sys.exit(1)

    raw_config = {}
    for section in parser.sections():
        raw_config[section] = dict(parser.items(section))

    for section, key, value in overrides or []:
        raw_config.setdefault(section, {})[key] = value

    return raw_config

def validate_config(raw_config, schemas, extensions=None):
    # Collect config schemas to validate against
    sections_and_schemas = schemas.items()
    for extension in extensions or []:
        sections_and_schemas.append(
            (extension.ext_name, extension.get_config_schema()))

    # Get validated config
    config = {}
    errors = {}
    for section_name, schema in sections_and_schemas:
        if section_name not in raw_config:
            errors[section_name] = {section_name: 'section not found'}
        try:
            items = raw_config[section_name].items()
            config[section_name] = schema.convert(items)
        except exceptions.ConfigError as error:
            errors[section_name] = error

    if errors:
        for section_name, error in errors.items():
            logger.error('[%s] config errors:', section_name)
            for key in error:
                logger.error('%s %s', key, error[key])
        sys.exit(1)

    return config

def setup_settings():
    path.get_or_create_folder(path.SETTINGS_PATH)
    path.get_or_create_folder(path.DATA_PATH)
    path.get_or_create_file(path.SETTINGS_FILE)
    try:
        settings.validate()
    except exceptions.SettingsError as ex:
        logger.error(ex.message)
        sys.exit(1)

def setup_audio(config):
    logger.info('Starting Mopidy audio')
    return Audio.start(config=config).proxy()

def stop_audio():
    logger.info('Stopping Mopidy audio')
    process.stop_actors_by_class(Audio)

def setup_backends(config, extensions, audio):
    logger.info('Starting Mopidy backends')
    backends = []
    for extension in extensions:
        for backend_class in extension.get_backend_classes():
            backend = backend_class.start(config=config, audio=audio).proxy()
            backends.append(backend)
    return backends

def stop_backends(extensions):
    logger.info('Stopping Mopidy backends')
    for extension in extensions:
        for backend_class in extension.get_backend_classes():
            process.stop_actors_by_class(backend_class)

def setup_core(audio, backends):
    logger.info('Starting Mopidy core')
    return Core.start(audio=audio, backends=backends).proxy()

def stop_core():
    logger.info('Stopping Mopidy core')
    process.stop_actors_by_class(Core)

def setup_frontends(config, extensions, core):
    logger.info('Starting Mopidy frontends')
    for extension in extensions:
        for frontend_class in extension.get_frontend_classes():
            frontend_class.start(config=config, core=core)

def stop_frontends(extensions):
    logger.info('Stopping Mopidy frontends')
    for extension in extensions:
        for frontend_class in extension.get_frontend_classes():
            process.stop_actors_by_class(frontend_class)

if __name__ == '__main__':
    main()
<commit_msg>main: Make it more obvious what helper to use for exiting<commit_after>from __future__ import unicode_literals

import codecs
import ConfigParser as configparser
import logging
import optparse
import os
import signal
import StringIO
import sys

import gobject
gobject.threads_init()

import pkg_resources
import pykka.debug

# Extract any command line arguments. This needs to be done before GStreamer is
# imported, so that GStreamer doesn't hijack e.g. ``--help``.
mopidy_args = sys.argv[1:]
sys.argv[1:] = []

# Add ../ to the path so we can run Mopidy from a Git checkout without
# installing it on the system.
sys.path.insert(
    0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))

from mopidy import exceptions, settings
from mopidy.audio import Audio
from mopidy.config import default_config, config_schemas
from mopidy.core import Core
from mopidy.utils import (
    config as config_utils, deps, log, path, process, versioning)

logger = logging.getLogger('mopidy.main')

def main():
    signal.signal(signal.SIGTERM, process.exit_handler)
    signal.signal(signal.SIGUSR1, pykka.debug.log_thread_tracebacks)

    loop = gobject.MainLoop()
    options = parse_options()
    config_files = options.config.split(':')
    config_overrides = options.overrides

    try:
        logging_config = load_config(config_files, config_overrides)
        log.setup_logging(
            logging_config, options.verbosity_level, options.save_debug_log)
        extensions = load_extensions()
        raw_config = load_config(config_files, config_overrides, extensions)
        extensions = filter_enabled_extensions(raw_config, extensions)
        config = validate_config(raw_config, config_schemas, extensions)
        log.setup_log_levels(config)
        check_old_folders()
        setup_settings()
        # Anything that wants to exit after this point must use
        # mopidy.utils.process.exit_process as actors have been started.
        audio = setup_audio(config)
        backends = setup_backends(config, extensions, audio)
        core = setup_core(audio, backends)
        setup_frontends(config, extensions, core)
        loop.run()
    except exceptions.SettingsError as ex:
        logger.error(ex.message)
    except KeyboardInterrupt:
        logger.info('Interrupted. Exiting...')
    except Exception as ex:
        logger.exception(ex)
    finally:
        loop.quit()
        stop_frontends(extensions)
        stop_core()
        stop_backends(extensions)
        stop_audio()
        process.stop_remaining_actors()

def check_config_override(option, opt, override):
    try:
        section, remainder = override.split('/', 1)
        key, value = remainder.split('=', 1)
        return (section, key, value)
    except ValueError:
        raise optparse.OptionValueError(
            'option %s: must have the format section/key=value' % opt)

def parse_options():
    parser = optparse.OptionParser(
        version='Mopidy %s' % versioning.get_version())

    # Ugly extension of optparse type checking magic :/
    optparse.Option.TYPES += ('config_override',)
    optparse.Option.TYPE_CHECKER['config_override'] = check_config_override

    # NOTE First argument to add_option must be bytestrings on Python < 2.6.2
    # See https://github.com/mopidy/mopidy/issues/302 for details
    parser.add_option(
        b'-q', '--quiet',
        action='store_const', const=0, dest='verbosity_level',
        help='less output (warning level)')
    parser.add_option(
        b'-v', '--verbose',
        action='count', default=1, dest='verbosity_level',
        help='more output (debug level)')
    parser.add_option(
        b'--save-debug-log',
        action='store_true', dest='save_debug_log',
        help='save debug log to "./mopidy.log"')
    parser.add_option(
        b'--show-config',
        action='callback', callback=show_config_callback,
        help='show current config')
    parser.add_option(
        b'--list-deps',
        action='callback', callback=deps.list_deps_optparse_callback,
        help='list dependencies and their versions')
    parser.add_option(
        b'--config',
        action='store', dest='config',
        default='/etc/mopidy/mopidy.conf:$XDG_CONFIG_DIR/mopidy/mopidy.conf',
        help='config files to use, colon seperated, later files override')
    parser.add_option(
        b'-o', b'--option',
        action='append', dest='overrides', type='config_override',
        help='`section/key=value` values to override config options')
    return parser.parse_args(args=mopidy_args)[0]

def show_config_callback(option, opt, value, parser):
    # TODO: don't use callback for this as --config or -o set after
    # --show-config will be ignored.
    files = getattr(parser.values, 'config', '').split(':')
    overrides = getattr(parser.values, 'overrides', [])

    extensions = load_extensions()
    raw_config = load_config(files, overrides, extensions)
    enabled_extensions = filter_enabled_extensions(raw_config, extensions)
    config = validate_config(raw_config, config_schemas, enabled_extensions)

    output = []
    for section_name, schema in config_schemas.items():
        options = config.get(section_name, {})
        if not options:
            continue
        output.append(schema.format(section_name, options))

    for extension in extensions:
        if extension in enabled_extensions:
            schema = extension.get_config_schema()
            options = config.get(extension.ext_name, {})
            output.append(schema.format(extension.ext_name, options))
        else:
            lines = ['[%s]' % extension.ext_name, 'enabled = false',
                     '# Config hidden as extension is disabled']
            output.append('\n'.join(lines))

    print '\n\n'.join(output)
    sys.exit(0)

def check_old_folders():
    # TODO: add old settings and pre extension storage locations?
    old_settings_folder = os.path.expanduser('~/.mopidy')

    if not os.path.isdir(old_settings_folder):
        return

    logger.warning(
        'Old settings folder found at %s, settings.py should be moved '
        'to %s, any cache data should be deleted. See release notes for '
        'further instructions.', old_settings_folder, path.SETTINGS_PATH)

def load_extensions():
    extensions = []
    for entry_point in pkg_resources.iter_entry_points('mopidy.ext'):
        logger.debug('Loading entry point: %s', entry_point)

        try:
            extension_class = entry_point.load()
        except pkg_resources.DistributionNotFound as ex:
            logger.info(
                'Disabled extension %s: Dependency %s not found',
                entry_point.name, ex)
            continue

        extension = extension_class()

        logger.debug(
            'Loaded extension: %s %s', extension.dist_name, extension.version)

        if entry_point.name != extension.ext_name:
            logger.warning(
                'Disabled extension %(ep)s: entry point name (%(ep)s) '
                'does not match extension name (%(ext)s)',
                {'ep': entry_point.name, 'ext': extension.ext_name})
            continue

        try:
            extension.validate_environment()
        except exceptions.ExtensionError as ex:
            logger.info(
                'Disabled extension %s: %s', entry_point.name, ex.message)
            continue

        extensions.append(extension)

    names = (e.ext_name for e in extensions)
    logging.debug('Discovered extensions: %s', ', '.join(names))
    return extensions

def filter_enabled_extensions(raw_config, extensions):
    boolean = config_utils.Boolean()
    enabled_extensions = []
    enabled_names = []
    disabled_names = []

    for extension in extensions:
        # TODO: handle key and value errors.
        enabled = raw_config[extension.ext_name]['enabled']
        if boolean.deserialize(enabled):
            enabled_extensions.append(extension)
            enabled_names.append(extension.ext_name)
        else:
            disabled_names.append(extension.ext_name)

    logging.info('Enabled extensions: %s', ', '.join(enabled_names))
    logging.info('Disabled extensions: %s', ', '.join(disabled_names))
    return enabled_extensions

def load_config(files, overrides, extensions=None):
    parser = configparser.RawConfigParser()

    files = [path.expand_path(f) for f in files]
    sources = ['builtin-defaults'] + files + ['command-line']
    logger.info('Loading config from: %s', ', '.join(sources))

    # Read default core config
    parser.readfp(StringIO.StringIO(default_config))

    # Read default extension config
    for extension in extensions or []:
        parser.readfp(StringIO.StringIO(extension.get_default_config()))

    # Load config from a series of config files
    for filename in files:
        # TODO: if this is the initial load of logging config we might not have
        # a logger at this point, we might want to handle this better.
        try:
            filehandle = codecs.open(filename, encoding='utf-8')
            parser.readfp(filehandle)
        except IOError:
            logger.debug('Config file %s not found; skipping', filename)
            continue
        except UnicodeDecodeError:
            logger.error('Config file %s is not UTF-8 encoded', filename)
            sys.exit(1)

    raw_config = {}
    for section in parser.sections():
        raw_config[section] = dict(parser.items(section))

    for section, key, value in overrides or []:
        raw_config.setdefault(section, {})[key] = value

    return raw_config

def validate_config(raw_config, schemas, extensions=None):
    # Collect config schemas to validate against
    sections_and_schemas = schemas.items()
    for extension in extensions or []:
        sections_and_schemas.append(
            (extension.ext_name, extension.get_config_schema()))

    # Get validated config
    config = {}
    errors = {}
    for section_name, schema in sections_and_schemas:
        if section_name not in raw_config:
            errors[section_name] = {section_name: 'section not found'}
        try:
            items = raw_config[section_name].items()
            config[section_name] = schema.convert(items)
        except exceptions.ConfigError as error:
            errors[section_name] = error

    if errors:
        for section_name, error in errors.items():
            logger.error('[%s] config errors:', section_name)
            for key in error:
                logger.error('%s %s', key, error[key])
        sys.exit(1)

    return config

def setup_settings():
    path.get_or_create_folder(path.SETTINGS_PATH)
    path.get_or_create_folder(path.DATA_PATH)
    path.get_or_create_file(path.SETTINGS_FILE)
    try:
        settings.validate()
    except exceptions.SettingsError as ex:
        logger.error(ex.message)
        sys.exit(1)

def setup_audio(config):
    logger.info('Starting Mopidy audio')
    return Audio.start(config=config).proxy()

def stop_audio():
    logger.info('Stopping Mopidy audio')
    process.stop_actors_by_class(Audio)

def setup_backends(config, extensions, audio):
    logger.info('Starting Mopidy backends')
    backends = []
    for extension in extensions:
        for backend_class in extension.get_backend_classes():
            backend = backend_class.start(config=config, audio=audio).proxy()
            backends.append(backend)
    return backends

def stop_backends(extensions):
    logger.info('Stopping Mopidy backends')
    for extension in extensions:
        for backend_class in extension.get_backend_classes():
            process.stop_actors_by_class(backend_class)

def setup_core(audio, backends):
    logger.info('Starting Mopidy core')
    return Core.start(audio=audio, backends=backends).proxy()

def stop_core():
    logger.info('Stopping Mopidy core')
    process.stop_actors_by_class(Core)

def setup_frontends(config, extensions, core):
    logger.info('Starting Mopidy frontends')
    for extension in extensions:
        for frontend_class in extension.get_frontend_classes():
            frontend_class.start(config=config, core=core)

def stop_frontends(extensions):
    logger.info('Stopping Mopidy frontends')
    for extension in extensions:
        for frontend_class in extension.get_frontend_classes():
            process.stop_actors_by_class(frontend_class)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from arches.app.utils.betterJSONSerializer import JSONDeserializer
from arches.app.search.elasticsearch_dsl_builder import Bool, Terms
from arches.app.search.components.base import BaseSearchFilter
from arches.app.models.models import Node
from arches.app.utils.permission_backend import get_resource_types_by_perm

details = {
    "searchcomponentid": "",
    "name": "Resource Type Filter",
    "icon": "",
    "modulename": "resource_type_filter.py",
    "classname": "ResourceTypeFilter",
    "type": "resource-type-filter",
    "componentpath": "views/components/search/resource-type-filter",
    "componentname": "resource-type-filter",
    "sortorder": "0",
    "enabled": True,
}

def get_permitted_graphids(permitted_nodegroups):
    permitted_graphids = set()
    for node in Node.objects.filter(nodegroup__in=permitted_nodegroups):
        permitted_graphids.add(str(node.graph_id))
    return permitted_graphids

class ResourceTypeFilter(BaseSearchFilter):
    def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional):
        search_query = Bool()
        querystring_params = self.request.GET.get(details["componentname"], "")
        graph_ids = []
        permitted_graphids = get_permitted_graphids(permitted_nodegroups)

        for resourceTypeFilter in JSONDeserializer().deserialize(querystring_params):
            graphid = str(resourceTypeFilter["graphid"])
            if resourceTypeFilter["inverted"] is True:
                try:
                    permitted_graphids.remove(graphid)
                except KeyError:
                    pass
            else:
                if graphid in permitted_graphids:
                    graph_ids.append(graphid)

        if resourceTypeFilter["inverted"] is True:
            terms = Terms(field="graph_id", terms=list(permitted_graphids))
        else:
            terms = Terms(field="graph_id", terms=graph_ids)

        search_query.filter(terms)

        search_results_object["query"].add_query(search_query)

    def view_data(self):
        return {"resources": get_resource_types_by_perm(self.request.user, "read_nodegroup")}
<commit_msg>Fixing style errors.<commit_after>from arches.app.utils.betterJSONSerializer import JSONDeserializer
from arches.app.search.elasticsearch_dsl_builder import Bool, Terms
from arches.app.search.components.base import BaseSearchFilter
from arches.app.models.models import Node
from arches.app.utils.permission_backend import get_resource_types_by_perm

details = {
    "searchcomponentid": "",
    "name": "Resource Type Filter",
    "icon": "",
    "modulename": "resource_type_filter.py",
    "classname": "ResourceTypeFilter",
    "type": "resource-type-filter",
    "componentpath": "views/components/search/resource-type-filter",
    "componentname": "resource-type-filter",
    "sortorder": "0",
    "enabled": True,
}

def get_permitted_graphids(permitted_nodegroups):
    permitted_graphids = set()
    for node in Node.objects.filter(nodegroup__in=permitted_nodegroups):
        permitted_graphids.add(str(node.graph_id))
    return permitted_graphids

class ResourceTypeFilter(BaseSearchFilter):
    def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional):
        search_query = Bool()
        querystring_params = self.request.GET.get(details["componentname"], "")
        graph_ids = []
        permitted_graphids = get_permitted_graphids(permitted_nodegroups)

        for resourceTypeFilter in JSONDeserializer().deserialize(querystring_params):
            graphid = str(resourceTypeFilter["graphid"])
            if resourceTypeFilter["inverted"] is True:
                try:
                    permitted_graphids.remove(graphid)
                except KeyError:
                    pass
            else:
                if graphid in permitted_graphids:
                    graph_ids.append(graphid)

        if resourceTypeFilter["inverted"] is True:
            terms = Terms(field="graph_id", terms=list(permitted_graphids))
        else:
            terms = Terms(field="graph_id", terms=graph_ids)

        search_query.filter(terms)

        search_results_object["query"].add_query(search_query)

    def view_data(self):
        return {"resources": get_resource_types_by_perm(self.request.user, "read_nodegroup")}
<|endoftext|>


<commit_before>import argparse
import time
import datetime
import json
import os
from urllib import urlencode
from urllib2 import urlopen, HTTPError, URLError

def parse_args():
    parser = argparse.ArgumentParser(description="Download all images from a subreddit")
    parser.add_argument('subreddit', help="The subreddit to download")
    parser.add_argument('-s', '--score', help='The minimum number of points a picture must have', type=int, default=0)
    parser.add_argument('-d', '--days', help="The number of days in the past to search through", type=int, default=365)
    parser.add_argument('-o', '--output', help="The directory to download into", type=str, default='downloads')
    parser.add_argument('-t', '--target', help="(Advanced) The target number of posts to download in each interval", type=int, default=20)
    parser.add_argument('-k', '--coefficient', help="(Advanced) The coefficient of change", type=float, default=2.0)
    return parser.parse_args()

def get_secs(t):
    return int(time.mktime(t.timetuple()))

def request(url, *ar, **kwa):
    _retries = kwa.pop('_retries', 4)
    _retry_pause = kwa.pop('_retry_pause', 0)
    res = None
    for _try in xrange(_retries):
        try:
            res = urlopen(url, *ar, **kwa)
        except Exception as exc:
            if _try == _retries - 1:
                return None
            print("Try %r err %r  (%r)" % (
                _try, exc, url))
        else:
            break
    return res

def do_search(subreddit, start, end):
    url = "https://reddit.com/{}/search.json?q=timestamp:{}..{}&sort=top&restrict_sr=on&syntax=cloudsearch".format(subreddit, get_secs(start), get_secs(end))
    try:
        r = request(url)
        if r is None:
            return []
        data = json.loads(r.read())
        return data["data"]["children"]
    except ValueError:
        return []

def extract_imgur_album_urls(album_url):
    response = request(album_url)
    if not response:
        return []
    info = response.info()

    # Rudimentary check to ensure the URL actually specifies an HTML file
    if 'content-type' in info and not info['content-type'].startswith('text/html'):
        return []

    filedata = response.read()
    # TODO: stop parsing HTML with regexes.
    match = re.compile(r'\"hash\":\"(.[^\"]*)\",\"title\"')
    items = []

    memfile = StringIO.StringIO(filedata)

    for line in memfile.readlines():
        results = re.findall(match, line)
        if not results:
            continue

        items += results

    memfile.close()
    # TODO : url may contain gif image.
    urls = ['http://i.imgur.com/%s.jpg' % (imghash) for imghash in items]

    return urls

def process_imgur_url(url):
    if 'imgur.com/a/' in url or 'imgur.com/gallery/' in url:
        return extract_imgur_album_urls(url)

    # Change .png to .jpg for imgur urls.
    if url.endswith('.png'):
        url = url.replace('.png', '.jpg')
    else:
        # Extract the file extension
        ext = pathsplitext(pathbasename(url))[1]
        if ext == '.gifv':
            url = url.replace('.gifv', '.gif')
        if not ext:
            # Append a default
            url += '.jpg'
    return [url]

def extract_urls(url):
    urls = []
    if 'imgur.com' in url:
        urls = process_imgur_url(url)
    else:
        urls = [url]

    return urls

def resize_interval(old_interval, matched_posts, target, coefficient):
    MAX_POSTS = 25
    if matched_posts <= target/4:
        return old_interval*(1+coefficient) # Safety
    difference = float(target - matched_posts)
    if difference > 0:
        return old_interval * ((difference/(3*target/4.0))*coefficient + 1)
    return old_interval * 1.0/(1 + coefficient/3.0*(difference/(target - MAX_POSTS)))

def find_images(subreddit, min_score, days, target, coefficient):
    end = datetime.datetime.now()
    current = end - datetime.timedelta(days=days)
    interval = 3.0
    while current < end:
        current_stop = current + datetime.timedelta(seconds=interval)
        matched_posts = 0
        for post in do_search(subreddit, current, current_stop):
            if post['data']['score'] > min_score:
                matched_posts += 1
                for url in extract_urls(post['data']['url']):
                    yield url
        print " === {} posts matched".format(matched_posts)
        interval = resize_interval(interval, matched_posts, target, coefficient)
        current = current_stop

def download_image(url, download_dir):
    if os.path.isdir(download_dir):
        os.makedirs(download_dir)

    dest_file = os.path.join(download_dir, os.urandom(10).encode('hex'))

    response = request(url)
    if not response:
        print "Error trying to download file"
        return

    info = response.info()
    actual_url = response.url
    if actual_url == 'http://i.imgur.com/removed.png':
        print "Imgur suggests the image was removed"
        return

    # Work out file type either from the response or the url.
    if 'content-type' in info.keys():
        filetype = info['content-type']
    elif url.endswith('.jpg') or url.endswith('.jpeg'):
        filetype = 'image/jpeg'
    elif url.endswith('.png'):
        filetype = 'image/png'
    elif url.endswith('.gif'):
        filetype = 'image/gif'
    else:
        filetype = 'unknown'

    # Only try to download acceptable image types
    if filetype == 'image/jpeg':
        dest_file += '.jpg'
    elif filetype == 'image/png':
        dest_file += '.png'
    elif filetype == 'image/gif':
        dest_file += '.gif'
    else:
        print 'WRONG FILE TYPE: %s has type: %s!' % (url, filetype)
        return

    filedata = response.read()
    filehandle = open(dest_file, 'wb')
    filehandle.write(filedata)
    filehandle.close()

def main():
    args = parse_args()
    for image_url in find_images(args.subreddit, args.score, args.days, args.target, args.coefficient):
        download_image(image_url, args.output)

if __name__ == "__main__":
    main()<commit_msg>Add WWW and headers<commit_after>import argparse
import time
import datetime
import json
import os
from urllib import urlencode
from urllib2 import urlopen, HTTPError, URLError, Request

def parse_args():
    parser = argparse.ArgumentParser(description="Download all images from a subreddit")
    parser.add_argument('subreddit', help="The subreddit to download")
    parser.add_argument('-s', '--score', help='The minimum number of points a picture must have', type=int, default=0)
    parser.add_argument('-d', '--days', help="The number of days in the past to search through", type=int, default=365)
    parser.add_argument('-o', '--output', help="The directory to download into", type=str, default='downloads')
    parser.add_argument('-t', '--target', help="(Advanced) The target number of posts to download in each interval", type=int, default=20)
    parser.add_argument('-k', '--coefficient', help="(Advanced) The coefficient of change", type=float, default=2.0)
    return parser.parse_args()

def get_secs(t):
    return int(time.mktime(t.timetuple()))

def request(url, *ar, **kwa):
    _retries = kwa.pop('_retries', 4)
    _retry_pause = kwa.pop('_retry_pause', 0)
    res = None
    for _try in xrange(_retries):
        try:
            req = Request(url, headers={"User-Agent" : "reddit-image-extractor by /u/Detry322 - " + ' '.join(os.uname())})
            res = urlopen(req, *ar, **kwa)
        except URLError as exc:
            if _try == _retries - 1:
                return None
            print("Try %r err %r  (%r)" % (
                _try, exc.reason, url))
        else:
            break
    return res

def do_search(subreddit, start, end):
    url = "https://www.reddit.com/r/{}/search.json?q=timestamp:{}..{}&sort=top&restrict_sr=on&syntax=cloudsearch".format(subreddit, get_secs(start), get_secs(end))
    try:
        r = request(url)
        if r is None:
            return []
        data = json.loads(r.read())
        return data["data"]["children"]
    except ValueError:
        return []

def extract_imgur_album_urls(album_url):
    response = request(album_url)
    if not response:
        return []
    info = response.info()

    # Rudimentary check to ensure the URL actually specifies an HTML file
    if 'content-type' in info and not info['content-type'].startswith('text/html'):
        return []

    filedata = response.read()
    # TODO: stop parsing HTML with regexes.
    match = re.compile(r'\"hash\":\"(.[^\"]*)\",\"title\"')
    items = []

    memfile = StringIO.StringIO(filedata)

    for line in memfile.readlines():
        results = re.findall(match, line)
        if not results:
            continue

        items += results

    memfile.close()
    # TODO : url may contain gif image.
    urls = ['http://i.imgur.com/%s.jpg' % (imghash) for imghash in items]

    return urls

def process_imgur_url(url):
    if 'imgur.com/a/' in url or 'imgur.com/gallery/' in url:
        return extract_imgur_album_urls(url)

    # Change .png to .jpg for imgur urls.
    if url.endswith('.png'):
        url = url.replace('.png', '.jpg')
    else:
        # Extract the file extension
        ext = os.path.splitext(os.path.basename(url))[1]
        if ext == '.gifv':
            url = url.replace('.gifv', '.gif')
        if not ext:
            # Append a default
            url += '.jpg'
    return [url]

def extract_urls(url):
    urls = []
    if 'imgur.com' in url:
        urls = process_imgur_url(url)
    else:
        urls = [url]

    return urls

def resize_interval(old_interval, matched_posts, target, coefficient):
    MAX_POSTS = 25
    if matched_posts <= target/4:
        return old_interval*(1+coefficient) # Safety
    difference = float(target - matched_posts)
    if difference > 0:
        return old_interval * ((difference/(3*target/4.0))*coefficient + 1)
    return old_interval * 1.0/(1 + coefficient/3.0*(difference/(target - MAX_POSTS)))

def find_images(subreddit, min_score, days, target, coefficient):
    end = datetime.datetime.now()
    current = end - datetime.timedelta(days=days)
    interval = 3.0
    while current < end:
        current_stop = current + datetime.timedelta(seconds=interval)
        matched_posts = 0
        for post in do_search(subreddit, current, current_stop):
            if post['data']['score'] > min_score:
                matched_posts += 1
                for url in extract_urls(post['data']['url']):
                    yield url
        print " === {} posts matched".format(matched_posts)
        interval = resize_interval(interval, matched_posts, target, coefficient)
        current = current_stop

def download_image(url, download_dir):
    if os.path.isdir(download_dir):
        os.makedirs(download_dir)

    dest_file = os.path.join(download_dir, os.urandom(10).encode('hex'))

    response = request(url)
    if not response:
        print "Error trying to download file"
        return

    info = response.info()
    actual_url = response.url
    if actual_url == 'http://i.imgur.com/removed.png':
        print "Imgur suggests the image was removed"
        return

    # Work out file type either from the response or the url.
    if 'content-type' in info.keys():
        filetype = info['content-type']
    elif url.endswith('.jpg') or url.endswith('.jpeg'):
        filetype = 'image/jpeg'
    elif url.endswith('.png'):
        filetype = 'image/png'
    elif url.endswith('.gif'):
        filetype = 'image/gif'
    else:
        filetype = 'unknown'

    # Only try to download acceptable image types
    if filetype == 'image/jpeg':
        dest_file += '.jpg'
    elif filetype == 'image/png':
        dest_file += '.png'
    elif filetype == 'image/gif':
        dest_file += '.gif'
    else:
        print 'WRONG FILE TYPE: %s has type: %s!' % (url, filetype)
        return

    filedata = response.read()
    filehandle = open(dest_file, 'wb')
    filehandle.write(filedata)
    filehandle.close()

def main():
    args = parse_args()
    for image_url in find_images(args.subreddit, args.score, args.days, args.target, args.coefficient):
        download_image(image_url, args.output)

if __name__ == "__main__":
    main()<|endoftext|>