<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
from frappe.website.utils import cleanup_page_name
from frappe.website.render import clear_cache
from frappe.modules import get_module_name

class WebsiteGenerator(Document):
	website = frappe._dict()

	def __init__(self, *args, **kwargs):
		self.route = None
		super(WebsiteGenerator, self).__init__(*args, **kwargs)

	def get_website_properties(self, key=None, default=None):
		out = getattr(self, '_website', None) or getattr(self, 'website', None) or {}
		if key:
			return out.get(key, default)
		else:
			return out

	def autoname(self):
		if not self.name and self.meta.autoname != "hash":
			self.name = self.scrubbed_title()

	def onload(self):
		self.get("__onload").update({
			"is_website_generator": True,
			"published": self.is_website_published()
		})

	def validate(self):
		if self.is_website_published() and not self.route:
			self.route = self.make_route()

		if self.route:
			self.route = self.route.strip('/.')[:139]

	def make_route(self):
		return self.scrubbed_title()

	def scrubbed_title(self):
		return self.scrub(self.get(self.get_website_properties('page_title_field', 'title')))

	def clear_cache(self):
		clear_cache(self.route)

	def scrub(self, text):
		return cleanup_page_name(text).replace('_', '-')

	def get_parents(self, context):
		'''Return breadcrumbs'''
		pass

	def on_trash(self):
		self.clear_cache()

	def is_website_published(self):
		"""Return true if published in website"""
		if self.get_website_properties('condition_field'):
			return self.get(self.get_website_properties('condition_field')) and True or False
		else:
			return True

	def get_page_info(self):
		route = frappe._dict()
		route.update({
			"doc": self,
			"page_or_generator": "Generator",
			"ref_doctype":self.doctype,
			"idx": self.idx,
			"docname": self.name,
			"controller": get_module_name(self.doctype, self.meta.module),
		})

		route.update(self.get_website_properties())

		if not route.page_title:
			route.page_title = self.get(self.get_website_properties('page_title'), 'title') \
				or self.get('name')

		return route
<commit_msg>[Fix] Webpage title fix<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
from frappe.website.utils import cleanup_page_name
from frappe.website.render import clear_cache
from frappe.modules import get_module_name

class WebsiteGenerator(Document):
	website = frappe._dict()

	def __init__(self, *args, **kwargs):
		self.route = None
		super(WebsiteGenerator, self).__init__(*args, **kwargs)

	def get_website_properties(self, key=None, default=None):
		out = getattr(self, '_website', None) or getattr(self, 'website', None) or {}
		if key:
			return out.get(key, default)
		else:
			return out

	def autoname(self):
		if not self.name and self.meta.autoname != "hash":
			self.name = self.scrubbed_title()

	def onload(self):
		self.get("__onload").update({
			"is_website_generator": True,
			"published": self.is_website_published()
		})

	def validate(self):
		if self.is_website_published() and not self.route:
			self.route = self.make_route()

		if self.route:
			self.route = self.route.strip('/.')[:139]

	def make_route(self):
		return self.scrubbed_title()

	def scrubbed_title(self):
		return self.scrub(self.get(self.get_website_properties('page_title_field', 'title')))

	def clear_cache(self):
		clear_cache(self.route)

	def scrub(self, text):
		return cleanup_page_name(text).replace('_', '-')

	def get_parents(self, context):
		'''Return breadcrumbs'''
		pass

	def on_trash(self):
		self.clear_cache()

	def is_website_published(self):
		"""Return true if published in website"""
		if self.get_website_properties('condition_field'):
			return self.get(self.get_website_properties('condition_field')) and True or False
		else:
			return True

	def get_page_info(self):
		route = frappe._dict()
		route.update({
			"doc": self,
			"page_or_generator": "Generator",
			"ref_doctype":self.doctype,
			"idx": self.idx,
			"docname": self.name,
			"controller": get_module_name(self.doctype, self.meta.module),
		})

		route.update(self.get_website_properties())

		if not route.page_title:
			route.page_title = self.get(self.get_website_properties('page_title_field'), 'title') \
				or self.get('name')

		return route
<|endoftext|>


<commit_before>from deployer.utils import esc1

def wget(url, target=None):
    """
    Download file using wget
    """
    if target:
        return "wget '%s' --output-document '%s'" %  (esc1(url), esc1(target))
    else:
        return "wget '%s'" % esc1(url)
<commit_msg>bashrc_append command added.<commit_after>from deployer.utils import esc1

def wget(url, target=None):
    """
    Download file using wget
    """
    if target:
        return "wget '%s' --output-document '%s'" %  (esc1(url), esc1(target))
    else:
        return "wget '%s'" % esc1(url)

def bashrc_append(line):
    """
    Create a command which appends something to .bashrc if this line was not yet added before.
    """
    return "grep '%s' ~/.bashrc || echo '%s' >> ~/.bashrc" % (esc1(line), esc1(line))
<|endoftext|>


<commit_before># Copyright (c) 2011 X.commerce, a business unit of eBay Inc.
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2014 IBM Corp.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_db.sqlalchemy import models
from oslo_utils import timeutils
from sqlalchemy import Column, Integer, String, Text, TIMESTAMP
from sqlalchemy import BLOB
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import ForeignKey, DateTime, Boolean
from sqlalchemy.orm import relationship

BASE = declarative_base()

class FreezerBase(models.TimestampMixin,
                  models.ModelBase):
    """Base class for Freezer Models."""

    __table_args__ = {'mysql_engine': 'InnoDB'}

    deleted_at = Column(DateTime)
    deleted = Column(Boolean, default=False)
    backup_metadata = None

    @staticmethod
    def delete_values():
        return {'deleted': True,
                'deleted_at': timeutils.utcnow()}

    def delete(self, session):
        """Delete this object."""
        updated_values = self.delete_values()
        self.update(updated_values)
        self.save(session=session)
        return updated_values

class Client(BASE, FreezerBase):
    """Represents a scheduler of the freezer action."""

    __tablename__ = 'clients'
    id = Column(String(255), primary_key=True)
    project_id = Column(String(36))
    user_id = Column(String(64), nullable=False)
    client_id = Column(String(255), nullable=False)
    hostname = Column(String(128))
    description = Column(String(255))
    uuid = Column(String(36), nullable=False)

class Action(BASE, FreezerBase):
    """Represents freezer action."""
    # The field backup_metadata is json, including :
    # hostname ,snapshot ,storage ,dry_run , lvm_auto_snap, lvm_dirmount,
    # lvm_snapname max_level , max_priority, max_segment_size , mode,
    # mysql_conf, path_to_backup,  remove_older_than restore_abs_path
    # restore_from_host, ssh_host , ssh_key , ssh_username, ssh_port , proxy
    # no_incremental, overwrite , nova_inst_id , engine_name
    # restore_from_date , command , incremental

    __tablename__ = 'actions'
    id = Column(String(36), primary_key=True)
    action = Column(String(255), nullable=False)
    project_id = Column(String(36))
    user_id = Column(String(64), nullable=False)
    mode = Column(String(255))
    src_file = Column(String(255))
    backup_name = Column(String(255))
    container = Column(String(255))
    timeout = Column(Integer)
    priority = Column(Integer)
    max_retries_interval = Column(Integer, default=6)
    max_retries = Column(Integer, default=5)
    mandatory = Column(Boolean, default=False)
    log_file = Column(String(255))
    backup_metadata = Column(Text)

class Session(BASE, FreezerBase):
    """Represents freezer session."""

    __tablename__ = 'sessions'
    id = Column(String(36), primary_key=True)
    session_tag = Column(String(255), default=0)
    description = Column(String(255))
    hold_off = Column(Integer, default=30)
    schedule = Column(Text)
    job = Column(Text)
    project_id = Column(String(36))
    user_id = Column(String(36), nullable=False)
    time_start = Column(Integer, default=-1)
    time_end = Column(Integer, default=-1)
    time_started = Column(Integer, default=-1)
    time_ended = Column(Integer, default=-1)
    status = Column(String(255))
    result = Column(String(255))

class Job(BASE, FreezerBase):
    """Represents freezer job."""

    __tablename__ = 'jobs'
    id = Column(String(36), primary_key=True)
    project_id = Column(String(36))
    user_id = Column(String(36), nullable=False)
    schedule = Column(Text)
    client_id = Column(String(255), ForeignKey('clients.id'), nullable=False)
    session_id = Column(String(36), ForeignKey('sessions.id'), nullable=False)
    session_tag = Column(String(255))
    description = Column(String(255))
    job_actions = Column(Text)
    client = relationship(Client, backref='jobs',
                          foreign_keys=client_id,
                          primaryjoin='and_('
                          'Job.client_id == Client.id,'
                          'Job.deleted == False)')
    session = relationship(Session, backref='jobs',
                           foreign_keys=session_id,
                           primaryjoin='and_('
                           'Job.session_id == Session.id,'
                           'Job.deleted == False)')

class ActionReport(BASE, FreezerBase):
    __tablename__ = 'action_reports'
    id = Column(String(36), primary_key=True)
    project_id = Column(String(36))
    user_id = Column(String(64), nullable=False)
    result = Column(String(255))
    time_elapsed = Column(String(255))
    report_date = Column(TIMESTAMP)
    log = Column(BLOB)

class Backup(BASE, FreezerBase):
    """Represents freezer Backup."""
    # The field backup_metadata is json, including :
    # nova_inst_id, engine_name, storage, remove_older_than, restore_from_date,
    # command, incremental, restore_abs_path, etc

    __tablename__ = 'backups'
    id = Column(String(36), primary_key=True)
    job_id = Column(String(36))
    project_id = Column(String(36))
    user_id = Column(String(64), nullable=False)
    user_name = Column(String(64), nullable=False)
    backup_metadata = Column(Text)

def register_models(engine):
    _models = (Client, Action, Job, Session,
               ActionReport, Backup)
    for _model in _models:
        _model.metadata.create_all(engine)

def unregister_models(engine):
    _models = (Client, Action, Job, Session,
               ActionReport, Backup)
    for _model in _models:
        _model.metadata.drop_all(engine)

def get_tables(engine):
    from sqlalchemy import MetaData
    _meta = MetaData()
    _meta.reflect(engine)
    return _meta.tables.keys()
<commit_msg>Correct session_tag field type error<commit_after># Copyright (c) 2011 X.commerce, a business unit of eBay Inc.
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2014 IBM Corp.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_db.sqlalchemy import models
from oslo_utils import timeutils
from sqlalchemy import Column, Integer, String, Text, TIMESTAMP
from sqlalchemy import BLOB
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import ForeignKey, DateTime, Boolean
from sqlalchemy.orm import relationship

BASE = declarative_base()

class FreezerBase(models.TimestampMixin,
                  models.ModelBase):
    """Base class for Freezer Models."""

    __table_args__ = {'mysql_engine': 'InnoDB'}

    deleted_at = Column(DateTime)
    deleted = Column(Boolean, default=False)
    backup_metadata = None

    @staticmethod
    def delete_values():
        return {'deleted': True,
                'deleted_at': timeutils.utcnow()}

    def delete(self, session):
        """Delete this object."""
        updated_values = self.delete_values()
        self.update(updated_values)
        self.save(session=session)
        return updated_values

class Client(BASE, FreezerBase):
    """Represents a scheduler of the freezer action."""

    __tablename__ = 'clients'
    id = Column(String(255), primary_key=True)
    project_id = Column(String(36))
    user_id = Column(String(64), nullable=False)
    client_id = Column(String(255), nullable=False)
    hostname = Column(String(128))
    description = Column(String(255))
    uuid = Column(String(36), nullable=False)

class Action(BASE, FreezerBase):
    """Represents freezer action."""
    # The field backup_metadata is json, including :
    # hostname ,snapshot ,storage ,dry_run , lvm_auto_snap, lvm_dirmount,
    # lvm_snapname max_level , max_priority, max_segment_size , mode,
    # mysql_conf, path_to_backup,  remove_older_than restore_abs_path
    # restore_from_host, ssh_host , ssh_key , ssh_username, ssh_port , proxy
    # no_incremental, overwrite , nova_inst_id , engine_name
    # restore_from_date , command , incremental

    __tablename__ = 'actions'
    id = Column(String(36), primary_key=True)
    action = Column(String(255), nullable=False)
    project_id = Column(String(36))
    user_id = Column(String(64), nullable=False)
    mode = Column(String(255))
    src_file = Column(String(255))
    backup_name = Column(String(255))
    container = Column(String(255))
    timeout = Column(Integer)
    priority = Column(Integer)
    max_retries_interval = Column(Integer, default=6)
    max_retries = Column(Integer, default=5)
    mandatory = Column(Boolean, default=False)
    log_file = Column(String(255))
    backup_metadata = Column(Text)

class Session(BASE, FreezerBase):
    """Represents freezer session."""

    __tablename__ = 'sessions'
    id = Column(String(36), primary_key=True)
    session_tag = Column(Integer, default=0)
    description = Column(String(255))
    hold_off = Column(Integer, default=30)
    schedule = Column(Text)
    job = Column(Text)
    project_id = Column(String(36))
    user_id = Column(String(36), nullable=False)
    time_start = Column(Integer, default=-1)
    time_end = Column(Integer, default=-1)
    time_started = Column(Integer, default=-1)
    time_ended = Column(Integer, default=-1)
    status = Column(String(255))
    result = Column(String(255))

class Job(BASE, FreezerBase):
    """Represents freezer job."""

    __tablename__ = 'jobs'
    id = Column(String(36), primary_key=True)
    project_id = Column(String(36))
    user_id = Column(String(36), nullable=False)
    schedule = Column(Text)
    client_id = Column(String(255), ForeignKey('clients.id'), nullable=False)
    session_id = Column(String(36), ForeignKey('sessions.id'), nullable=False)
    session_tag = Column(String(255))
    description = Column(String(255))
    job_actions = Column(Text)
    client = relationship(Client, backref='jobs',
                          foreign_keys=client_id,
                          primaryjoin='and_('
                          'Job.client_id == Client.id,'
                          'Job.deleted == False)')
    session = relationship(Session, backref='jobs',
                           foreign_keys=session_id,
                           primaryjoin='and_('
                           'Job.session_id == Session.id,'
                           'Job.deleted == False)')

class ActionReport(BASE, FreezerBase):
    __tablename__ = 'action_reports'
    id = Column(String(36), primary_key=True)
    project_id = Column(String(36))
    user_id = Column(String(64), nullable=False)
    result = Column(String(255))
    time_elapsed = Column(String(255))
    report_date = Column(TIMESTAMP)
    log = Column(BLOB)

class Backup(BASE, FreezerBase):
    """Represents freezer Backup."""
    # The field backup_metadata is json, including :
    # nova_inst_id, engine_name, storage, remove_older_than, restore_from_date,
    # command, incremental, restore_abs_path, etc

    __tablename__ = 'backups'
    id = Column(String(36), primary_key=True)
    job_id = Column(String(36))
    project_id = Column(String(36))
    user_id = Column(String(64), nullable=False)
    user_name = Column(String(64), nullable=False)
    backup_metadata = Column(Text)

def register_models(engine):
    _models = (Client, Action, Job, Session,
               ActionReport, Backup)
    for _model in _models:
        _model.metadata.create_all(engine)

def unregister_models(engine):
    _models = (Client, Action, Job, Session,
               ActionReport, Backup)
    for _model in _models:
        _model.metadata.drop_all(engine)

def get_tables(engine):
    from sqlalchemy import MetaData
    _meta = MetaData()
    _meta.reflect(engine)
    return _meta.tables.keys()
<|endoftext|>


<commit_before># Copyright (C) 2007, Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import os
import sys
import ConfigParser

import gtk

from hulahop import config

sys.path.insert(0, os.path.join(config.libxul_dir, 'python'))

from hulahop._hulahop import shutdown
from hulahop import _hulahop

_XO_DPI = 200

_app_version = ''

def set_app_version(version):
    global _app_version
    _app_version = version

def startup(profile_path, components_dirs=[]):
    _check_compreg(profile_path)

    _hulahop.set_profile_path(profile_path)
    if not os.path.isdir(profile_path):
        try:
            os.makedirs(profile_path)
        except OSError, exc:
            raise RuntimeError('Could not create user directory.')

    for path in components_dirs:
        _hulahop.add_components_path(path)

    _hulahop.startup()

    from xpcom import components

    cls = components.classes["@mozilla.org/preferences-service;1"]
    prefService = cls.getService(components.interfaces.nsIPrefService)
    branch = prefService.getBranch('')
    branch.setIntPref('layout.css.dpi', _get_layout_dpi())

def _check_compreg(profile_path):
    comp_path = os.path.join(profile_path, 'compatibility.ini')
    existant = True
    valid = True

    try:
        cp = ConfigParser.ConfigParser(defaults={'app_version' : ''})
        if not cp.read(comp_path):
            existant = False
        else:
            valid = cp.get('main', 'app_version') == _app_version and \
                    cp.get('main', 'libxul_dir') == config.libxul_dir
    except ConfigParser.Error:
        valid = False

    if not valid and existant:
        compreg = os.path.join(profile_path, 'compreg.dat')

        os.unlink(comp_path)
        os.unlink(compreg)

    cp = ConfigParser.ConfigParser()
    cp.add_section('main')
    cp.set('main', 'app_version', _app_version)
    cp.set('main', 'libxul_dir', config.libxul_dir)

    f = open(comp_path, 'w')
    cp.write(f)
    f.close()

def _get_layout_dpi():
    gtk_xft_dpi = gtk.settings_get_default().get_property('gtk-xft-dpi')
    screen_dpi = gtk_xft_dpi / 1024
    screen_width = gtk.gdk.screen_width()

    if screen_dpi != _XO_DPI or screen_width != 1200:
        return -1

    # Layout:
    # 1024x768 -> (96 * 6) / 1024 * 201 = 113 dpi
    # 800x600  -> (96 * 6) /  800 * 201 = 144 dpi
    #
    # Fonts:
    # 7  pt ->  7 / 12 * 201 = 117 dpi
    # 8  pt ->  8 / 12 * 201 = 134 dpi
    # 9  pt ->  9 / 12 * 201 = 150 dpi

    return 134
<commit_msg>check compreg only when the profile is available #8318<commit_after># Copyright (C) 2007, Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import os
import sys
import ConfigParser

import gtk

from hulahop import config

sys.path.insert(0, os.path.join(config.libxul_dir, 'python'))

from hulahop._hulahop import shutdown
from hulahop import _hulahop

_XO_DPI = 200

_app_version = ''

def set_app_version(version):
    global _app_version
    _app_version = version

def startup(profile_path, components_dirs=[]):
    _hulahop.set_profile_path(profile_path)
    if not os.path.isdir(profile_path):
        try:
            os.makedirs(profile_path)
        except OSError, exc:
            raise RuntimeError('Could not create user directory.')

    _check_compreg(profile_path)

    for path in components_dirs:
        _hulahop.add_components_path(path)

    _hulahop.startup()

    from xpcom import components

    cls = components.classes["@mozilla.org/preferences-service;1"]
    prefService = cls.getService(components.interfaces.nsIPrefService)
    branch = prefService.getBranch('')
    branch.setIntPref('layout.css.dpi', _get_layout_dpi())

def _check_compreg(profile_path):
    comp_path = os.path.join(profile_path, 'compatibility.ini')
    existant = True
    valid = True

    try:
        cp = ConfigParser.ConfigParser(defaults={'app_version' : ''})
        if not cp.read(comp_path):
            existant = False
        else:
            valid = cp.get('main', 'app_version') == _app_version and \
                    cp.get('main', 'libxul_dir') == config.libxul_dir
    except ConfigParser.Error:
        valid = False

    if not valid and existant:
        compreg = os.path.join(profile_path, 'compreg.dat')

        os.unlink(comp_path)
        os.unlink(compreg)

    cp = ConfigParser.ConfigParser()
    cp.add_section('main')
    cp.set('main', 'app_version', _app_version)
    cp.set('main', 'libxul_dir', config.libxul_dir)

    f = open(comp_path, 'w')
    cp.write(f)
    f.close()

def _get_layout_dpi():
    gtk_xft_dpi = gtk.settings_get_default().get_property('gtk-xft-dpi')
    screen_dpi = gtk_xft_dpi / 1024
    screen_width = gtk.gdk.screen_width()

    if screen_dpi != _XO_DPI or screen_width != 1200:
        return -1

    # Layout:
    # 1024x768 -> (96 * 6) / 1024 * 201 = 113 dpi
    # 800x600  -> (96 * 6) /  800 * 201 = 144 dpi
    #
    # Fonts:
    # 7  pt ->  7 / 12 * 201 = 117 dpi
    # 8  pt ->  8 / 12 * 201 = 134 dpi
    # 9  pt ->  9 / 12 * 201 = 150 dpi

    return 134
<|endoftext|>


<commit_before>from pyxbigfile import BigBlock
from pyxbigfile import BigFileError
from pyxbigfile import BigFile as BigFileBase
from pyxbigfile import set_buffer_size
import bigfilempi

import os
import numpy

class BigFile(BigFileBase):
    def mpi_create(self, comm, block, dtype=None, size=None,
            Nfile=1):
        return bigfilempi.create_block(comm, self, block, dtype, size, Nfile)

    def mpi_create_from_data(self, comm, block, localdata, Nfile=1):
        bigfilempi.write_block(comm, self, block, localdata, Nfile)

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.close()

    def __getitem__(self, key):
        if key.endswith('/'):
            return BigFile(os.path.join(self.basename, key))

        return self.open(key)

    def __contains__(self, key):
        return key in self.blocks

    def __iter__(self):
        return iter(self.blocks)

class BigData:
    def __init__(self, file, blocks):
        self.blocknames = blocks
        self.blocks = dict([
            (block, file[block]) for block in self.blocknames])

        dtype = []
        size = None
        for block in self.blocknames:
            bb = self.blocks[block]
            dtype.append((block, bb.dtype))
            if size is None: size = bb.size
            elif bb.size != size:
                raise BigFileError('Dataset length is inconsistent on %s' %block)

        self.size = size
        self.dtype = numpy.dtype(dtype)

    def __getitem__(self, sl):
        if isinstance(sl, slice):
            start, end, stop = sl.indices(self.size)
            assert stop == 1
            result = numpy.empty(end - start, dtype=self.dtype)
            for block in self.blocknames:
                result[block][:] = self.blocks[block][sl]
            return result
        elif numpy.isscalar(sl):
            sl = slice(sl, sl + 1)
            return self[sl][0]
        else:
            raise TypeError('Expecting a slice or a scalar, got a `%s`' %
                    str(type(sl)))

<commit_msg>BigData[column]<commit_after>from pyxbigfile import BigBlock
from pyxbigfile import BigFileError
from pyxbigfile import BigFile as BigFileBase
from pyxbigfile import set_buffer_size
import bigfilempi

import os
import numpy

try:
    basestring  # attempt to evaluate basestring
    def isstr(s):
        return isinstance(s, basestring)
except NameError:
    def isstr(s):
        return isinstance(s, str)

class BigFile(BigFileBase):
    def mpi_create(self, comm, block, dtype=None, size=None,
            Nfile=1):
        return bigfilempi.create_block(comm, self, block, dtype, size, Nfile)

    def mpi_create_from_data(self, comm, block, localdata, Nfile=1):
        bigfilempi.write_block(comm, self, block, localdata, Nfile)

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.close()

    def __getitem__(self, key):
        if key.endswith('/'):
            return BigFile(os.path.join(self.basename, key))

        return self.open(key)

    def __contains__(self, key):
        return key in self.blocks

    def __iter__(self):
        return iter(self.blocks)

class BigData:
    def __init__(self, file, blocks):
        self.blocknames = blocks
        self.blocks = dict([
            (block, file[block]) for block in self.blocknames])

        dtype = []
        size = None
        for block in self.blocknames:
            bb = self.blocks[block]
            dtype.append((block, bb.dtype))
            if size is None: size = bb.size
            elif bb.size != size:
                raise BigFileError('Dataset length is inconsistent on %s' %block)

        self.size = size
        self.dtype = numpy.dtype(dtype)

    def __getitem__(self, sl):
        if isinstance(sl, slice):
            start, end, stop = sl.indices(self.size)
            assert stop == 1
            result = numpy.empty(end - start, dtype=self.dtype)
            for block in self.blocknames:
                result[block][:] = self.blocks[block][sl]
            return result
        elif isstr(sl):
            return self.blocks[sl]
        elif numpy.isscalar(sl):
            sl = slice(sl, sl + 1)
            return self[sl][0]
        else:
            raise TypeError('Expecting a slice or a scalar, got a `%s`' %
                    str(type(sl)))

<|endoftext|>


<commit_before># -*- coding=utf -*-

import os.path
import json
from collections import namedtuple
from .extensions import get_namespace, initialize_namespace
from .browser import Cell
from .errors import *
from .common import read_json_file, sorted_dependencies

__all__ = (
    "create_authorizer",
    "Authorizer",
    "SimpleAuthorizer",
    "NotAuthorized"
)

ALL_CUBES_WILDCARD = '*'

class NotAuthorized(UserError):
    """Raised when user is not authorized for the request."""
    # Note: This is not called NotAuthorizedError as it is not in fact an
    # error, it is just type of signal.

def create_authorizer(name, **options):
    """Gets a new instance of an authorizer with name `name`."""

    ns = get_namespace("authorizers")
    if not ns:
        ns = initialize_namespace("authorizers", root_class=Authorizer,
                                  suffix="_authorizer",
                                  option_checking=True)
    try:
        factory = ns[name]
    except KeyError:
        raise ConfigurationError("Unknown authorizer '%s'" % name)

    return factory(**options)

class Authorizer(object):
    def authorize(self, token, cubes):
        """Returns list of authorized cubes from `cubes`. If none of the cubes
        are authorized an empty list is returned.

        Default implementation returs the same `cubes` list as provided.
        """
        return cubes

    def restricted_cell(self, token, cube, cell=None):
        """Restricts the `cell` for `cube` according to authorization by
        `token`. If no cell is provided or the cell is empty then returns
        the restriction cell. If there is no restriction, returns the original
        `cell` if provided or `None`.
        """
        return cell

class NoopAuthorizer(Authorizer):
    def __init__(self):
        super(NoopAuthorizer, self).__init__()

class _SimpleAccessRight(object):
    def __init__(self, roles, allow_cubes, deny_cubes, cube_restrictions):
        self.roles = roles or []
        self.allow_cubes = set(allow_cubes) if allow_cubes else set([])
        self.deny_cubes = set(deny_cubes) if deny_cubes else set([])
        self.cube_restrictions = cube_restrictions or {}

    def merge(self, other):
        """Merge `right` with the receiver:

        * `allow_cubes` are merged (union)
        * `deny_cubes` are merged (union)
        * `cube_restrictions` from `other` with same cube replace restrictions
          from the receiver"""

        self.allow_cubes |= other.allow_cubes
        self.deny_cubes |= other.deny_cubes

        for cube, restrictions in other.cube_restrictions:
            if not cube in self.cube_restrictions:
                self.cube_restrictions = list(other.cube_restrictions)
            else:
                mine = self.cube_restrictions.get(cube)
                mine += restrictions

    def is_allowed(self, cube_name):
        return (self.allow_cubes and
                (cube_name in self.allow_cubes or ALL_CUBES_WILDCARD in self.allow_cubes)) \
                or \
                (self.deny_cubes and
                (cube_name not in self.deny_cubes and ALL_CUBES_WILDCARD not in self.deny_cubes))

class NoopAuthorizer(Authorizer):
    def __init__(self):
        super(NoopAuthorizer, self).__init__()

    def authorize(self, token, cube, cell=None):
        if cell is not None:
            return cell
        return Cell(cube)

class _SimpleAccessRight(object):
    def __init__(self, roles, allow_cubes, deny_cubes, cube_restrictions):
        self.roles = roles or []
        self.allow_cubes = set(allow_cubes) if allow_cubes else set()
        self.deny_cubes = set(deny_cubes) if deny_cubes else set()
        self.cube_restrictions = cube_restrictions or {}

    def merge(self, other):
        """Merge `right` with the receiver:

        * `allow_cubes` are merged (union)
        * `deny_cubes` are merged (union)
        * `cube_restrictions` from `other` with same cube replace restrictions
          from the receiver"""

        self.allow_cubes |= other.allow_cubes
        self.deny_cubes |= other.deny_cubes

        for cube, restrictions in other.cube_restrictions:
            if not cube in self.cube_restrictions:
                self.cube_restrictions = list(other.cube_restrictions)
            else:
                mine = self.cube_restrictions.get(cube)
                mine += restritions

class NoopAuthorizer(Authorizer):
    def __init__(self):
        super(NoopAuthorizer, self).__init__()

    def authorize(self, token, cube, cell=None):
        if cell is not None:
            return cell
        return Cell(cube)

class SimpleAuthorizer(Authorizer):
    __options__ = [
        {
            "name": "rights_file",
            "description": "JSON file with access rights",
            "type": "string"
        },
        {
            "name": "roles_file",
            "description": "JSON file with access right roles",
            "type": "string"
        },

    ]

    def __init__(self, rights_file=None, roles_file=None, roles=None,
                 rights=None, **options):
        """Creates a simple JSON-file based authorizer. Reads data from
        `rights_file` and `roles_file` and merge them with `roles` and
        `rights` dictionaries respectively."""

        super(SimpleAuthorizer, self).__init__()

        roles = roles or {}
        rights = rights or {}

        if roles_file:
            content = read_json_file(roles_file, "access roles")
            roles.update(content)

        if rights_file:
            content = read_json_file(rights_file, "access rights")
            rights.update(content)

        self.roles = {}
        self.rights = {}

        # Process the roles
        for key, info in roles.items():
            role = _SimpleAccessRight(roles=info.get("roles"),
                                      allow_cubes=info.get("allowed_cubes"),
                                      deny_cubes=info.get("denied_cubes"),
                                      cube_restrictions=info.get("cube_restrictions"))
            self.roles[key] = role

        deps = dict((name, role.roles) for name, role in self.roles.items())
        order = sorted_dependencies(deps)

        for name in order:
            role = self.roles[name]
            for parent_name in role.roles:
                parent = self.roles[parent_name]
                role.merge(parent)

        # Process rights
        for key, info in rights.items():
            right = _SimpleAccessRight(roles=info.get("roles"),
                                       allow_cubes=info.get("allowed_cubes"),
                                       deny_cubes=info.get("denied_cubes"),
                                       cube_restrictions=info.get("cube_restrictions"))
            self.rights[key] = right

            for role_name in right.roles:
                role = self.roles[role_name]
                right.merge(role)

    def right(self, token):
        try:
            right = self.rights[token]
        except KeyError:
            raise NotAuthorized("Unknown access right '%s'" % token)
        return right

    def authorize(self, token, cubes):
        try:
            right = self.right(token)
        except NotAuthorized:
            return []

        authorized = []

        for cube in cubes:
            cube_name = str(cube)

            if right.is_allowed(cube_name):
                authorized.append(cube)

        return authorized

    def restricted_cell(self, token, cube, cell):
        right = self.right(token)

        cuts = right.cube_restrictions.get(cube.name)

        if cuts:
            cuts = [cut_from_dict(cut) for cut in cuts]
            restriction = Cell(cube, cuts)
        else:
            restriction = None

        if not restriction:
            return cell
        elif cell:
            return cell + restriction
        else:
            return restriction

<commit_msg>Allow string based cuts; better restriction merging<commit_after># -*- coding=utf -*-

import os.path
import json
from collections import namedtuple
from .extensions import get_namespace, initialize_namespace
from .browser import Cell
from .errors import *
from .common import read_json_file, sorted_dependencies

__all__ = (
    "create_authorizer",
    "Authorizer",
    "SimpleAuthorizer",
    "NotAuthorized"
)

ALL_CUBES_WILDCARD = '*'

class NotAuthorized(UserError):
    """Raised when user is not authorized for the request."""
    # Note: This is not called NotAuthorizedError as it is not in fact an
    # error, it is just type of signal.

def create_authorizer(name, **options):
    """Gets a new instance of an authorizer with name `name`."""

    ns = get_namespace("authorizers")
    if not ns:
        ns = initialize_namespace("authorizers", root_class=Authorizer,
                                  suffix="_authorizer",
                                  option_checking=True)
    try:
        factory = ns[name]
    except KeyError:
        raise ConfigurationError("Unknown authorizer '%s'" % name)

    return factory(**options)

class Authorizer(object):
    def authorize(self, token, cubes):
        """Returns list of authorized cubes from `cubes`. If none of the cubes
        are authorized an empty list is returned.

        Default implementation returs the same `cubes` list as provided.
        """
        return cubes

    def restricted_cell(self, token, cube, cell=None):
        """Restricts the `cell` for `cube` according to authorization by
        `token`. If no cell is provided or the cell is empty then returns
        the restriction cell. If there is no restriction, returns the original
        `cell` if provided or `None`.
        """
        return cell

class NoopAuthorizer(Authorizer):
    def __init__(self):
        super(NoopAuthorizer, self).__init__()

class _SimpleAccessRight(object):
    def __init__(self, roles, allow_cubes, deny_cubes, cube_restrictions):
        self.roles = roles or []
        self.allow_cubes = set(allow_cubes) if allow_cubes else set([])
        self.deny_cubes = set(deny_cubes) if deny_cubes else set([])
        self.cube_restrictions = cube_restrictions or {}

    def merge(self, other):
        """Merge `right` with the receiver:

        * `allow_cubes` are merged (union)
        * `deny_cubes` are merged (union)
        * `cube_restrictions` from `other` with same cube replace restrictions
          from the receiver"""

        self.allow_cubes |= other.allow_cubes
        self.deny_cubes |= other.deny_cubes

        for cube, restrictions in other.cube_restrictions:
            if not cube in self.cube_restrictions:
                self.cube_restrictions[cube] = restrictions
            else:
                mine = self.cube_restrictions[cube]
                mine += restrictions

    def is_allowed(self, cube_name):
        return (self.allow_cubes \
                    and (cube_name in self.allow_cubes \
                            or ALL_CUBES_WILDCARD in self.allow_cubes)) \
                or \
                    (self.deny_cubes \
                    and (cube_name not in self.deny_cubes \
                            and ALL_CUBES_WILDCARD not in self.deny_cubes))

class NoopAuthorizer(Authorizer):
    def __init__(self):
        super(NoopAuthorizer, self).__init__()

    def authorize(self, token, cube, cell=None):
        if cell is not None:
            return cell
        return Cell(cube)

class _SimpleAccessRight(object):
    def __init__(self, roles, allow_cubes, deny_cubes, cube_restrictions):
        self.roles = roles or []
        self.allow_cubes = set(allow_cubes) if allow_cubes else set()
        self.deny_cubes = set(deny_cubes) if deny_cubes else set()
        self.cube_restrictions = cube_restrictions or {}

    def merge(self, other):
        """Merge `right` with the receiver:

        * `allow_cubes` are merged (union)
        * `deny_cubes` are merged (union)
        * `cube_restrictions` from `other` with same cube replace restrictions
          from the receiver"""

        self.allow_cubes |= other.allow_cubes
        self.deny_cubes |= other.deny_cubes

        for cube, restrictions in other.cube_restrictions:
            if not cube in self.cube_restrictions:
                self.cube_restrictions = list(other.cube_restrictions)
            else:
                mine = self.cube_restrictions.get(cube)
                mine += restritions

class NoopAuthorizer(Authorizer):
    def __init__(self):
        super(NoopAuthorizer, self).__init__()

    def authorize(self, token, cube, cell=None):
        if cell is not None:
            return cell
        return Cell(cube)

class SimpleAuthorizer(Authorizer):
    __options__ = [
        {
            "name": "rights_file",
            "description": "JSON file with access rights",
            "type": "string"
        },
        {
            "name": "roles_file",
            "description": "JSON file with access right roles",
            "type": "string"
        },

    ]

    def __init__(self, rights_file=None, roles_file=None, roles=None,
                 rights=None, **options):
        """Creates a simple JSON-file based authorizer. Reads data from
        `rights_file` and `roles_file` and merge them with `roles` and
        `rights` dictionaries respectively."""

        super(SimpleAuthorizer, self).__init__()

        roles = roles or {}
        rights = rights or {}

        if roles_file:
            content = read_json_file(roles_file, "access roles")
            roles.update(content)

        if rights_file:
            content = read_json_file(rights_file, "access rights")
            rights.update(content)

        self.roles = {}
        self.rights = {}

        # Process the roles
        for key, info in roles.items():
            role = _SimpleAccessRight(roles=info.get("roles"),
                                      allow_cubes=info.get("allowed_cubes"),
                                      deny_cubes=info.get("denied_cubes"),
                                      cube_restrictions=info.get("cube_restrictions"))
            self.roles[key] = role

        deps = dict((name, role.roles) for name, role in self.roles.items())
        order = sorted_dependencies(deps)

        for name in order:
            role = self.roles[name]
            for parent_name in role.roles:
                parent = self.roles[parent_name]
                role.merge(parent)

        # Process rights
        for key, info in rights.items():
            right = _SimpleAccessRight(roles=info.get("roles"),
                                       allow_cubes=info.get("allowed_cubes"),
                                       deny_cubes=info.get("denied_cubes"),
                                       cube_restrictions=info.get("cube_restrictions"))
            self.rights[key] = right

            for role_name in right.roles:
                role = self.roles[role_name]
                right.merge(role)

    def right(self, token):
        try:
            right = self.rights[token]
        except KeyError:
            raise NotAuthorized("Unknown access right '%s'" % token)
        return right

    def authorize(self, token, cubes):
        try:
            right = self.right(token)
        except NotAuthorized:
            return []

        authorized = []

        for cube in cubes:
            cube_name = str(cube)

            if right.is_allowed(cube_name):
                authorized.append(cube)

        return authorized

    def restricted_cell(self, token, cube, cell):
        right = self.right(token)

        cuts = right.cube_restrictions.get(cube.name)

        if cuts:
            if isinstance(cuts, basestring):
                cuts = cuts_from_string(cube, cuts)
            else:
                cuts = [cut_from_dict(cut) for cut in cuts]

            restriction = Cell(cube, cuts)
        else:
            restriction = None

        if not restriction:
            return cell
        elif cell:
            return cell + restriction
        else:
            return restriction

<|endoftext|>


<commit_before>cue_lookup = {
    'Escape': 1,
    'F1': 2,
    'F2': 3,
    'F3': 4,
    'F4': 5,
    'F5': 6,
    'F6': 7,
    'F7': 8,
    'F8': 9,
    'F9': 10,
    'F10': 11,
    'F11': 12,
    'GraveAccentAndTilde': 13,
    '1': 14,
    '2': 15,
    '3': 16,
    '4': 17,
    '5': 18,
    '6': 19,
    '7': 20,
    '8': 21,
    '9': 22,
    '0': 23,
    'MinusAndUnderscore': 24,
    'Tab': 25,
    'Q': 26,
    'W': 27,
    'E': 28,
    'R': 29,
    'T': 30,
    'Y': 31,
    'U': 32,
    'I': 33,
    'O': 34,
    'P': 35,
    'BracketLeft': 36,
    'CapsLock': 37,
    'A': 38,
    'S': 39,
    'D': 40,
    'F': 41,
    'G': 42,
    'H': 43,
    'J': 44,
    'K': 45,
    'L': 46,
    'SemicolonAndColon': 47,
    'ApostropheAndDoubleQuote': 48,
    'LeftShift': 49,
    'NonUsBackslash': 50,
    'Z': 51,
    'X': 52,
    'C': 53,
    'V': 54,
    'B': 55,
    'N': 56,
    'M': 57,
    'CommaAndLessThan': 58,
    'PeriodAndBiggerThan': 59,
    'SlashAndQuestionMark': 60,
    'LeftCtrl': 61,
    'LeftGui': 62,
    'LeftAlt': 63,
    'Lang2': 64,
    'Space': 65,
    'Lang1': 66,
    'International2': 67,
    'RightAlt': 68,
    'RightGui': 69,
    'Application': 70,
    'LedProgramming': 71,
    'Brightness': 72,
    'F12': 73,
    'PrintScreen': 74,
    'ScrollLock': 75,
    'PauseBreak': 76,
    'Insert': 77,
    'Home': 78,
    'PageUp': 79,
    'BracketRight': 80,
    'Backslash': 81,
    'NonUsTilde': 82,
    'Enter': 83,
    'International1': 84,
    'EqualsAndPlus': 85,
    'International3': 86,
    'Backspace': 87,
    'Delete': 88,
    'End': 89,
    'PageDown': 90,
    'RightShift': 91,
    'RightCtrl': 92,
    'UpArrow': 93,
    'LeftArrow': 94,
    'DownArrow': 95,
    'RightArrow': 96,
    'WinLock': 97,
    'Mute': 98,
    'Stop': 99,
    'ScanPreviousTrack': 100,
    'PlayPause': 101,
    'ScanNextTrack': 102,
    'NumLock': 103,
    'KeypadSlash': 104,
    'KeypadAsterisk': 105,
    'KeypadMinus': 106,
    'KeypadPlus': 107,
    'KeypadEnter': 108,
    'Keypad7': 109,
    'Keypad8': 110,
    'Keypad9': 111,
    'KeypadComma': 112,
    'Keypad4': 113,
    'Keypad5': 114,
    'Keypad6': 115,
    'Keypad1': 116,
    'Keypad2': 117,
    'Keypad3': 118,
    'Keypad0': 119,
    'KeypadPeriodAndDelete': 120,
    'G1': 121,
    'G2': 122,
    'G3': 123,
    'G4': 124,
    'G5': 125,
    'G6': 126,
    'G7': 127,
    'G8': 128,
    'G9': 129,
    'G10': 130,
    'VolumeUp': 131,
    'VolumeDown': 132,
    'MR': 133,
    'M1': 134,
    'M2': 135,
    'M3': 136,
    'G11': 137,
    'G12': 138,
    'G13': 139,
    'G14': 140,
    'G15': 141,
    'G16': 142,
    'G17': 143,
    'G18': 144,
    'International5': 145,
    'International4': 146,
    'Fn': 147,
    'CLM_1': 148,
    'CLM_2': 149,
    'CLM_3': 150,
    'CLM_4': 151,
    'CLH_LeftLogo': 152,
    'CLH_RightLogo': 153,
    'Logo': 154}
<commit_msg>Remove obsoleted cue lookup dict.<commit_after><|endoftext|>


<commit_before>from IPython.display import HTML, display
import os
import uuid
import imolecule.json_formatter as json
from imolecule import format_converter

filename = "imolecule.min.js"
file_path = os.path.normpath(os.path.dirname(__file__))
local_path = os.path.join("nbextensions", filename)
remote_path = ("https://rawgit.com/patrickfuller/imolecule/master/"
               "js/build/imolecule.min.js")

def draw(data, format="auto", size=(400, 300), drawing_type="ball and stick",
         camera_type="perspective", shader="toon", display_html=True):
    """Draws an interactive 3D visualization of the inputted chemical.

    Args:
        data: A string or file representing a chemical.
        format: The format of the `data` variable (default is "auto").
        size: Starting dimensions of visualization, in pixels.
        drawing_type: Specifies the molecular representation. Can be "ball and
            stick", "wireframe", or "space filling".
        camera_type: Can be "perspective" or "orthographic".
        shader: Specifies shading algorithm to use. Can be "toon", "basic",
            "phong", or "lambert".

    The `format` can be any value specified by Open Babel
    (http://openbabel.org/docs/2.3.1/FileFormats/Overview.html). The "auto"
    option uses the extension for files (ie. my_file.mol -> mol) and defaults
    to SMILES (smi) for strings.
    """
    # Catch errors on string-based input before getting js involved
    draw_options = ["ball and stick", "wireframe", "space filling"]
    camera_options = ["perspective", "orthographic"]
    shader_options = ["toon", "basic", "phong", "lambert"]
    if drawing_type not in draw_options:
        raise Exception("Invalid drawing type! Please use one of: "
                        + ", ".join(draw_options))
    if camera_type not in camera_options:
        raise Exception("Invalid camera type! Please use one of: "
                        + ", ".join(camera_options))
    if shader not in shader_options:
        raise Exception("Invalid shader! Please use one of: "
                        + ", ".join(shader_options))

    # Try using IPython >=2.0 to install js locally
    try:
        from IPython.html.nbextensions import install_nbextension
        install_nbextension([os.path.join(file_path,
                             "js/build", filename)], verbose=0)
    except:
        pass

    json_mol = generate(data, format)
    div_id = uuid.uuid4()
    html = """

           """ % (div_id, local_path[:-3], remote_path[:-3],
                           div_id, size[0], size[1], drawing_type,
                           camera_type, shader, json_mol, size[0], size[1])

    # Execute js and display the results in a div (see script for more)
    if display_html is True:
        display(HTML(html))
    else:
        return html

def generate(data, format="auto"):
    """Converts input chemical formats to json and optimizes structure.

    Args:
        data: A string or file representing a chemical
        format: The format of the `data` variable (default is "auto")

    The `format` can be any value specified by Open Babel
    (http://openbabel.org/docs/2.3.1/FileFormats/Overview.html). The "auto"
    option uses the extension for files (ie. my_file.mol -> mol) and defaults
    to SMILES (smi) for strings.
    """
    # Support both files and strings and attempt to infer file type
    try:
        with open(data) as in_file:
            if format == "auto":
                format = data.split(".")[-1]
            data = in_file.read()
    except:
        if format == "auto":
            format = "smi"
    return format_converter.convert(data, format, "json")

def to_json(data, compress=False):
    """Converts the output of `generate(...)` to formatted json.

    Floats are rounded to three decimals and positional vectors are printed on
    one line with some whitespace buffer.
    """
    return json.compress(data) if compress else json.dumps(data)
<commit_msg>Conform to PEP8 and add the display_html to Args<commit_after>from IPython.display import HTML, display
import os
import uuid
import imolecule.json_formatter as json
from imolecule import format_converter

filename = "imolecule.min.js"
file_path = os.path.normpath(os.path.dirname(__file__))
local_path = os.path.join("nbextensions", filename)
remote_path = ("https://rawgit.com/patrickfuller/imolecule/master/"
               "js/build/imolecule.min.js")

def draw(data, format="auto", size=(400, 300), drawing_type="ball and stick",
         camera_type="perspective", shader="toon", display_html=True):
    """Draws an interactive 3D visualization of the inputted chemical.

    Args:
        data: A string or file representing a chemical.
        format: The format of the `data` variable (default is "auto").
        size: Starting dimensions of visualization, in pixels.
        drawing_type: Specifies the molecular representation. Can be "ball and
            stick", "wireframe", or "space filling".
        camera_type: Can be "perspective" or "orthographic".
        shader: Specifies shading algorithm to use. Can be "toon", "basic",
            "phong", or "lambert".
        display_html: If True (default), embed the html in a IPython display,
            if False, returns the html as a string.

    The `format` can be any value specified by Open Babel
    (http://openbabel.org/docs/2.3.1/FileFormats/Overview.html). The "auto"
    option uses the extension for files (ie. my_file.mol -> mol) and defaults
    to SMILES (smi) for strings.
    """
    # Catch errors on string-based input before getting js involved
    draw_options = ["ball and stick", "wireframe", "space filling"]
    camera_options = ["perspective", "orthographic"]
    shader_options = ["toon", "basic", "phong", "lambert"]
    if drawing_type not in draw_options:
        raise Exception("Invalid drawing type! Please use one of: "
                        + ", ".join(draw_options))
    if camera_type not in camera_options:
        raise Exception("Invalid camera type! Please use one of: "
                        + ", ".join(camera_options))
    if shader not in shader_options:
        raise Exception("Invalid shader! Please use one of: "
                        + ", ".join(shader_options))

    # Try using IPython >=2.0 to install js locally
    try:
        from IPython.html.nbextensions import install_nbextension
        install_nbextension([os.path.join(file_path,
                             "js/build", filename)], verbose=0)
    except:
        pass

    json_mol = generate(data, format)
    div_id = uuid.uuid4()
    html = """

           """ % (div_id, local_path[:-3], remote_path[:-3],
                           div_id, size[0], size[1], drawing_type,
                           camera_type, shader, json_mol, size[0], size[1])

    # Execute js and display the results in a div (see script for more)
    if display_html:
        display(HTML(html))
    else:
        return html

def generate(data, format="auto"):
    """Converts input chemical formats to json and optimizes structure.

    Args:
        data: A string or file representing a chemical
        format: The format of the `data` variable (default is "auto")

    The `format` can be any value specified by Open Babel
    (http://openbabel.org/docs/2.3.1/FileFormats/Overview.html). The "auto"
    option uses the extension for files (ie. my_file.mol -> mol) and defaults
    to SMILES (smi) for strings.
    """
    # Support both files and strings and attempt to infer file type
    try:
        with open(data) as in_file:
            if format == "auto":
                format = data.split(".")[-1]
            data = in_file.read()
    except:
        if format == "auto":
            format = "smi"
    return format_converter.convert(data, format, "json")

def to_json(data, compress=False):
    """Converts the output of `generate(...)` to formatted json.

    Floats are rounded to three decimals and positional vectors are printed on
    one line with some whitespace buffer.
    """
    return json.compress(data) if compress else json.dumps(data)
<|endoftext|>


<commit_before>from abc import ABCMeta, abstractmethod
from alembic.migration import MigrationContext
from alembic.operations import Operations
from cum import db
from sqlalchemy import inspect
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.declarative.clsregistry import _ModuleMarker
from sqlalchemy.orm import RelationshipProperty
from sqlalchemy.sql import sqltypes

class DatabaseSanity(object):
    """Class designed for database validation and repair."""

    def __init__(self, base, engine):
        self.base = base
        self.engine = engine
        self.errors = []
        self.inspection_engine = inspect(self.engine)

    @property
    def database_tables(self):
        """Generator that returns table names from the database file."""
        for table_name in self.inspection_engine.get_table_names():
            yield table_name

    @property
    def is_sane(self):
        """Returns a boolean value that indicates whether the database file
        passes the sanity checks or not based on the length of the error list.
        """
        return not len(self.errors)

    @property
    def model_tables(self):
        """Generator that returns tuples containing model name and model class.
        Models with classes inherited from _ModuleMarker class are skipped as
        they are not actual models.
        """
        for name, class_ in self.base._decl_class_registry.items():
            if isinstance(class_, _ModuleMarker):
                continue
            yield (name, class_)

    def test(self):
        """Runs through all the database sanity tests."""
        self.test_tables()
        for table in self.database_tables:
            self.test_columns(table)
        self.test_madokami_url()  # TODO: Remove in the future.

    def test_columns(self, table):
        """Tests the columns in database table. If a column exists in the
        table, check that the column has correct datatype. If the column
        doesn't exist, add a MissingColumn object to the error list.
        """
        columns = self.inspection_engine.get_columns(table)
        column_names = [c["name"] for c in columns]

        # Search for correct class from models for the mapper.
        mapper = None
        for name, class_ in self.model_tables:
            if class_.__tablename__ == table:
                mapper = inspect(class_)
                break
        if not mapper:
            return

        for column_property in mapper.attrs:
            if isinstance(column_property, RelationshipProperty):
                # TODO: Add RelationshipProperty sanity checking.
                pass
            else:
                for column in column_property.columns:
                    if column.key in column_names:
                        self.test_datatype(table, column)
                    else:
                        self.errors.append(MissingColumn(table, column.key,
                                                         parent=self))

    def test_datatype(self, table, column):
        """Tests that database column datatype matches the one defined in the
        models.
        """

        # Search a matching database column for the model.
        for c in self.inspection_engine.get_columns(table):
            if c['name'] == column.key:
                database_column = c
                break

        if isinstance(column.type, sqltypes.String):
            expected_type = sqltypes.VARCHAR
        elif isinstance(column.type, sqltypes.Integer):
            expected_type = sqltypes.INTEGER
        elif isinstance(column.type, sqltypes.Boolean):
            expected_type = sqltypes.BOOLEAN
        elif isinstance(column.type, sqltypes.DateTime):
            expected_type = sqltypes.DATETIME

        if not isinstance(database_column['type'], expected_type):
            self.errors.append(
                DatatypeMismatch(table, database_column, expected_type,
                                 parent=self)
            )

    def test_madokami_url(self):
        """Temporary function to test if Madokami series and chapters have the
        correct URL after the top-level domain change. Will be removed after an
        appropriate time has passed for people have made the switch.
        """
        old_domain = 'manga.madokami.com'
        new_domain = 'manga.madokami.al'
        models = [db.Series, db.Chapter]
        for model in models:
            condition = model.url.ilike('%{}%'.format(old_domain))
            try:
                results = db.session.query(model).filter(condition).all()
            except SQLAlchemyError:
                continue
            if results:
                error = IncorrectDomain(model, old_domain, new_domain)
                self.errors.append(error)

    def test_tables(self):
        """Tests if all the defined models are found in the database. If a
        table is missing, a MissingTable object is added to the error list.
        """
        for name, class_ in self.model_tables:
            table = class_.__tablename__
            if table not in self.database_tables:
                self.errors.append(MissingTable(table, parent=self))

class SanityError(metaclass=ABCMeta):
    """Base class for all sanity errors."""

    @abstractmethod
    def __str__():
        """Returns an error message that can be displayed to the user."""
        raise NotImplementedError

    @abstractmethod
    def fix(self):
        """Does appropriate changes to the database to rectify the issue."""
        raise NotImplementedError

class DatatypeMismatch(SanityError):
    """Class used for columns where the database datatype is different from the
    model datatype.
    """

    def __init__(self, table, column, expected_type, parent=None):
        self.column = column
        self.expected_type = expected_type
        self.parent = parent
        self.table = table

    def __str__(self):
        return ("{}.{} column has inappropriate datatype {} (should be {})"
                .format(self.table, self.column['name'],
                        self.column['type'], self.expected_type.__name__))

    def fix(self):
        """Uses Alembic batch operations to alter the column datatype in the table.
        """
        context = MigrationContext.configure(self.parent.engine.connect())
        op = Operations(context)
        for table in self.parent.base.metadata.sorted_tables:
            if table.name == self.table:
                for column in table.columns:
                    if column.name == self.column['name']:
                        with op.batch_alter_table(table.name) as batch_op:
                            batch_op.alter_column(column.name,
                                                  type_=column.type)
                        return

class IncorrectDomain(SanityError):
    """Class used for database entries that have an outdated domain."""

    def __init__(self, model, old_domain, new_domain):
        self.model = model
        self.old_domain = old_domain
        self.new_domain = new_domain

    def __str__(self):
        return ('{s.model.__tablename__} has entries with incorrect domain '
                '({s.old_domain} -> {s.new_domain})'
                .format(s=self))

    def fix(self):
        condition = self.model.url.ilike('%{}%'.format(self.old_domain))
        results = db.session.query(self.model).filter(condition).all()
        for result in results:
            result.url = result.url.replace(self.old_domain, self.new_domain)
        db.session.commit()

class MissingColumn(SanityError):
    """Class used for columns that are missing from the database tables."""

    def __init__(self, table, name, parent=None):
        self.name = name
        self.parent = parent
        self.table = table

    def __str__(self):
        return ('{s.table}.{s.name} column is missing from database'
                .format(s=self))

    def fix(self):
        context = MigrationContext.configure(self.parent.engine.connect())
        op = Operations(context)
        for table in self.parent.base.metadata.sorted_tables:
            if table.name == self.table:
                for column in table.columns:
                    if column.name == self.name:
                        with op.batch_alter_table(table.name) as batch_op:
                            batch_op.add_column(column.copy())
                        return

class MissingTable(SanityError):
    """Class used for tables that are missing from the database file."""

    def __init__(self, name, parent=None):
        self.name = name
        self.parent = parent

    def __str__(self):
        return ('{} table is missing from database'
                .format(self.name))

    def fix(self):
        """Searches for the schema.Table object from SQLAlchemy metadata and
        creates it using the engine bind.
        """
        for table in self.parent.base.metadata.sorted_tables:
            if table.name == self.name:
                table.create(bind=self.parent.engine)
                break
<commit_msg>Remove circular import to support Python 3.3 and 3.4.<commit_after>from abc import ABCMeta, abstractmethod
from alembic.migration import MigrationContext
from alembic.operations import Operations
from sqlalchemy import inspect
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.declarative.clsregistry import _ModuleMarker
from sqlalchemy.orm import RelationshipProperty
from sqlalchemy.sql import sqltypes

class DatabaseSanity(object):
    """Class designed for database validation and repair."""

    def __init__(self, base, engine):
        self.base = base
        self.engine = engine
        self.errors = []
        self.inspection_engine = inspect(self.engine)

    @property
    def database_tables(self):
        """Generator that returns table names from the database file."""
        for table_name in self.inspection_engine.get_table_names():
            yield table_name

    @property
    def is_sane(self):
        """Returns a boolean value that indicates whether the database file
        passes the sanity checks or not based on the length of the error list.
        """
        return not len(self.errors)

    @property
    def model_tables(self):
        """Generator that returns tuples containing model name and model class.
        Models with classes inherited from _ModuleMarker class are skipped as
        they are not actual models.
        """
        for name, class_ in self.base._decl_class_registry.items():
            if isinstance(class_, _ModuleMarker):
                continue
            yield (name, class_)

    def test(self):
        """Runs through all the database sanity tests."""
        self.test_tables()
        for table in self.database_tables:
            self.test_columns(table)
        self.test_madokami_url()  # TODO: Remove in the future.

    def test_columns(self, table):
        """Tests the columns in database table. If a column exists in the
        table, check that the column has correct datatype. If the column
        doesn't exist, add a MissingColumn object to the error list.
        """
        columns = self.inspection_engine.get_columns(table)
        column_names = [c["name"] for c in columns]

        # Search for correct class from models for the mapper.
        mapper = None
        for name, class_ in self.model_tables:
            if class_.__tablename__ == table:
                mapper = inspect(class_)
                break
        if not mapper:
            return

        for column_property in mapper.attrs:
            if isinstance(column_property, RelationshipProperty):
                # TODO: Add RelationshipProperty sanity checking.
                pass
            else:
                for column in column_property.columns:
                    if column.key in column_names:
                        self.test_datatype(table, column)
                    else:
                        self.errors.append(MissingColumn(table, column.key,
                                                         parent=self))

    def test_datatype(self, table, column):
        """Tests that database column datatype matches the one defined in the
        models.
        """

        # Search a matching database column for the model.
        for c in self.inspection_engine.get_columns(table):
            if c['name'] == column.key:
                database_column = c
                break

        if isinstance(column.type, sqltypes.String):
            expected_type = sqltypes.VARCHAR
        elif isinstance(column.type, sqltypes.Integer):
            expected_type = sqltypes.INTEGER
        elif isinstance(column.type, sqltypes.Boolean):
            expected_type = sqltypes.BOOLEAN
        elif isinstance(column.type, sqltypes.DateTime):
            expected_type = sqltypes.DATETIME

        if not isinstance(database_column['type'], expected_type):
            self.errors.append(
                DatatypeMismatch(table, database_column, expected_type,
                                 parent=self)
            )

    def test_madokami_url(self):
        """Temporary function to test if Madokami series and chapters have the
        correct URL after the top-level domain change. Will be removed after an
        appropriate time has passed for people have made the switch.
        """
        global db
        from cum import db
        old_domain = 'manga.madokami.com'
        new_domain = 'manga.madokami.al'
        models = [db.Series, db.Chapter]
        for model in models:
            condition = model.url.ilike('%{}%'.format(old_domain))
            try:
                results = db.session.query(model).filter(condition).all()
            except SQLAlchemyError:
                continue
            if results:
                error = IncorrectDomain(model, old_domain, new_domain)
                self.errors.append(error)

    def test_tables(self):
        """Tests if all the defined models are found in the database. If a
        table is missing, a MissingTable object is added to the error list.
        """
        for name, class_ in self.model_tables:
            table = class_.__tablename__
            if table not in self.database_tables:
                self.errors.append(MissingTable(table, parent=self))

class SanityError(metaclass=ABCMeta):
    """Base class for all sanity errors."""

    @abstractmethod
    def __str__():
        """Returns an error message that can be displayed to the user."""
        raise NotImplementedError

    @abstractmethod
    def fix(self):
        """Does appropriate changes to the database to rectify the issue."""
        raise NotImplementedError

class DatatypeMismatch(SanityError):
    """Class used for columns where the database datatype is different from the
    model datatype.
    """

    def __init__(self, table, column, expected_type, parent=None):
        self.column = column
        self.expected_type = expected_type
        self.parent = parent
        self.table = table

    def __str__(self):
        return ("{}.{} column has inappropriate datatype {} (should be {})"
                .format(self.table, self.column['name'],
                        self.column['type'], self.expected_type.__name__))

    def fix(self):
        """Uses Alembic batch operations to alter the column datatype in the table.
        """
        context = MigrationContext.configure(self.parent.engine.connect())
        op = Operations(context)
        for table in self.parent.base.metadata.sorted_tables:
            if table.name == self.table:
                for column in table.columns:
                    if column.name == self.column['name']:
                        with op.batch_alter_table(table.name) as batch_op:
                            batch_op.alter_column(column.name,
                                                  type_=column.type)
                        return

class IncorrectDomain(SanityError):
    """Class used for database entries that have an outdated domain."""

    def __init__(self, model, old_domain, new_domain):
        self.model = model
        self.old_domain = old_domain
        self.new_domain = new_domain

    def __str__(self):
        return ('{s.model.__tablename__} has entries with incorrect domain '
                '({s.old_domain} -> {s.new_domain})'
                .format(s=self))

    def fix(self):
        condition = self.model.url.ilike('%{}%'.format(self.old_domain))
        results = db.session.query(self.model).filter(condition).all()
        for result in results:
            result.url = result.url.replace(self.old_domain, self.new_domain)
        db.session.commit()

class MissingColumn(SanityError):
    """Class used for columns that are missing from the database tables."""

    def __init__(self, table, name, parent=None):
        self.name = name
        self.parent = parent
        self.table = table

    def __str__(self):
        return ('{s.table}.{s.name} column is missing from database'
                .format(s=self))

    def fix(self):
        context = MigrationContext.configure(self.parent.engine.connect())
        op = Operations(context)
        for table in self.parent.base.metadata.sorted_tables:
            if table.name == self.table:
                for column in table.columns:
                    if column.name == self.name:
                        with op.batch_alter_table(table.name) as batch_op:
                            batch_op.add_column(column.copy())
                        return

class MissingTable(SanityError):
    """Class used for tables that are missing from the database file."""

    def __init__(self, name, parent=None):
        self.name = name
        self.parent = parent

    def __str__(self):
        return ('{} table is missing from database'
                .format(self.name))

    def fix(self):
        """Searches for the schema.Table object from SQLAlchemy metadata and
        creates it using the engine bind.
        """
        for table in self.parent.base.metadata.sorted_tables:
            if table.name == self.name:
                table.create(bind=self.parent.engine)
                break
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# flake8: noqa: F401
# noreorder
"""
Pytube: a very serious Python library for downloading YouTube Videos.
"""
__title__ = "pytube3"
__author__ = "Nick Ficano, Harold Martin"
__license__ = "MIT License"
__copyright__ = "Copyright 2019 Nick Ficano"
__js__ = None
__js_url__ = None

from pytube.version import __version__
from pytube.streams import Stream
from pytube.captions import Caption
from pytube.query import CaptionQuery
from pytube.query import StreamQuery
from pytube.__main__ import YouTube
from pytube.contrib.playlist import Playlist
<commit_msg>Update __init__.py<commit_after># -*- coding: utf-8 -*-
# flake8: noqa: F401
# noreorder
"""
Pytube: a very serious Python library for downloading YouTube Videos.
"""
__title__ = "pytube"
__author__ = "Nick Ficano, Harold Martin"
__license__ = "MIT License"
__copyright__ = "Copyright 2019 Nick Ficano"
__js__ = None
__js_url__ = None

from pytube.version import __version__
from pytube.streams import Stream
from pytube.captions import Caption
from pytube.query import CaptionQuery
from pytube.query import StreamQuery
from pytube.__main__ import YouTube
from pytube.contrib.playlist import Playlist
<|endoftext|>


<commit_before>import os
import sys
import errno
import argparse
from subprocess import call, PIPE, Popen

HOME_DIR = os.environ.get('HOME')

def install_pysimm(prefix):
    os.chdir(prefix)
    call('git clone https://github.com/polysimtools/pysimm', shell=True)
    call("echo export PYTHONPATH='$PYTHONPATH':{} >> {}".format(prefix,
                                                                os.path.join(HOME_DIR, '.bashrc')),
         shell=True)
    call("echo export PATH='$PATH':{} >> {}".format(os.path.join(prefix, 'bin'),
                                                    os.path.join(HOME_DIR, '.bashrc')),
         shell=True)

def apt_update():
    call('sudo apt-get update', shell=True)

def apt_install(*packages):
    call('sudo apt-get -y install {}'.format(' '.join(packages)),
         shell=True)

def install_lammps(prefix, *packages):
    os.chdir(prefix)
    call('git clone git://git.lammps.org/lammps-ro.git lammps', shell=True)
    os.chdir(os.path.join(prefix,'lammps','src'))
    for package in packages:
        call('make yes-{}'.format(package), shell=True)
    call('make mpi', shell=True)
    call("echo export PATH='$PATH':{} >> {}".format(os.path.join(prefix, 'lammps', 'src'),
                                                    os.path.join(HOME_DIR,'.bashrc')),
         shell=True)
    call("echo export LAMMPS_EXEC={} >> {}".format(os.path.join(prefix, 'lammps', 'src', 'lmp_mpi'),
                                                   os.path.join(HOME_DIR,'.bashrc')),
         shell=True)

def install_ambertools(dir_):
    os.chdir(dir_)
    call("echo export AMBERHOME={} >> {}".format(dir_, os.path.join(HOME_DIR,'.bashrc')),
         shell=True)
    os.environ['AMBERHOME'] = dir_
    call('./configure gnu', shell=True)
    call('make install', shell=True)
    call("echo export ANTECHAMBER_EXEC={} >> {}".format(os.path.join(dir_, 'bin', 'antechamber'),
                                                   os.path.join(HOME_DIR,'.bashrc')),
         shell=True)

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--apt-update', dest='apt_update', default=True)
    parser.add_argument('--pysimm', dest='pysimm_prefix', default=HOME_DIR)
    parser.add_argument('--lammps', dest='lammps_prefix', default=None)
    parser.add_argument('--lammps-packages', dest='lammps_packages', nargs='*',
                        default=['molecule', 'class2', 'kspace', 'user-misc', 'qeq', 'manybody'])
    parser.add_argument('--amber-tools', dest='ambertools_dir', default=None)
    return parser.parse_args()

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

if __name__ == '__main__':

    args = parse_args()

    if bool(args.apt_update):
        apt_update()

    if args.pysimm_prefix:
        apt_install('git', 'python-numpy', 'python-matplotlib')
        mkdir_p(args.pysimm_prefix)
        install_pysimm(args.pysimm_prefix)

    if args.lammps_prefix:
        apt_install('make git g++', 'libopenmpi-dev', 'openmpi-bin')
        mkdir_p(args.lammps_prefix)
        install_lammps(args.lammps_prefix, *args.lammps_packages)

    if args.ambertools_dir:
        apt_install('make', 'csh', 'gfortran', 'libopenmpi-dev', 'openmpi-bin', 'xorg-dev', 'xserver-xorg')
        install_ambertools(args.ambertools_dir)

    os.chdir(HOME_DIR)
<commit_msg>updated AMBERHOME export<commit_after>import os
import sys
import errno
import argparse
from subprocess import call, PIPE, Popen

HOME_DIR = os.environ.get('HOME')

def install_pysimm(prefix):
    os.chdir(prefix)
    call('git clone https://github.com/polysimtools/pysimm', shell=True)
    call("echo export PYTHONPATH='$PYTHONPATH':{} >> {}".format(os.path.join(prefix, 'pysimm'),
                                                                os.path.join(HOME_DIR, '.bashrc')),
         shell=True)
    call("echo export PATH='$PATH':{} >> {}".format(os.path.join(prefix, 'pysimm', 'bin'),
                                                    os.path.join(HOME_DIR, '.bashrc')),
         shell=True)

def apt_update():
    call('sudo apt-get update', shell=True)

def apt_install(*packages):
    call('sudo apt-get -y install {}'.format(' '.join(packages)),
         shell=True)

def install_lammps(prefix, *packages):
    os.chdir(prefix)
    call('git clone git://git.lammps.org/lammps-ro.git lammps', shell=True)
    os.chdir(os.path.join(prefix,'lammps','src'))
    for package in packages:
        call('make yes-{}'.format(package), shell=True)
    call('make mpi', shell=True)
    call("echo export PATH='$PATH':{} >> {}".format(os.path.join(prefix, 'lammps', 'src'),
                                                    os.path.join(HOME_DIR,'.bashrc')),
         shell=True)
    call("echo export LAMMPS_EXEC={} >> {}".format(os.path.join(prefix, 'lammps', 'src', 'lmp_mpi'),
                                                   os.path.join(HOME_DIR,'.bashrc')),
         shell=True)

def install_ambertools(dir_):
    os.chdir(dir_)
    call("echo export AMBERHOME={} >> {}".format(dir_, os.path.join(HOME_DIR,'.bashrc')),
         shell=True)
    os.environ['AMBERHOME'] = dir_
    call('./configure gnu', shell=True)
    call('make install', shell=True)
    call("echo export ANTECHAMBER_EXEC={} >> {}".format(os.path.join(dir_, 'bin', 'antechamber'),
                                                   os.path.join(HOME_DIR,'.bashrc')),
         shell=True)

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--apt-update', dest='apt_update', action="store_true", default=True)
    parser.add_argument('--no-apt-update', dest='apt_update', action="store_false")
    parser.add_argument('--pysimm', dest='pysimm_prefix', default=HOME_DIR)
    parser.add_argument('--lammps', dest='lammps_prefix', default=None)
    parser.add_argument('--lammps-packages', dest='lammps_packages', nargs='*',
                        default=['molecule', 'class2', 'kspace', 'user-misc', 'qeq', 'manybody'])
    parser.add_argument('--amber-tools', dest='ambertools_dir', default=None)
    return parser.parse_args()

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

if __name__ == '__main__':

    args = parse_args()

    if bool(args.apt_update):
        apt_update()

    if args.pysimm_prefix:
        apt_install('git', 'python-numpy', 'python-matplotlib')
        mkdir_p(args.pysimm_prefix)
        install_pysimm(args.pysimm_prefix)

    if args.lammps_prefix:
        apt_install('make git g++', 'libopenmpi-dev', 'openmpi-bin')
        mkdir_p(args.lammps_prefix)
        install_lammps(args.lammps_prefix, *args.lammps_packages)

    if args.ambertools_dir:
        apt_install('make', 'csh', 'gfortran', 'libopenmpi-dev', 'openmpi-bin', 'xorg-dev', 'xserver-xorg')
        install_ambertools(args.ambertools_dir)

    os.chdir(HOME_DIR)
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()  # NOQA

import chainer

from chainerrl.agents import categorical_dqn
from chainerrl.agents.categorical_dqn import _apply_categorical_projection

class CategoricalDoubleDQN(categorical_dqn.CategoricalDQN):
    """Categorical Double DQN.

    """

    def _compute_target_values(self, exp_batch):
        """Compute a batch of target return distributions."""

        batch_next_state = exp_batch['next_state']
        batch_rewards = exp_batch['reward']
        batch_terminal = exp_batch['is_state_terminal']

        with chainer.using_config('train', False):
            if self.recurrent:
                target_next_qout, _ = self.target_model.n_step_forward(
                    batch_next_state, exp_batch['next_recurrent_state'],
                    output_mode='concat')
                next_qout, _ = self.model.n_step_forward(
                    batch_next_state, exp_batch['next_recurrent_state'],
                    output_mode='concat')
            else:
                target_next_qout = self.target_model(batch_next_state)
                next_qout = self.model(batch_next_state)

        next_q_max = target_next_qout.evaluate_actions(
            next_qout.greedy_actions)

        batch_size = batch_rewards.shape[0]
        z_values = target_next_qout.z_values
        n_atoms = z_values.size

        # next_q_max: (batch_size, n_atoms)
        next_q_max = target_next_qout.max_as_distribution.array
        assert next_q_max.shape == (batch_size, n_atoms), next_q_max.shape

        # Tz: (batch_size, n_atoms)
        Tz = (batch_rewards[..., None]
              + (1.0 - batch_terminal[..., None])
              * self.xp.expand_dims(exp_batch['discount'], 1)
              * z_values[None])
        return _apply_categorical_projection(Tz, next_q_max, z_values)
<commit_msg>corrects double update error<commit_after>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()  # NOQA

import chainer

from chainerrl.agents import categorical_dqn
from chainerrl.agents.categorical_dqn import _apply_categorical_projection

class CategoricalDoubleDQN(categorical_dqn.CategoricalDQN):
    """Categorical Double DQN.

    """

        # next_q_max: (batch_size, n_atoms)
        next_q_max = target_next_qout.max_as_distribution.array
        assert next_q_max.shape == (batch_size, n_atoms), next_q_max.shape

        # Tz: (batch_size, n_atoms)
        Tz = (batch_rewards[..., None]
              + (1.0 - batch_terminal[..., None])
              * self.xp.expand_dims(exp_batch['discount'], 1)
              * z_values[None])
        return _apply_categorical_projection(Tz, next_q_max, z_values)

    def _compute_target_values(self, exp_batch):
        """Compute a batch of target return distributions."""

        batch_next_state = exp_batch['next_state']
        batch_rewards = exp_batch['reward']
        batch_terminal = exp_batch['is_state_terminal']

        with chainer.using_config('train', False):
            if self.recurrent:
                target_next_qout, _ = self.target_model.n_step_forward(
                    batch_next_state, exp_batch['next_recurrent_state'],
                    output_mode='concat')
                next_qout, _ = self.model.n_step_forward(
                    batch_next_state, exp_batch['next_recurrent_state'],
                    output_mode='concat')
            else:
                target_next_qout = self.target_model(batch_next_state)
                next_qout = self.model(batch_next_state)

        batch_size = batch_rewards.shape[0]
        z_values = target_next_qout.z_values
        n_atoms = z_values.size

        # next_q_max: (batch_size, n_atoms)
        next_q_max = target_next_qout.evaluate_actions_as_distribution(
            next_qout.greedy_actions)
        assert next_q_max.shape == (batch_size, n_atoms), next_q_max.shape

        # Tz: (batch_size, n_atoms)
        Tz = (batch_rewards[..., None]
              + (1.0 - batch_terminal[..., None])
              * self.xp.expand_dims(exp_batch['discount'], 1)
              * z_values[None])
        return _apply_categorical_projection(Tz, next_q_max, z_values)
<|endoftext|>


<commit_before>import re

# Ordered list of UNTL elements
UNTL_XML_ORDER = [
    'title',
    'creator',
    'contributor',
    'publisher',
    'date',
    'language',
    'description',
    'subject',
    'primarySource',
    'coverage',
    'source',
    'citation',
    'relation',
    'collection',
    'institution',
    'rights',
    'resourceType',
    'format',
    'identifier',
    'degree',
    'note',
    'meta',
]

UNTL_PTH_ORDER = [
    'hidden',
    'title',
    'creator',
    'contributor',
    'publisher',
    'date',
    'language',
    'description',
    'subject',
    'primarySource',
    'coverage',
    'source',
    'citation',
    'relation',
    'collection',
    'institution',
    'rights',
    'resourceType',
    'format',
    'identifier',
    'degree',
    'note',
    'meta',
]

DC_ORDER = [
    'title',
    'creator',
    'contributor',
    'publisher',
    'date',
    'language',
    'description',
    'subject',
    'coverage',
    'source',
    'relation',
    'rights',
    'type',
    'format',
    'identifier',
]

ETD_MS_ORDER = [
    'title',
    'creator',
    'subject',
    'description',
    'publisher',
    'contributor',
    'date',
    'type',
    'identifier',
    'language',
    'coverage',
    'rights',
    'degree',
]

HIGHWIRE_ORDER = [
    'citation_title',
    'citation_author',
    'citation_publisher',
    'citation_publication_date',
    'citation_online_date',
    'citation_journal_title',
    'citation_conference_title',
    'citation_issn',
    'citation_isbn',
    'citation_doi',
    'citation_volume',
    'citation_issue',
    'citation_firstpage',
    'citation_lastpage',
    'citation_dissertation_insitution',
    'citation_technical_report_insitution',
    'citation_technical_report_number',
]

# Namespaces for the UNTL xml
UNTL_NAMESPACES = {
    'untl': 'http://digitalprojects.library.unt.edu/'
}

# All Vocabularies Location
VOCABULARIES_URL = 'http://libdigidev02.library.unt.edu:92/vocabularies/all.json'

# URL of the UNTL metadata usage page
UNTL_USAGE_LINK = 'http://www.library.unt.edu/digital-projects-unit/metadata/fields/'

# Creation Year/Month/Day regexes
CREATION_DATE_REGEX = re.compile(
    r'^(\d\d\d\d)[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])$'
)

CREATION_MONTH_REGEX = re.compile(r'^(\d\d\d\d)[- /.](0[1-9]|1[012])$')

CREATION_YEAR_REGEX = re.compile(r'(\d\d\d\d)')

META_CREATION_DATE_REGEX = re.compile(
    r'(\d\d\d\d)[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])'
)
<commit_msg>Update the vocabularies link.<commit_after>import re

# Ordered list of UNTL elements
UNTL_XML_ORDER = [
    'title',
    'creator',
    'contributor',
    'publisher',
    'date',
    'language',
    'description',
    'subject',
    'primarySource',
    'coverage',
    'source',
    'citation',
    'relation',
    'collection',
    'institution',
    'rights',
    'resourceType',
    'format',
    'identifier',
    'degree',
    'note',
    'meta',
]

UNTL_PTH_ORDER = [
    'hidden',
    'title',
    'creator',
    'contributor',
    'publisher',
    'date',
    'language',
    'description',
    'subject',
    'primarySource',
    'coverage',
    'source',
    'citation',
    'relation',
    'collection',
    'institution',
    'rights',
    'resourceType',
    'format',
    'identifier',
    'degree',
    'note',
    'meta',
]

DC_ORDER = [
    'title',
    'creator',
    'contributor',
    'publisher',
    'date',
    'language',
    'description',
    'subject',
    'coverage',
    'source',
    'relation',
    'rights',
    'type',
    'format',
    'identifier',
]

ETD_MS_ORDER = [
    'title',
    'creator',
    'subject',
    'description',
    'publisher',
    'contributor',
    'date',
    'type',
    'identifier',
    'language',
    'coverage',
    'rights',
    'degree',
]

HIGHWIRE_ORDER = [
    'citation_title',
    'citation_author',
    'citation_publisher',
    'citation_publication_date',
    'citation_online_date',
    'citation_journal_title',
    'citation_conference_title',
    'citation_issn',
    'citation_isbn',
    'citation_doi',
    'citation_volume',
    'citation_issue',
    'citation_firstpage',
    'citation_lastpage',
    'citation_dissertation_insitution',
    'citation_technical_report_insitution',
    'citation_technical_report_number',
]

# Namespaces for the UNTL xml
UNTL_NAMESPACES = {
    'untl': 'http://digitalprojects.library.unt.edu/'
}

# All Vocabularies Location
VOCABULARIES_URL = 'https://digital2.library.unt.edu/vocabularies/all.json'

# URL of the UNTL metadata usage page
UNTL_USAGE_LINK = 'http://www.library.unt.edu/digital-projects-unit/metadata/fields/'

# Creation Year/Month/Day regexes
CREATION_DATE_REGEX = re.compile(
    r'^(\d\d\d\d)[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])$'
)

CREATION_MONTH_REGEX = re.compile(r'^(\d\d\d\d)[- /.](0[1-9]|1[012])$')

CREATION_YEAR_REGEX = re.compile(r'(\d\d\d\d)')

META_CREATION_DATE_REGEX = re.compile(
    r'(\d\d\d\d)[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])'
)
<|endoftext|>


<commit_before>__version__ = "0.8.2"

from .Restful import Restful

def connect(url, username, password):

    rest_client = Restful(url, username, password)
    target_uri = "%s/univmax/restapi/system/version" % (url)
    version_response = rest_client.get(target_uri)
    univmax_version = None
    if 'version' in version_response:
        univmax_version = version_response.get('version')

# import the correct API module for the version of Unisphere discovered
    if univmax_version == 'V8.2.0.5':
        from .VmaxApi_v82 import VmaxApi
    elif univmax_version == 'V8.0.1.7':
        from .VmaxApi_v80 import VmaxApi
    else:
        from .VmaxApi_v82 import VmaxApi
        print('Unsupported VMAX API Version {}'.format(univmax_version))
        print('Using latest VMAX API Version')

    return VmaxApi(rest_client, url)

__all__ = ['connect']
<commit_msg>parsed version string more robustly<commit_after>__version__ = "0.8.2"

from pyvmax.Restful import Restful

def connect(url, username, password):

    rest_client = Restful(url, username, password)
    target_uri = "%s/univmax/restapi/system/version" % (url)
    version_response = rest_client.get(target_uri)
    if 'version' in version_response:
        univmax_version = version_response.get('version', '')

# import the correct API module for the version of Unisphere discovered
    version_major, version_minor = ""
    if '.' in univmax_version:
        _version_parts = univmax_version.split('.')
        version_major, version_minor = '.'.join(_version_parts[:n]), '.'.join(_version_parts[n:])
    if version_major == 'V8.2':
        from pyvmax.VmaxApi_v82 import VmaxApi
    elif version_major == 'V8.0':
        from pyvmax.VmaxApi_v80 import VmaxApi
    else:
        from pyvmax.VmaxApi_v82 import VmaxApi
        print('Unsupported VMAX API Version', univmax_version)
        print('Using latest VMAX API Version')

    return VmaxApi(rest_client, url)

__all__ = ['connect']
<|endoftext|>