<commit_before>from datetime import datetime
from dateutil import parser as datetime_parser
from dateutil.tz import tzutc
from werkzeug.security import generate_password_hash, check_password_hash
from itsdangerous import TimedJSONWebSignatureSerializer as Serializer
from flask import g, url_for, current_app
from . import db
from .exceptions import ValidationError, ConflictError
from .utils import split_url

class User(db.Model):
    ''' Users DB interface '''
    __tablename__ = 'users'
    __table_args__ = {'extend_existing': True}

    id = db.Column(db.Integer, primary_key=True)
    username = db.Column(db.String(64), index=True, unique=True)
    password_hash = db.Column(db.String(128))

    def set_password(self, password):
        self.password_hash = generate_password_hash(password)

    def verify_password(self, password):
        return check_password_hash(self.password_hash, password)

    def generate_auth_token(self, expires_in=3600):
        s = Serializer(current_app.config['SECRET_KEY'], expires_in=expires_in)
        return s.dumps({'id': self.id}).decode('utf-8')

    @staticmethod
    def verify_auth_token(token):
        s = Serializer(current_app.config['SECRET_KEY'])
        try:
            data = s.loads(token)
        except:
            return None
        return User.query.get(data['id'])

    def import_data(self, data):
        try:
            if not self.query.filter_by(username=data['username']).count():
                self.username = data['username']
                self.password_hash = generate_password_hash(data['password'])
            else:
                raise ConflictError('That username is taken.')
        except KeyError as e:
            raise ValidationError('Invalid user: missing ' + e.args[0])
        return self

class Bucketlist(db.Model):
    ''' Bucketlists DB interface '''
    __tablename__ = 'bucketlists'

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(64), index=True)
    description = db.Column(db.Text)
    date_created = db.Column(db.DateTime, default=datetime.now)
    date_modified = db.Column(db.DateTime, onupdate=datetime.now)
    created_by = db.Column(db.Integer, db.ForeignKey('users.id'), index=True)
    items = db.relationship('BucketlistItem', backref='bucketlist',
                            lazy='dynamic',
                            cascade='all, delete-orphan')

    def get_url(self):
        return url_for('api.get_bucketlist', id=self.id, _external=True)

    def export_data(self):
        items = BucketlistItem.query.filter_by(bucketlist_id=self.id).all()
        return {
            'id': self.id,
            'name': self.name,
            'description': self.description,
            # change to export_data() if data of individual items is preferred
            'items': [item.get_url() for item in items],
            'date_created': self.date_created,
            'date_modified': self.date_modified,
            'created_by': self.created_by,
            'self_url': self.get_url(),
            'items_url': url_for('api.get_items', id=self.id,
                                 _external=True)
        }

    def import_data(self, data):
        try:
            if not self.query.filter_by(name=data['name']) \
                             .filter_by(created_by=g.user.id).count():
                self.name = data['name']
                if 'description' in data:
                    self.description = data['description']
            else:
                raise ConflictError(
                    'You already have a bucketlist with that name.'
                )
        except KeyError as e:
            raise ValidationError('Invalid bucketlist: missing ' + e.args[0])
        return self

class BucketlistItem(db.Model):
    ''' Bucketlist Items DB interface '''
    __tablename__ = 'items'
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(64))
    description = db.Column(db.Text)
    bucketlist_id = db.Column(db.Integer, db.ForeignKey('bucketlists.id'))
    # created_by = db.Column(db.Integer, db.ForeignKey('users.id'))
    date_created = db.Column(db.DateTime, default=datetime.now)
    date_modified = db.Column(db.DateTime, onupdate=datetime.now)
    # user = db.relationship('User',
    #                        backref=db.backref('bucketlistitems',
    #                                           lazy='dynamic'))
    done = db.Column(db.Boolean, default=False)

    def get_url(self):
        return url_for(
                'api.get_item',
                item_id=self.id,
                id=self.bucketlist_id,
                _external=True
               )

    def export_data(self):
        return {
            'id': self.id,
            'name': self.name,
            'date_created': self.date_created,
            'date_modified': self.date_modified,
            'done': self.done,
            'self_url': self.get_url()
        }

    def import_data(self, data):
        try:
            # self.name = data['name']
            # if 'description' in data:
            #     self.description = data['description']
            if not self.query.filter_by(name=data['name']) \
                             .filter_by(
                                bucketlist_id=data['bucketlist_id']
                             ).count():
                self.name = data['name']
                if 'description' in data:
                    self.description = data['description']
            else:
                raise ConflictError(
                    'You already have an item with that name.'
                )
        except KeyError as e:
            raise ValidationError('Invalid item: missing ' + e.args[0])
        return self
<commit_msg>[Fix] Disallowed the creation of resources with an empty string for name<commit_after>from datetime import datetime
from dateutil import parser as datetime_parser
from dateutil.tz import tzutc
from werkzeug.security import generate_password_hash, check_password_hash
from itsdangerous import TimedJSONWebSignatureSerializer as Serializer
from flask import g, url_for, current_app
from . import db
from .exceptions import ValidationError, ConflictError
from .utils import split_url

class User(db.Model):
    ''' Users DB interface '''
    __tablename__ = 'users'
    __table_args__ = {'extend_existing': True}

    id = db.Column(db.Integer, primary_key=True)
    username = db.Column(db.String(64), index=True, unique=True)
    password_hash = db.Column(db.String(128))

    def set_password(self, password):
        self.password_hash = generate_password_hash(password)

    def verify_password(self, password):
        return check_password_hash(self.password_hash, password)

    def generate_auth_token(self, expires_in=3600):
        s = Serializer(current_app.config['SECRET_KEY'], expires_in=expires_in)
        return s.dumps({'id': self.id}).decode('utf-8')

    @staticmethod
    def verify_auth_token(token):
        s = Serializer(current_app.config['SECRET_KEY'])
        try:
            data = s.loads(token)
        except:
            return None
        return User.query.get(data['id'])

    def import_data(self, data):
        try:
            if not self.query.filter_by(username=data['username']).count():
                self.username = data['username']
                self.password_hash = generate_password_hash(data['password'])
            else:
                raise ConflictError('That username is taken.')
        except KeyError as e:
            raise ValidationError('Invalid user: missing ' + e.args[0])
        return self

class Bucketlist(db.Model):
    ''' Bucketlists DB interface '''
    __tablename__ = 'bucketlists'

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(64), index=True)
    description = db.Column(db.Text)
    date_created = db.Column(db.DateTime, default=datetime.now)
    date_modified = db.Column(db.DateTime, onupdate=datetime.now)
    created_by = db.Column(db.Integer, db.ForeignKey('users.id'), index=True)
    items = db.relationship('BucketlistItem', backref='bucketlist',
                            lazy='dynamic',
                            cascade='all, delete-orphan')

    def get_url(self):
        return url_for('api.get_bucketlist', id=self.id, _external=True)

    def export_data(self):
        items = BucketlistItem.query.filter_by(bucketlist_id=self.id).all()
        return {
            'id': self.id,
            'name': self.name,
            'description': self.description,
            # change to export_data() if data of individual items is preferred
            'items': [item.get_url() for item in items],
            'date_created': self.date_created,
            'date_modified': self.date_modified,
            'created_by': self.created_by,
            'self_url': self.get_url(),
            'items_url': url_for('api.get_items', id=self.id,
                                 _external=True)
        }

    def import_data(self, data):
        try:
            if not self.query.filter_by(name=data['name']) \
                             .filter_by(created_by=g.user.id).count():
                if len(data['name']) != 0:
                    self.name = data['name']
                else:
                    raise ValidationError('name is empty')
                if 'description' in data:
                    self.description = data['description']
            else:
                raise ConflictError(
                    'You already have a bucketlist with that name.'
                )
        except KeyError as e:
            raise ValidationError('Invalid bucketlist: missing ' + e.args[0])
        return self

class BucketlistItem(db.Model):
    ''' Bucketlist Items DB interface '''
    __tablename__ = 'items'
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(64))
    description = db.Column(db.Text)
    bucketlist_id = db.Column(db.Integer, db.ForeignKey('bucketlists.id'))
    # created_by = db.Column(db.Integer, db.ForeignKey('users.id'))
    date_created = db.Column(db.DateTime, default=datetime.now)
    date_modified = db.Column(db.DateTime, onupdate=datetime.now)
    # user = db.relationship('User',
    #                        backref=db.backref('bucketlistitems',
    #                                           lazy='dynamic'))
    done = db.Column(db.Boolean, default=False)

    def get_url(self):
        return url_for(
                'api.get_item',
                item_id=self.id,
                id=self.bucketlist_id,
                _external=True
               )

    def export_data(self):
        return {
            'id': self.id,
            'name': self.name,
            'date_created': self.date_created,
            'date_modified': self.date_modified,
            'done': self.done,
            'self_url': self.get_url()
        }

    def import_data(self, data):
        try:
            # self.name = data['name']
            # if 'description' in data:
            #     self.description = data['description']
            if not self.query.filter_by(name=data['name']) \
                             .filter_by(
                                bucketlist_id=data['bucketlist_id']
                             ).count():
                if len(data['name']) != 0:
                    self.name = data['name']
                else:
                    raise ValidationError('name is empty')
                if 'description' in data:
                    self.description = data['description']
            else:
                raise ConflictError(
                    'You already have an item with that name.'
                )
        except KeyError as e:
            raise ValidationError('Invalid item: missing ' + e.args[0])
        return self
<|endoftext|>


<commit_before>"""
Contains the models to be used with the SQLAlchemy database interface.\n
Created by Abigail Franz on 3/12/2018.\n
Last modified by Abigail Franz on 4/16/2018.
"""

from datetime import datetime
from werkzeug.security import generate_password_hash, check_password_hash
from flask_login import UserMixin
from app import db, login

@login.user_loader
def load_admin(id):
	return Admin.query.get(int(id))

class Admin(UserMixin, db.Model):
	"""
	Represents a PowerToken administrator, capable of viewing the admin
	dashboard and supervising user progress.
	"""
	id = db.Column(db.Integer, primary_key=True)
	username = db.Column(db.String(64), index=True, unique=True)
	email = db.Column(db.String(120), index=True, unique=True)
	password_hash = db.Column(db.String(128))

	def set_password(self, password):
		self.password_hash = generate_password_hash(password)

	def check_password(self, password):
		return check_password_hash(self.password_hash, password)

	def __repr__(self):
		return "<Admin {}>".format(self.username)

class User(db.Model):
	"""
	Represents a PowerToken user who is in recovery.
	"""
	id = db.Column(db.Integer, primary_key=True)
	username = db.Column(db.String(64), index=True, unique=True)
	registered_on = db.Column(db.DateTime, index=True, default=datetime.now())
	goal_period = db.Column(db.String(16), default="daily")
	wc_id = db.Column(db.Integer, unique=True)
	wc_token = db.Column(db.String(128))
	fb_token = db.Column(db.String(256))
	logs = db.relationship("Log", backref="user", lazy="dynamic")
	activities = db.relationship("Activity", backref="user", lazy="dynamic")
	errors = db.relationship("Error", backref="user", lazy="dynamic")
	days = db.relationship("Day", backref="user", lazy="dynamic")

	def __repr__(self):
		return "<User {}>".format(self.username)

class Log(db.Model):
	"""
	Represents a WEconnect-Fitbit progress log.
	"""
	id = db.Column(db.Integer, primary_key=True)
	timestamp = db.Column(db.DateTime, index=True, default=datetime.now())
	daily_progress = db.Column(db.Float)
	weekly_progress = db.Column(db.Float)
	step_count = db.Column(db.Integer)
	user_id = db.Column(db.Integer, db.ForeignKey("user.id"))

	def __repr__(self):
		timestr = self.timestamp.strftime("%Y-%m-%d %I:%M %p")
		return "<Log {} at {}>".format(self.user.username, timestr)

class Activity(db.Model):
	"""
	Represents a WEconnect activity.
	"""
	id = db.Column(db.Integer, primary_key=True)
	wc_act_id = db.Column(db.Integer, index=True, unique=True)
	name = db.Column(db.String(256))
	expiration = db.Column(db.DateTime, index=True)
	weight = db.Column(db.Integer, default=1)
	user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
	events = db.relationship("Event", backref="activity", lazy="dynamic")

	def __repr__(self):
		return "<Activity '{}'>".format(self.name)

class Error(db.Model):
	"""
	Represents an error that occurred somewhere in the application(s).
	"""
	id = db.Column(db.Integer, primary_key=True)
	timestamp = db.Column(db.DateTime, default=datetime.now())
	summary = db.Column(db.String(64))
	origin = db.Column(db.String(256))
	message = db.Column(db.String(256))
	traceback = db.Column(db.String(1048))
	user_id = db.Column(db.Integer, db.ForeignKey("user.id"))

	def __repr__(self):
		return "<Error '{}', '{}'>".format(self.summary, self.message)

class Day(db.Model):
	"""
	Represents a day of progress (which activities are completed, etc).
	"""
	id = db.Column(db.Integer, primary_key=True)
	date = db.Column(db.DateTime, index=True)
	computed_progress = db.Column(db.Float, default=0.0)
	user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
	events = db.relationship("Event", backref="day", lazy="dynamic")

	def __repr__(self):
		return "<Day {}>".format(self.date.strftime("%Y-%m-%d"))

class Event(db.Model):
	"""
	Represents a WEconnect event (an activity on a particular date).
	"""
	id = db.Column(db.Integer, primary_key=True)
	eid = db.Column(db.String, index=True)
	start_time = db.Column(db.DateTime)	# Date portion is ignored
	end_time = db.Column(db.DateTime)	# Date portion is ignored
	completed = db.Column(db.Boolean)
	day_id = db.Column(db.Integer, db.ForeignKey("day.id"))
	activity_id = db.Column(db.Integer, db.ForeignKey("activity.id"))

	def __repr__(self):
		output = "<Event '{}' at {}>".format(self.activity.name,
				self.start_time.strftime("%I:%M %p"))<commit_msg>Drastic improvements to background.maintenance.<commit_after>"""
Contains the models to be used with the SQLAlchemy database interface.\n
Created by Abigail Franz on 3/12/2018.\n
Last modified by Abigail Franz on 4/16/2018.
"""

from datetime import datetime
from werkzeug.security import generate_password_hash, check_password_hash
from flask_login import UserMixin
from app import db, login

@login.user_loader
def load_admin(id):
	return Admin.query.get(int(id))

class Admin(UserMixin, db.Model):
	"""
	Represents a PowerToken administrator, capable of viewing the admin
	dashboard and supervising user progress.
	"""
	id = db.Column(db.Integer, primary_key=True)
	username = db.Column(db.String(64), index=True, unique=True)
	email = db.Column(db.String(120), index=True, unique=True)
	password_hash = db.Column(db.String(128))

	def set_password(self, password):
		self.password_hash = generate_password_hash(password)

	def check_password(self, password):
		return check_password_hash(self.password_hash, password)

	def __repr__(self):
		return "<Admin {}>".format(self.username)

class User(db.Model):
	"""
	Represents a PowerToken user who is in recovery.
	"""
	id = db.Column(db.Integer, primary_key=True)
	username = db.Column(db.String(64), index=True, unique=True)
	registered_on = db.Column(db.DateTime, index=True, default=datetime.now())
	goal_period = db.Column(db.String(16), default="daily")
	wc_id = db.Column(db.Integer, unique=True)
	wc_token = db.Column(db.String(128))
	fb_token = db.Column(db.String(256))
	logs = db.relationship("Log", backref="user", lazy="dynamic")
	activities = db.relationship("Activity", backref="user", lazy="dynamic")
	errors = db.relationship("Error", backref="user", lazy="dynamic")
	days = db.relationship("Day", backref="user", lazy="dynamic")

	def __repr__(self):
		return "<User {}>".format(self.username)

class Log(db.Model):
	"""
	Represents a WEconnect-Fitbit progress log.
	"""
	id = db.Column(db.Integer, primary_key=True)
	timestamp = db.Column(db.DateTime, index=True, default=datetime.now())
	daily_progress = db.Column(db.Float)
	weekly_progress = db.Column(db.Float)
	step_count = db.Column(db.Integer)
	user_id = db.Column(db.Integer, db.ForeignKey("user.id"))

	def __repr__(self):
		timestr = self.timestamp.strftime("%Y-%m-%d %I:%M %p")
		return "<Log {} at {}>".format(self.user.username, timestr)

class Activity(db.Model):
	"""
	Represents a WEconnect activity.
	"""
	id = db.Column(db.Integer, primary_key=True)
	wc_act_id = db.Column(db.Integer, index=True, unique=True)
	name = db.Column(db.String(256))
	expiration = db.Column(db.DateTime, index=True)
	weight = db.Column(db.Integer, default=1)
	user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
	events = db.relationship("Event", backref="activity", lazy="dynamic")

	def __repr__(self):
		return "<Activity '{}'>".format(self.name)

class Error(db.Model):
	"""
	Represents an error that occurred somewhere in the application(s).
	"""
	id = db.Column(db.Integer, primary_key=True)
	timestamp = db.Column(db.DateTime, default=datetime.now())
	summary = db.Column(db.String(64))
	origin = db.Column(db.String(256))
	message = db.Column(db.String(256))
	traceback = db.Column(db.String(1048))
	user_id = db.Column(db.Integer, db.ForeignKey("user.id"))

	def __repr__(self):
		return "<Error '{}', '{}'>".format(self.summary, self.message)

class Day(db.Model):
	"""
	Represents a day of progress (which activities are completed, etc).
	"""
	id = db.Column(db.Integer, primary_key=True)
	date = db.Column(db.DateTime, index=True)
	computed_progress = db.Column(db.Float, default=0.0)
	user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
	events = db.relationship("Event", backref="day", lazy="dynamic")

	def __repr__(self):
		return "<Day {}>".format(self.date.strftime("%Y-%m-%d"))

class Event(db.Model):
	"""
	Represents a WEconnect event (an activity on a particular date).
	"""
	id = db.Column(db.Integer, primary_key=True)
	eid = db.Column(db.String, index=True)
	start_time = db.Column(db.DateTime)	# Date portion is ignored
	end_time = db.Column(db.DateTime)	# Date portion is ignored
	completed = db.Column(db.Boolean)
	day_id = db.Column(db.Integer, db.ForeignKey("day.id"))
	activity_id = db.Column(db.Integer, db.ForeignKey("activity.id"))

	def __repr__(self):
		output = "<Event '{}' at {}>".format(self.activity.name,
				self.start_time.strftime("%I:%M %p"))
		return output<|endoftext|>


<commit_before># Copyright 2016 ACSONE SA/NV
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Alfodoo CMIS Web Proxy",
    "summary": """
        Odoo as proxy server for your cmis requests.""",
    "category": "Document Management",
    "version": "14.0.1.0.0",
    "license": "AGPL-3",
    "author": "ACSONE SA/NV",
    "website": "https://alfodoo.org",
    "depends": ["cmis_web"],
    "data": ["views/cmis_backend.xml", "views/cmis_web_proxy.xml"],
    "external_dependencies": {"python": ["requests", "cmislib"]},
    "images": ["static/description/main_icon.png"],
    "installable": True,
}
<commit_msg>cmis_web_proxy 14.0.1.0.1<commit_after># Copyright 2016 ACSONE SA/NV
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Alfodoo CMIS Web Proxy",
    "summary": """
        Odoo as proxy server for your cmis requests.""",
    "category": "Document Management",
    "version": "14.0.1.0.1",
    "license": "AGPL-3",
    "author": "ACSONE SA/NV",
    "website": "https://alfodoo.org",
    "depends": ["cmis_web"],
    "data": ["views/cmis_backend.xml", "views/cmis_web_proxy.xml"],
    "external_dependencies": {"python": ["requests", "cmislib"]},
    "images": ["static/description/main_icon.png"],
    "installable": True,
}
<|endoftext|>


<commit_before>from datetime import datetime

from sqlalchemy import (
    Column, Integer, Unicode, UnicodeText, Boolean, DateTime,
    create_engine
)
from sqlalchemy.ext import declarative
from sqlalchemy.orm import sessionmaker

Base = declarative.declarative_base()
engine = create_engine('sqlite:///db.sqlite3', echo=True)

class Task(Base):
    __tablename__ = 'tasks'
    id = Column(Integer, primary_key=True)
    title = Column(Unicode(255), nullable=False)
    detail = Column(UnicodeText)
    done = Column(Boolean, nullable=False, default=False)
    updated_at = Column(DateTime, default=datetime.now(), nullable=False)
    created_at = Column(DateTime, default=datetime.now(), nullable=False)

    def __repr__(self):
        return "<Task (title='%s')>" % self.title

    def update(self, title=None, detail=None, done=None, **kwargs):
        if title:
            self.title = title
        if detail:
            self.detail = detail
        if done:
            self.done = done
        self.updated_at = datetime.now()

    @property
    def serialize(self):
        return {
            'id': self.id,
            'title': self.title,
            'detail': self.detail,
            'done': self.done,
            'updated_at': self.updated_at.strftime('%Y-%m-%d'),
            'created_at': self.created_at.strftime('%Y-%m-%d'),
        }

Session = sessionmaker()
Session.configure(bind=engine)
Base.metadata.create_all(engine)

session = Session()
<commit_msg>Fix a bug that doesn't work done checkbox<commit_after>from datetime import datetime

from sqlalchemy import (
    Column, Integer, Unicode, UnicodeText, Boolean, DateTime,
    create_engine
)
from sqlalchemy.ext import declarative
from sqlalchemy.orm import sessionmaker

Base = declarative.declarative_base()
engine = create_engine('sqlite:///db.sqlite3', echo=True)

class Task(Base):
    __tablename__ = 'tasks'
    id = Column(Integer, primary_key=True)
    title = Column(Unicode(255), nullable=False)
    detail = Column(UnicodeText)
    done = Column(Boolean, nullable=False, default=False)
    updated_at = Column(DateTime, default=datetime.now(), nullable=False)
    created_at = Column(DateTime, default=datetime.now(), nullable=False)

    def __repr__(self):
        return "<Task (title='%s')>" % self.title

    def update(self, title=None, detail=None, done=None, **kwargs):
        if title is not None:
            self.title = title
        if detail is not None:
            self.detail = detail
        if done is not None:
            self.done = done
        self.updated_at = datetime.now()

    @property
    def serialize(self):
        return {
            'id': self.id,
            'title': self.title,
            'detail': self.detail,
            'done': self.done,
            'updated_at': self.updated_at.strftime('%Y-%m-%d'),
            'created_at': self.created_at.strftime('%Y-%m-%d'),
        }

Session = sessionmaker()
Session.configure(bind=engine)
Base.metadata.create_all(engine)

session = Session()
<|endoftext|>


<commit_before>import datetime
import os
import unicodedata

from whoosh.fields import Schema, ID, KEYWORD, STORED, TEXT

from rst import rst_to_html
from settings import DIRECTIVES, INDEX

SCHEMA = Schema(
    content=TEXT(stored=True),
    tags=KEYWORD,
    title=ID(unique=True, stored=True),
    time=STORED,
)

class Document(object):
    """
    A document consists of body or block-level elements, and may be structured
    into sections.

    :param title:
        A string representing the path to the file, e.g.
        ``"src/Andy_Warhol.rst"``.

    :param time:
        A float representing the time of last modification of *title* as
        measured in numbers of seconds since the last epoch, e.g.
        ``1405237014.0``.

    :param tags:
        A set of strings representing keywords that describe the file.
    """
    def __init__(self, title, time=None, tags=None):
        self._title = title
        self.time = time
        self.tags = tags or []

    @property
    def content(self):
        with open(self.title, 'rU') as f:
            return unicode(self._decode(f.read()))

    @property
    def datetime(self):
        """The date and time the document was modified."""
        return datetime.datetime.fromtimestamp(self.time)

    @property
    def filename(self):
        return os.path.splitext(os.path.split(self.title)[-1])[0]

    @property
    def html(self):
        with open(INDEX, 'rU') as index, open(DIRECTIVES, 'rU') as directives:
            body = index.read() + directives.read() + self.content
            return rst_to_html(body)

    def gen_elements(self):
        line_buffer = []
        with open(self.title, 'rU') as f:
            for line in f:
                if line.strip():
                    line_buffer.append(line.rstrip())
                else:
                    if line_buffer:
                        yield self._decode("\n".join(line_buffer))
                    line_buffer = []
        if line_buffer:
            yield self._decode("\n".join(line_buffer))

    def gen_paragraphs(self):
        for element in self.gen_elements():
            if self._element_is_paragraph(element):
                yield element

    @property
    def is_stale(self):
        return os.path.getmtime(self.title) > self.time

    @property
    def paragraphs(self):
        return list(self.gen_paragraphs())

    @property
    def reference_name(self):
        return self.filename.replace("_", " ")

    @property
    def title(self):
        return unicode(self._title)

    def _element_is_section_title(self, element):
        lines = element.split('\n')
        return any(len(set(line)) == 1 for line in lines)

    def _element_is_paragraph(self, element):
        if self._element_is_section_title(element):
            return False
        else:
            return element[0].isalpha() or element[0] == "*"

    def _decode(self, text, encoding="UTF-8"):
        """
        Encode a Unicode string as ascii, ignoring any errors.
        """
        unistr = text.decode(encoding)
        form = unicodedata.normalize('NFKD', unistr)
        return form.encode('ascii', 'ignore')

    def __repr__(self):
        return "Document(title=%s, time=%s, tags=%s)" % (self.title, self.time,
                                                         self.tags)

def gen_documents(src):
    for dirpath, _, filenames in os.walk(src):
        for filename in filenames:
            if not filename.startswith("."):
                filepath = os.path.join(dirpath, filename)
                modtime = os.path.getmtime(filepath)
                yield Document(title=filepath, time=modtime)
<commit_msg>Document gen_documents<commit_after>import datetime
import os
import unicodedata

from whoosh.fields import Schema, ID, KEYWORD, STORED, TEXT

from rst import rst_to_html
from settings import DIRECTIVES, INDEX

SCHEMA = Schema(
    content=TEXT(stored=True),
    tags=KEYWORD,
    title=ID(unique=True, stored=True),
    time=STORED,
)

class Document(object):
    """
    A document consists of body or block-level elements, and may be structured
    into sections.

    :param title:
        A string representing the path to the file, e.g.
        ``"src/Andy_Warhol.rst"``.

    :param time:
        A float representing the time of last modification of *title* as
        measured in numbers of seconds since the last epoch, e.g.
        ``1405237014.0``.

    :param tags:
        A set of strings representing keywords that describe the file.
    """
    def __init__(self, title, time=None, tags=None):
        self._title = title
        self.time = time
        self.tags = tags or []

    @property
    def content(self):
        with open(self.title, 'rU') as f:
            return unicode(self._decode(f.read()))

    @property
    def datetime(self):
        """The date and time the document was modified."""
        return datetime.datetime.fromtimestamp(self.time)

    @property
    def filename(self):
        return os.path.splitext(os.path.split(self.title)[-1])[0]

    @property
    def html(self):
        with open(INDEX, 'rU') as index, open(DIRECTIVES, 'rU') as directives:
            body = index.read() + directives.read() + self.content
            return rst_to_html(body)

    def gen_elements(self):
        line_buffer = []
        with open(self.title, 'rU') as f:
            for line in f:
                if line.strip():
                    line_buffer.append(line.rstrip())
                else:
                    if line_buffer:
                        yield self._decode("\n".join(line_buffer))
                    line_buffer = []
        if line_buffer:
            yield self._decode("\n".join(line_buffer))

    def gen_paragraphs(self):
        for element in self.gen_elements():
            if self._element_is_paragraph(element):
                yield element

    @property
    def is_stale(self):
        return os.path.getmtime(self.title) > self.time

    @property
    def paragraphs(self):
        return list(self.gen_paragraphs())

    @property
    def reference_name(self):
        return self.filename.replace("_", " ")

    @property
    def title(self):
        return unicode(self._title)

    def _element_is_section_title(self, element):
        lines = element.split('\n')
        return any(len(set(line)) == 1 for line in lines)

    def _element_is_paragraph(self, element):
        if self._element_is_section_title(element):
            return False
        else:
            return element[0].isalpha() or element[0] == "*"

    def _decode(self, text, encoding="UTF-8"):
        """
        Encode a Unicode string as ascii, ignoring any errors.
        """
        unistr = text.decode(encoding)
        form = unicodedata.normalize('NFKD', unistr)
        return form.encode('ascii', 'ignore')

    def __repr__(self):
        return "Document(title=%s, time=%s, tags=%s)" % (self.title, self.time,
                                                         self.tags)

def gen_documents(top):
    """
    Generate the Documents in a directory tree rooted at directory *top* by
    walking the tree top-down (including *top* itself).
    """
    for dirpath, _, filenames in os.walk(top):
        for filename in filenames:
            if not filename.startswith("."):
                filepath = os.path.join(dirpath, filename)
                modtime = os.path.getmtime(filepath)
                yield Document(title=filepath, time=modtime)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from cms.models import CMSPlugin, Placeholder
from cms.utils.compat.dj import python_2_unicode_compatible
from django.db import models

@python_2_unicode_compatible
class AliasPluginModel(CMSPlugin):
    plugin = models.ForeignKey(CMSPlugin, editable=False, related_name="alias_reference", null=True)
    alias_placeholder = models.ForeignKey(Placeholder, editable=False, related_name="alias_placeholder", null=True)

    class Meta:
        app_label = 'cms'

    def __str__(self):
        if self.plugin_id:
            return self.plugin.get_plugin_name()
        else:
            return self.alias_placeholder.get_label()
<commit_msg>Ensure that __str__ really only returns text (unicode)<commit_after># -*- coding: utf-8 -*-
from cms.models import CMSPlugin, Placeholder
from cms.utils.compat.dj import python_2_unicode_compatible
from django.utils.encoding import force_text
from django.db import models

@python_2_unicode_compatible
class AliasPluginModel(CMSPlugin):
    plugin = models.ForeignKey(CMSPlugin, editable=False, related_name="alias_reference", null=True)
    alias_placeholder = models.ForeignKey(Placeholder, editable=False, related_name="alias_placeholder", null=True)

    class Meta:
        app_label = 'cms'

    def __str__(self):
        if self.plugin_id:
            return force_text(self.plugin.get_plugin_name())
        else:
            return force_text(self.alias_placeholder.get_label())
<|endoftext|>


<commit_before>import inspect
import os

from coalib.misc.i18n import _

class StringConstants:
    THIS_IS_A_BUG = _("This is a bug. We are sorry for the inconvenience. "
                      "Please contact the developers for assistance.")

    OBJ_NOT_ACCESSIBLE = _("{} is not accessible and will be ignored!")

    TRUE_STRINGS = ['1',
                    "on",
                    'y',
                    'yes',
                    "yeah",
                    "sure",
                    'true',
                    'definitely',
                    'yup',
                    "right"]

    FALSE_STRINGS = ['0',
                     'off',
                     'n',
                     'no',
                     'nope',
                     'nah',
                     'false',
                     "wrong"]

    # This string contains many unicode characters to challenge tests.
    COMPLEX_TEST_STRING = ("4 r34l ch4ll3n63: 123 ÄÖü ABc @€¥ §&% {[( ←↓→↑ "
                           "ĦŊħ ß°^ \\\n\u2192")

    # Results from coverage for unittests are stored here.
    COVERAGE_DIR = "./.coverageresults"

    # Path to the coalib directory
    coalib_root = os.path.join(os.path.dirname(inspect.getfile(_)),
                               os.path.pardir)

    # Path to the directory containing the default bears
    coalib_bears_root = os.path.join(coalib_root, os.path.pardir, "bears")

    # Path to the language definition files
    language_definitions = os.path.join(coalib_root,
                                        "bearlib",
                                        "languages",
                                        "definitions")

    system_coafile = os.path.join(coalib_root, "default_coafile")

    user_coafile = os.path.join(os.path.expanduser("~"), ".coarc")
<commit_msg>StringConstants: Add python_executable<commit_after>import inspect
import os
import sys

from coalib.misc.i18n import _

class StringConstants:
    THIS_IS_A_BUG = _("This is a bug. We are sorry for the inconvenience. "
                      "Please contact the developers for assistance.")

    OBJ_NOT_ACCESSIBLE = _("{} is not accessible and will be ignored!")

    TRUE_STRINGS = ['1',
                    "on",
                    'y',
                    'yes',
                    "yeah",
                    "sure",
                    'true',
                    'definitely',
                    'yup',
                    "right"]

    FALSE_STRINGS = ['0',
                     'off',
                     'n',
                     'no',
                     'nope',
                     'nah',
                     'false',
                     "wrong"]

    # This string contains many unicode characters to challenge tests.
    COMPLEX_TEST_STRING = ("4 r34l ch4ll3n63: 123 ÄÖü ABc @€¥ §&% {[( ←↓→↑ "
                           "ĦŊħ ß°^ \\\n\u2192")

    # Results from coverage for unittests are stored here.
    COVERAGE_DIR = "./.coverageresults"

    # Path to the coalib directory
    coalib_root = os.path.join(os.path.dirname(inspect.getfile(_)),
                               os.path.pardir)

    # Path to the directory containing the default bears
    coalib_bears_root = os.path.join(coalib_root, os.path.pardir, "bears")

    # Path to the language definition files
    language_definitions = os.path.join(coalib_root,
                                        "bearlib",
                                        "languages",
                                        "definitions")

    system_coafile = os.path.join(coalib_root, "default_coafile")

    user_coafile = os.path.join(os.path.expanduser("~"), ".coarc")

    if sys.platform.startswith("win"):  # pragma: no cover
        python_executable = "python"
    else:
        python_executable = "python3"
<|endoftext|>


<commit_before>import argparse
import subprocess
import sys
import os

CLUSTER_SETTINGS = '" -q all.q -V -cwd -e ./log -o ./log {params.sge_opts} -w n -S /bin/bash"'
CLUSTER_FLAG = ("--drmaa", CLUSTER_SETTINGS, "-w", "30")

def _get_dist_dir():
    dirname, filename = os.path.split(os.path.abspath(__file__))
    return dirname

def _build_prefix(args):
    prefix = ["snakemake", "--snakefile", os.path.join(_get_dist_dir(), "Snakefile"), "-pq", "-j", str(args.jobs)]
    if args.dryrun:
        prefix.append("-n")

    if args.distribute:
        prefix.extend(CLUSTER_FLAG)

    return tuple(prefix)

def index(args):
    prefix = _build_prefix(args)
    command = prefix + ("prepare_reference", "--config", "reference=%s" % args.reference)
    return subprocess.call(" ".join(command), shell=True)

def align(args):
    prefix = _build_prefix(args)
    command = prefix + (
        "align_reads",
        "--config",
        "reference=%s" % args.reference,
        "reads=%s" % args.reads,
        "alignments=%s" % args.alignments,
        "alignments_dir=%s" % args.alignments_dir,
        "batches=%s" % args.batches,
        "threads=%s" % args.threads
    )
    return subprocess.call(" ".join(command), shell=True)

def detect(args):
    """
    Detect SVs from signatures in read alignments.
    """
    # Find candidate regions in alignments.
    sys.stdout.write("Searching for candidate regions\n")
    prefix = _build_prefix(args)

    command = prefix + (
        "get_regions",
        "--config",
        "reference=%s" % args.reference,
        "alignments=%s" % args.alignments
    )

    if args.exclude:
        command = command + ("regions_to_exclude=%s" % args.exclude,)

    if args.candidates:
        command = command + ("candidates=%s" % args.candidates,)

    return subprocess.call(" ".join(command), shell=True)

def assemble(args):
    """
    Assemble candidate regions from raw reads aligned to regions.
    """
    # Generate local assemblies across the genome.
    sys.stdout.write("Starting local assemblies\n")
    prefix = _build_prefix(args)

    command = prefix + (
        "collect_assembly_alignments",
        "--config",
        "reference=%s" % args.reference,
        "alignments=%s" % args.alignments,
        "reads=%s" % args.reads
    )

    if args.regions:
        command = command + ("regions_to_assemble=%s" % args.regions,)

    if args.assembly_alignments:
        command = command + ("assembly_alignments=%s" % args.assembly_alignments,)

    return subprocess.call(" ".join(command), shell=True)

def call(args):
    # Call SVs, indels, and inversions.
    sys.stdout.write("Calling variants\n")
    prefix = _build_prefix(args)
    command = prefix + (
        "call_variants",
        "--config",
        "reference=%s" % args.reference,
        "alignments=%s" % args.alignments,
        "local_assembly_alignments=%s" % args.assembly_alignments,
        "variants=%s" % args.variants
    )
    return_code = subprocess.call(" ".join(command), shell=True)

    if return_code != 0:
        sys.stderr.write("Failed to call variants\n")
        return return_code

def run(args):
    # Detect SVs.
    return_code = detect(args)
    if return_code != 0:
        sys.stderr.write("Failed to identify candidate regions\n")
        return return_code

    # Run local assemblies.
    return_code = assemble(args)
    if return_code != 0:
        sys.stderr.write("Failed to generate local assemblies\n")
        return return_code

    # Call SVs, indels, and inversions.
    return_code = call(args)
    if return_code != 0:
        sys.stderr.write("Failed to call variants\n")
        return return_code

def genotype(args):
    print("Genotype")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--dryrun", "-n", action="store_true", help="Print commands that will run without running them")
    parser.add_argument("--distribute", action="store_true", help="Distribute analysis to Grid Engine-style cluster")
    parser.add_argument("--jobs", help="number of jobs to run simultaneously", type=int, default=1)
    subparsers = parser.add_subparsers()

    # Index a reference for use by BLASR.
    parser_index = subparsers.add_parser("index", help="index a reference sequence for use by BLASR")
    parser_index.add_argument("reference", help="FASTA file of reference to index")
    parser_index.set_defaults(func=index)

    # Align PacBio reads to an indexed reference with BLASR.
    parser_align = subparsers.add_parser("align", help="align PacBio reads to an indexed reference with BLASR")
    parser_align.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_align.add_argument("reads", help="text file with one absolute path to a PacBio reads file (.bax.h5) per line")
    parser_align.add_argument("--alignments", help="text file with one absolute path to a BLASR alignments file (.bam) per line", default="alignments.fofn")
    parser_align.add_argument("--alignments_dir", help="absolute path of directory for BLASR alignment files", default="alignments")
    parser_align.add_argument("--batches", help="number of batches to split input reads into such that there will be one BAM output file per batch", type=int, default=1)
    parser_align.add_argument("--threads", help="number of threads to use for each BLASR alignment job", type=int, default=1)
    parser_align.set_defaults(func=align)

    # Detect SV signatures in BLASR alignments and build sliding windows to assemble.
    parser_detector = subparsers.add_parser("detect", help="detect SV signatures in BLASR-aligned reads")
    parser_detector.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_detector.add_argument("alignments", help="text file with one absolute path to a BLASR raw reads alignments file (.bam) per line")
    parser_detector.add_argument("candidates", help="BED file of candidates detected in read alignments")
    parser_detector.add_argument("--exclude", help="BED file of regions to exclude from local assembly (e.g., heterochromatic sequences, etc.)")
    parser_detector.set_defaults(func=detect)

    # Assemble candidate regions and align assemblies back to the reference.
    parser_assembler = subparsers.add_parser("assemble", help="assemble candidate regions and align assemblies back to the reference")
    parser_assembler.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_assembler.add_argument("reads", help="text file with one absolute path to a PacBio reads file (.bax.h5) per line")
    parser_assembler.add_argument("alignments", help="text file with one absolute path to a BLASR raw reads alignments file (.bam) per line")
    parser_assembler.add_argument("regions", help="BED file of regions to assemble from raw read alignments")
    parser_assembler.add_argument("assembly_alignments", help="BAM file with BLASR alignments of local assemblies against the reference")
    parser_assembler.set_defaults(func=assemble)

    # Call SVs and indels from BLASR alignments of local assemblies.
    parser_caller = subparsers.add_parser("call", help="call SVs and indels by BLASR alignments of local or whole genome assemblies")
    parser_caller.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_caller.add_argument("alignments", help="text file with one absolute path to a BLASR raw reads alignments file (.bam) per line")
    parser_caller.add_argument("assembly_alignments", help="BAM file with BLASR alignments of local assemblies against the reference")
    parser_caller.add_argument("variants", help="VCF of variants called by local assembly alignments")
    parser_caller.set_defaults(func=call)

    # Call SVs and indels from BLASR alignments of raw reads.
    parser_runner = subparsers.add_parser("run", help="call SVs and indels by local assembly of BLASR-aligned reads")
    parser_runner.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_runner.add_argument("reads", help="text file with one absolute path to a PacBio reads file (.bax.h5) per line")
    parser_runner.add_argument("alignments", help="text file with one absolute path to a BLASR alignments file (.bam) per line")
    parser_runner.add_argument("variants", help="VCF of variants called by local assembly alignments")
    parser_runner.add_argument("--exclude", help="BED file of regions to exclude from local assembly (e.g., heterochromatic sequences, etc.)")
    parser_runner.set_defaults(func=call)

    # Genotype SVs with Illumina reads.
    parser_genotyper = subparsers.add_parser("genotype", help="Genotype SVs with Illumina reads")
    parser_genotyper.add_argument("variants", help="VCF of SMRT SV variants to genotype")
    parser_genotyper.add_argument("genotyped_variants", help="VCF of SMRT SV variant genotypes for the given sample-level BAMs")
    parser_genotyper.add_argument("samples", nargs="+", help="one or more sample-level BAMs to genotype for the given variants")
    parser_genotyper.set_defaults(func=genotype)

    args = parser.parse_args()
    return_code = args.func(args)
    sys.exit(return_code)
<commit_msg>Added cluster-sync support.<commit_after>import argparse
import subprocess
import sys
import os

#CLUSTER_SETTINGS = '" -q all.q -V -cwd -e ./log -o ./log {params.sge_opts} -w n -S /bin/bash"'
#CLUSTER_FLAG = ("--drmaa", CLUSTER_SETTINGS, "-w", "120")
CLUSTER_SETTINGS = '"qsub -sync y -q all.q -V -cwd -e ./log -o ./log {params.sge_opts} -S /bin/bash"'
CLUSTER_FLAG = ("--cluster-sync", CLUSTER_SETTINGS, "-w", "120")

def _get_dist_dir():
    dirname, filename = os.path.split(os.path.abspath(__file__))
    return dirname

def _build_prefix(args):
    prefix = ["snakemake", "--snakefile", os.path.join(_get_dist_dir(), "Snakefile"), "-pq", "-j", str(args.jobs)]
    if args.dryrun:
        prefix.append("-n")

    if args.distribute:
        prefix.extend(CLUSTER_FLAG)

    return tuple(prefix)

def index(args):
    prefix = _build_prefix(args)
    command = prefix + ("prepare_reference", "--config", "reference=%s" % args.reference)
    return subprocess.call(" ".join(command), shell=True)

def align(args):
    prefix = _build_prefix(args)
    command = prefix + (
        "align_reads",
        "--config",
        "reference=%s" % args.reference,
        "reads=%s" % args.reads,
        "alignments=%s" % args.alignments,
        "alignments_dir=%s" % args.alignments_dir,
        "batches=%s" % args.batches,
        "threads=%s" % args.threads
    )
    return subprocess.call(" ".join(command), shell=True)

def detect(args):
    """
    Detect SVs from signatures in read alignments.
    """
    # Find candidate regions in alignments.
    sys.stdout.write("Searching for candidate regions\n")
    prefix = _build_prefix(args)

    command = prefix + (
        "get_regions",
        "--config",
        "reference=%s" % args.reference,
        "alignments=%s" % args.alignments
    )

    if args.exclude:
        command = command + ("regions_to_exclude=%s" % args.exclude,)

    if args.candidates:
        command = command + ("candidates=%s" % args.candidates,)

    return subprocess.call(" ".join(command), shell=True)

def assemble(args):
    """
    Assemble candidate regions from raw reads aligned to regions.
    """
    # Generate local assemblies across the genome.
    sys.stdout.write("Starting local assemblies\n")
    prefix = _build_prefix(args)

    command = prefix + (
        "collect_assembly_alignments",
        "--config",
        "reference=%s" % args.reference,
        "alignments=%s" % args.alignments,
        "reads=%s" % args.reads
    )

    if args.regions:
        command = command + ("regions_to_assemble=%s" % args.regions,)

    if args.assembly_alignments:
        command = command + ("assembly_alignments=%s" % args.assembly_alignments,)

    return subprocess.call(" ".join(command), shell=True)

def call(args):
    # Call SVs, indels, and inversions.
    sys.stdout.write("Calling variants\n")
    prefix = _build_prefix(args)
    command = prefix + (
        "call_variants",
        "--config",
        "reference=%s" % args.reference,
        "alignments=%s" % args.alignments,
        "local_assembly_alignments=%s" % args.assembly_alignments,
        "variants=%s" % args.variants
    )
    return_code = subprocess.call(" ".join(command), shell=True)

    if return_code != 0:
        sys.stderr.write("Failed to call variants\n")
        return return_code

def run(args):
    # Detect SVs.
    return_code = detect(args)
    if return_code != 0:
        sys.stderr.write("Failed to identify candidate regions\n")
        return return_code

    # Run local assemblies.
    return_code = assemble(args)
    if return_code != 0:
        sys.stderr.write("Failed to generate local assemblies\n")
        return return_code

    # Call SVs, indels, and inversions.
    return_code = call(args)
    if return_code != 0:
        sys.stderr.write("Failed to call variants\n")
        return return_code

def genotype(args):
    print("Genotype")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--dryrun", "-n", action="store_true", help="Print commands that will run without running them")
    parser.add_argument("--distribute", action="store_true", help="Distribute analysis to Grid Engine-style cluster")
    parser.add_argument("--jobs", help="number of jobs to run simultaneously", type=int, default=1)
    subparsers = parser.add_subparsers()

    # Index a reference for use by BLASR.
    parser_index = subparsers.add_parser("index", help="index a reference sequence for use by BLASR")
    parser_index.add_argument("reference", help="FASTA file of reference to index")
    parser_index.set_defaults(func=index)

    # Align PacBio reads to an indexed reference with BLASR.
    parser_align = subparsers.add_parser("align", help="align PacBio reads to an indexed reference with BLASR")
    parser_align.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_align.add_argument("reads", help="text file with one absolute path to a PacBio reads file (.bax.h5) per line")
    parser_align.add_argument("--alignments", help="text file with one absolute path to a BLASR alignments file (.bam) per line", default="alignments.fofn")
    parser_align.add_argument("--alignments_dir", help="absolute path of directory for BLASR alignment files", default="alignments")
    parser_align.add_argument("--batches", help="number of batches to split input reads into such that there will be one BAM output file per batch", type=int, default=1)
    parser_align.add_argument("--threads", help="number of threads to use for each BLASR alignment job", type=int, default=1)
    parser_align.set_defaults(func=align)

    # Detect SV signatures in BLASR alignments and build sliding windows to assemble.
    parser_detector = subparsers.add_parser("detect", help="detect SV signatures in BLASR-aligned reads")
    parser_detector.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_detector.add_argument("alignments", help="text file with one absolute path to a BLASR raw reads alignments file (.bam) per line")
    parser_detector.add_argument("candidates", help="BED file of candidates detected in read alignments")
    parser_detector.add_argument("--exclude", help="BED file of regions to exclude from local assembly (e.g., heterochromatic sequences, etc.)")
    parser_detector.set_defaults(func=detect)

    # Assemble candidate regions and align assemblies back to the reference.
    parser_assembler = subparsers.add_parser("assemble", help="assemble candidate regions and align assemblies back to the reference")
    parser_assembler.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_assembler.add_argument("reads", help="text file with one absolute path to a PacBio reads file (.bax.h5) per line")
    parser_assembler.add_argument("alignments", help="text file with one absolute path to a BLASR raw reads alignments file (.bam) per line")
    parser_assembler.add_argument("regions", help="BED file of regions to assemble from raw read alignments")
    parser_assembler.add_argument("assembly_alignments", help="BAM file with BLASR alignments of local assemblies against the reference")
    parser_assembler.set_defaults(func=assemble)

    # Call SVs and indels from BLASR alignments of local assemblies.
    parser_caller = subparsers.add_parser("call", help="call SVs and indels by BLASR alignments of local or whole genome assemblies")
    parser_caller.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_caller.add_argument("alignments", help="text file with one absolute path to a BLASR raw reads alignments file (.bam) per line")
    parser_caller.add_argument("assembly_alignments", help="BAM file with BLASR alignments of local assemblies against the reference")
    parser_caller.add_argument("variants", help="VCF of variants called by local assembly alignments")
    parser_caller.set_defaults(func=call)

    # Call SVs and indels from BLASR alignments of raw reads.
    parser_runner = subparsers.add_parser("run", help="call SVs and indels by local assembly of BLASR-aligned reads")
    parser_runner.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_runner.add_argument("reads", help="text file with one absolute path to a PacBio reads file (.bax.h5) per line")
    parser_runner.add_argument("alignments", help="text file with one absolute path to a BLASR alignments file (.bam) per line")
    parser_runner.add_argument("variants", help="VCF of variants called by local assembly alignments")
    parser_runner.add_argument("--exclude", help="BED file of regions to exclude from local assembly (e.g., heterochromatic sequences, etc.)")
    parser_runner.set_defaults(func=call)

    # Genotype SVs with Illumina reads.
    parser_genotyper = subparsers.add_parser("genotype", help="Genotype SVs with Illumina reads")
    parser_genotyper.add_argument("variants", help="VCF of SMRT SV variants to genotype")
    parser_genotyper.add_argument("genotyped_variants", help="VCF of SMRT SV variant genotypes for the given sample-level BAMs")
    parser_genotyper.add_argument("samples", nargs="+", help="one or more sample-level BAMs to genotype for the given variants")
    parser_genotyper.set_defaults(func=genotype)

    args = parser.parse_args()
    return_code = args.func(args)
    sys.exit(return_code)
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" ForwardingRuleMigration class: It is the handler to migrate
a forwarding rule based on the type of it.

"""
import warnings

from vm_network_migration.errors import *
from vm_network_migration.handler_helper.selfLink_executor import SelfLinkExecutor
from vm_network_migration.module_helpers.forwarding_rule_helper import ForwardingRuleHelper
from vm_network_migration.modules.forwarding_rule_modules.external_regional_forwarding_rule import ExternalRegionalForwardingRule
from vm_network_migration.modules.forwarding_rule_modules.global_forwarding_rule import GlobalForwardingRule
from vm_network_migration.modules.forwarding_rule_modules.internal_regional_forwarding_rule import InternalRegionalForwardingRule
from vm_network_migration.utils import initializer
from vm_network_migration.handlers.compute_engine_resource_migration import ComputeEngineResourceMigration
from vm_network_migration.handlers.backend_service_migration import BackendServiceMigration
class ForwardingRuleMigration(ComputeEngineResourceMigration):
    @initializer
    def __init__(self, compute, project, forwarding_rule_name,
                 network_name, subnetwork_name,
                 preserve_instance_external_ip, region=None):
        """ Initialize a InstanceNetworkMigration object

        Args:
            project: project ID
            forwarding_rule_name: name of the forwarding rule
            network: target network
            subnetwork: target subnet
            preserve_instance_external_ip: whether preserve the external IP
            of the instances which serves this load balancer
            region: region of the internal load balancer
        """
        super(ForwardingRuleMigration, self).__init__()
        self.forwarding_rule = self.build_forwarding_rule()
        self.backends_migration_handlers = []

    def build_forwarding_rule(self):
        """ Use a helper class to create a ForwardingRule object

        Returns: a ForwardingRule object

        """
        forwarding_rule_helper = ForwardingRuleHelper(self.compute,
                                                      self.project,
                                                      self.forwarding_rule_name,
                                                      self.network_name,
                                                      self.subnetwork_name,
                                                      self.region)
        return forwarding_rule_helper.build_a_forwarding_rule()

    def migrate_an_external_regional_forwarding_rule(self):
        """ Network migration for an external regional forwarding rule.
        The external regional forwarding rule is in use by a target pool.
        So the tool will run the target pool migration.

        Returns:

        """
        target_pool_selfLink = self.forwarding_rule.target_pool_selfLink
        if target_pool_selfLink == None:
            print('No backends need to be migrated. Terminating the migration.')
            return
        selfLink_executor = SelfLinkExecutor(self.compute, target_pool_selfLink,
                                             self.network_name,
                                             self.subnetwork_name,
                                             self.preserve_instance_external_ip)
        backends_migration_handler = selfLink_executor.build_target_pool_migration_handler()
        self.backends_migration_handlers.append(backends_migration_handler)
        print('Migrating the target pool.')
        backends_migration_handler.network_migration()

    def migrate_an_internal_regional_forwarding_rule(self):
        """ Network migration for an internal regional forwarding rule.
        The internal regional forwarding rule will be in use by a single
        backend service. The forwarding rule will be deleted first.
        Then, the tool will migrate the backend service.
        Finally, recreate the forwarding rule in the target subnet.

        Returns:

        """
        backend_service_selfLink = self.forwarding_rule.backend_service_selfLink
        if backend_service_selfLink != None:

            selfLink_executor = SelfLinkExecutor(self.compute,
                                                 backend_service_selfLink,
                                                 self.network_name,
                                                 self.subnetwork_name,
                                                 self.preserve_instance_external_ip)
            # the backends can be a target instance or an internal backend service
            backends_migration_handler = selfLink_executor.build_migration_handler()
            if backends_migration_handler != None:
                self.backends_migration_handlers.append(
                    backends_migration_handler)
                if isinstance(backends_migration_handler,
                              BackendServiceMigration):
                    backend_service = backends_migration_handler.backend_service
                    if backend_service != None and backend_service.count_forwarding_rules() > 1:
                        print(
                            'The backend service is associated with two or more forwarding rules, \n'
                            'so it can not be migrated. \n'
                            'Terminating. ')
                        return

        print('Deleting the forwarding rule %s.' %(self.forwarding_rule_name))
        self.forwarding_rule.delete_forwarding_rule()
        for backend_service in self.backends_migration_handlers:
            print('Migrating the backends of the forwarding rule.')
            backend_service.network_migration()
        print('Recreating the forwarding rule in the target subnet.')
        self.forwarding_rule.insert_forwarding_rule(
            self.forwarding_rule.new_forwarding_rule_configs)

    def migrate_a_global_forwarding_rule(self):
        """ Network migration for a global forwarding rule. The global
        forwarding rule points to a target proxy. Through the target proxy,
        the tool can find the backend services information.
        The tool will migrate these backends one by one without deleting
        and recreating.

        Returns:

        """
        backend_service_selfLinks = self.forwarding_rule.backend_service_selfLinks
        if backend_service_selfLinks == []:
            print('No backends need to be migrated. Terminating the migration.')
            return
        for selfLink in backend_service_selfLinks:
            selfLink_executor = SelfLinkExecutor(self.compute, selfLink,
                                                 self.network_name,
                                                 self.subnetwork_name,
                                                 self.preserve_instance_external_ip)
            backend_service_migration_handler = selfLink_executor.build_backend_service_migration_handler()
            # Save handlers for rollback purpose
            self.backends_migration_handlers.append(
                backend_service_migration_handler)
            backend_service_migration_handler.network_migration()

    def rollback_a_global_forwarding_rule(self):
        """ Rollback a global forwarding rule

        Returns:

        """
        for backend_service_migration_handler in self.backends_migration_handlers:
            backend_service_migration_handler.rollback()

    def rollback_an_internal_regional_forwarding_rule(self):
        """ Rollback an internal regional forwarding rule

        Returns:

        """
        if not self.forwarding_rule.migrated and \
                self.forwarding_rule.check_forwarding_rule_exists():
            return
        if self.forwarding_rule.migrated:
            print('Deleting the migrated forwarding rule.')
            self.forwarding_rule.delete_forwarding_rule()
            for backend_service_migration_handler in self.backends_migration_handlers:
                backend_service_migration_handler.rollback()
            self.forwarding_rule.insert_forwarding_rule(
                self.forwarding_rule.forwarding_rule_configs)
        else:
            for backend_service_migration_handler in self.backends_migration_handlers:
                backend_service_migration_handler.rollback()
            self.forwarding_rule.insert_forwarding_rule(
                self.forwarding_rule.forwarding_rule_configs)

    def rollback_an_external_regional_forwarding_rule(self):
        """ Rollback an external regional forwarding rule

        Returns:

        """
        for target_pool_migration_handler in self.backends_migration_handlers:
            target_pool_migration_handler.rollback()

    def network_migration(self):
        """ Select correct network migration functions based on the type of the
        forwarding rule.

        Returns:

        """
        print('Migrating: %s' %(self.forwarding_rule_name))
        try:
            if isinstance(self.forwarding_rule, ExternalRegionalForwardingRule):
                self.migrate_an_external_regional_forwarding_rule()
            elif isinstance(self.forwarding_rule,
                            InternalRegionalForwardingRule):
                self.migrate_an_internal_regional_forwarding_rule()
            elif isinstance(self.forwarding_rule, GlobalForwardingRule):
                self.migrate_a_global_forwarding_rule()
        except Exception as e:
            warnings.warn(e, Warning)
            self.rollback()
            raise MigrationFailed('Rollback has been finished.')

    def rollback(self):
        """ Error happens. Rollback to the original status.

        """
        print(
            'The migration was failed. Rolling back to the original forwarding rule settings.')
        if isinstance(self.forwarding_rule, ExternalRegionalForwardingRule):
            self.rollback_an_external_regional_forwarding_rule()
        elif isinstance(self.forwarding_rule, InternalRegionalForwardingRule):
            self.rollback_an_internal_regional_forwarding_rule()
        elif isinstance(self.forwarding_rule, GlobalForwardingRule):
            self.rollback_a_global_forwarding_rule()
        print('Rollback has been finished.')
<commit_msg>fixed: find all backend services selfLinks through urlmapping config<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" ForwardingRuleMigration class: It is the handler to migrate
a forwarding rule based on the type of it.

"""
import warnings

from vm_network_migration.errors import *
from vm_network_migration.handler_helper.selfLink_executor import SelfLinkExecutor
from vm_network_migration.module_helpers.forwarding_rule_helper import ForwardingRuleHelper
from vm_network_migration.modules.forwarding_rule_modules.external_regional_forwarding_rule import ExternalRegionalForwardingRule
from vm_network_migration.modules.forwarding_rule_modules.global_forwarding_rule import GlobalForwardingRule
from vm_network_migration.modules.forwarding_rule_modules.internal_regional_forwarding_rule import InternalRegionalForwardingRule
from vm_network_migration.utils import initializer
from vm_network_migration.handlers.compute_engine_resource_migration import ComputeEngineResourceMigration
from vm_network_migration.handlers.backend_service_migration import BackendServiceMigration
class ForwardingRuleMigration(ComputeEngineResourceMigration):
    @initializer
    def __init__(self, compute, project, forwarding_rule_name,
                 network_name, subnetwork_name,
                 preserve_instance_external_ip, region=None):
        """ Initialize a InstanceNetworkMigration object

        Args:
            project: project ID
            forwarding_rule_name: name of the forwarding rule
            network: target network
            subnetwork: target subnet
            preserve_instance_external_ip: whether preserve the external IP
            of the instances which serves this load balancer
            region: region of the internal load balancer
        """
        super(ForwardingRuleMigration, self).__init__()
        self.forwarding_rule = self.build_forwarding_rule()
        self.backends_migration_handlers = []

    def build_forwarding_rule(self):
        """ Use a helper class to create a ForwardingRule object

        Returns: a ForwardingRule object

        """
        forwarding_rule_helper = ForwardingRuleHelper(self.compute,
                                                      self.project,
                                                      self.forwarding_rule_name,
                                                      self.network_name,
                                                      self.subnetwork_name,
                                                      self.region)
        return forwarding_rule_helper.build_a_forwarding_rule()

    def migrate_an_external_regional_forwarding_rule(self):
        """ Network migration for an external regional forwarding rule.
        The external regional forwarding rule is in use by a target pool.
        So the tool will run the target pool migration.

        Returns:

        """
        target_pool_selfLink = self.forwarding_rule.target_pool_selfLink
        if target_pool_selfLink == None:
            print('No backends need to be migrated. Terminating the migration.')
            return
        selfLink_executor = SelfLinkExecutor(self.compute, target_pool_selfLink,
                                             self.network_name,
                                             self.subnetwork_name,
                                             self.preserve_instance_external_ip)
        backends_migration_handler = selfLink_executor.build_migration_handler()
        if backends_migration_handler != None:
            self.backends_migration_handlers.append(backends_migration_handler)
            print('Migrating the target pool.')
            backends_migration_handler.network_migration()

    def migrate_an_internal_regional_forwarding_rule(self):
        """ Network migration for an internal regional forwarding rule.
        The internal regional forwarding rule will be in use by a single
        backend service. The forwarding rule will be deleted first.
        Then, the tool will migrate the backend service.
        Finally, recreate the forwarding rule in the target subnet.

        Returns:

        """
        backend_service_selfLink = self.forwarding_rule.backend_service_selfLink
        if backend_service_selfLink != None:

            selfLink_executor = SelfLinkExecutor(self.compute,
                                                 backend_service_selfLink,
                                                 self.network_name,
                                                 self.subnetwork_name,
                                                 self.preserve_instance_external_ip)
            # the backends can be a target instance or an internal backend service
            backends_migration_handler = selfLink_executor.build_migration_handler()
            if backends_migration_handler != None:
                self.backends_migration_handlers.append(
                    backends_migration_handler)
                if isinstance(backends_migration_handler,
                              BackendServiceMigration):
                    backend_service = backends_migration_handler.backend_service
                    if backend_service != None and backend_service.count_forwarding_rules() > 1:
                        print(
                            'The backend service is associated with two or more forwarding rules, \n'
                            'so it can not be migrated. \n'
                            'Terminating. ')
                        return

        print('Deleting the forwarding rule %s.' %(self.forwarding_rule_name))
        self.forwarding_rule.delete_forwarding_rule()
        for backend_service in self.backends_migration_handlers:
            print('Migrating the backends of the forwarding rule.')
            backend_service.network_migration()
        print('Recreating the forwarding rule in the target subnet.')
        self.forwarding_rule.insert_forwarding_rule(
            self.forwarding_rule.new_forwarding_rule_configs)

    def migrate_a_global_forwarding_rule(self):
        """ Network migration for a global forwarding rule. The global
        forwarding rule points to a target proxy. Through the target proxy,
        the tool can find the backend services information.
        The tool will migrate these backends one by one without deleting
        and recreating.

        Returns:

        """
        backend_service_selfLinks = self.forwarding_rule.backend_service_selfLinks
        if backend_service_selfLinks == []:
            print('No backends need to be migrated. Terminating the migration.')
            return
        for selfLink in backend_service_selfLinks:
            selfLink_executor = SelfLinkExecutor(self.compute, selfLink,
                                                 self.network_name,
                                                 self.subnetwork_name,
                                                 self.preserve_instance_external_ip)
            backend_service_migration_handler = selfLink_executor.build_migration_handler()
            # Save handlers for rollback purpose
            if backend_service_migration_handler != None:
                self.backends_migration_handlers.append(
                    backend_service_migration_handler)
                backend_service_migration_handler.network_migration()

    def rollback_a_global_forwarding_rule(self):
        """ Rollback a global forwarding rule

        Returns:

        """
        for backend_service_migration_handler in self.backends_migration_handlers:
            backend_service_migration_handler.rollback()

    def rollback_an_internal_regional_forwarding_rule(self):
        """ Rollback an internal regional forwarding rule

        Returns:

        """
        if not self.forwarding_rule.migrated and \
                self.forwarding_rule.check_forwarding_rule_exists():
            return
        if self.forwarding_rule.migrated:
            print('Deleting the migrated forwarding rule.')
            self.forwarding_rule.delete_forwarding_rule()
            for backend_service_migration_handler in self.backends_migration_handlers:
                backend_service_migration_handler.rollback()
            self.forwarding_rule.insert_forwarding_rule(
                self.forwarding_rule.forwarding_rule_configs)
        else:
            for backend_service_migration_handler in self.backends_migration_handlers:
                backend_service_migration_handler.rollback()
            self.forwarding_rule.insert_forwarding_rule(
                self.forwarding_rule.forwarding_rule_configs)

    def rollback_an_external_regional_forwarding_rule(self):
        """ Rollback an external regional forwarding rule

        Returns:

        """
        for target_pool_migration_handler in self.backends_migration_handlers:
            target_pool_migration_handler.rollback()

    def network_migration(self):
        """ Select correct network migration functions based on the type of the
        forwarding rule.

        Returns:

        """
        print('Migrating: %s' %(self.forwarding_rule_name))
        try:
            if isinstance(self.forwarding_rule, ExternalRegionalForwardingRule):
                self.migrate_an_external_regional_forwarding_rule()
            elif isinstance(self.forwarding_rule,
                            InternalRegionalForwardingRule):
                self.migrate_an_internal_regional_forwarding_rule()
            elif isinstance(self.forwarding_rule, GlobalForwardingRule):
                self.migrate_a_global_forwarding_rule()
        except Exception as e:
            warnings.warn(e, Warning)
            self.rollback()
            raise MigrationFailed('Rollback has been finished.')

    def rollback(self):
        """ Error happens. Rollback to the original status.

        """
        print(
            'The migration was failed. Rolling back to the original forwarding rule settings.')
        if isinstance(self.forwarding_rule, ExternalRegionalForwardingRule):
            self.rollback_an_external_regional_forwarding_rule()
        elif isinstance(self.forwarding_rule, InternalRegionalForwardingRule):
            self.rollback_an_internal_regional_forwarding_rule()
        elif isinstance(self.forwarding_rule, GlobalForwardingRule):
            self.rollback_a_global_forwarding_rule()
        print('Rollback has been finished.')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import io
import os

class _EmbeddedCSV(object):
    """Internal use class used to encapsulate a DwcA-enclosed CSV file and its metadata."""
    # TODO: Test this class
    # Not done yet cause issues there will probably make DwCAReader tests fails anyway
    # In the future it could make sense to make it public
    def __init__(self, metadata_section, unzipped_folder_path):
        #metadata_section: <core> or <extension> section of metaxml concerning the file to iterate.
        #unzipped_folder_path: absolute path to the directory containing the unzipped archive.

        self._metadata_section = metadata_section
        self._unzipped_folder_path = unzipped_folder_path

        self._core_fhandler = io.open(self.filepath,
                                      mode='r',
                                      encoding=self.encoding,
                                      newline=self.newline_str,
                                      errors='replace')

    @property
    def headers(self):
        """Returns a list of (ordered) column names that can be used to create a header line."""
        field_tags = self._metadata_section.find_all('field')

        columns = {}

        for tag in field_tags:
            columns[int(tag['index'])] = tag['term']

        # In addition to DwC terms, we may also have id or core_id columns
        if self._metadata_section.id:
            columns[int(self._metadata_section.id['index'])] = 'id'
        if self._metadata_section.coreid:
            columns[int(self._metadata_section.id['coreindex'])] = 'coreid'

        return [columns[f] for f in sorted(columns.iterkeys())]

    @property
    def filepath(self):
        """Returns the absolute path to the 'subject' file."""
        return os.path.join(self._unzipped_folder_path,
                            self._metadata_section.files.location.string)

    @property
    def encoding(self):
        return self._metadata_section['encoding']

    @property
    def newline_str(self):
        return self._metadata_section['linesTerminatedBy'].decode("string-escape")

    def _position_file_after_header(self):
        self._core_fhandler.seek(0, 0)
        if self.lines_to_ignore > 0:
            self._core_fhandler.readlines(self.lines_to_ignore)

    def __iter__(self):
        self._position_file_after_header()
        return self

    def next(self):
        for line in self._core_fhandler:
            return line

        raise StopIteration

    # TODO: Optimize for large files ?
    # ideas: http://stackoverflow.com/questions/620367/python-how-to-jump-to-a-particular-line-in-a-huge-text-file
    def get_row_by_index(self, index):
        self._position_file_after_header()

        for (i, row) in enumerate(self._core_fhandler):
            if i == index:
                return row
        else:
            return None  # Reached end of file

    @property
    def lines_to_ignore(self):
        try:
            return int(self._metadata_section['ignoreHeaderLines'])
        except KeyError:
            return 0
<commit_msg>Much better performance for access to random line (and indirectly looping) for large CSV files, at the cost of an 'index building step' on file open.<commit_after># -*- coding: utf-8 -*-

import io
import os

class _EmbeddedCSV(object):
    """Internal use class used to encapsulate a DwcA-enclosed CSV file and its metadata."""
    # TODO: Test this class
    # Not done yet cause issues there will probably make DwCAReader tests fails anyway
    # In the future it could make sense to make it public
    def __init__(self, metadata_section, unzipped_folder_path):
        #metadata_section: <core> or <extension> section of metaxml concerning the file to iterate.
        #unzipped_folder_path: absolute path to the directory containing the unzipped archive.

        self._metadata_section = metadata_section
        self._unzipped_folder_path = unzipped_folder_path

        self._core_fhandler = io.open(self.filepath,
                                      mode='r',
                                      encoding=self.encoding,
                                      newline=self.newline_str,
                                      errors='replace')

        # On init, we parse the file once to build an index of newlines (including lines to ignore)
        # that will make random access faster later on...
        self._line_offsets = get_all_line_offsets(self._core_fhandler, self.encoding)

    @property
    def headers(self):
        """Returns a list of (ordered) column names that can be used to create a header line."""
        field_tags = self._metadata_section.find_all('field')

        columns = {}

        for tag in field_tags:
            columns[int(tag['index'])] = tag['term']

        # In addition to DwC terms, we may also have id or core_id columns
        if self._metadata_section.id:
            columns[int(self._metadata_section.id['index'])] = 'id'
        if self._metadata_section.coreid:
            columns[int(self._metadata_section.id['coreindex'])] = 'coreid'

        return [columns[f] for f in sorted(columns.iterkeys())]

    @property
    def filepath(self):
        """Returns the absolute path to the 'subject' file."""
        return os.path.join(self._unzipped_folder_path,
                            self._metadata_section.files.location.string)

    @property
    def encoding(self):
        return self._metadata_section['encoding']

    @property
    def newline_str(self):
        return self._metadata_section['linesTerminatedBy'].decode("string-escape")

    def _position_file_after_header(self):
        self._core_fhandler.seek(0, 0)
        if self.lines_to_ignore > 0:
            self._core_fhandler.readlines(self.lines_to_ignore)

    def __iter__(self):
        self._position_file_after_header()
        return self

    def next(self):
        for line in self._core_fhandler:
            return line

        raise StopIteration

    def get_row_by_index(self, index):
        if index < 0:
            return None
        try:
            self._core_fhandler.seek(self._line_offsets[index + self.lines_to_ignore], 0)
            return self._core_fhandler.readline()
        except IndexError:
            return None

    @property
    def lines_to_ignore(self):
        try:
            return int(self._metadata_section['ignoreHeaderLines'])
        except KeyError:
            return 0

def get_all_line_offsets(f, encoding):
    """ Parse the file whose handler is given and return a list of each line beginning positions.

        The value returned is suitable for seek() operations.
        This can take long for large files.

        It needs to know the encoding to properly count the bytes in a given string.
    """

    f.seek(0, 0)
    line_offsets = []
    offset = 0
    for line in f:
        line_offsets.append(offset)
        offset += len(line.encode(encoding))

    f.seek(0, 0)
    return line_offsets
<|endoftext|>


<commit_before>import os
import json
import time
import shutil
import logging
import subprocess

from pathlib import Path
from copy import deepcopy
from tempfile import NamedTemporaryFile
from collections import defaultdict

from . import config

SEPARATORS = (',', ':')
TIMESTAMP_KEYS = {'dump': 'dumpDate',
                  'classes': 'classUpdate',
                  'properties': 'propertyUpdate',
                  }
ISO8601_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
logger = logging.getLogger(__name__)

def get_current_timestamp():
    """Returns the current timestamp in suitable (ISO 8601) format."""
    return time.strftime(ISO8601_TIMESTAMP_FORMAT)

def _json_file_name(name):
    """Returns the name of the statistics file for `name`."""
    return name + '.json'

def get_json_data(name):
    """Load statistics file "`name`.json"."""
    try:
        with open(_json_file_name(name), 'r') as file:
            return json.load(file)
    except OSError:
        return {}

def update_json_data(name, data, timestamp=None):
    """Atomically replaces the data in file "`name'.json"."""
    try:
        with NamedTemporaryFile(mode='w', delete=False) as file:
            logger.info('Writing new JSON %s data ...', name)
            json.dump(data, file, separators=SEPARATORS)
            logger.info('Writing new %s JSON file ...', name)
    except RuntimeError as err:
        logger.error('Writing dump failed: %s', err)
    else:
        os.chmod(file.name, 0o644)
        shutil.move(file.name, _json_file_name(name))
        if timestamp is not None:
            update_timestamp(name, timestamp)
        logger.info('Update for %s complete.', name)

def update_split_json_data(name, data, chunk_size):
    """Updates split JSON data for file "`name.json",
       with customisable chunk size."""

    chunks = defaultdict(dict)

    for entity, record in data.items():
        chunk = int(entity) // chunk_size
        chunks[chunk][entity] = record

    for index, chunk in chunks.items():
        update_json_data('{}-{}'.format(name, index), chunk)

def get_timestamp(name):
  """Retrieves the statistics timestamp `name`."""
  data = get_json_data('statistics')
  return data[TIMESTAMP_KEYS[name]]

def update_timestamp(name, timestamp):
    """Sets the statistics timestamp `name` to `timestamp`."""
    data = get_json_data('statistics')
    data[TIMESTAMP_KEYS[name]] = timestamp

    with open(_json_file_name('statistics'), 'w') as file:
        json.dump(data, file)

def merge(old, new, default_others=None):
    """Merges two statistics documents `old` and `new`.

    `default_others` is a dict that will be applied to all items
    not updated.
    """
    merged = defaultdict(dict, deepcopy(old))

    for key in new.keys():
        merged[key].update(new[key])

    if default_others is not None:
        others = old.keys() - new.keys()

        for key in others:
            merged[key].update(default_others)

    return dict(merged)

def check_new_dump(script_path):
  """Check for a new dump file. If present, queue a job on the grid to
  rebuild the full statistics."""
  dumpdate = get_timestamp('dump')
  basedir = Path(config.DUMP_LOCATION)
  dumps = {}

  logger.debug("Enumerating dumps in `%s'", basedir)
  for subdir in (path for path in basedir.iterdir() if path.is_dir()):
    date = subdir.parts[-1]
    filename = config.DUMP_BASENAME.format(date=date)
    dumpfile = subdir.joinpath(filename)

    if dumpfile.exists():
      logger.debug("Found dump `%s', dated `%s'", filename, date)
      dumps[date] = dumpfile

  latest = sorted(dumps.keys())[-1]
  logger.debug("Most recent dump is dated `%s', statistics based on `%s'",
               latest, dumpdate)

  if latest > dumpdate:
    logger.info("Found more recent dump `%s'", latest)

    link = Path(config.DUMP_LINK.format(date=latest))
    link.parent.mkdir(parents=True, exist_ok=True)
    if not link.exists():
      link.symlink_to(dumps[latest])

    job = [script_path, '--only=process-dump']
    logger.debug("submitting job: `%s'", repr(job))
    _queue_job(config.DUMP_PROCESS_MEMORY, *job)

def process_dump(script_path):
  """Generate statistics from a dump file. After success, move the statistics into place."""
  pass

def _queue_job(memory, *args):
  return subprocess.run([config.GRID_SUBMIT,
                         config.GRID_MEMORY.format(memory=memory),
                         *args])
<commit_msg>Explicitly invoke python3<commit_after>import os
import json
import time
import shutil
import logging
import subprocess

from pathlib import Path
from copy import deepcopy
from tempfile import NamedTemporaryFile
from collections import defaultdict

from . import config

SEPARATORS = (',', ':')
TIMESTAMP_KEYS = {'dump': 'dumpDate',
                  'classes': 'classUpdate',
                  'properties': 'propertyUpdate',
                  }
ISO8601_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
logger = logging.getLogger(__name__)

def get_current_timestamp():
    """Returns the current timestamp in suitable (ISO 8601) format."""
    return time.strftime(ISO8601_TIMESTAMP_FORMAT)

def _json_file_name(name):
    """Returns the name of the statistics file for `name`."""
    return name + '.json'

def get_json_data(name):
    """Load statistics file "`name`.json"."""
    try:
        with open(_json_file_name(name), 'r') as file:
            return json.load(file)
    except OSError:
        return {}

def update_json_data(name, data, timestamp=None):
    """Atomically replaces the data in file "`name'.json"."""
    try:
        with NamedTemporaryFile(mode='w', delete=False) as file:
            logger.info('Writing new JSON %s data ...', name)
            json.dump(data, file, separators=SEPARATORS)
            logger.info('Writing new %s JSON file ...', name)
    except RuntimeError as err:
        logger.error('Writing dump failed: %s', err)
    else:
        os.chmod(file.name, 0o644)
        shutil.move(file.name, _json_file_name(name))
        if timestamp is not None:
            update_timestamp(name, timestamp)
        logger.info('Update for %s complete.', name)

def update_split_json_data(name, data, chunk_size):
    """Updates split JSON data for file "`name.json",
       with customisable chunk size."""

    chunks = defaultdict(dict)

    for entity, record in data.items():
        chunk = int(entity) // chunk_size
        chunks[chunk][entity] = record

    for index, chunk in chunks.items():
        update_json_data('{}-{}'.format(name, index), chunk)

def get_timestamp(name):
  """Retrieves the statistics timestamp `name`."""
  data = get_json_data('statistics')
  return data[TIMESTAMP_KEYS[name]]

def update_timestamp(name, timestamp):
    """Sets the statistics timestamp `name` to `timestamp`."""
    data = get_json_data('statistics')
    data[TIMESTAMP_KEYS[name]] = timestamp

    with open(_json_file_name('statistics'), 'w') as file:
        json.dump(data, file)

def merge(old, new, default_others=None):
    """Merges two statistics documents `old` and `new`.

    `default_others` is a dict that will be applied to all items
    not updated.
    """
    merged = defaultdict(dict, deepcopy(old))

    for key in new.keys():
        merged[key].update(new[key])

    if default_others is not None:
        others = old.keys() - new.keys()

        for key in others:
            merged[key].update(default_others)

    return dict(merged)

def check_new_dump(script_path):
  """Check for a new dump file. If present, queue a job on the grid to
  rebuild the full statistics."""
  dumpdate = get_timestamp('dump')
  basedir = Path(config.DUMP_LOCATION)
  dumps = {}

  logger.debug("Enumerating dumps in `%s'", basedir)
  for subdir in (path for path in basedir.iterdir() if path.is_dir()):
    date = subdir.parts[-1]
    filename = config.DUMP_BASENAME.format(date=date)
    dumpfile = subdir.joinpath(filename)

    if dumpfile.exists():
      logger.debug("Found dump `%s', dated `%s'", filename, date)
      dumps[date] = dumpfile

  latest = sorted(dumps.keys())[-1]
  logger.debug("Most recent dump is dated `%s', statistics based on `%s'",
               latest, dumpdate)

  if latest > dumpdate:
    logger.info("Found more recent dump `%s'", latest)

    link = Path(config.DUMP_LINK.format(date=latest))
    link.parent.mkdir(parents=True, exist_ok=True)
    if not link.exists():
      link.symlink_to(dumps[latest])

    job = ['python3', script_path, '--only=process-dump']
    logger.debug("submitting job: `%s'", repr(job))
    _queue_job(config.DUMP_PROCESS_MEMORY, *job)

def process_dump(script_path):
  """Generate statistics from a dump file. After success, move the statistics into place."""
  pass

def _queue_job(memory, *args):
  return subprocess.run([config.GRID_SUBMIT,
                         config.GRID_MEMORY.format(memory=memory),
                         *args])
<|endoftext|>