<commit_before>import datetime

from flask import (
    render_template, session, redirect, url_for, current_app, flash, request,
    abort,
)
from flask_login import login_required

from .. import db
from ..models import (
    User,
    Admission,
    Patient, Address,
    Vaccine, Hospitalization, UTIHospitalization, ClinicalEvolution,
    Symptom, ObservedSymptom,
    Sample, Method, CdcExam,
)
from . import main
from .forms import NameForm, AdmissionForm, VaccineForm

NONE = 9
TRUE = 1
FALSE = 0

@main.route('/secret')
@login_required
def secret():
    return 'Only authenticated users are allowed!'

@main.route('/', methods=['GET'])
def index():
    return render_template('index.html')

@main.route('/admissions', methods=['GET'])
def list_admissions():
    admissions = Admission.query.all()
    return render_template('list-admissions.html', admissions=admissions)

@main.route('/admissions/<int:id>/detail', methods=['GET', 'POST'])
@main.route('/admissions/<int:id>/edit', methods=['GET', 'POST'])
@main.route('/admissions/<int:id>', methods=['GET', 'POST'])
def edit_admission(id):
    admission = Admission.query.get_or_404(id)

    symptoms = [{'symptom_id': s.id, 'symptom_name': s.name}
                for s in Symptom.get_primary_symptoms()]
    sec_symptoms = [{'symptom_id': s.id, 'symptom_name': s.name}
                    for s in Symptom.get_secondary_symptoms()]

    for obs_symptom in admission.symptoms:
        for symptom in symptoms:
            if symptom['symptom_id'] == obs_symptom.symptom.id:
                symptom['observed'] = obs_symptom.observed
                symptom['details'] = obs_symptom.details
        for sec_symptom in sec_symptoms:
            if sec_symptom['symptom_id'] == obs_symptom.symptom.id:
                sec_symptom['observed'] = obs_symptom.observed
                sec_symptom['details'] = obs_symptom.details

    form = AdmissionForm(
        id_lvrs_intern=admission.id_lvrs_intern,
        first_symptoms_date=admission.first_symptoms_date,
        semepi_symptom=admission.semepi_symptom,
        state_id=admission.state_id,
        city_id=admission.city_id,
        health_unit=admission.health_unit,
        requesting_institution=admission.requesting_institution,
        details=admission.details,
        patient=admission.patient,
        vaccine=admission.vaccine,
        hospitalization=admission.hospitalization,
        uti_hospitalization=admission.uti_hospitalization,
        clinical_evolution=admission.clinical_evolution,
        symptoms=symptoms,
        sec_symptoms=sec_symptoms,
        samples=admission.samples,
    )

    return render_template('create-admission.html', form=form,)

@main.route('/admissions/<int:id>/delete', methods=['GET'])
def delete_admission(id):
    admission = Admission.query.get(id)
    if admission is None:
        flash('Admissão com esse id não existe.')
        abort(404)
    return ('Delete not implemented yet.')

@main.route('/admissions/create', methods=['GET', 'POST'])
def create_admission():
    form = AdmissionForm(
        symptoms=[{'symptom_id': s.id, 'symptom_name': s.name}
                  for s in Symptom.get_primary_symptoms()],
        sec_symptoms=[{'symptom_id': s.id, 'symptom_name': s.name}
                      for s in Symptom.get_secondary_symptoms()],
    )

    # POST and valid
    if form.validate_on_submit():
        admission = Admission.query.filter_by(
            id_lvrs_intern=form.id_lvrs_intern.data).first()
        if admission is None:
            patient = Patient(
                name=form.patient.data['name'],
                birth_date=form.patient.data['birth_date'],
                age=form.patient.data['age'],
                age_unit=form.patient.data['age_unit'],
                gender=form.patient.data['gender'],
            )

            patient_residence = Address(
                patient=patient,
                country_id = form.patient.residence.data['country_id']
                    if form.patient.residence.data['country_id'] is not -1 else None,
                state_id=form.patient.residence.data['state_id']
                    if form.patient.residence.data['state_id'] is not -1 else None,
                city_id=form.patient.residence.data['city_id']
                    if form.patient.residence.data['city_id'] is not -1 else None,
                neighborhood=form.patient.residence.data['neighborhood'],
                zone=form.patient.residence.data['zone'],
                details=form.patient.residence.data['details']
            )

            admission = Admission(
                id_lvrs_intern=form.id_lvrs_intern.data,
                first_symptoms_date=form.first_symptoms_date.data,
                semepi_symptom=form.semepi_symptom.data,
                state_id=form.state_id.data
                    if form.state_id.data is not -1 else None,
                city_id=form.city_id.data
                    if form.city_id.data is not -1 else None,
                health_unit=form.health_unit.data,
                requesting_institution=form.requesting_institution.data,
                details=form.details.data,
                patient=patient,
            )

            if form.vaccine.data['applied'] is not NONE:
                Vaccine(
                    applied=bool(form.vaccine.data['applied']),
                    last_dose_date=form.vaccine.data['last_dose_date'],
                    admission=admission,
                )

            if form.hospitalization.data['occurred'] is not NONE:
                Hospitalization(
                    occurred=bool(form.hospitalization.data['occurred']),
                    date=form.hospitalization.data['date'],
                    admission=admission,
                )

            if form.uti_hospitalization.data['occurred'] is not NONE:
                UTIHospitalization(
                    occurred=bool(form.uti_hospitalization.data['occurred']),
                    date=form.uti_hospitalization.data['date'],
                    admission=admission,
                )

            if form.clinical_evolution.data['death'] is not NONE:
                ClinicalEvolution(
                    death=bool(form.clinical_evolution.data['death']),
                    date=form.vaccine.data['date'],
                    admission=admission,
                )

            for symptom_form in form.symptoms:
                if symptom_form.data['observed'] is not NONE:
                    ObservedSymptom(
                        observed=bool(symptom_form.data['observed']),
                        details=symptom_form.data['details'],
                        symptom_id=symptom_form.data['symptom_id'],
                        admission=admission,
                    )

            for sec_symptom_form in form.sec_symptoms:
                if sec_symptom_form.data['observed'] is True:
                    ObservedSymptom(
                        observed=True,
                        details=sec_symptom_form.data['details'],
                        symptom_id=sec_symptom_form.data['symptom_id'],
                        admission=admission,
                    )

            for sample_form in form.samples:
                sample = Sample(
                    admission_date=sample_form.data['admission_date'],
                    collection_date=sample_form.data['collection_date'],
                    semepi=sample_form.data['semepi'],
                    method_id=sample_form.data['method_id']
                        if sample_form.data['method_id'] is not -1 else None,
                    admission=admission,
                )
                CdcExam(
                    flu_type=sample_form.cdc_exam.data['flu_type'],
                    flu_subtype=sample_form.cdc_exam.data['flu_subtype'],
                    dominant_ct=sample_form.cdc_exam.data['dominant_ct'],
                    details=sample_form.cdc_exam.data['details'],
                    sample=sample,
                )

            db.session.add(admission)
            flash('Admissão criada com sucesso!')
        else:
            flash('Número Interno já cadastrado!')
        return redirect(url_for('.create_admission'))

    return render_template('create-admission.html', form=form,)
<commit_msg>:rocket::poop: Add edit form but still has issues (multiple observed symptoms)<commit_after>import datetime

from flask import (
    render_template, session, redirect, url_for, current_app, flash, request,
    abort,
)
from flask_login import login_required

from .. import db
from ..models import (
    User,
    Admission,
    Patient, Address,
    Vaccine, Hospitalization, UTIHospitalization, ClinicalEvolution,
    Symptom, ObservedSymptom,
    Sample, Method, CdcExam,
)
from . import main
from .forms import NameForm, AdmissionForm, VaccineForm

IGNORED = 9
TRUE = 1
FALSE = 0

@main.route('/secret')
@login_required
def secret():
    return 'Only authenticated users are allowed!'

@main.route('/', methods=['GET'])
def index():
    return render_template('index.html')

@main.route('/admissions', methods=['GET'])
def list_admissions():
    admissions = Admission.query.all()
    return render_template('list-admissions.html', admissions=admissions)

def symptom_in_admission_symptoms(symptom_id, admission):
    found = None
    for obs_symptom in admission.symptoms:
        if obs_symptom.symptom.id == symptom_id:
            return obs_symptom

    return found

@main.route('/admissions/<int:id>/detail', methods=['GET', 'POST'])
@main.route('/admissions/<int:id>/edit', methods=['GET', 'POST'])
@main.route('/admissions/<int:id>', methods=['GET', 'POST'])
def edit_admission(id):
    admission = Admission.query.get_or_404(id)

    symptoms = [{'symptom_id': s.id, 'symptom_name': s.name}
                for s in Symptom.get_primary_symptoms()]
    sec_symptoms = [{'symptom_id': s.id, 'symptom_name': s.name}
                    for s in Symptom.get_secondary_symptoms()]

    for obs_symptom in admission.symptoms:
        for symptom in symptoms:
            if symptom['symptom_id'] == obs_symptom.symptom.id:
                symptom['observed'] = obs_symptom.observed
                symptom['details'] = obs_symptom.details
        for sec_symptom in sec_symptoms:
            if sec_symptom['symptom_id'] == obs_symptom.symptom.id:
                sec_symptom['observed'] = obs_symptom.observed
                sec_symptom['details'] = obs_symptom.details

    form = AdmissionForm(
        id_lvrs_intern=admission.id_lvrs_intern,
        first_symptoms_date=admission.first_symptoms_date,
        semepi_symptom=admission.semepi_symptom,
        state_id=admission.state_id,
        city_id=admission.city_id,
        health_unit=admission.health_unit,
        requesting_institution=admission.requesting_institution,
        details=admission.details,
        patient=admission.patient,
        vaccine=admission.vaccine,
        hospitalization=admission.hospitalization,
        uti_hospitalization=admission.uti_hospitalization,
        clinical_evolution=admission.clinical_evolution,
        symptoms=symptoms,
        sec_symptoms=sec_symptoms,
        samples=admission.samples,
    )
    form.submit.label.text = 'Editar'

    if form.validate_on_submit():
        if form.id_lvrs_intern.data != admission.id_lvrs_intern and \
                Admission.query.filter_by(
                    id_lvrs_intern=form.id_lvrs_intern.data).first() \
                    is not None:
            flash('Número Interno já cadastrado! Escolha outro!')
        else:
            admission.patient.name = form.patient.data['name']
            admission.patient.birth_date = form.patient.data['birth_date']
            admission.patient.age = form.patient.data['age']
            admission.patient.age_unit = form.patient.data['age_unit']
            admission.patient.gender = form.patient.data['gender']

            admission.patient.residence.country_id = form.patient.residence.data['country_id']\
                if form.patient.residence.data['country_id'] is not -1 else None
            admission.patient.residence.state_id = form.patient.residence.data['state_id']\
                if form.patient.residence.data['state_id'] is not -1 else None
            admission.patient.residence.city_id = form.patient.residence.data['city_id']\
                if form.patient.residence.data['city_id'] is not -1 else None
            admission.patient.residence.neighborhood = form.patient.residence.data['neighborhood']
            admission.patient.residence.zone = form.patient.residence.data['zone']
            admission.patient.residence.details = form.patient.residence.data['details']

            admission.id_lvrs_intern = form.id_lvrs_intern.data
            admission.first_symptoms_date = form.first_symptoms_date.data
            admission.semepi_symptom = form.semepi_symptom.data
            admission.state_id = form.state_id.data \
                if form.state_id.data is not -1 else None
            admission.city_id = form.city_id.data \
                if form.city_id.data is not -1 else None
            admission.health_unit = form.health_unit.data
            admission.requesting_institution = form.requesting_institution.data
            admission.details = form.details.data

            if form.vaccine.data['applied'] is not IGNORED:
                admission.vaccine.applied = bool(form.vaccine.data['applied'])
                admission.vaccine.last_dose_date = \
                    form.vaccine.data['last_dose_date']
            else:
                admission.vaccine = None

            if form.hospitalization.data['occurred'] is not IGNORED:
                admission.hospitalization.occurred = \
                    bool(form.hospitalization.data['occurred'])
                admission.hospitalization.date = form.hospitalization.data['date']
            else:
                admission.hospitalization = None

            if form.uti_hospitalization.data['occurred'] is not IGNORED:
                admission.uti_hospitalization.occurred = \
                    bool(form.uti_hospitalization.data['occurred'])
                admission.uti_hospitalization.date = \
                    form.uti_hospitalization.data['date']
            else:
                admission.uti_hospitalization = None

            if form.clinical_evolution.data['death'] is not IGNORED:
                admission.clinical_evolution.death = \
                    bool(form.clinical_evolution.data['death'])
                admission.clinical_evolution.date = \
                    form.clinical_evolution.data['date']
            else:
                admission.clinical_evolution = None

            for symptom_form in form.symptoms:
                observed = symptom_in_admission_symptoms(
                    symptom_form['symptom_id'], admission)
                if observed is None:
                    if symptom_form.data['observed'] is not IGNORED:
                        observed = ObservedSymptom(
                            observed=bool(symptom_form.data['observed']),
                            details=symptom_form.data['details'],
                            symptom_id=symptom_form.data['symptom_id'],
                            admission=admission,
                        )
                else:
                    os = ObservedSymptom.query.get(observed.id)
                    if symptom_form.data['observed'] is IGNORED:
                        db.session.delete(os)
                    else:
                        os.observed = bool(symptom_form.data['observed'])
                        os.details = symptom_form.data['details']
                        db.session.add(os)

            for symptom_form in form.sec_symptoms:
                observed = symptom_in_admission_symptoms(
                    symptom_form['symptom_id'], admission)
                if observed is None:
                    if symptom_form.data['observed'] is not IGNORED:
                        observed = ObservedSymptom(
                            observed=bool(symptom_form.data['observed']),
                            details=symptom_form.data['details'],
                            symptom_id=symptom_form.data['symptom_id'],
                            admission=admission,
                        )
                else:
                    os = ObservedSymptom.query.get(observed.id)
                    if symptom_form.data['observed'] is IGNORED:
                        db.session.delete(os)
                    else:
                        os.observed = bool(symptom_form.data['observed'])
                        os.details = symptom_form.data['details']
                        db.session.add(os)

            for sample_form in form.samples:
                sample = admission.samples.first()
                sample.admission_date = sample_form.data['admission_date']
                sample.collection_date = sample_form.data['collection_date']
                sample.semepi = sample_form.data['semepi']
                sample.method_id = sample_form.data['method_id'] \
                    if sample_form.data['method_id'] is not -1 else None
                sample_form.cdc_exam.form.populate_obj(sample.cdc_exam)
                    # flu_type=sample_form.cdc_exam.data['flu_type'],
                    # flu_subtype=sample_form.cdc_exam.data['flu_subtype'],
                    # dominant_ct=sample_form.cdc_exam.data['dominant_ct'],
                    # details=sample_form.cdc_exam.data['details'],
                    # sample=sample,
                # )

            db.session.add(admission)
            flash('Admissão editada com sucesso!')
        return redirect(url_for('main.edit_admission', id=admission.id))

    return render_template('create-admission.html', form=form)

@main.route('/admissions/<int:id>/delete', methods=['GET'])
def delete_admission(id):
    admission = Admission.query.get(id)
    if admission is None:
        flash('Admissão com esse id não existe.')
        abort(404)
    return ('Delete not implemented yet.')

@main.route('/admissions/create', methods=['GET', 'POST'])
def create_admission():
    form = AdmissionForm(
        symptoms=[{'symptom_id': s.id, 'symptom_name': s.name}
                  for s in Symptom.get_primary_symptoms()],
        sec_symptoms=[{'symptom_id': s.id, 'symptom_name': s.name}
                      for s in Symptom.get_secondary_symptoms()],
    )

    # POST and valid
    if form.validate_on_submit():
        admission = Admission.query.filter_by(
            id_lvrs_intern=form.id_lvrs_intern.data).first()
        if admission is None:
            patient = Patient(
                name=form.patient.data['name'],
                birth_date=form.patient.data['birth_date'],
                age=form.patient.data['age'],
                age_unit=form.patient.data['age_unit'],
                gender=form.patient.data['gender'],
            )

            patient_residence = Address(
                patient=patient,
                country_id = form.patient.residence.data['country_id']
                    if form.patient.residence.data['country_id'] is not -1 else None,
                state_id=form.patient.residence.data['state_id']
                    if form.patient.residence.data['state_id'] is not -1 else None,
                city_id=form.patient.residence.data['city_id']
                    if form.patient.residence.data['city_id'] is not -1 else None,
                neighborhood=form.patient.residence.data['neighborhood'],
                zone=form.patient.residence.data['zone'],
                details=form.patient.residence.data['details']
            )

            admission = Admission(
                id_lvrs_intern=form.id_lvrs_intern.data,
                first_symptoms_date=form.first_symptoms_date.data,
                semepi_symptom=form.semepi_symptom.data,
                state_id=form.state_id.data
                    if form.state_id.data is not -1 else None,
                city_id=form.city_id.data
                    if form.city_id.data is not -1 else None,
                health_unit=form.health_unit.data,
                requesting_institution=form.requesting_institution.data,
                details=form.details.data,
                patient=patient,
            )

            if form.vaccine.data['applied'] is not IGNORED:
                Vaccine(
                    applied=bool(form.vaccine.data['applied']),
                    last_dose_date=form.vaccine.data['last_dose_date'],
                    admission=admission,
                )

            if form.hospitalization.data['occurred'] is not IGNORED:
                Hospitalization(
                    occurred=bool(form.hospitalization.data['occurred']),
                    date=form.hospitalization.data['date'],
                    admission=admission,
                )

            if form.uti_hospitalization.data['occurred'] is not IGNORED:
                UTIHospitalization(
                    occurred=bool(form.uti_hospitalization.data['occurred']),
                    date=form.uti_hospitalization.data['date'],
                    admission=admission,
                )

            if form.clinical_evolution.data['death'] is not IGNORED:
                ClinicalEvolution(
                    death=bool(form.clinical_evolution.data['death']),
                    date=form.vaccine.data['date'],
                    admission=admission,
                )

            for symptom_form in form.symptoms:
                if symptom_form.data['observed'] is not IGNORED:
                    ObservedSymptom(
                        observed=bool(symptom_form.data['observed']),
                        details=symptom_form.data['details'],
                        symptom_id=symptom_form.data['symptom_id'],
                        admission=admission,
                    )

            for sec_symptom_form in form.sec_symptoms:
                if sec_symptom_form.data['observed'] is True:
                    ObservedSymptom(
                        observed=True,
                        details=sec_symptom_form.data['details'],
                        symptom_id=sec_symptom_form.data['symptom_id'],
                        admission=admission,
                    )

            for sample_form in form.samples:
                sample = Sample(
                    admission_date=sample_form.data['admission_date'],
                    collection_date=sample_form.data['collection_date'],
                    semepi=sample_form.data['semepi'],
                    method_id=sample_form.data['method_id']
                        if sample_form.data['method_id'] is not -1 else None,
                    admission=admission,
                )
                CdcExam(
                    flu_type=sample_form.cdc_exam.data['flu_type'],
                    flu_subtype=sample_form.cdc_exam.data['flu_subtype'],
                    dominant_ct=sample_form.cdc_exam.data['dominant_ct'],
                    details=sample_form.cdc_exam.data['details'],
                    sample=sample,
                )

            db.session.add(admission)
            flash('Admissão criada com sucesso!')
        else:
            flash('Número Interno já cadastrado! Escolha outro!')
        return redirect(url_for('.create_admission'))

    return render_template('create-admission.html', form=form,)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from django.db import models

class HistoryDescriptor(object):
    def __init__(self, model):
        self.model = model

    def __get__(self, instance, owner):
        if instance is None:
            return HistoryManager(self.model)
        return HistoryManager(self.model, instance)

class HistoryManager(models.Manager, object):
    def __init__(self, model, instance=None):
        super(HistoryManager, self).__init__()
        self.model = model
        self.instance = instance

    def get_super_queryset(self):
        try:
            return super(HistoryManager, self).get_queryset()
        except AttributeError:
            return super(HistoryManager, self).get_query_set()

    def get_queryset(self):
        qs = self.get_super_queryset()
        if self.instance is None:
            return qs

        if isinstance(self.instance._meta.pk, models.OneToOneField):
            filter = {self.instance._meta.pk.name + "_id": self.instance.pk}
        else:
            filter = {self.instance._meta.pk.name: self.instance.pk}
        return self.get_super_queryset().filter(**filter)

    get_query_set = get_queryset

    def most_recent(self):
        """
        Returns the most recent copy of the instance available in the history.
        """
        if not self.instance:
            raise TypeError("Can't use most_recent() without a %s instance." %
                            self.model._meta.object_name)
        tmp = []
        for field in self.instance._meta.fields:
            if isinstance(field, models.ForeignKey):
                tmp.append(field.name + "_id")
            else:
                tmp.append(field.name)
        fields = tuple(tmp)
        try:
            values = self.values_list(*fields)[0]
        except IndexError:
            raise self.instance.DoesNotExist("%s has no historical record." %
                                             self.instance._meta.object_name)
        return self.instance.__class__(*values)

    def as_of(self, date):
        """Get a snapshot as of a specific date.

        Returns an instance, or an iterable of the instances, of the
        original model with all the attributes set according to what
        was present on the object on the date provided.
        """
        if not self.instance:
            return self._as_of_set(date)
        queryset = self.filter(history_date__lte=date)
        try:
            history_obj = queryset[0]
        except IndexError:
            raise self.instance.DoesNotExist(
                "%s had not yet been created." %
                self.instance._meta.object_name)
        if history_obj.history_type == '-':
            raise self.instance.DoesNotExist(
                "%s had already been deleted." %
                self.instance._meta.object_name)
        return history_obj.instance

    def _as_of_set(self, date):
        model = type(self.model().instance)  # a bit of a hack to get the model
        pk_attr = model._meta.pk.name
        queryset = self.filter(history_date__lte=date)
        for original_pk in set(
                queryset.order_by().values_list(pk_attr, flat=True)):
            changes = queryset.filter(**{pk_attr: original_pk})
            last_change = changes.latest('history_date')
            if changes.filter(history_date=last_change.history_date,
                              history_type='-').exists():
                continue
            yield last_change.instance
<commit_msg>Explicitly get queryset when using query methods<commit_after>from __future__ import unicode_literals

from django.db import models

class HistoryDescriptor(object):
    def __init__(self, model):
        self.model = model

    def __get__(self, instance, owner):
        if instance is None:
            return HistoryManager(self.model)
        return HistoryManager(self.model, instance)

class HistoryManager(models.Manager, object):
    def __init__(self, model, instance=None):
        super(HistoryManager, self).__init__()
        self.model = model
        self.instance = instance

    def get_super_queryset(self):
        try:
            return super(HistoryManager, self).get_queryset()
        except AttributeError:
            return super(HistoryManager, self).get_query_set()

    def get_queryset(self):
        qs = self.get_super_queryset()
        if self.instance is None:
            return qs

        if isinstance(self.instance._meta.pk, models.OneToOneField):
            filter = {self.instance._meta.pk.name + "_id": self.instance.pk}
        else:
            filter = {self.instance._meta.pk.name: self.instance.pk}
        return self.get_super_queryset().filter(**filter)

    get_query_set = get_queryset

    def most_recent(self):
        """
        Returns the most recent copy of the instance available in the history.
        """
        if not self.instance:
            raise TypeError("Can't use most_recent() without a %s instance." %
                            self.model._meta.object_name)
        tmp = []
        for field in self.instance._meta.fields:
            if isinstance(field, models.ForeignKey):
                tmp.append(field.name + "_id")
            else:
                tmp.append(field.name)
        fields = tuple(tmp)
        try:
            values = self.get_queryset().values_list(*fields)[0]
        except IndexError:
            raise self.instance.DoesNotExist("%s has no historical record." %
                                             self.instance._meta.object_name)
        return self.instance.__class__(*values)

    def as_of(self, date):
        """Get a snapshot as of a specific date.

        Returns an instance, or an iterable of the instances, of the
        original model with all the attributes set according to what
        was present on the object on the date provided.
        """
        if not self.instance:
            return self._as_of_set(date)
        queryset = self.get_queryset().filter(history_date__lte=date)
        try:
            history_obj = queryset[0]
        except IndexError:
            raise self.instance.DoesNotExist(
                "%s had not yet been created." %
                self.instance._meta.object_name)
        if history_obj.history_type == '-':
            raise self.instance.DoesNotExist(
                "%s had already been deleted." %
                self.instance._meta.object_name)
        return history_obj.instance

    def _as_of_set(self, date):
        model = type(self.model().instance)  # a bit of a hack to get the model
        pk_attr = model._meta.pk.name
        queryset = self.get_queryset().filter(history_date__lte=date)
        for original_pk in set(
                queryset.order_by().values_list(pk_attr, flat=True)):
            changes = queryset.filter(**{pk_attr: original_pk})
            last_change = changes.latest('history_date')
            if changes.filter(history_date=last_change.history_date,
                              history_type='-').exists():
                continue
            yield last_change.instance
<|endoftext|>


<commit_before>import discord
from discord.ext import commands
# import asyncio
import json
# import io
# import random
# import time
# import random
import aiohttp

class Watashi(commands.Bot):
    def __init__(self, config_path: str, *args, **kwargs):
        self.config_path = config_path
        with open(self.config_path) as f:
            self.configs = json.load(f)

        self.default_cogs = ('cogs.base', 'cogs.coding', 'cogs.emoji')

        self.embed_colour = int(self.configs['embed_colour'], 16)

        super().__init__(command_prefix=self.configs['prefix'], self_bot=True)

        #Colours
        self.gold = 16766720
        self.red = 16711680
        self.orange = 16753920
        self.yellow = 16776960
        self.darkgreen = 25600
        self.lightgreen = 589568
        self.lightblue = 58879
        self.darkblue = 255
        self.blurple = 7506394

        self.aiohttp_session = aiohttp.ClientSession(loop=self.loop)

    def run(self):
        super().run(self.configs['token'], bot=False)

    async def on_ready(self):
            print('\n'*100)
            print("<----------------->\nWatashi SelfBot\n<----------------->\nCoded by:\nPrestige#9162\nDemo#9465\nnaught0#4417\n<----------------->\nWarning:\nUnder the MIT\nlicense we are\nnot liable for any\ndamage caused/\naction taken\nagainst you for\nusing a selfbot\nwhich is in violation\nof Discord's TOS")
            for cog in self.default_cogs:
                self.load_extension(cog)<commit_msg>Wiki Cog Support<commit_after>import discord
from discord.ext import commands
# import asyncio
import json
# import io
# import random
# import time
# import random
import aiohttp

class Watashi(commands.Bot):
    def __init__(self, config_path: str, *args, **kwargs):
        self.config_path = config_path
        with open(self.config_path) as f:
            self.configs = json.load(f)

        self.default_cogs = ('cogs.base', 'cogs.coding', 'cogs.emoji', 'cogs.wiki')

        self.embed_colour = int(self.configs['embed_colour'], 16)

        super().__init__(command_prefix=self.configs['prefix'], self_bot=True)

        #Colours
        self.gold = 16766720
        self.red = 16711680
        self.orange = 16753920
        self.yellow = 16776960
        self.darkgreen = 25600
        self.lightgreen = 589568
        self.lightblue = 58879
        self.darkblue = 255
        self.blurple = 7506394

        self.aiohttp_session = aiohttp.ClientSession(loop=self.loop)

    def run(self):
        super().run(self.configs['token'], bot=False)

    async def on_ready(self):
            print('\n'*100)
            print("<----------------->\nWatashi SelfBot\n<----------------->\nCoded by:\nPrestige#9162\nDemo#9465\nnaught0#4417\n<----------------->\nWarning:\nUnder the MIT\nlicense we are\nnot liable for any\ndamage caused/\naction taken\nagainst you for\nusing a selfbot\nwhich is in violation\nof Discord's TOS")
            for cog in self.default_cogs:
                self.load_extension(cog)<|endoftext|>


<commit_before>import tensorflow as tf
from wavenet_ops import causal_conv

class WaveNet(object):
    '''Implements the WaveNet network for generative audio.

    Usage (with the architecture as in the DeepMind paper):
        dilations = [2**i for i in range(N)] * M
        channels = 2**8  # Quantize to 256 possible amplitude values.
        net = WaveNet(batch_size, channels, dilations)
        loss = net.loss(input_batch)
    '''

    def __init__(self,
                 batch_size,
                 channels,
                 dilations,
                 filter_width,
                 residual_channels,
                 dilation_channels):
        self.batch_size = batch_size
        self.channels = channels
        self.dilations = dilations
        self.filter_width = filter_width
        self.residual_channels = residual_channels
        self.dilation_channels = dilation_channels

    # A single causal convolution layer that can change the number of channels.
    def _create_causal_layer(self, input_batch, in_channels, out_channels):
        with tf.name_scope('causal_layer'):
            weights_filter = tf.Variable(tf.truncated_normal(
                [self.filter_width, in_channels, out_channels],
                stddev=0.2,
                name="filter"))
            return causal_conv(input_batch, weights_filter, 1)

    def _create_dilation_layer(self, input_batch, layer_index, dilation, in_channels, dilation_channels):
        '''Adds a single causal dilated convolution layer.'''

        weights_filter = tf.Variable(tf.truncated_normal(
            [self.filter_width, in_channels, dilation_channels],
            stddev=0.2,
            name="filter"))
        weights_gate = tf.Variable(tf.truncated_normal(
            [self.filter_width, in_channels, dilation_channels],
            stddev=0.2, name="gate"))

        conv_filter = causal_conv(input_batch, weights_filter, dilation)
        conv_gate = causal_conv(input_batch, weights_gate, dilation)

        out = tf.tanh(conv_filter) * tf.sigmoid(conv_gate)

        weights_dense = tf.Variable(tf.truncated_normal(
            [1, dilation_channels, in_channels], stddev=0.2, name="dense"))
        transformed = tf.nn.conv1d(out, weights_dense, stride=1,
                                   padding="SAME", name="dense")
        layer = 'layer{}'.format(layer_index)
        tf.histogram_summary(layer + '_filter', weights_filter)
        tf.histogram_summary(layer + '_gate', weights_gate)
        tf.histogram_summary(layer + '_dense', weights_dense)

        return transformed, input_batch + transformed

    def _preprocess(self, audio):
        '''Quantizes waveform amplitudes.'''
        with tf.name_scope('preprocessing'):
            mu = self.channels - 1
            # Perform mu-law companding transformation (ITU-T, 1988).
            magnitude = tf.log(1 + mu * tf.abs(audio)) / tf.log(1. + mu)
            signal = tf.sign(audio) * magnitude
            # Quantize signal to the specified number of levels
            quantized = tf.cast((signal + 1) / 2 * mu, tf.int32)

        return quantized

    def decode(self, output):
        mu = self.channels
        output = tf.cast(output, tf.float32)
        y = (2 * output  - 1) / mu
        x = tf.sign(y) * (tf.exp(y * tf.log(1. + mu)) - 1) / mu

        return x

    def _create_network(self, input_batch):
        outputs = []
        current_layer = input_batch

        current_layer = self._create_causal_layer(current_layer, self.channels, self.residual_channels)

        # Add all defined dilation layers.
        with tf.name_scope('dilated_stack'):
            for layer_index, dilation in enumerate(self.dilations):
                with tf.name_scope('layer{}'.format(layer_index)):
                    output, current_layer = self._create_dilation_layer(
                        current_layer,
                        layer_index,
                        dilation,
                        self.residual_channels,
                        self.dilation_channels)
                    outputs.append(output)

        with tf.name_scope('postprocessing'):
            # Perform (+) -> ReLU -> 1x1 conv -> ReLU -> 1x1 conv to
            # postprocess the output.
            w1 = tf.Variable(tf.truncated_normal(
                [1, self.residual_channels, int(self.channels / 2)], stddev=0.3,
                name="postprocess1"))
            w2 = tf.Variable(tf.truncated_normal(
                [1, int(self.channels / 2), self.channels], stddev=0.3,
                name="postprocess2"))

            tf.histogram_summary('postprocess1_weights', w1)
            tf.histogram_summary('postprocess2_weights', w2)

            # We skip connections from the outputs of each layer, adding them
            # all up here.
            total = sum(outputs)
            transformed1 = tf.nn.relu(total)
            conv1 = tf.nn.conv1d(transformed1, w1, stride=1, padding="SAME")
            transformed2 = tf.nn.relu(conv1)
            conv2 = tf.nn.conv1d(transformed2, w2, stride=1, padding="SAME")

        return conv2

    def _one_hot(self, input_batch):
        # One-hot encode waveform amplitudes, so we can define the network
        # as a categorical distribution over possible amplitudes.
        with tf.name_scope('one_hot_encode'):
            encoded = tf.one_hot(input_batch, depth=self.channels,
                                 dtype=tf.float32)
            encoded = tf.reshape(encoded,
                                 [self.batch_size, -1, self.channels])

        return encoded

    def predict_proba(self, waveform, name='wavenet'):
        with tf.variable_scope(name):
            encoded = self._one_hot(waveform)
            raw_output = self._create_network(encoded)
            out = tf.reshape(raw_output, [-1, self.channels])
            proba = tf.nn.softmax(tf.cast(out, tf.float64))
            last = tf.slice(proba, [tf.shape(proba)[0] - 1, 0], [1, self.channels])
            return tf.reshape(last, [-1])

    def loss(self, input_batch, name='wavenet'):
        with tf.variable_scope(name):
            input_batch = self._preprocess(input_batch)
            encoded = self._one_hot(input_batch)
            raw_output = self._create_network(encoded)

            with tf.name_scope('loss'):
                # Shift original input left by one sample, which means that
                # each output sample has to predict the next input sample.
                shifted = tf.slice(encoded, [0, 1, 0],
                                   [-1, tf.shape(encoded)[1] - 1, -1])
                shifted = tf.pad(shifted, [[0, 0], [0, 1], [0, 0]])

                prediction = tf.reshape(raw_output, [-1, self.channels])
                loss = tf.nn.softmax_cross_entropy_with_logits(
                    prediction,
                    tf.reshape(shifted, [-1, self.channels]))
                reduced_loss = tf.reduce_mean(loss)

                tf.scalar_summary('loss', reduced_loss)

        return reduced_loss
<commit_msg>Fix inverse of mu law<commit_after>import tensorflow as tf
from wavenet_ops import causal_conv

class WaveNet(object):
    '''Implements the WaveNet network for generative audio.

    Usage (with the architecture as in the DeepMind paper):
        dilations = [2**i for i in range(N)] * M
        channels = 2**8  # Quantize to 256 possible amplitude values.
        net = WaveNet(batch_size, channels, dilations)
        loss = net.loss(input_batch)
    '''

    def __init__(self,
                 batch_size,
                 channels,
                 dilations,
                 filter_width,
                 residual_channels,
                 dilation_channels):
        self.batch_size = batch_size
        self.channels = channels
        self.dilations = dilations
        self.filter_width = filter_width
        self.residual_channels = residual_channels
        self.dilation_channels = dilation_channels

    # A single causal convolution layer that can change the number of channels.
    def _create_causal_layer(self, input_batch, in_channels, out_channels):
        with tf.name_scope('causal_layer'):
            weights_filter = tf.Variable(tf.truncated_normal(
                [self.filter_width, in_channels, out_channels],
                stddev=0.2,
                name="filter"))
            return causal_conv(input_batch, weights_filter, 1)

    def _create_dilation_layer(self, input_batch, layer_index, dilation, in_channels, dilation_channels):
        '''Adds a single causal dilated convolution layer.'''

        weights_filter = tf.Variable(tf.truncated_normal(
            [self.filter_width, in_channels, dilation_channels],
            stddev=0.2,
            name="filter"))
        weights_gate = tf.Variable(tf.truncated_normal(
            [self.filter_width, in_channels, dilation_channels],
            stddev=0.2, name="gate"))

        conv_filter = causal_conv(input_batch, weights_filter, dilation)
        conv_gate = causal_conv(input_batch, weights_gate, dilation)

        out = tf.tanh(conv_filter) * tf.sigmoid(conv_gate)

        weights_dense = tf.Variable(tf.truncated_normal(
            [1, dilation_channels, in_channels], stddev=0.2, name="dense"))
        transformed = tf.nn.conv1d(out, weights_dense, stride=1,
                                   padding="SAME", name="dense")
        layer = 'layer{}'.format(layer_index)
        tf.histogram_summary(layer + '_filter', weights_filter)
        tf.histogram_summary(layer + '_gate', weights_gate)
        tf.histogram_summary(layer + '_dense', weights_dense)

        return transformed, input_batch + transformed

    def _preprocess(self, audio):
        '''Quantizes waveform amplitudes.'''
        with tf.name_scope('preprocessing'):
            mu = self.channels - 1
            # Perform mu-law companding transformation (ITU-T, 1988).
            magnitude = tf.log(1 + mu * tf.abs(audio)) / tf.log(1. + mu)
            signal = tf.sign(audio) * magnitude
            # Quantize signal to the specified number of levels
            quantized = tf.cast((signal + 1) / 2 * mu, tf.int32)

        return quantized

    def decode(self, output):
        mu = self.channels - 1
        y = tf.cast(output, tf.float32)
        y = 2 * (output / mu) - 1
        x = tf.sign(y) * (1 / mu) * ((1 + mu)**abs(y) - 1)
        return x

    def _create_network(self, input_batch):
        outputs = []
        current_layer = input_batch

        current_layer = self._create_causal_layer(current_layer, self.channels, self.residual_channels)

        # Add all defined dilation layers.
        with tf.name_scope('dilated_stack'):
            for layer_index, dilation in enumerate(self.dilations):
                with tf.name_scope('layer{}'.format(layer_index)):
                    output, current_layer = self._create_dilation_layer(
                        current_layer,
                        layer_index,
                        dilation,
                        self.residual_channels,
                        self.dilation_channels)
                    outputs.append(output)

        with tf.name_scope('postprocessing'):
            # Perform (+) -> ReLU -> 1x1 conv -> ReLU -> 1x1 conv to
            # postprocess the output.
            w1 = tf.Variable(tf.truncated_normal(
                [1, self.residual_channels, int(self.channels / 2)], stddev=0.3,
                name="postprocess1"))
            w2 = tf.Variable(tf.truncated_normal(
                [1, int(self.channels / 2), self.channels], stddev=0.3,
                name="postprocess2"))

            tf.histogram_summary('postprocess1_weights', w1)
            tf.histogram_summary('postprocess2_weights', w2)

            # We skip connections from the outputs of each layer, adding them
            # all up here.
            total = sum(outputs)
            transformed1 = tf.nn.relu(total)
            conv1 = tf.nn.conv1d(transformed1, w1, stride=1, padding="SAME")
            transformed2 = tf.nn.relu(conv1)
            conv2 = tf.nn.conv1d(transformed2, w2, stride=1, padding="SAME")

        return conv2

    def _one_hot(self, input_batch):
        # One-hot encode waveform amplitudes, so we can define the network
        # as a categorical distribution over possible amplitudes.
        with tf.name_scope('one_hot_encode'):
            encoded = tf.one_hot(input_batch, depth=self.channels,
                                 dtype=tf.float32)
            encoded = tf.reshape(encoded,
                                 [self.batch_size, -1, self.channels])

        return encoded

    def predict_proba(self, waveform, name='wavenet'):
        with tf.variable_scope(name):
            encoded = self._one_hot(waveform)
            raw_output = self._create_network(encoded)
            out = tf.reshape(raw_output, [-1, self.channels])
            proba = tf.nn.softmax(tf.cast(out, tf.float64))
            last = tf.slice(proba, [tf.shape(proba)[0] - 1, 0], [1, self.channels])
            return tf.reshape(last, [-1])

    def loss(self, input_batch, name='wavenet'):
        with tf.variable_scope(name):
            input_batch = self._preprocess(input_batch)
            encoded = self._one_hot(input_batch)
            raw_output = self._create_network(encoded)

            with tf.name_scope('loss'):
                # Shift original input left by one sample, which means that
                # each output sample has to predict the next input sample.
                shifted = tf.slice(encoded, [0, 1, 0],
                                   [-1, tf.shape(encoded)[1] - 1, -1])
                shifted = tf.pad(shifted, [[0, 0], [0, 1], [0, 0]])

                prediction = tf.reshape(raw_output, [-1, self.channels])
                loss = tf.nn.softmax_cross_entropy_with_logits(
                    prediction,
                    tf.reshape(shifted, [-1, self.channels]))
                reduced_loss = tf.reduce_mean(loss)

                tf.scalar_summary('loss', reduced_loss)

        return reduced_loss
<|endoftext|>


<commit_before>from __future__ import division
from hurry.filesize import size

from hammr.lib.argumentParser import ArgumentParser, ArgumentParserError
from hammr.lib.cmdHamr import Cmd, HammrGlobal
from hammr.utils import *

__author__="UShareSoft"

class Quota(Cmd, HammrGlobal):
        """List the status of all the quotas that can be set for the user (disk usage, generations, scans and number of templates)"""

        cmd_name="quota"

        def __init__(self):
                super(Quota, self).__init__()

        def arg_list(self):
                doParser = ArgumentParser(prog=self.cmd_name+" list", add_help = True, description="Displays the user’s quota information")
                return doParser

        def do_list(self, args):
                try:
                        #call UForge API
                        printer.out("Getting quotas for ["+self.login+"] ...")
                        quotas = self.api.Users(self.login).Quotas.Get()
                        if quotas is None or not hasattr(quotas, 'get_quota'):
                                printer.out("No quotas available")
                        else:
                                values={};
                                for quota in quotas.get_quota():
                                        if quota.limit==-1:
                                                nb=" ("+str(quota.nb)+")"
                                        else:
                                                nb=" ("+str(quota.nb)+"/"+str(quota.limit)+")"

                                        if quota.type_ == constants.QUOTAS_SCAN:
                                                text="Scan"+ ("s" if quota.nb>1 else "") +nb
                                        elif quota.type_ == constants.QUOTAS_TEMPLATE:
                                                text="Template"+ ("s" if quota.nb>1 else "") +nb
                                        elif quota.type_ == constants.QUOTAS_GENERATION:
                                                text="Generation"+ ("s" if quota.nb>1 else "") +nb
                                        elif quota.type_ == constants.QUOTAS_DISK_USAGE:
                                                text="Disk usage"+" ("+size(quota.nb)+")"

                                        if quota.limit!=-1:
                                                values[text]=int(quota.nb/quota.limit*50)
                                        else:
                                                values[text]=-1

                                ascii_bar_graph.print_graph(values)
                        return 0

                except ArgumentParserError as e:
                        printer.out("ERROR: In Arguments: "+str(e), printer.ERROR)
                        self.help_list()
                except Exception as e:
                        generics_utils.print_uforge_exception(e)

        def help_list(self):
                doParser = self.arg_list()
                doParser.print_help()

        <commit_msg>Fix for issue #32<commit_after>from __future__ import division
from hurry.filesize import size

from hammr.lib.argumentParser import ArgumentParser, ArgumentParserError
from hammr.lib.cmdHamr import Cmd, HammrGlobal
from hammr.utils import *

__author__="UShareSoft"

class Quota(Cmd, HammrGlobal):
        """List the status of all the quotas that can be set for the user (disk usage, generations, scans and number of templates)"""

        cmd_name="quota"

        def __init__(self):
                super(Quota, self).__init__()

        def arg_list(self):
                doParser = ArgumentParser(prog=self.cmd_name+" list", add_help = True, description="Displays the user's quota information")
                return doParser

        def do_list(self, args):
                try:
                        #call UForge API
                        printer.out("Getting quotas for ["+self.login+"] ...")
                        quotas = self.api.Users(self.login).Quotas.Get()
                        if quotas is None or not hasattr(quotas, 'get_quota'):
                                printer.out("No quotas available")
                        else:
                                values={};
                                for quota in quotas.get_quota():
                                        if quota.limit==-1:
                                                nb=" ("+str(quota.nb)+")"
                                        else:
                                                nb=" ("+str(quota.nb)+"/"+str(quota.limit)+")"

                                        if quota.type_ == constants.QUOTAS_SCAN:
                                                text="Scan"+ ("s" if quota.nb>1 else "") +nb
                                        elif quota.type_ == constants.QUOTAS_TEMPLATE:
                                                text="Template"+ ("s" if quota.nb>1 else "") +nb
                                        elif quota.type_ == constants.QUOTAS_GENERATION:
                                                text="Generation"+ ("s" if quota.nb>1 else "") +nb
                                        elif quota.type_ == constants.QUOTAS_DISK_USAGE:
                                                text="Disk usage"+" ("+size(quota.nb)+")"

                                        if quota.limit!=-1:
                                                values[text]=int(quota.nb/quota.limit*50)
                                        else:
                                                values[text]=-1

                                ascii_bar_graph.print_graph(values)
                        return 0

                except ArgumentParserError as e:
                        printer.out("ERROR: In Arguments: "+str(e), printer.ERROR)
                        self.help_list()
                except Exception as e:
                        generics_utils.print_uforge_exception(e)

        def help_list(self):
                doParser = self.arg_list()
                doParser.print_help()

<|endoftext|>


<commit_before>from rest_framework.serializers import (DateTimeField, ModelSerializer,
                                        PrimaryKeyRelatedField)

from threads.models import Post

from .models import User

class UserSerializer(ModelSerializer):
    posts = PrimaryKeyRelatedField(many=True,
                                   queryset=Post.objects.all())

    class Meta:
        model=User
        fields=('username', 'email', 'date_joined', 'last_login', 'posts')
<commit_msg>split user serializer into public/private serializer.<commit_after>from rest_framework.serializers import (DateTimeField, ModelSerializer,
                                        PrimaryKeyRelatedField)

from threads.models import Post

from .models import User

class PublicUserSerializer(ModelSerializer):
    posts = PrimaryKeyRelatedField(many=True,
                                   queryset=Post.objects.all())

    class Meta:
        model=User
        fields=('username', 'date_joined', 'posts')

class PrivateUserSerializer(ModelSerializer):
    posts = PrimaryKeyRelatedField(many=True,
                                   queryset=Post.objects.all())

    class Meta:
        model=User
        fields=('username','email', 'date_joined', 'last_login', 'posts')
<|endoftext|>


<commit_before>import os
import re

from .utils import run_cmd
from wand.image import Image
from PyPDF2 import PdfFileMerger

def rotate_image(filename, degrees):
    """Rotate image given amount of degrees.

    :param string filename: file to rotate
    :param int degrees: amount of degrees to rotate

    """
    with Image(filename=filename) as image:
        with image.clone() as rotated:
            rotated.rotate(degrees)
            rotated.save(filename=filename)

def rotate_all_images_in_dir(dirname, degrees):
    """Rotate all files in directory.

    :param string dirname: name of directory in which files should be rotated
    :param int degrees: number of degrees to rotate

    """
    for f in os.scandir(dirname):
        if not f.is_file():
            continue
        rotate_image(os.path.join(dirname, f.name), degrees)

def is_blank(filename):
    """
    Check if image is blank.

    Return true if filename is a blank image. This is a slightly modified
    version of Vinatha Ekanayake's is_blank(), which is part of Scanpdf
    (https://github.com/virantha/scanpdf) and licensed under the Apache
    license.

    :param string filename: file name of image to check
    :returns: True if image is blank, False otherwise.

    """
    if not os.path.exists(filename):
        return True

    c = 'identify -verbose %s' % filename
    result = run_cmd(c)
    mStdDev = re.compile(
        b'\s*standard deviation:\s*\d+\.\d+\s*\((?P<percent>\d+\.\d+)\).*')

    for line in result.splitlines():
        match = mStdDev.search(line)
        if match:
            stdev = float(match.group('percent'))
            if stdev > 0.05:
                return False

    return True

def remove_if_blank(filename):
    """Remove file if it is blank.

    This is useful when scanning in duplex mode using a backend that doesn't
    support skipping blank pages.

    :param string filename: name of file to remove, if blank

    """
    if is_blank(filename):
        print('Removing (probably) blank page {}'.format(filename))
        os.remove(filename)

def merge_pdfs(inputs, output):
    """Merge selected pdfs.

    :param list inputs: files to concatenate
    :param string output: name of file to write

    """
    merger = PdfFileMerger()
    input_fds = dict()

    out = open(output, 'wb')

    for filename in inputs:
        try:
            input_fds[filename] = open(filename, 'rb')
        except OSError as e:
            print('Error opening {}: {}'.format(filename, e))
        merger.append(input_fds[filename])

    merger.write(out)

def merge_pdfs_in_dir(directory, output):
    """Read all pdf files in directory and create one merged output.

    :param string directory: directory containing pdf files to be merged
    :param string output: filename of new merged pdf
    """
    files_to_merge = []

    for file in os.scandir(directory):
        if not file.is_file():
            continue
        if file.name[-4:] != '.pdf':
            continue

        files_to_merge.append(os.path.join(directory, file.name))

    merge_pdfs(files_to_merge, output)

def ocr(filename, language):
    """Perform OCR on file using Tesseract.

    :param string filename: file to perform OCR on
    :param string language: language(s) expected to be used in file

    """
    base_output_name = filename[:-4]
    command = 'tesseract {} {} -l {} pdf'.format(filename,
                                                 base_output_name,
                                                 language)
    run_cmd(command)

def ocr_pnms_in_dir(directory, language):
    """Perform OCR on all pnm files in given directory.

    :param string directory: directory in which all pnm files will be OCR:ed
    :param string language: language(s) expected to be used in files

    """
    for file in os.scandir(directory):
        if not file.is_file():
            continue
        if file.name[-4:] != '.pnm':
            continue
        ocr(os.path.join(directory, file.name), language)
<commit_msg>Add move_blanks to postprocessing<commit_after>import os
import re
import shutil

from .utils import run_cmd
from wand.image import Image
from PyPDF2 import PdfFileMerger

def rotate_image(filename, degrees):
    """Rotate image given amount of degrees.

    :param string filename: file to rotate
    :param int degrees: amount of degrees to rotate

    """
    with Image(filename=filename) as image:
        with image.clone() as rotated:
            rotated.rotate(degrees)
            rotated.save(filename=filename)

def rotate_all_images_in_dir(dirname, degrees):
    """Rotate all files in directory.

    :param string dirname: name of directory in which files should be rotated
    :param int degrees: number of degrees to rotate

    """
    for f in os.scandir(dirname):
        if not f.is_file():
            continue
        rotate_image(os.path.join(dirname, f.name), degrees)

def is_blank(filename):
    """
    Check if image is blank.

    Return true if filename is a blank image. This is a slightly modified
    version of Vinatha Ekanayake's is_blank(), which is part of Scanpdf
    (https://github.com/virantha/scanpdf) and licensed under the Apache
    license.

    :param string filename: file name of image to check
    :returns: True if image is blank, False otherwise.

    """
    if not os.path.exists(filename):
        return True

    c = 'identify -verbose %s' % filename
    result = run_cmd(c)
    mStdDev = re.compile(
        b'\s*standard deviation:\s*\d+\.\d+\s*\((?P<percent>\d+\.\d+)\).*')

    for line in result.splitlines():
        match = mStdDev.search(line)
        if match:
            stdev = float(match.group('percent'))
            if stdev > 0.05:
                return False

    return True

def move_blanks(input_dir, output_dir):
    """Move blank .pnm's in input_dir to output_dir

    :param string input_dir: directory to check for blank .pnm files
    :param string output_dir: where to move blank .pnm files
    :returns: number of blank pages moved
    :rtype: int

    """
    number_of_blanks = 0

    for entry in os.scandir(input_dir):
        if not entry.is_file() or not entry.name.endswith('.pnm'):
            continue

        image = os.path.join(input_dir, entry.name)

        if is_blank(image):
            try:
                os.mkdir(output_dir)
            except:
                pass  # Assume directory exists.

            shutil.move(image, output_dir)
            number_of_blanks += 1

    return number_of_blanks

def remove_if_blank(filename):
    """Remove file if it is blank.

    This is useful when scanning in duplex mode using a backend that doesn't
    support skipping blank pages.

    :param string filename: name of file to remove, if blank

    """
    if is_blank(filename):
        print('Removing (probably) blank page {}'.format(filename))
        os.remove(filename)

def merge_pdfs(inputs, output):
    """Merge selected pdfs.

    :param list inputs: files to concatenate
    :param string output: name of file to write

    """
    merger = PdfFileMerger()
    input_fds = dict()

    out = open(output, 'wb')

    for filename in inputs:
        try:
            input_fds[filename] = open(filename, 'rb')
        except OSError as e:
            print('Error opening {}: {}'.format(filename, e))
        merger.append(input_fds[filename])

    merger.write(out)

def merge_pdfs_in_dir(directory, output):
    """Read all pdf files in directory and create one merged output.

    :param string directory: directory containing pdf files to be merged
    :param string output: filename of new merged pdf
    """
    files_to_merge = []

    for file in os.scandir(directory):
        if not file.is_file():
            continue
        if file.name[-4:] != '.pdf':
            continue

        files_to_merge.append(os.path.join(directory, file.name))

    merge_pdfs(files_to_merge, output)

def ocr(filename, language):
    """Perform OCR on file using Tesseract.

    :param string filename: file to perform OCR on
    :param string language: language(s) expected to be used in file

    """
    base_output_name = filename[:-4]
    command = 'tesseract {} {} -l {} pdf'.format(filename,
                                                 base_output_name,
                                                 language)
    run_cmd(command)

def ocr_pnms_in_dir(directory, language):
    """Perform OCR on all pnm files in given directory.

    :param string directory: directory in which all pnm files will be OCR:ed
    :param string language: language(s) expected to be used in files

    """
    for file in os.scandir(directory):
        if not file.is_file():
            continue
        if file.name[-4:] != '.pnm':
            continue
        ocr(os.path.join(directory, file.name), language)
<|endoftext|>


<commit_before>import os
import re
import shutil
import time

from .config import config, load_config
from .utils import run_cmd, files
from PyPDF2 import PdfFileMerger
from wand.image import Image

def rotate_image(filename, degrees):
    """Rotate image given amount of degrees.

    :param string filename: file to rotate
    :param int degrees: amount of degrees to rotate

    """
    with Image(filename=filename) as image:
        with image.clone() as rotated:
            rotated.rotate(degrees)
            rotated.save(filename=filename)

def rotate_all_images_in_dir(dirname, degrees):
    """Rotate all files in directory.

    :param string dirname: name of directory in which files should be rotated
    :param int degrees: number of degrees to rotate

    """
    for f in files(dirname):
        rotate_image(os.path.join(dirname, f.name), degrees)

def unpaper(filename):
    """Process file with unpaper and delete original.

    :param filename: TODO

    """
    unpapered_filename = filename + '.unpapered'
    # TODO: We don't use unpaper's --overwrite because it currently seems to be
    # broken. Once it's been fixed, just --overwrite the original.
    run_cmd('unpaper --size a4 "{}" "{}"'.format(filename, unpapered_filename))
    shutil.move(unpapered_filename, filename)

def unpaper_dir(directory, extension=None):
    """Run unpaper on all files with given extension in directory

    :param string directory: directory to process
    :param string extension: extension of files to run unpaper on

    """

    for f in files(directory, extension):
        unpaper(os.path.oin(directory, f.name))

def is_blank(filename):
    """
    Check if image is blank.

    Return true if filename is a blank image. This is a slightly modified
    version of Vinatha Ekanayake's is_blank(), which is part of Scanpdf
    (https://github.com/virantha/scanpdf) and licensed under the Apache
    license.

    :param string filename: file name of image to check
    :returns: True if image is blank, False otherwise.

    """
    if not os.path.exists(filename):
        return True

    c = 'identify -verbose %s' % filename
    result = run_cmd(c)
    mStdDev = re.compile(
        b'\s*standard deviation:\s*\d+\.\d+\s*\((?P<percent>\d+\.\d+)\).*')

    for line in result.splitlines():
        match = mStdDev.search(line)
        if match:
            stdev = float(match.group('percent'))
            if stdev > 0.05:
                return False

    return True

def move_blanks(input_dir, output_dir):
    """Move blank .pnm's in input_dir to output_dir

    :param string input_dir: directory to check for blank .pnm files
    :param string output_dir: where to move blank .pnm files
    :returns: number of blank pages moved
    :rtype: int

    """
    number_of_blanks = 0

    for entry in os.scandir(input_dir):
        if not entry.is_file() or not entry.name.endswith('.pnm'):
            continue

        image = os.path.join(input_dir, entry.name)

        if is_blank(image):
            try:
                os.mkdir(output_dir)
            except:
                pass  # Assume directory exists.

            shutil.move(image, output_dir)
            number_of_blanks += 1

    return number_of_blanks

def remove_if_blank(filename):
    """Remove file if it is blank.

    This is useful when scanning in duplex mode using a backend that doesn't
    support skipping blank pages.

    :param string filename: name of file to remove, if blank

    """
    if is_blank(filename):
        print('Removing (probably) blank page {}'.format(filename))
        os.remove(filename)

def merge_pdfs(inputs, output):
    """Merge selected pdfs.

    :param list inputs: files to concatenate
    :param string output: name of file to write

    """
    merger = PdfFileMerger()
    input_fds = dict()

    out = open(output, 'wb')

    for filename in inputs:
        try:
            input_fds[filename] = open(filename, 'rb')
        except OSError as e:
            print('Error opening {}: {}'.format(filename, e))
        merger.append(input_fds[filename])

    merger.write(out)

def merge_pdfs_in_dir(directory, output):
    """Read all pdf files in directory and create one merged output.

    :param string directory: directory containing pdf files to be merged
    :param string output: filename of new merged pdf
    """
    files_to_merge = []

    for file in os.scandir(directory):
        if not file.is_file():
            continue
        if file.name[-4:] != '.pdf':
            continue

        files_to_merge.append(os.path.join(directory, file.name))

    merge_pdfs(files_to_merge, output)

def ocr(filename, language):
    """Perform OCR on file using Tesseract.

    :param string filename: file to perform OCR on
    :param string language: language(s) expected to be used in file

    """
    base_output_name = filename[:-4]
    command = 'tesseract {} {} -l {} pdf'.format(filename,
                                                 base_output_name,
                                                 language)
    run_cmd(command)

def ocr_pnms_in_dir(directory, language):
    """Perform OCR on all pnm files in given directory.

    :param string directory: directory in which all pnm files will be OCR:ed
    :param string language: language(s) expected to be used in files

    """
    for file in os.scandir(directory):
        if not file.is_file():
            continue
        if file.name[-4:] != '.pnm':
            continue
        ocr(os.path.join(directory, file.name), language)

def scand():
    """
    Polls DATA_DIR for finished scans. Once found, scand will:

        - Move blank images to subdir blank/
        - Rotate remaining images
        - OCR remaining images
        - Merge resulting pdf files
        - Move the directory to INBOX

    """

    load_config()

    while True:
        for entry in os.scandir(config['Paths']['data']):
            if entry.name.endswith('.unfinished') or not entry.is_dir() or entry.name == 'INBOX':
                continue

            scan_dir = os.path.join(config['Paths']['data'], entry.name)

            move_blanks(scan_dir, os.path.join(scan_dir, 'blank'))

            rotate_all_images_in_dir(scan_dir, 180)

            unpaper_dir(scan_dir, 'pnm')

            ocr_pnms_in_dir(scan_dir, 'swe')

            pdf_output = os.path.join(scan_dir, 'output.pdf')
            merge_pdfs_in_dir(scan_dir, pdf_output)

            inbox_dir = os.path.join(config['Paths']['data'], 'INBOX')

            try:
                os.mkdir(inbox_dir)
            except:
                pass  # Assume directory exists.

            shutil.move(scan_dir, inbox_dir)

        time.sleep(1)
<commit_msg>Make move_blanks use utils.files<commit_after>import os
import re
import shutil
import time

from .config import config, load_config
from .utils import run_cmd, files
from PyPDF2 import PdfFileMerger
from wand.image import Image

def rotate_image(filename, degrees):
    """Rotate image given amount of degrees.

    :param string filename: file to rotate
    :param int degrees: amount of degrees to rotate

    """
    with Image(filename=filename) as image:
        with image.clone() as rotated:
            rotated.rotate(degrees)
            rotated.save(filename=filename)

def rotate_all_images_in_dir(dirname, degrees):
    """Rotate all files in directory.

    :param string dirname: name of directory in which files should be rotated
    :param int degrees: number of degrees to rotate

    """
    for f in files(dirname):
        rotate_image(os.path.join(dirname, f.name), degrees)

def unpaper(filename):
    """Process file with unpaper and delete original.

    :param filename: TODO

    """
    unpapered_filename = filename + '.unpapered'
    # TODO: We don't use unpaper's --overwrite because it currently seems to be
    # broken. Once it's been fixed, just --overwrite the original.
    run_cmd('unpaper --size a4 "{}" "{}"'.format(filename, unpapered_filename))
    shutil.move(unpapered_filename, filename)

def unpaper_dir(directory, extension=None):
    """Run unpaper on all files with given extension in directory

    :param string directory: directory to process
    :param string extension: extension of files to run unpaper on

    """

    for f in files(directory, extension):
        unpaper(os.path.oin(directory, f.name))

def is_blank(filename):
    """
    Check if image is blank.

    Return true if filename is a blank image. This is a slightly modified
    version of Vinatha Ekanayake's is_blank(), which is part of Scanpdf
    (https://github.com/virantha/scanpdf) and licensed under the Apache
    license.

    :param string filename: file name of image to check
    :returns: True if image is blank, False otherwise.

    """
    if not os.path.exists(filename):
        return True

    c = 'identify -verbose %s' % filename
    result = run_cmd(c)
    mStdDev = re.compile(
        b'\s*standard deviation:\s*\d+\.\d+\s*\((?P<percent>\d+\.\d+)\).*')

    for line in result.splitlines():
        match = mStdDev.search(line)
        if match:
            stdev = float(match.group('percent'))
            if stdev > 0.05:
                return False

    return True

def move_blanks(input_dir, output_dir):
    """Move blank .pnm's in input_dir to output_dir

    :param string input_dir: directory to check for blank .pnm files
    :param string output_dir: where to move blank .pnm files
    :returns: number of blank pages moved
    :rtype: int

    """
    number_of_blanks = 0

    for file in files(input_dir, 'pnm'):
        image = os.path.join(input_dir, file)

        if is_blank(image):
            try:
                os.mkdir(output_dir)
            except:
                pass  # Assume directory exists.

            shutil.move(image, output_dir)
            number_of_blanks += 1

    return number_of_blanks

def remove_if_blank(filename):
    """Remove file if it is blank.

    This is useful when scanning in duplex mode using a backend that doesn't
    support skipping blank pages.

    :param string filename: name of file to remove, if blank

    """
    if is_blank(filename):
        print('Removing (probably) blank page {}'.format(filename))
        os.remove(filename)

def merge_pdfs(inputs, output):
    """Merge selected pdfs.

    :param list inputs: files to concatenate
    :param string output: name of file to write

    """
    merger = PdfFileMerger()
    input_fds = dict()

    out = open(output, 'wb')

    for filename in inputs:
        try:
            input_fds[filename] = open(filename, 'rb')
        except OSError as e:
            print('Error opening {}: {}'.format(filename, e))
        merger.append(input_fds[filename])

    merger.write(out)

def merge_pdfs_in_dir(directory, output):
    """Read all pdf files in directory and create one merged output.

    :param string directory: directory containing pdf files to be merged
    :param string output: filename of new merged pdf
    """
    files_to_merge = []

    for file in os.scandir(directory):
        if not file.is_file():
            continue
        if file.name[-4:] != '.pdf':
            continue

        files_to_merge.append(os.path.join(directory, file.name))

    merge_pdfs(files_to_merge, output)

def ocr(filename, language):
    """Perform OCR on file using Tesseract.

    :param string filename: file to perform OCR on
    :param string language: language(s) expected to be used in file

    """
    base_output_name = filename[:-4]
    command = 'tesseract {} {} -l {} pdf'.format(filename,
                                                 base_output_name,
                                                 language)
    run_cmd(command)

def ocr_pnms_in_dir(directory, language):
    """Perform OCR on all pnm files in given directory.

    :param string directory: directory in which all pnm files will be OCR:ed
    :param string language: language(s) expected to be used in files

    """
    for file in os.scandir(directory):
        if not file.is_file():
            continue
        if file.name[-4:] != '.pnm':
            continue
        ocr(os.path.join(directory, file.name), language)

def scand():
    """
    Polls DATA_DIR for finished scans. Once found, scand will:

        - Move blank images to subdir blank/
        - Rotate remaining images
        - OCR remaining images
        - Merge resulting pdf files
        - Move the directory to INBOX

    """

    load_config()

    while True:
        for entry in os.scandir(config['Paths']['data']):
            if entry.name.endswith('.unfinished') or not entry.is_dir() or entry.name == 'INBOX':
                continue

            scan_dir = os.path.join(config['Paths']['data'], entry.name)

            move_blanks(scan_dir, os.path.join(scan_dir, 'blank'))

            rotate_all_images_in_dir(scan_dir, 180)

            unpaper_dir(scan_dir, 'pnm')

            ocr_pnms_in_dir(scan_dir, 'swe')

            pdf_output = os.path.join(scan_dir, 'output.pdf')
            merge_pdfs_in_dir(scan_dir, pdf_output)

            inbox_dir = os.path.join(config['Paths']['data'], 'INBOX')

            try:
                os.mkdir(inbox_dir)
            except:
                pass  # Assume directory exists.

            shutil.move(scan_dir, inbox_dir)

        time.sleep(1)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# ----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

import os
from itertools import chain, izip
from gzip import open as gzip_open
from skbio.core.iterator import FastaIterator, FastqIterator

FILEEXT_MAP = {'fna': (FastaIterator, open),
               'fna.gz': (FastaIterator, gzip_open),
               'fasta': (FastaIterator, open),
               'fasta.gz': (FastaIterator, gzip_open),
               'qual': (FastaIterator, open),
               'qual.gz': (FastaIterator, gzip_open),
               'fastq': (FastqIterator, open),
               'fastq.gz': (FastqIterator, gzip_open),
               'fq': (FastqIterator, open),
               'fq.gz': (FastqIterator, gzip_open)}

def _determine_types_and_openers(files):
    """Attempt to determine the appropriate iterators and openers"""
    if files is None:
        return [], []

    iters = []
    openers = []
    for fpath in files:
        if fpath.endswith('.gz'):
            ext = '.'.join(fpath.rsplit('.', 2)[-2:])
        else:
            ext = fpath.rsplit('.', 1)[-1]

        i, o = FILEEXT_MAP.get(ext, (None, None))
        if i is None:
            raise IOError("Unknown filetype for %s" % fpath)

        iters.append(i)
        openers.append(o)

    return iters, openers

def _is_single_iterator_type(iters):
    """Determine if there is a single or multiple type of iterator

    If iters is [], this method returns True it considers the null case to be
    a single iterator type.
    """
    if iters:
        return len(set(iters)) == 1
    else:
        return True

def _open_or_none(opener, f):
    """Open a file or returns None"""
    if not opener:
        return None
    else:
        name = opener.__name__

        if not os.path.exists(f):
            raise IOError("%s does not appear to exist!" % f)

        try:
            opened = opener(f)
        except IOError:
            raise IOError("Could not open %s with %s!" % (f, name))

        return opened

def factory(seqs, qual=None, **kwargs):
    """Construct the appropriate iterator for all your processing needs

    This method will attempt to open all files correctly and to feed the
    appropriate objects into the correct iterators.

    Seqs can list multiple types of files (e.g., FASTA and FASTQ), but if
    multiple file types are specified, qual must be None

    Parameters
    ----------
    seqs : str or list of sequence file paths
    qual : str or list of qual file paths or None
    kwargs : dict
        passed into the subsequent generators.

    Returns
    -------
    SequenceIterator instance
        the return is ``Iterable``

    See Also
    --------
    skbio.core.iterator.SequenceIterator
    skbio.core.iterator.FastaIterator
    skbio.core.iterator.FastqIterator
    """
    if not seqs:
        raise ValueError("Must pass in sequences!")

    if isinstance(seqs, str):
        seqs = [seqs]

    if isinstance(qual, str):
        qual = [qual]

    # i -> iters, o -> openers
    i_seqs, o_seqs = _determine_types_and_openers(seqs)
    i_qual, o_qual = _determine_types_and_openers(qual)

    seqs = [_open_or_none(o, f) for f, o in izip(seqs, o_seqs)]
    qual = [_open_or_none(o, f) for f, o in izip(qual or [], o_qual or [])]

    if not qual:
        qual = None

    if not _is_single_iterator_type(i_seqs) and qual is not None:
        # chaining Fasta/Fastq for sequence is easy, but it gets nasty quick
        # if seqs is a mix of fasta/fastq, with qual coming in as there aren't
        # 1-1 mappings. This could be addressed if necessary, but seems like
        # an unnecessary block of code right now
        raise ValueError("Cannot resolve iterators!")

    if _is_single_iterator_type(i_seqs):
        seqs_constructor = i_seqs[0]
        gen = seqs_constructor(seq=seqs, qual=qual, **kwargs)
    else:
        gen = chain(*[c(seq=[fp], **kwargs) for c, fp in izip(i_seqs, seqs)])

    return gen
<commit_msg>module docstring<commit_after>#!/usr/bin/env python
r"""
Sequence factory (:mod:`skbio.factory.sequence`)
================================================

.. currentmodule:: skbio.factory.sequence

The sequence factory provides a standard mechanism to iterate over sequence
files regardless of file type or whether they are compressed. The method
will take a single or list of file paths, resolve the necessary file openers,
necessary iterator objects and the corresponding parsers.

Methods
-------

.. autosummary::
   :toctree: generated/

   factory

See Also
--------

skbio.core.iterator.SequenceIterator

Examples
--------

>>> pass

"""

# ----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

import os
from itertools import chain, izip
from gzip import open as gzip_open
from skbio.core.iterator import FastaIterator, FastqIterator

FILEEXT_MAP = {'fna': (FastaIterator, open),
               'fna.gz': (FastaIterator, gzip_open),
               'fasta': (FastaIterator, open),
               'fasta.gz': (FastaIterator, gzip_open),
               'qual': (FastaIterator, open),
               'qual.gz': (FastaIterator, gzip_open),
               'fastq': (FastqIterator, open),
               'fastq.gz': (FastqIterator, gzip_open),
               'fq': (FastqIterator, open),
               'fq.gz': (FastqIterator, gzip_open)}

def _determine_types_and_openers(files):
    """Attempt to determine the appropriate iterators and openers"""
    if files is None:
        return [], []

    iters = []
    openers = []
    for fpath in files:
        if fpath.endswith('.gz'):
            ext = '.'.join(fpath.rsplit('.', 2)[-2:])
        else:
            ext = fpath.rsplit('.', 1)[-1]

        i, o = FILEEXT_MAP.get(ext, (None, None))
        if i is None:
            raise IOError("Unknown filetype for %s" % fpath)

        iters.append(i)
        openers.append(o)

    return iters, openers

def _is_single_iterator_type(iters):
    """Determine if there is a single or multiple type of iterator

    If iters is [], this method returns True it considers the null case to be
    a single iterator type.
    """
    if iters:
        return len(set(iters)) == 1
    else:
        return True

def _open_or_none(opener, f):
    """Open a file or returns None"""
    if not opener:
        return None
    else:
        name = opener.__name__

        if not os.path.exists(f):
            raise IOError("%s does not appear to exist!" % f)

        try:
            opened = opener(f)
        except IOError:
            raise IOError("Could not open %s with %s!" % (f, name))

        return opened

def factory(seqs, qual=None, **kwargs):
    """Construct the appropriate iterator for all your processing needs

    This method will attempt to open all files correctly and to feed the
    appropriate objects into the correct iterators.

    Seqs can list multiple types of files (e.g., FASTA and FASTQ), but if
    multiple file types are specified, qual must be None

    Parameters
    ----------
    seqs : str or list of sequence file paths
    qual : str or list of qual file paths or None
    kwargs : dict
        passed into the subsequent generators.

    Returns
    -------
    SequenceIterator instance
        the return is ``Iterable``

    See Also
    --------
    skbio.core.iterator.SequenceIterator
    skbio.core.iterator.FastaIterator
    skbio.core.iterator.FastqIterator
    """
    if not seqs:
        raise ValueError("Must pass in sequences!")

    if isinstance(seqs, str):
        seqs = [seqs]

    if isinstance(qual, str):
        qual = [qual]

    # i -> iters, o -> openers
    i_seqs, o_seqs = _determine_types_and_openers(seqs)
    i_qual, o_qual = _determine_types_and_openers(qual)

    seqs = [_open_or_none(o, f) for f, o in izip(seqs, o_seqs)]
    qual = [_open_or_none(o, f) for f, o in izip(qual or [], o_qual or [])]

    if not qual:
        qual = None

    if not _is_single_iterator_type(i_seqs) and qual is not None:
        # chaining Fasta/Fastq for sequence is easy, but it gets nasty quick
        # if seqs is a mix of fasta/fastq, with qual coming in as there aren't
        # 1-1 mappings. This could be addressed if necessary, but seems like
        # an unnecessary block of code right now
        raise ValueError("Cannot resolve iterators!")

    if _is_single_iterator_type(i_seqs):
        seqs_constructor = i_seqs[0]
        gen = seqs_constructor(seq=seqs, qual=qual, **kwargs)
    else:
        gen = chain(*[c(seq=[fp], **kwargs) for c, fp in izip(i_seqs, seqs)])

    return gen
<|endoftext|>