<commit_before>"""Utilities for fitting modles.
"""

from warnings import warn
import sys

from datetime import datetime
from docopt import docopt
from pprint import pformat
from sklearn.model_selection import GridSearchCV

from .interfaces import annotate
from .util import apply_kwargs
from .util import args_from_config
from .util import initialize_config
from .util import logger
from .util import PluggableDecorator
from .util import timer

@PluggableDecorator('fit_decorators')
@args_from_config
def fit(dataset_loader_train, model, model_persister, persist=True,
        activate=True, dataset_loader_test=None, evaluate=False,
        persist_if_better_than=None):

    if persist_if_better_than is not None:
        evaluate = True
        if dataset_loader_test is None:
            raise ValueError(
                "When using 'persist_if_better_than', make sure you also "
                "provide a 'dataset_loader_test'."
                )

    with timer(logger.info, "Loading data"):
        X, y = dataset_loader_train()

    with timer(logger.info, "Fitting model"):
        model.fit(X, y)

    if evaluate:
        with timer(logger.debug, "Evaluating model on train set"):
            score_train = model.score(X, y)
            annotate(model, {'score_train': score_train})
            logger.info("Train score: {}".format(score_train))

    score_test = None
    if evaluate and dataset_loader_test is not None:
        X_test, y_test = dataset_loader_test()
        with timer(logger.debug, "Evaluating model on test set"):
            score_test = model.score(X_test, y_test)
            annotate(model, {'score_test': score_test})
            logger.info("Test score:  {}".format(score_test))

    if persist:
        if (persist_if_better_than is not None and
            score_test < persist_if_better_than):
            logger.info("Not persisting model that has a test score "
                        "{} < {}".format(score_test, persist_if_better_than))
        else:
            annotate(model, {'train_timestamp': datetime.now().isoformat()})
            with timer(logger.info, "Writing model"):
                version = model_persister.write(model)
            logger.info("Wrote model with version {}.".format(version))
            if activate:
                model_persister.activate(version)

    return model

def fit_cmd(argv=sys.argv[1:]):  # pragma: no cover
    """\
Fit a model and save to database.

Will use 'dataset_loader_train', 'model', and 'model_perister' from
the configuration file, to load a dataset to train a model with, and
persist it.

Usage:
  pld-fit [options]

Options:
  -n --no-save              Don't persist the fitted model to disk.

  --no-activate             Don't activate the fitted model.

  --save-if-better-than=<k> Persist only if test score better than given
                            value.

  -e --evaluate             Evaluate fitted model on train and test set and
                            print out results.

  -h --help                 Show this screen.
"""
    arguments = docopt(fit_cmd.__doc__, argv=argv)
    no_save = arguments['--no-save']
    no_activate = arguments['--no-activate']
    save_if_better_than = arguments['--save-if-better-than']
    evaluate = arguments['--evaluate'] or bool(save_if_better_than)
    if save_if_better_than is not None:
        save_if_better_than = float(save_if_better_than)
    initialize_config(__mode__='fit')
    fit(
        persist=not no_save,
        activate=not no_activate,
        evaluate=evaluate,
        persist_if_better_than=save_if_better_than,
        )

@args_from_config
def activate(model_persister, model_version):
    model_persister.activate(model_version)
    logger.info("Activated model with version {}.".format(model_version))

@args_from_config
def delete(model_persister, model_version):
    model_persister.delete(model_version)
    logger.info("Deleted model with version {}.".format(model_version))

def admin_cmd(argv=sys.argv[1:]):  # pragma: no cover
    """\
Activate or delete models.

Models are usually made active right after fitting (see command
pld-fit).  The 'activate' command allows you to explicitly set the
currently active model.  Use 'pld-list' to get an overview of all
available models along with their version identifiers.

Deleting a model will simply remove it from the database.

Usage:
  pld-admin activate <version> [options]
  pld-admin delete <version> [options]

Options:
  -h --help                 Show this screen.
"""
    arguments = docopt(admin_cmd.__doc__, argv=argv)
    initialize_config(__mode__='fit')
    if arguments['activate']:
        activate(model_version=int(arguments['<version>']))
    elif arguments['delete']:
        delete(model_version=int(arguments['<version>']))

@args_from_config
def grid_search(dataset_loader_train, model, grid_search, scoring=None):
    with timer(logger.info, "Loading data"):
        X, y = dataset_loader_train()

    grid_search_kwargs = {
        'refit': False,
        }
    grid_search_kwargs.update(grid_search)

    cv = grid_search_kwargs.get('cv', None)
    if callable(cv):
        grid_search_kwargs['cv'] = apply_kwargs(cv, n=len(y), X=X, y=y)

    if 'scoring' in grid_search_kwargs:
        warn("Use of 'scoring' inside of 'grid_search' is deprecated. "
             "To fix, move 'scoring' up to the top level of the configuration "
             "dict.", DeprecationWarning)
        if scoring is not None:
            raise ValueError("You cannot define 'scoring' in 'grid_search' "
                             "and globally.")
        scoring = grid_search_kwargs['scoring']
    elif scoring is not None:
        grid_search_kwargs['scoring'] = scoring

    if not (hasattr(model, 'score') or scoring is not None):
        raise ValueError(
            "Your model doesn't seem to implement a 'score' method.  You may "
            "want to define a 'scoring' option in the configuration."
            )

    with timer(logger.info, "Running grid search"):
        gs = GridSearchCV(model, **grid_search_kwargs)
        gs.fit(X, y)

    scores = []
    means = gs.cv_results_['mean_test_score']
    stds = gs.cv_results_['std_test_score']
    params = gs.cv_results_['params']
    for mean, std, param in zip(means, stds, params):
        scores.append("mean: {0:.5f}, std: {1:.5f}, params: {2}".format(mean, std, param))
    logger.info('\n{}'.format(
        pformat(sorted(scores, reverse=True)).replace('"', '')))
    return scores

def grid_search_cmd(argv=sys.argv[1:]):  # pragma: no cover
    """\
Grid search parameters for the model.

Uses 'dataset_loader_train', 'model', and 'grid_search' from the
configuration to load a training dataset, and run a grid search on the
model using the grid of hyperparameters.

Usage:
  pld-grid-search [options]

Options:
  -h --help                Show this screen.
"""
    docopt(grid_search_cmd.__doc__, argv=argv)
    initialize_config(__mode__='fit')
    grid_search()
<commit_msg>Also use custom scorer if available for fit --evaluate<commit_after>"""Utilities for fitting modles.
"""

from warnings import warn
import sys

from datetime import datetime
from docopt import docopt
from pprint import pformat
from sklearn.metrics import get_scorer
from sklearn.model_selection import GridSearchCV

from .interfaces import annotate
from .util import apply_kwargs
from .util import args_from_config
from .util import initialize_config
from .util import logger
from .util import PluggableDecorator
from .util import timer

@PluggableDecorator('fit_decorators')
@args_from_config
def fit(dataset_loader_train, model, model_persister, persist=True,
        activate=True, dataset_loader_test=None, evaluate=False,
        persist_if_better_than=None, scoring=None):

    if persist_if_better_than is not None:
        evaluate = True
        if dataset_loader_test is None:
            raise ValueError(
                "When using 'persist_if_better_than', make sure you also "
                "provide a 'dataset_loader_test'."
                )

    if evaluate and not (hasattr(model, 'score') or scoring is not None):
        raise ValueError(
            "Your model doesn't seem to implement a 'score' method.  You may "
            "want to define a 'scoring' option in the configuration."
            )

    if scoring is not None:
        scorer = get_scorer(scoring)
    else:
        def scorer(model, X, y):
            return model.score(X, y)

    with timer(logger.info, "Loading data"):
        X, y = dataset_loader_train()

    with timer(logger.info, "Fitting model"):
        model.fit(X, y)

    if evaluate:
        with timer(logger.debug, "Evaluating model on train set"):
            score_train = scorer(model, X, y)
            annotate(model, {'score_train': score_train})
            logger.info("Train score: {}".format(score_train))

    score_test = None
    if evaluate and dataset_loader_test is not None:
        X_test, y_test = dataset_loader_test()
        with timer(logger.debug, "Evaluating model on test set"):
            score_test = scorer(model, X_test, y_test)
            annotate(model, {'score_test': score_test})
            logger.info("Test score:  {}".format(score_test))

    if persist:
        if (persist_if_better_than is not None and
            score_test < persist_if_better_than):
            logger.info("Not persisting model that has a test score "
                        "{} < {}".format(score_test, persist_if_better_than))
        else:
            annotate(model, {'train_timestamp': datetime.now().isoformat()})
            with timer(logger.info, "Writing model"):
                version = model_persister.write(model)
            logger.info("Wrote model with version {}.".format(version))
            if activate:
                model_persister.activate(version)

    return model

def fit_cmd(argv=sys.argv[1:]):  # pragma: no cover
    """\
Fit a model and save to database.

Will use 'dataset_loader_train', 'model', and 'model_perister' from
the configuration file, to load a dataset to train a model with, and
persist it.

Usage:
  pld-fit [options]

Options:
  -n --no-save              Don't persist the fitted model to disk.

  --no-activate             Don't activate the fitted model.

  --save-if-better-than=<k> Persist only if test score better than given
                            value.

  -e --evaluate             Evaluate fitted model on train and test set and
                            print out results.

  -h --help                 Show this screen.
"""
    arguments = docopt(fit_cmd.__doc__, argv=argv)
    no_save = arguments['--no-save']
    no_activate = arguments['--no-activate']
    save_if_better_than = arguments['--save-if-better-than']
    evaluate = arguments['--evaluate'] or bool(save_if_better_than)
    if save_if_better_than is not None:
        save_if_better_than = float(save_if_better_than)
    initialize_config(__mode__='fit')
    fit(
        persist=not no_save,
        activate=not no_activate,
        evaluate=evaluate,
        persist_if_better_than=save_if_better_than,
        )

@args_from_config
def activate(model_persister, model_version):
    model_persister.activate(model_version)
    logger.info("Activated model with version {}.".format(model_version))

@args_from_config
def delete(model_persister, model_version):
    model_persister.delete(model_version)
    logger.info("Deleted model with version {}.".format(model_version))

def admin_cmd(argv=sys.argv[1:]):  # pragma: no cover
    """\
Activate or delete models.

Models are usually made active right after fitting (see command
pld-fit).  The 'activate' command allows you to explicitly set the
currently active model.  Use 'pld-list' to get an overview of all
available models along with their version identifiers.

Deleting a model will simply remove it from the database.

Usage:
  pld-admin activate <version> [options]
  pld-admin delete <version> [options]

Options:
  -h --help                 Show this screen.
"""
    arguments = docopt(admin_cmd.__doc__, argv=argv)
    initialize_config(__mode__='fit')
    if arguments['activate']:
        activate(model_version=int(arguments['<version>']))
    elif arguments['delete']:
        delete(model_version=int(arguments['<version>']))

@args_from_config
def grid_search(dataset_loader_train, model, grid_search, scoring=None):
    with timer(logger.info, "Loading data"):
        X, y = dataset_loader_train()

    grid_search_kwargs = {
        'refit': False,
        }
    grid_search_kwargs.update(grid_search)

    cv = grid_search_kwargs.get('cv', None)
    if callable(cv):
        grid_search_kwargs['cv'] = apply_kwargs(cv, n=len(y), X=X, y=y)

    if 'scoring' in grid_search_kwargs:
        warn("Use of 'scoring' inside of 'grid_search' is deprecated. "
             "To fix, move 'scoring' up to the top level of the configuration "
             "dict.", DeprecationWarning)
        if scoring is not None:
            raise ValueError("You cannot define 'scoring' in 'grid_search' "
                             "and globally.")
        scoring = grid_search_kwargs['scoring']
    elif scoring is not None:
        grid_search_kwargs['scoring'] = scoring

    if not (hasattr(model, 'score') or scoring is not None):
        raise ValueError(
            "Your model doesn't seem to implement a 'score' method.  You may "
            "want to define a 'scoring' option in the configuration."
            )

    with timer(logger.info, "Running grid search"):
        gs = GridSearchCV(model, **grid_search_kwargs)
        gs.fit(X, y)

    scores = []
    means = gs.cv_results_['mean_test_score']
    stds = gs.cv_results_['std_test_score']
    params = gs.cv_results_['params']
    for mean, std, param in zip(means, stds, params):
        scores.append("mean: {0:.5f}, std: {1:.5f}, params: {2}".format(mean, std, param))
    logger.info('\n{}'.format(
        pformat(sorted(scores, reverse=True)).replace('"', '')))
    return scores

def grid_search_cmd(argv=sys.argv[1:]):  # pragma: no cover
    """\
Grid search parameters for the model.

Uses 'dataset_loader_train', 'model', and 'grid_search' from the
configuration to load a training dataset, and run a grid search on the
model using the grid of hyperparameters.

Usage:
  pld-grid-search [options]

Options:
  -h --help                Show this screen.
"""
    docopt(grid_search_cmd.__doc__, argv=argv)
    initialize_config(__mode__='fit')
    grid_search()
<|endoftext|>


<commit_before>from django.db import models
from django.core.exceptions import ObjectDoesNotExist
from papers.utils import nstr, iunaccent
from django.utils.translation import ugettext_lazy as _

OA_STATUS_CHOICES = (
        ('OA', _('Open Access')),
        ('OK', _('Allows pre/post prints')),
        ('NOK', _('Forbids pre/post prints')),
        ('UNK', _('Policy unclear')),
   )

PDF_STATUS_CHOICES = [('OK', _('Available')),
                      ('NOK', _('Unavailable'))]

VISIBILITY_CHOICES = [('VISIBLE', _('Visible')),
                      ('CANDIDATE', _('Candidate')),
                      ('DELETED', _('Deleted')),
                      ]

# Information about the researchers and their groups
class Department(models.Model):
    name = models.CharField(max_length=300)

    @property
    def sorted_researchers(self):
        return self.researcher_set.order_by('name')

    def __unicode__(self):
        return self.name

class ResearchGroup(models.Model):
    name = models.CharField(max_length=300)

    def __unicode__(self):
        return self.name

class Researcher(models.Model):
    department = models.ForeignKey(Department)
    groups = models.ManyToManyField(ResearchGroup,blank=True,null=True)
    email = models.EmailField(blank=True,null=True)
    homepage = models.URLField(blank=True, null=True)
    role = models.CharField(max_length=128, null=True, blank=True)

    # DOI search
    last_doi_search = models.DateTimeField(null=True,blank=True)
    status = models.CharField(max_length=512, blank=True, null=True)
    last_status_update = models.DateTimeField(auto_now=True)

    def __unicode__(self):
        first_name = Name.objects.filter(researcher_id=self.id).order_by('id').first()
        if first_name:
            return unicode(first_name)
        return _("Anonymous researcher")

    @property
    def authors_by_year(self):
        return Author.objects.filter(name__researcher_id=self.id).order_by('-paper__year')
    @property
    def names(self):
        return self.name_set.order_by('id')
    @property
    def name(self):
        name = self.names[0]
        if name:
            return name
        else:
            return _("Anonymous researcher")
    @property
    def aka(self):
        return self.names[:2]

class Name(models.Model):
    researcher = models.ForeignKey(Researcher, blank=True, null=True, on_delete=models.SET_NULL)
    first = models.CharField(max_length=256)
    last = models.CharField(max_length=256)
    full = models.CharField(max_length=513, db_index=True)

    unique_together = ('first','last')# TODO Two researchers with the same name is not supported

    class Meta:
        ordering = ['last','first']

    @classmethod
    def create(cls, first, last):
        full = iunaccent(first+' '+last)
        return cls(first=first,last=last,full=full)
    def __unicode__(self):
        return '%s %s' % (self.first,self.last)
    @property
    def is_known(self):
        return self.researcher != None

# Papers matching one or more researchers
class Paper(models.Model):
    title = models.CharField(max_length=1024)
    fingerprint = models.CharField(max_length=64)
    year = models.IntegerField()
    last_modified = models.DateField(auto_now=True)
    visibility = models.CharField(max_length=32, default='VISIBLE')

    def __unicode__(self):
        return self.title

    # The two following fields need to be updated after the relevant changes
    # using the methods below.
    oa_status = models.CharField(max_length=32, null=True, blank=True)
    pdf_url = models.URLField(max_length=2048, null=True, blank=True)

    def update_oa_status(self):
        # Look for publications
        qs = list(self.publication_set.all()[:1])
        if qs:
            self.oa_status = qs[0].oa_status()
        else:
            self.oa_status = 'UNK'
        # Look for automatic OA OAI sources
        records = list(self.oairecord_set.filter(source__oa=True)[:1])
        if records:
            self.oa_status = 'OA'
        self.save()

    # TODO merge these two functions !

    def update_pdf_url(self):
        # TODO: create an oa_status field in each publication so that we optimize queries
        # and can deal with hybrid OA

        # If it is an open access publisher, keep the publisher version
        if self.oa_status == 'OA':
            publications = Publication.objects.filter(journal__publisher__oa_status='OA')[:1]
            if publications:
                self.pdf_url = publications[0].splash_url()
                self.save()
                return
        # otherwise try the OAI sources
        matches = OaiRecord.objects.filter(about=self.id,pdf_url__isnull=False)[:1]
        if matches:
            self.pdf_url = matches[0].pdf_url
        self.save()

# Researcher / Paper binary relation
# TODO: it could be a ManyToMany field...
class Author(models.Model):
    paper = models.ForeignKey(Paper)
    name = models.ForeignKey(Name)
    def __unicode__(self):
        return unicode(self.name)

# Publisher associated with a journal
class Publisher(models.Model):
    romeo_id = models.CharField(max_length=64)
    name = models.CharField(max_length=256)
    alias = models.CharField(max_length=256,null=True,blank=True)
    url = models.URLField(null=True,blank=True)
    preprint = models.CharField(max_length=32)
    postprint = models.CharField(max_length=32)
    pdfversion = models.CharField(max_length=32)
    oa_status = models.CharField(max_length=32, choices=OA_STATUS_CHOICES)
    def __unicode__(self):
        return self.name
    @property
    def publi_count(self):
        count = 0
        for journal in self.journal_set.all():
            count += journal.publication_set.filter(paper__visibility='VISIBLE').count()
        return count
    @property
    def preprint_conditions(self):
        return self.publisherrestrictiondetail_set.filter(applies_to='preprint')
    @property
    def postprint_conditions(self):
        return self.publisherrestrictiondetail_set.filter(applies_to='postprint')
    @property
    def pdfversion_conditions(self):
        return self.publisherrestrictiondetail_set.filter(applies_to='pdfversion')
    def change_oa_status(self, new_oa_status):
        self.oa_status = new_oa_status
        self.save()
        papers = Paper.objects.filter(publication__journal__publisher=self.pk)
        for p in papers:
            p.update_oa_status()

# Journal data retrieved from RoMEO
class Journal(models.Model):
    title = models.CharField(max_length=256)
    last_updated = models.DateTimeField(auto_now=True)
    issn = models.CharField(max_length=10, blank=True, null=True, unique=True)
    publisher = models.ForeignKey(Publisher)
    def __unicode__(self):
        return self.title

class PublisherCondition(models.Model):
    publisher = models.ForeignKey(Publisher)
    text = models.CharField(max_length=1024)
    def __unicode__(self):
        return self.text

class PublisherCopyrightLink(models.Model):
    publisher = models.ForeignKey(Publisher)
    text = models.CharField(max_length=256)
    url = models.URLField()
    def __unicode__(self):
        return self.text

class PublisherRestrictionDetail(models.Model):
    publisher = models.ForeignKey(Publisher)
    text = models.CharField(max_length=256)
    applies_to = models.CharField(max_length=32)
    def __unicode__(self):
        return self.text

class Disambiguation(models.Model):
    publications = models.ManyToManyField('Publication')
    title = models.CharField(max_length=512)
    issn = models.CharField(max_length=128)
    unique_together = ('title', 'issn')

class DisambiguationChoice(models.Model):
    about = models.ForeignKey(Disambiguation)
    title = models.CharField(max_length=512)
    issn = models.CharField(max_length=128)

# Publication of these papers (in journals or conference proceedings)
class Publication(models.Model):
    paper = models.ForeignKey(Paper)
    pubtype = models.CharField(max_length=64)
    title = models.CharField(max_length=256) # this is actually the *journal* title
    journal = models.ForeignKey(Journal, blank=True, null=True)
    issue = models.CharField(max_length=64, blank=True, null=True)
    volume = models.CharField(max_length=64, blank=True, null=True)
    pages = models.CharField(max_length=64, blank=True, null=True)
    date = models.CharField(max_length=128, blank=True, null=True)
    publisher = models.CharField(max_length=256, blank=True, null=True)
    doi = models.CharField(max_length=1024, unique=True, blank=True, null=True) # in theory, there is no limit
    def oa_status(self):
        if self.journal:
            return self.journal.publisher.oa_status
        else:
            return 'UNK'

    def splash_url(self):
        if self.doi:
            return 'http://dx.doi.org/'+self.doi

    def details_to_str(self):
        result = ''
        if self.issue or self.volume or self.pages or self.date:
            result += ', '
        if self.issue:
            result += self.issue
        if self.volume:
            result += '('+self.volume+')'
        if self.issue or self.volume:
            result += ', '
        if self.pages:
            result += self.pages+', '
        if self.date:
            result += self.date
        return result

    def __unicode__(self):
        return self.title+self.details_to_str()

# Rough data extracted through OAI-PMH
class OaiSource(models.Model):
    identifier = models.CharField(max_length=300)
    name = models.CharField(max_length=100)
    oa = models.BooleanField(default=False)

    # Fetching properties
    last_status_update = models.DateTimeField(auto_now=True)
    def __unicode__(self):
        return self.name

class OaiRecord(models.Model):
    source = models.ForeignKey(OaiSource)
    identifier = models.CharField(max_length=512, unique=True)
    splash_url = models.URLField(max_length=1024, null=True, blank=True)
    pdf_url = models.URLField(max_length=1024, null=True, blank=True)
    about = models.ForeignKey(Paper)
    description = models.TextField(null=True,blank=True)
    def __unicode__(self):
        return self.identifier

<commit_msg>Fixed a big bug in PDF url generation<commit_after>from django.db import models
from django.core.exceptions import ObjectDoesNotExist
from papers.utils import nstr, iunaccent
from django.utils.translation import ugettext_lazy as _

OA_STATUS_CHOICES = (
        ('OA', _('Open Access')),
        ('OK', _('Allows pre/post prints')),
        ('NOK', _('Forbids pre/post prints')),
        ('UNK', _('Policy unclear')),
   )

PDF_STATUS_CHOICES = [('OK', _('Available')),
                      ('NOK', _('Unavailable'))]

VISIBILITY_CHOICES = [('VISIBLE', _('Visible')),
                      ('CANDIDATE', _('Candidate')),
                      ('DELETED', _('Deleted')),
                      ]

# Information about the researchers and their groups
class Department(models.Model):
    name = models.CharField(max_length=300)

    @property
    def sorted_researchers(self):
        return self.researcher_set.order_by('name')

    def __unicode__(self):
        return self.name

class ResearchGroup(models.Model):
    name = models.CharField(max_length=300)

    def __unicode__(self):
        return self.name

class Researcher(models.Model):
    department = models.ForeignKey(Department)
    groups = models.ManyToManyField(ResearchGroup,blank=True,null=True)
    email = models.EmailField(blank=True,null=True)
    homepage = models.URLField(blank=True, null=True)
    role = models.CharField(max_length=128, null=True, blank=True)

    # DOI search
    last_doi_search = models.DateTimeField(null=True,blank=True)
    status = models.CharField(max_length=512, blank=True, null=True)
    last_status_update = models.DateTimeField(auto_now=True)

    def __unicode__(self):
        first_name = Name.objects.filter(researcher_id=self.id).order_by('id').first()
        if first_name:
            return unicode(first_name)
        return _("Anonymous researcher")

    @property
    def authors_by_year(self):
        return Author.objects.filter(name__researcher_id=self.id).order_by('-paper__year')
    @property
    def names(self):
        return self.name_set.order_by('id')
    @property
    def name(self):
        name = self.names[0]
        if name:
            return name
        else:
            return _("Anonymous researcher")
    @property
    def aka(self):
        return self.names[:2]

class Name(models.Model):
    researcher = models.ForeignKey(Researcher, blank=True, null=True, on_delete=models.SET_NULL)
    first = models.CharField(max_length=256)
    last = models.CharField(max_length=256)
    full = models.CharField(max_length=513, db_index=True)

    unique_together = ('first','last')# TODO Two researchers with the same name is not supported

    class Meta:
        ordering = ['last','first']

    @classmethod
    def create(cls, first, last):
        full = iunaccent(first+' '+last)
        return cls(first=first,last=last,full=full)
    def __unicode__(self):
        return '%s %s' % (self.first,self.last)
    @property
    def is_known(self):
        return self.researcher != None

# Papers matching one or more researchers
class Paper(models.Model):
    title = models.CharField(max_length=1024)
    fingerprint = models.CharField(max_length=64)
    year = models.IntegerField()
    last_modified = models.DateField(auto_now=True)
    visibility = models.CharField(max_length=32, default='VISIBLE')

    def __unicode__(self):
        return self.title

    # The two following fields need to be updated after the relevant changes
    # using the methods below.
    oa_status = models.CharField(max_length=32, null=True, blank=True)
    pdf_url = models.URLField(max_length=2048, null=True, blank=True)

    def update_oa_status(self):
        # Look for publications
        qs = list(self.publication_set.all()[:1])
        if qs:
            self.oa_status = qs[0].oa_status()
        else:
            self.oa_status = 'UNK'
        # Look for automatic OA OAI sources
        records = list(self.oairecord_set.filter(source__oa=True)[:1])
        if records:
            self.oa_status = 'OA'
        self.save()

    # TODO merge these two functions !

    def update_pdf_url(self):
        # TODO: create an oa_status field in each publication so that we optimize queries
        # and can deal with hybrid OA

        # If it is an open access publisher, keep the publisher version
        if self.oa_status == 'OA':
            publications = self.publication_set.filter(journal__publisher__oa_status='OA')[:1]
            if publications:
                self.pdf_url = publications[0].splash_url()
                self.save()
                return
        # otherwise try the OAI sources
        matches = OaiRecord.objects.filter(about=self.id,pdf_url__isnull=False)[:1]
        if matches:
            self.pdf_url = matches[0].pdf_url
        self.save()

# Researcher / Paper binary relation
# TODO: it could be a ManyToMany field...
class Author(models.Model):
    paper = models.ForeignKey(Paper)
    name = models.ForeignKey(Name)
    def __unicode__(self):
        return unicode(self.name)

# Publisher associated with a journal
class Publisher(models.Model):
    romeo_id = models.CharField(max_length=64)
    name = models.CharField(max_length=256)
    alias = models.CharField(max_length=256,null=True,blank=True)
    url = models.URLField(null=True,blank=True)
    preprint = models.CharField(max_length=32)
    postprint = models.CharField(max_length=32)
    pdfversion = models.CharField(max_length=32)
    oa_status = models.CharField(max_length=32, choices=OA_STATUS_CHOICES)
    def __unicode__(self):
        return self.name
    @property
    def publi_count(self):
        count = 0
        for journal in self.journal_set.all():
            count += journal.publication_set.filter(paper__visibility='VISIBLE').count()
        return count
    @property
    def preprint_conditions(self):
        return self.publisherrestrictiondetail_set.filter(applies_to='preprint')
    @property
    def postprint_conditions(self):
        return self.publisherrestrictiondetail_set.filter(applies_to='postprint')
    @property
    def pdfversion_conditions(self):
        return self.publisherrestrictiondetail_set.filter(applies_to='pdfversion')
    def change_oa_status(self, new_oa_status):
        self.oa_status = new_oa_status
        self.save()
        papers = Paper.objects.filter(publication__journal__publisher=self.pk)
        for p in papers:
            p.update_oa_status()

# Journal data retrieved from RoMEO
class Journal(models.Model):
    title = models.CharField(max_length=256)
    last_updated = models.DateTimeField(auto_now=True)
    issn = models.CharField(max_length=10, blank=True, null=True, unique=True)
    publisher = models.ForeignKey(Publisher)
    def __unicode__(self):
        return self.title

class PublisherCondition(models.Model):
    publisher = models.ForeignKey(Publisher)
    text = models.CharField(max_length=1024)
    def __unicode__(self):
        return self.text

class PublisherCopyrightLink(models.Model):
    publisher = models.ForeignKey(Publisher)
    text = models.CharField(max_length=256)
    url = models.URLField()
    def __unicode__(self):
        return self.text

class PublisherRestrictionDetail(models.Model):
    publisher = models.ForeignKey(Publisher)
    text = models.CharField(max_length=256)
    applies_to = models.CharField(max_length=32)
    def __unicode__(self):
        return self.text

class Disambiguation(models.Model):
    publications = models.ManyToManyField('Publication')
    title = models.CharField(max_length=512)
    issn = models.CharField(max_length=128)
    unique_together = ('title', 'issn')

class DisambiguationChoice(models.Model):
    about = models.ForeignKey(Disambiguation)
    title = models.CharField(max_length=512)
    issn = models.CharField(max_length=128)

# Publication of these papers (in journals or conference proceedings)
class Publication(models.Model):
    paper = models.ForeignKey(Paper)
    pubtype = models.CharField(max_length=64)
    title = models.CharField(max_length=256) # this is actually the *journal* title
    journal = models.ForeignKey(Journal, blank=True, null=True)
    issue = models.CharField(max_length=64, blank=True, null=True)
    volume = models.CharField(max_length=64, blank=True, null=True)
    pages = models.CharField(max_length=64, blank=True, null=True)
    date = models.CharField(max_length=128, blank=True, null=True)
    publisher = models.CharField(max_length=256, blank=True, null=True)
    doi = models.CharField(max_length=1024, unique=True, blank=True, null=True) # in theory, there is no limit
    def oa_status(self):
        if self.journal:
            return self.journal.publisher.oa_status
        else:
            return 'UNK'

    def splash_url(self):
        if self.doi:
            return 'http://dx.doi.org/'+self.doi

    def details_to_str(self):
        result = ''
        if self.issue or self.volume or self.pages or self.date:
            result += ', '
        if self.issue:
            result += self.issue
        if self.volume:
            result += '('+self.volume+')'
        if self.issue or self.volume:
            result += ', '
        if self.pages:
            result += self.pages+', '
        if self.date:
            result += self.date
        return result

    def __unicode__(self):
        return self.title+self.details_to_str()

# Rough data extracted through OAI-PMH
class OaiSource(models.Model):
    identifier = models.CharField(max_length=300)
    name = models.CharField(max_length=100)
    oa = models.BooleanField(default=False)

    # Fetching properties
    last_status_update = models.DateTimeField(auto_now=True)
    def __unicode__(self):
        return self.name

class OaiRecord(models.Model):
    source = models.ForeignKey(OaiSource)
    identifier = models.CharField(max_length=512, unique=True)
    splash_url = models.URLField(max_length=1024, null=True, blank=True)
    pdf_url = models.URLField(max_length=1024, null=True, blank=True)
    about = models.ForeignKey(Paper)
    description = models.TextField(null=True,blank=True)
    def __unicode__(self):
        return self.identifier

<|endoftext|>


<commit_before># CB: this file is difficult to understand because it's been built up
# over a long time. Now that it's clearer what we want it to achieve,
# I could simplify it significantly.

# A complication is that as the buildbot setup has changed, the shell
# command that is checked for the timings has changed number. This
# means the timing data can change directory and filename. Therefore,
# this script cannot be used to get all historical data in one
# go. There's a MIN_BUILD variable that specifies the first build to
# check. That can be changed together with the filename_pattern
# variable to do chunks of the historical data, if desired. Once the
# script has processed data, it's stored in a pickle, so the data
# doesn't need to be retrieved again. As it is, we have a timings
# pickle on doozy that has quite a lot of historical data (i.e. I have
# changed MIN_BUILD and filename_pattern as time has gone on, to keep
# up to date with changes to the buildbot config).

import sys
import re
import pickle
from glob import glob

from sys import argv

#logfile = 'LOG'
script = "lissom_oo_or.ty"

# Replace this mechanism with bars under the x-axis, so that
# ranges can be annotated.
annotations = {
    (8100,66.5):((0,0),"A"),
    (8600,58):((0,4),"B"),
    (8250,53.5):((0,0),"C"),
    (8395,54.3):((20,-2),"D"),
    (8100,56.5):((-20,-2),"E"),
    (8425,55):((0,2),"F")
    }
# CB: the above's just approx for now; needs updating.

## annotations_key = {'A':'something',
##                    'B':'something else',
##                    'C':'another thing'}

V=re.compile(r'[0-9]*-log')
def get_build_no(logfilename):
    return int(V.findall(logfilename)[0].rstrip('-log'))

def create_timings(i_am_sure=False):
    if i_am_sure:
        timings = {}
        f = open('/home/ceball/buildbot/timings.pkl','w')
        pickle.dump(timings,f,0)
        f.close()

def create_startups(i_am_sure=False):
    if i_am_sure:
        startups = {}
        f = open('/home/ceball/buildbot/startups.pkl','w')
        pickle.dump(startups,f,0)
        f.close()

def get_timings():
    f = open('/home/ceball/buildbot/timings.pkl')
    timings = pickle.load(f)
    f.close()
    return timings

def save_timings(timings):
    f = open('/home/ceball/buildbot/timings.pkl','w')
    pickle.dump(timings,f,0)
    f.close()

def get_startups():
    f = open('/home/ceball/buildbot/startups.pkl')
    startups = pickle.load(f)
    f.close()
    return startups

def save_startups(startups):
    f = open('/home/ceball/buildbot/startups.pkl','w')
    pickle.dump(startups,f,0)
    f.close()

# exclude these builds
exclusions = []
exclusions.append(153) # doc why this is excluded
exclusions.append(425) # ran with wrong no. of iterations

def get_date_version_time(logfile,timings=None,startups=None):

    build = get_build_no(logfile)

    f = open(logfile)
    all_lines = f.readlines()
    f.close()

    ok = ok2 = False

    datei=versioni=timingi=startupi=None

    i = 0;
    for line in all_lines:
        if line.find("Running at")>=0:

            datei=i
        elif line.find("svnversion")>=0:

            versioni=i
        elif line.startswith("[examples/%s]"%script):
            timingi=i

        elif line.find("[examples/%s startup]"%script)>=0:
            startupi=i

        elif line.find("Results from examples/%s have not changed."%script)>=0:
            ok2=True
        elif line.find('program finished')>=0:
            ok=True

        i+=1;

    if not ok:
        print "...build %s currently incomplete"%build
        return None

    if not ok2:
        print "...speed test invalid because results didn't match"
        if timings:
            timings[script][build] = None
        if startups:
            startups[script][build] = None
        return None

    if datei is None or versioni is None or timingi is None or startupi is None:
        print "...not all data available - build didn't complete?"
        if timings:
            timings[script][build] = None
        if startups:
            startups[script][build] = None
        return None

    datel= all_lines[datei]
    d=re.compile(r'at [0-9]*')
    date = float(d.findall(datel)[0][3::])

    versionl = all_lines[versioni]
    v=re.compile(r'n [0-9]*')
    version = int(v.findall(versionl)[0][2::])

    timel = all_lines[timingi]
    t=re.compile(r'Now: [0-9.]* ')
    start,stop = t.search(timel).span()
    timing = float(timel[start+5:stop-1])

    cpusel = all_lines[timingi+1]
    start,stop = cpusel.index('elapsed')+8,cpusel.index('%CPU')
    cpu_usage = float(cpusel[start:stop])

    startupl = all_lines[startupi]
    t=re.compile(r'Now: [0-9.]* ')
    start,stop = t.search(startupl).span()
    startup = float(startupl[start+5:stop-1])

    startcpusel = all_lines[startupi+1]

    try:
        start,stop = startcpusel.index('elapsed')+8,startcpusel.index('%CPU')
        startcpusage = float(startcpusel[start:stop])
    except ValueError:
        startcpusage = 99  # HACK: for cases where it's missing, assume 99!

    if timings:

        if cpu_usage>95:
            timings[script][build] = (date,version,timing,cpu_usage)
        else:
            print "...build %s had %s percent cpu during timing (not >95)"%(build,cpu_usage)
            timings[script][build] = None

    if startups:

        if startcpusage>95:
            startups[script][build] = (date,version,startup,startcpusage)
        else:
            print "...build %s had %s percent cpu during startup (not >95)"%(build,startcpusage)
            timings[script][build] = None

    return (build,date,version,timing,startup,cpu_usage)

MIN_BUILD=153 # where to start analysis
filename_pattern = '*-log-shell_3-stdio'
def update_timings(location="/home/ceball/buildbot/buildmaster/slow-tests_x86_ubuntu7.04/"):

    timings = get_timings()

    if script not in timings:
        timings[script]={}

    startups = get_startups()

    if script not in startups:
        startups[script]={}

    filenames = glob(location+filename_pattern)
    for filename in filenames:

        build = get_build_no(filename)

        if build>=MIN_BUILD:

	    do_timings=do_startups=False

            if build not in timings[script] and build not in exclusions:
                print "Adding timing for build...",build
		do_timings=True
            elif build in exclusions and build not in timings[script]:
                print "Build %s excluded; timing skipped."%build
                timings[script][build]=None

            if build not in startups[script] and build not in exclusions:
                print "Adding startup time for build...",build
            	do_startups=True
            elif build in exclusions and build not in startups[script]:
                print "Build %s excluded; startup timing skipped."%build
                startups[script][build]=None

	    if do_timings and do_startups:
                get_date_version_time(filename,timings,startups)
            if do_timings and not do_startups:
                get_date_version_time(filename,timings,None)
            if not do_timings and do_startups:
                get_date_version_time(filename,None,startups)

    save_timings(timings)
    save_startups(startups)
    #print timings

def plot_timings():
    timings=get_timings()
    t=timings['lissom_oo_or.ty']
    tytle="lissom_oo_or.ty, 250 iterations"
    filename="/home/ceball/buildbot/buildmaster/public_html/p/lissom_oo_or_250"
    plott(t,tytle,filename)

def plot_startups():
    timings=get_startups()
    t=timings['lissom_oo_or.ty']
    tytle="lissom_oo_or.ty, startup"
    filename="/home/ceball/buildbot/buildmaster/public_html/p/lissom_oo_or_startup"
    plott(t,tytle,filename)

import matplotlib;matplotlib.use('Agg')

def sgn(x):
    return +1 if x>=0.0 else -1

def plott(t,tytle,filename):

    from pylab import title,xlabel,ylabel,savefig,figure,annotate,figtext

    builds=[]
    versions=[]
    times=[]
    for build,data in t.items():
        if data is not None:
            builds.append(build)
            versions.append(data[1])
            times.append(data[2]*data[3]/99.0)

    from topo.commands.pylabplots import vectorplot
    figure()
    vectorplot(times,versions,style='bx')

    title(tytle)
    xlabel("svnversion")
    ylabel("time /s")

    for k,v in annotations.items():
        if v[0]==(0,0):
            arrowprops=None
        else:
            arrowprops={'width':0.25,'frac':0.05,'headwidth':5}

        #CEBALERT:necessary cos of bug in matplotlib where arrow can\t be vertical or horizontal?
        xshift=sgn(v[0][0])*max(abs(v[0][0]),0.01)
        yshift=sgn(v[0][1])*max(abs(v[0][1]),0.01)

        annotate(v[1],k,xytext=(k[0]+xshift,k[1]+yshift),arrowprops=arrowprops)

    savefig(filename+"_svnversion.png")

    figure()
    vectorplot(times,builds,style='bx')

    title("lissom_oo_or.ty, 250 iterations")
    xlabel("build")
    ylabel("time /s")

##     key = ""
##     for k,v in annotations_key.items():
##         key+="%s: %s\n"%(k,v)

##     figtext(0,0,key)

    savefig(filename+"_buildno.png")

## if __name__=='__main__':

##     if len(sys.argv)>1:
##         if sys.argv[1]=='update':
##             update_timings()
<commit_msg>Added function to write out results to a web page.<commit_after># CB: this file is difficult to understand because it's been built up
# over a long time. Now that it's clearer what we want it to achieve,
# I could simplify it significantly.

# A complication is that as the buildbot setup has changed, the shell
# command that is checked for the timings has changed number. This
# means the timing data can change directory and filename. Therefore,
# this script cannot be used to get all historical data in one
# go. There's a MIN_BUILD variable that specifies the first build to
# check. That can be changed together with the filename_pattern
# variable to do chunks of the historical data, if desired. Once the
# script has processed data, it's stored in a pickle, so the data
# doesn't need to be retrieved again. As it is, we have a timings
# pickle on doozy that has quite a lot of historical data (i.e. I have
# changed MIN_BUILD and filename_pattern as time has gone on, to keep
# up to date with changes to the buildbot config).

import sys
import re
import pickle
from glob import glob

from sys import argv

#logfile = 'LOG'
script = "lissom_oo_or.ty"

# Replace this mechanism with bars under the x-axis, so that
# ranges can be annotated.
annotations = {
    (8100,66.5):((0,0),"A"),
    (8600,58):((0,4),"B"),
    (8250,53.5):((0,0),"C"),
    (8395,54.3):((20,-2),"D"),
    (8100,56.5):((-20,-2),"E"),
    (8425,55):((0,2),"F")
    }
# CB: the above's just approx for now; needs updating.

annotations_key = {'A':(0.0,'something'),
                   'B':(0.0,'something else'),
                   'C':(0.0,'another thing')}

# CB: in progress - obviously the page construction should
# be separated out, etc
def write_page():
    locn = "/home/ceball/buildbot/buildmaster/public_html/p/"
    imgfile = "lissom_oo_or_250_svnversion.png"

    key = "
"
    for k,v in annotations_key.items():
        key+="
%s	%s	%s
"%(k,v[0],v[1])
    key+="
"

    s = "

%s
"%(locn+imgfile,key)

    f = open(locn+"running.html",'w')
    f.write(s)
    f.close()

V=re.compile(r'[0-9]*-log')
def get_build_no(logfilename):
    return int(V.findall(logfilename)[0].rstrip('-log'))

def create_timings(i_am_sure=False):
    if i_am_sure:
        timings = {}
        f = open('/home/ceball/buildbot/timings.pkl','w')
        pickle.dump(timings,f,0)
        f.close()

def create_startups(i_am_sure=False):
    if i_am_sure:
        startups = {}
        f = open('/home/ceball/buildbot/startups.pkl','w')
        pickle.dump(startups,f,0)
        f.close()

def get_timings():
    f = open('/home/ceball/buildbot/timings.pkl')
    timings = pickle.load(f)
    f.close()
    return timings

def save_timings(timings):
    f = open('/home/ceball/buildbot/timings.pkl','w')
    pickle.dump(timings,f,0)
    f.close()

def get_startups():
    f = open('/home/ceball/buildbot/startups.pkl')
    startups = pickle.load(f)
    f.close()
    return startups

def save_startups(startups):
    f = open('/home/ceball/buildbot/startups.pkl','w')
    pickle.dump(startups,f,0)
    f.close()

# exclude these builds
exclusions = []
exclusions.append(153) # doc why this is excluded
exclusions.append(425) # ran with wrong no. of iterations

def get_date_version_time(logfile,timings=None,startups=None):

    build = get_build_no(logfile)

    f = open(logfile)
    all_lines = f.readlines()
    f.close()

    ok = ok2 = False

    datei=versioni=timingi=startupi=None

    i = 0;
    for line in all_lines:
        if line.find("Running at")>=0:

            datei=i
        elif line.find("svnversion")>=0:

            versioni=i
        elif line.startswith("[examples/%s]"%script):
            timingi=i

        elif line.find("[examples/%s startup]"%script)>=0:
            startupi=i

        elif line.find("Results from examples/%s have not changed."%script)>=0:
            ok2=True
        elif line.find('program finished')>=0:
            ok=True

        i+=1;

    if not ok:
        print "...build %s currently incomplete"%build
        return None

    if not ok2:
        print "...speed test invalid because results didn't match"
        if timings:
            timings[script][build] = None
        if startups:
            startups[script][build] = None
        return None

    if datei is None or versioni is None or timingi is None or startupi is None:
        print "...not all data available - build didn't complete?"
        if timings:
            timings[script][build] = None
        if startups:
            startups[script][build] = None
        return None

    datel= all_lines[datei]
    d=re.compile(r'at [0-9]*')
    date = float(d.findall(datel)[0][3::])

    versionl = all_lines[versioni]
    v=re.compile(r'n [0-9]*')
    version = int(v.findall(versionl)[0][2::])

    timel = all_lines[timingi]
    t=re.compile(r'Now: [0-9.]* ')
    start,stop = t.search(timel).span()
    timing = float(timel[start+5:stop-1])

    cpusel = all_lines[timingi+1]
    start,stop = cpusel.index('elapsed')+8,cpusel.index('%CPU')
    cpu_usage = float(cpusel[start:stop])

    startupl = all_lines[startupi]
    t=re.compile(r'Now: [0-9.]* ')
    start,stop = t.search(startupl).span()
    startup = float(startupl[start+5:stop-1])

    startcpusel = all_lines[startupi+1]

    try:
        start,stop = startcpusel.index('elapsed')+8,startcpusel.index('%CPU')
        startcpusage = float(startcpusel[start:stop])
    except ValueError:
        startcpusage = 99  # HACK: for cases where it's missing, assume 99!

    if timings:

        if cpu_usage>95:
            timings[script][build] = (date,version,timing,cpu_usage)
        else:
            print "...build %s had %s percent cpu during timing (not >95)"%(build,cpu_usage)
            timings[script][build] = None

    if startups:

        if startcpusage>95:
            startups[script][build] = (date,version,startup,startcpusage)
        else:
            print "...build %s had %s percent cpu during startup (not >95)"%(build,startcpusage)
            timings[script][build] = None

    return (build,date,version,timing,startup,cpu_usage)

MIN_BUILD=153 # where to start analysis
filename_pattern = '*-log-shell_3-stdio'
def update_timings(location="/home/ceball/buildbot/buildmaster/slow-tests_x86_ubuntu7.04/"):

    timings = get_timings()

    if script not in timings:
        timings[script]={}

    startups = get_startups()

    if script not in startups:
        startups[script]={}

    filenames = glob(location+filename_pattern)
    for filename in filenames:

        build = get_build_no(filename)

        if build>=MIN_BUILD:

	    do_timings=do_startups=False

            if build not in timings[script] and build not in exclusions:
                print "Adding timing for build...",build
		do_timings=True
            elif build in exclusions and build not in timings[script]:
                print "Build %s excluded; timing skipped."%build
                timings[script][build]=None

            if build not in startups[script] and build not in exclusions:
                print "Adding startup time for build...",build
            	do_startups=True
            elif build in exclusions and build not in startups[script]:
                print "Build %s excluded; startup timing skipped."%build
                startups[script][build]=None

	    if do_timings and do_startups:
                get_date_version_time(filename,timings,startups)
            if do_timings and not do_startups:
                get_date_version_time(filename,timings,None)
            if not do_timings and do_startups:
                get_date_version_time(filename,None,startups)

    save_timings(timings)
    save_startups(startups)
    #print timings

def plot_timings():
    timings=get_timings()
    t=timings['lissom_oo_or.ty']
    tytle="lissom_oo_or.ty, 250 iterations"
    filename="/home/ceball/buildbot/buildmaster/public_html/p/lissom_oo_or_250"
    plott(t,tytle,filename)

def plot_startups():
    timings=get_startups()
    t=timings['lissom_oo_or.ty']
    tytle="lissom_oo_or.ty, startup"
    filename="/home/ceball/buildbot/buildmaster/public_html/p/lissom_oo_or_startup"
    plott(t,tytle,filename)

import matplotlib;matplotlib.use('Agg')

def sgn(x):
    return +1 if x>=0.0 else -1

def plott(t,tytle,filename):

    from pylab import title,xlabel,ylabel,savefig,figure,annotate,figtext

    builds=[]
    versions=[]
    times=[]
    for build,data in t.items():
        if data is not None:
            builds.append(build)
            versions.append(data[1])
            times.append(data[2]*data[3]/99.0)

    from topo.commands.pylabplots import vectorplot
    figure()
    vectorplot(times,versions,style='bx')

    title(tytle)
    xlabel("svnversion")
    ylabel("time /s")

    for k,v in annotations.items():
        if v[0]==(0,0):
            arrowprops=None
        else:
            arrowprops={'width':0.25,'frac':0.05,'headwidth':5}

        #CEBALERT:necessary cos of bug in matplotlib where arrow can\t be vertical or horizontal?
        xshift=sgn(v[0][0])*max(abs(v[0][0]),0.01)
        yshift=sgn(v[0][1])*max(abs(v[0][1]),0.01)

        annotate(v[1],k,xytext=(k[0]+xshift,k[1]+yshift),arrowprops=arrowprops)

    savefig(filename+"_svnversion.png")

    figure()
    vectorplot(times,builds,style='bx')

    title("lissom_oo_or.ty, 250 iterations")
    xlabel("build")
    ylabel("time /s")

##     key = ""
##     for k,v in annotations_key.items():
##         key+="%s: %s\n"%(k,v)

##     figtext(0,0,key)

    savefig(filename+"_buildno.png")

## if __name__=='__main__':

##     if len(sys.argv)>1:
##         if sys.argv[1]=='update':
##             update_timings()
<|endoftext|>


<commit_before>"""
Script that parsing poldnev.ru, checking students factors
using Emotion API and creating 'visits.csv'.

@author: Seva Zhidkov
@license: MIT
"""

import csv
import json
import re
import requests
import lxml.html

POLDNEV_BASE_URL = 'http://poldnev.ru/lksh/id{}'
STUDENT_IMAGE_REGEX = re.compile(r'href=\\"https?://img-fotki.yandex.ru/get/.*_XXL')

print('Starting analysing data from poldnev.ru')
visits_file = open('visits.csv', 'w')
visits_writer = csv.writer(visits_file)
# First student id
student_id = 1
while True:
    student_page = requests.get(POLDNEV_BASE_URL.format(student_id)).text
    student_html = lxml.html.document_fromstring(student_page)
    student_name = student_html.find_class('header-center').pop().text_content()
    if 'Ошибка' in student_name:
        break
    student_photos = []
    for photo in re.findall(STUDENT_IMAGE_REGEX, student_page):
        # Delete a beginning ('href="') of the string
        student_photos.append(photo[7:])
    print(student_id, student_name, student_photos)
    student_id += 1

visits_file.close()
<commit_msg>Add visits parsing<commit_after>"""
Script that parsing poldnev.ru, checking students factors
using Emotion API and creating 'visits.csv'.

@author: Seva Zhidkov
@license: MIT
"""

import csv
import json
import re
import requests
import lxml.html

def detect_emotions(photo_url):
    pass

POLDNEV_BASE_URL = 'http://poldnev.ru/lksh/id{}'
STUDENT_IMAGE_REGEX = re.compile(r'href=\\"https?://img-fotki.yandex.ru/get/.*_XXL')

print('Starting analysing data from poldnev.ru')
visits_file = open('visits.csv', 'w')
visits_writer = csv.writer(visits_file)
# First student id
student_id = 1
while True:
    student_page = requests.get(POLDNEV_BASE_URL.format(student_id)).text
    student_html = lxml.html.document_fromstring(student_page)
    student_name = str(student_html.find_class('header-center').pop().text_content())
    if 'Ошибка' in student_name:
        break
    student_photos = []
    for photo in re.findall(STUDENT_IMAGE_REGEX, student_page):
        # Delete a beginning ('href="') of the string
        student_photos.append(photo[7:])
    # First row - names of columns
    visits_rows = student_html.get_element_by_id('person-table').getchildren()[1:]
    photo_num = 0
    for row in visits_rows:
        # row[1] - year and month, row[2] - position
        visit_date = row[1].text_content()
        visit_position = row[2].text_content()
        # If there is no photo for this visit - continue to next student visit
        if 'class' not in row.attrib:
            continue
        print(student_id, student_name, visit_date,
              visit_position, student_photos[photo_num])
        photo_num += 1
    student_id += 1

visits_file.close()
<|endoftext|>


<commit_before>"""Definitions for the @App decorator and the App classes.

The App class encapsulates a generic leaf task that can be executed asynchronously.
"""
import logging
from inspect import signature

logger = logging.getLogger(__name__)

class AppBase(object):
    """This is the base class that defines the two external facing functions that an App must define.

    The  __init__ () which is called when the interpreter sees the definition of the decorated
    function, and the __call__ () which is invoked when a decorated function is called by the user.

    """

    def __init__(self, func, data_flow_kernel=None, walltime=60, executors='all', cache=False, exec_type="bash"):
        """Construct the App object.

        Args:
             - func (function): Takes the function to be made into an App

        Kwargs:
             - data_flow_kernel (DataFlowKernel): The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for
               managing this app. This can be omitted only
               after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`.
             - walltime (int) : Walltime in seconds for the app execution.
             - executors (str|list) : Labels of the executors that this app can execute over. Default is 'all'.
             - exec_type (string) : App type (bash|python)
             - cache (Bool) : Enable caching of this app ?

        Returns:
             - App object.

        """
        self.__name__ = func.__name__
        self.func = func
        self.data_flow_kernel = data_flow_kernel
        self.exec_type = exec_type
        self.status = 'created'
        self.executors = executors
        if not (isinstance(executors, list) or isinstance(executors, str)):
            logger.error("App {} specifies invalid executor option, expects string or list".format(
                func.__name__))
        self.cache = cache

        params = signature(func).parameters

        self.kwargs = {}
        if 'stdout' in params:
            self.kwargs['stdout'] = params['stdout'].default
        if 'stderr' in params:
            self.kwargs['stderr'] = params['stderr'].default
        self.outputs = params['outputs'].default if 'outputs' in params else []
        self.inputs = params['inputs'].default if 'inputs' in params else []

    def __call__(self, *args, **kwargs):
        """The __call__ function must be implemented in the subclasses."""
        raise NotImplementedError

def app_wrapper(func):

    def wrapper(*args, **kwargs):
        logger.debug("App wrapper begins")
        x = func(*args, **kwargs)
        logger.debug("App wrapper ends")
        return x

    return wrapper

def App(apptype, data_flow_kernel=None, walltime=60, cache=False, executors='all'):
    """The App decorator function.

    Args:
        - apptype (string) : Apptype can be bash|python

    Kwargs:
        - data_flow_kernel (DataFlowKernel): The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for
          managing this app. This can be omitted only
          after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`.
        - walltime (int) : Walltime for app in seconds,
             default=60
        - executors (str|list) : Labels of the executors that this app can execute over. Default is 'all'.
        - cache (Bool) : Enable caching of the app call
             default=False

    Returns:
         An AppFactory object, which when called runs the apps through the executor.
    """
    from parsl import APP_FACTORY_FACTORY

    def wrapper(f):
        return APP_FACTORY_FACTORY.make(apptype, f,
                                        data_flow_kernel=data_flow_kernel,
                                        executors=executors,
                                        cache=cache,
                                        walltime=walltime)

    return wrapper

def python_app(function=None, data_flow_kernel=None, walltime=60, cache=False, executors='all'):
    """Decorator function for making python apps.

    Parameters
    ----------
    function : function
        Do not pass this keyword argument directly. This is needed in order to allow for omitted parenthesis,
        for example, `@python_app` if using all defaults or `@python_app(walltime=120)`. If the
        decorator is used alone, function will be the actual function being decorated, whereas if it
        is called with arguments, function will be None. Default is None.
    data_flow_kernel : DataFlowKernel
        The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for managing this app. This can
        be omitted only after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`. Default is None.
    walltime : int
        Walltime for app in seconds. Default is 60.
    executors : string or list
        Labels of the executors that this app can execute over. Default is 'all'.
    cache : bool
        Enable caching of the app call. Default is False.
    """

    from parsl import APP_FACTORY_FACTORY

    def decorator(func):
        def wrapper(f):
            return APP_FACTORY_FACTORY.make('python', f,
                                            data_flow_kernel=data_flow_kernel,
                                            executors=executors,
                                            cache=cache,
                                            walltime=walltime)

        return wrapper(func)
    if function is not None:
        return decorator(function)
    return decorator

def bash_app(function=None, data_flow_kernel=None, walltime=60, cache=False, executors='all'):
    """Decorator function for making python apps.

    Parameters
    ----------
    function : function
        Do not pass this keyword argument directly. This is needed in order to allow for omitted parenthesis,
        for example, `@bash_app` if using all defaults or `@bash_app(walltime=120)`. If the
        decorator is used alone, function will be the actual function being decorated, whereas if it
        is called with arguments, function will be None. Default is None.
    data_flow_kernel : DataFlowKernel
        The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for managing this app. This can
        be omitted only after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`. Default is None.
    walltime : int
        Walltime for app in seconds. Default is 60.
    executors : string or list
        Labels of the executors that this app can execute over. Default is 'all'.
    cache : bool
        Enable caching of the app call. Default is False.
    """
    from parsl import APP_FACTORY_FACTORY

    def decorator(func):
        def wrapper(f):
            return APP_FACTORY_FACTORY.make('bash', f,
                                            data_flow_kernel=data_flow_kernel,
                                            executors=executors,
                                            cache=cache,
                                            walltime=walltime)

        return wrapper(func)
    if function is not None:
        return decorator(function)
    return decorator
<commit_msg>Correct bash name in docstring<commit_after>"""Definitions for the @App decorator and the App classes.

The App class encapsulates a generic leaf task that can be executed asynchronously.
"""
import logging
from inspect import signature

logger = logging.getLogger(__name__)

class AppBase(object):
    """This is the base class that defines the two external facing functions that an App must define.

    The  __init__ () which is called when the interpreter sees the definition of the decorated
    function, and the __call__ () which is invoked when a decorated function is called by the user.

    """

    def __init__(self, func, data_flow_kernel=None, walltime=60, executors='all', cache=False, exec_type="bash"):
        """Construct the App object.

        Args:
             - func (function): Takes the function to be made into an App

        Kwargs:
             - data_flow_kernel (DataFlowKernel): The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for
               managing this app. This can be omitted only
               after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`.
             - walltime (int) : Walltime in seconds for the app execution.
             - executors (str|list) : Labels of the executors that this app can execute over. Default is 'all'.
             - exec_type (string) : App type (bash|python)
             - cache (Bool) : Enable caching of this app ?

        Returns:
             - App object.

        """
        self.__name__ = func.__name__
        self.func = func
        self.data_flow_kernel = data_flow_kernel
        self.exec_type = exec_type
        self.status = 'created'
        self.executors = executors
        if not (isinstance(executors, list) or isinstance(executors, str)):
            logger.error("App {} specifies invalid executor option, expects string or list".format(
                func.__name__))
        self.cache = cache

        params = signature(func).parameters

        self.kwargs = {}
        if 'stdout' in params:
            self.kwargs['stdout'] = params['stdout'].default
        if 'stderr' in params:
            self.kwargs['stderr'] = params['stderr'].default
        self.outputs = params['outputs'].default if 'outputs' in params else []
        self.inputs = params['inputs'].default if 'inputs' in params else []

    def __call__(self, *args, **kwargs):
        """The __call__ function must be implemented in the subclasses."""
        raise NotImplementedError

def app_wrapper(func):

    def wrapper(*args, **kwargs):
        logger.debug("App wrapper begins")
        x = func(*args, **kwargs)
        logger.debug("App wrapper ends")
        return x

    return wrapper

def App(apptype, data_flow_kernel=None, walltime=60, cache=False, executors='all'):
    """The App decorator function.

    Args:
        - apptype (string) : Apptype can be bash|python

    Kwargs:
        - data_flow_kernel (DataFlowKernel): The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for
          managing this app. This can be omitted only
          after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`.
        - walltime (int) : Walltime for app in seconds,
             default=60
        - executors (str|list) : Labels of the executors that this app can execute over. Default is 'all'.
        - cache (Bool) : Enable caching of the app call
             default=False

    Returns:
         An AppFactory object, which when called runs the apps through the executor.
    """
    from parsl import APP_FACTORY_FACTORY

    def wrapper(f):
        return APP_FACTORY_FACTORY.make(apptype, f,
                                        data_flow_kernel=data_flow_kernel,
                                        executors=executors,
                                        cache=cache,
                                        walltime=walltime)

    return wrapper

def python_app(function=None, data_flow_kernel=None, walltime=60, cache=False, executors='all'):
    """Decorator function for making python apps.

    Parameters
    ----------
    function : function
        Do not pass this keyword argument directly. This is needed in order to allow for omitted parenthesis,
        for example, `@python_app` if using all defaults or `@python_app(walltime=120)`. If the
        decorator is used alone, function will be the actual function being decorated, whereas if it
        is called with arguments, function will be None. Default is None.
    data_flow_kernel : DataFlowKernel
        The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for managing this app. This can
        be omitted only after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`. Default is None.
    walltime : int
        Walltime for app in seconds. Default is 60.
    executors : string or list
        Labels of the executors that this app can execute over. Default is 'all'.
    cache : bool
        Enable caching of the app call. Default is False.
    """

    from parsl import APP_FACTORY_FACTORY

    def decorator(func):
        def wrapper(f):
            return APP_FACTORY_FACTORY.make('python', f,
                                            data_flow_kernel=data_flow_kernel,
                                            executors=executors,
                                            cache=cache,
                                            walltime=walltime)

        return wrapper(func)
    if function is not None:
        return decorator(function)
    return decorator

def bash_app(function=None, data_flow_kernel=None, walltime=60, cache=False, executors='all'):
    """Decorator function for making bash apps.

    Parameters
    ----------
    function : function
        Do not pass this keyword argument directly. This is needed in order to allow for omitted parenthesis,
        for example, `@bash_app` if using all defaults or `@bash_app(walltime=120)`. If the
        decorator is used alone, function will be the actual function being decorated, whereas if it
        is called with arguments, function will be None. Default is None.
    data_flow_kernel : DataFlowKernel
        The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for managing this app. This can
        be omitted only after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`. Default is None.
    walltime : int
        Walltime for app in seconds. Default is 60.
    executors : string or list
        Labels of the executors that this app can execute over. Default is 'all'.
    cache : bool
        Enable caching of the app call. Default is False.
    """
    from parsl import APP_FACTORY_FACTORY

    def decorator(func):
        def wrapper(f):
            return APP_FACTORY_FACTORY.make('bash', f,
                                            data_flow_kernel=data_flow_kernel,
                                            executors=executors,
                                            cache=cache,
                                            walltime=walltime)

        return wrapper(func)
    if function is not None:
        return decorator(function)
    return decorator
<|endoftext|>


<commit_before>

class Compound():
	def __init__(self, compound_string):
		pass<commit_msg>Added compounded parser<commit_after>from chemlib import getMolarMass as ptable_getMolarMass
from re import findall, search

class Compound():
	def __init__(self, compound_string):
		self.elements = {}
		raw_elements = findall("[A-Z][a-z]?[a-z]?[1-9]?[0-9]?", compound_string)
		for element in raw_elements:
			if element[-1].isdigit():
				self.elements[search("[A-Z][a-z]?[a-z]?", element).group(0)] = int(search("[1-9][0-9]?", element).group(0))
			else:
				self.elements[search("[A-Z][a-z]?[a-z]?", element).group(0)] = 1
		print self.elements

	def getMolarMass(self):
		mm = 0.0
		for element in self.elements.keys():
			mm += ptable_getMolarMass(element) * self.elements[element]
		return mm<|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys

def passwd_change(keys_file='keys.txt', passwd_orig='passwd',
                  passwd_new='passwd_new', passwd_log='passwd.log',
                  shadow_orig='shadow', shadow_new='shadow_new',
                  shadow_log='shadow.log'):
    try:
        with open(keys_file, 'r') as k:
            keys = k.readlines()
            keys = [key.strip().split('@')[0] for key in keys]
            keys = [key for key in keys if key != '']

        with open(passwd_orig, 'r') as po:
            passwd_lines = po.readlines()

        passwd_log = open(passwd_log, 'w')
        passwd_new_keys = []

        with open(passwd_new, 'w') as pn:
            for line in passwd_lines:
                if line.split(':')[0] in keys or line.split(':')[3] != '12':
                    pn.write(line)
                    passwd_new_keys.append(line.split(':')[0])
                else:
                    passwd_log.write(line)

        passwd_log.close()

        with open(shadow_orig, 'r') as so:
            shadow_lines = so.readlines()

        shadow_log = open(shadow_log, 'w')

        with open(shadow_new, 'w') as sn:
            for line in shadow_lines:
                if line.split(':')[0] in passwd_new_keys:
                    sn.write(line)
                else:
                    shadow_log.write(line)

        shadow_log.close()

    except Exception as e:
        print(str(e))
        sys.exit()

if __name__ == "__main__":
    passwd_change()
<commit_msg>Add missing users from input keys file log.<commit_after>#!/usr/bin/env python3

import sys

def passwd_change(keys_file='keys.txt', passwd_orig='passwd',
                  passwd_new='passwd_new', passwd_log='passwd.log',
                  shadow_orig='shadow', shadow_new='shadow_new',
                  shadow_log='shadow.log', missing_log='missing.log'):
    try:
        with open(keys_file, 'r') as k:
            keys = k.readlines()
            keys = [key.strip().split('@')[0] for key in keys]
            keys = [key for key in keys if key != '']

        with open(passwd_orig, 'r') as po:
            passwd_lines = po.readlines()

        passwd_log = open(passwd_log, 'w')
        passwd_new_keys = []

        with open(passwd_new, 'w') as pn:
            for line in passwd_lines:
                if line.split(':')[0] in keys or line.split(':')[3] != '12':
                    pn.write(line)
                    passwd_new_keys.append(line.split(':')[0])
                else:
                    passwd_log.write(line)

        passwd_log.close()

        with open(shadow_orig, 'r') as so:
            shadow_lines = so.readlines()

        shadow_log = open(shadow_log, 'w')

        with open(shadow_new, 'w') as sn:
            for line in shadow_lines:
                if line.split(':')[0] in passwd_new_keys:
                    sn.write(line)
                else:
                    shadow_log.write(line)

        shadow_log.close()

        passwd_names = [line.split(':')[0] for line in passwd_lines]

        missing = open(missing_log, 'w')

        for key in keys:
            if key not in passwd_names:
                missing.write(key + '\n')

        missing.close()

    except Exception as e:
        print(str(e))
        sys.exit()

if __name__ == "__main__":
    passwd_change()
<|endoftext|>


<commit_before>"""This file contains the import method for pas-files.

Import a .pas file (basically a json format)

R.A. Collenteur - August 2017

"""

import json
from collections import OrderedDict

import pandas as pd
import pastas as ps

def load(fname):
    data = json.load(open(fname), object_hook=pastas_hook)
    return data

def pastas_hook(obj):
    for key, value in obj.items():
        if key in ["tmin", "tmax", "date_modified", "date_created"]:
            val = pd.Timestamp(value)
            if val is pd.NaT:
                val = None
            obj[key] = val
        elif key == "stress":
            try:
                obj[key] = list()
                for ts in value:
                    obj[key].append(pd.read_json(ts, typ='series',
                                                 orient="split"))
            except:
                obj[key] = value
        elif key in ["series"]:
            try:
                obj[key] = pd.read_json(value, typ='series', orient="split")
            except:
                try:
                     obj[key] = ps.TimeSeries(**value)
                except:
                    obj[key] = value
        elif key in ["time_offset"]:
            obj[key] = pd.Timedelta(value)
        elif key in ["parameters"]:
            # Necessary to maintain order when using the JSON format!
            value = json.loads(value, object_pairs_hook=OrderedDict)
            obj[key] = pd.DataFrame(data=value, columns=value.keys()).T
        else:
            try:
                obj[key] = json.loads(value, object_hook=pastas_hook)
            except:
                obj[key] = value
    return obj

def dump(fname, data):
    json.dump(data, open(fname, 'w'), indent=4, cls=PastasEncoder)
    return print("%s file succesfully exported" % fname)

class PastasEncoder(json.JSONEncoder):
    """Enhanced encoder to deal with the pandas formats used
    throughout PASTAS.

    Notes
    -----
    Currently supported formats are: DataFrame, Series,
    Timedelta, TimeStamps.

    see: https://docs.python.org/3/library/json.html

    """

    def default(self, obj):
        if isinstance(obj, pd.Timestamp):
            return obj.isoformat()
        elif isinstance(obj, pd.Series):
            return obj.to_json(date_format="iso", orient="split")
        elif isinstance(obj, pd.DataFrame):
            # Necessary to maintain order when using the JSON format!
            return obj.to_json(orient="index")
        elif isinstance(obj, pd.Timedelta):
            return obj.to_timedelta64().__str__()
        else:
            return super(PastasEncoder, self).default(obj)
<commit_msg>Deal with NaT in pas-files<commit_after>"""This file contains the import method for pas-files.

Import a .pas file (basically a json format)

R.A. Collenteur - August 2017

"""

import json
from collections import OrderedDict

import pandas as pd
import pastas as ps

def load(fname):
    data = json.load(open(fname), object_hook=pastas_hook)
    return data

def pastas_hook(obj):
    for key, value in obj.items():
        if key in ["tmin", "tmax", "date_modified", "date_created"]:
            val = pd.Timestamp(value)
            if val is pd.NaT:
                val = None
            obj[key] = val
        elif key == "stress":
            try:
                obj[key] = list()
                for ts in value:
                    obj[key].append(pd.read_json(ts, typ='series',
                                                 orient="split"))
            except:
                obj[key] = value
        elif key in ["series"]:
            try:
                obj[key] = pd.read_json(value, typ='series', orient="split")
            except:
                try:
                     obj[key] = ps.TimeSeries(**value)
                except:
                    obj[key] = value
        elif key in ["time_offset"]:
            obj[key] = pd.Timedelta(value)
        elif key in ["parameters"]:
            # Necessary to maintain order when using the JSON format!
            value = json.loads(value, object_pairs_hook=OrderedDict)
            obj[key] = pd.DataFrame(data=value, columns=value.keys()).T
        else:
            try:
                obj[key] = json.loads(value, object_hook=pastas_hook)
            except:
                obj[key] = value
    return obj

def dump(fname, data):
    json.dump(data, open(fname, 'w'), indent=4, cls=PastasEncoder)
    return print("%s file succesfully exported" % fname)

class PastasEncoder(json.JSONEncoder):
    """Enhanced encoder to deal with the pandas formats used
    throughout PASTAS.

    Notes
    -----
    Currently supported formats are: DataFrame, Series,
    Timedelta, TimeStamps.

    see: https://docs.python.org/3/library/json.html

    """

    def default(self, obj):

        if isinstance(obj, pd.Timestamp):
            return obj.isoformat()
        elif isinstance(obj, pd.Series):
            return obj.to_json(date_format="iso", orient="split")
        elif isinstance(obj, pd.DataFrame):
            # Necessary to maintain order when using the JSON format!
            return obj.to_json(orient="index")
        elif isinstance(obj, pd.Timedelta):
            return obj.to_timedelta64().__str__()
        elif isinstance(obj, pd._libs.tslib.NaTType):
            return None
        else:
            return super(PastasEncoder, self).default(obj)
<|endoftext|>