<commit_before>"""
BioMed Central harvester of public projects for the SHARE Notification Service
Note: At the moment, this harvester only harvests basic data on each article, and does
not make a seperate request for additional metadata for each record.

Example API query: http://www.biomedcentral.com/search/results?terms=*&format=json&drpAddedInLast=1&itemsPerPage=250#2015-04-23
"""

from __future__ import unicode_literals

import json
import logging
from datetime import date, timedelta, datetime

from nameparser import HumanName

from scrapi import requests
from scrapi import settings
from scrapi.base import JSONHarvester
from scrapi.linter.document import RawDocument
from scrapi.base.helpers import build_properties, date_formatter

logger = logging.getLogger(__name__)

def process_contributors(authors):

    authors = authors.strip().replace('', '').replace('', '')
    authors = authors.split(',')

    if ' and ' in authors[-1] or ' et al.' in authors[-1]:
        split_name = authors.pop(-1).replace(' et al.', '').split(' and ')
        authors.extend(split_name)

    contributor_list = []
    for person in authors:
        name = HumanName(person)
        contributor = {
            'name': person,
            'givenName': name.first,
            'additionalName': name.middle,
            'familyName': name.last,
            'sameAs': [],
        }
        contributor_list.append(contributor)

    return contributor_list

def join_uris(*args):
    return [arg for arg in args]

class BiomedCentralHarvester(JSONHarvester):
    short_name = 'biomedcentral'
    long_name = 'BioMed Central'
    url = 'http://www.biomedcentral.com/'
    count = 0

    URL = 'http://www.biomedcentral.com/search/results?terms=*&format=json&drpAddedInLast={}&itemsPerPage=250'

    @property
    def schema(self):
        return {
            'contributors': ('/authorNames', process_contributors),
            'uris': {
                'canonicalUri': '/articleFullUrl',
                'objectUris': ('/doi', lambda x: ['http://dx.doi.org/' + x]),
                'providerUris': ('/articleFullUrl', '/abstractPath', join_uris)
            },
            'title': ('/bibliographyTitle', '/blurbTitle', lambda x, y: x or y),
            'providerUpdatedDateTime': ('/published Date', lambda x: date_formatter(x)),
            'description': '/blurbText',
            'freeToRead': {
                'startDate': ('/is_free', '/published Date', lambda x, y: y if x else None)
            },
            'otherProperties': build_properties(
                ('imageURL', '/imageUrl', {'description': 'a image url'}),
                ('type', '/type'),
                ('isOpenAccess', '/isOpenAccess'),
                ('articleUrl', '/articleUrl'),
                ('articleFullUrl', '/articleFullUrl'),
                ('isFree', '/isFree'),
                ('isHighlyAccessed', '/isHighlyAccessed'),
                ('status', '/status'),
                ('abstractPath', '/abstractPath'),
                ('journal Id', '/journal Id'),
                ('article_host', '/article_host'),
                ('longCitation', '/longCitation'),
                ('is_subscription', '/is_subscription')
            )
        }

    def harvest(self, start_date=None, end_date=None):

        start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)

        # Biomed central can only have a start date
        end_date = date.today()
        date_number = end_date - start_date

        search_url = self.URL.format(date_number.days)
        records = self.get_records(search_url)

        record_list = []
        for record in records:
            doc_id = record['arxId']

            record_list.append(
                RawDocument(
                    {
                        'doc': json.dumps(record),
                        'source': self.short_name,
                        'docID': doc_id,
                        'filetype': 'json'
                    }
                )
            )

        return record_list

    def get_records(self, search_url):
        now = datetime.now()
        records = requests.get(search_url + "#{}".format(date.today()))
        page = 1

        all_records = []
        current_records = len(records.json()['entries'])
        while current_records > 0:
            record_list = records.json()['entries']

            for record in record_list:
                # Check to see if this is from the future, don't grab if so
                published_date = record.get('published Date')
                if datetime.strptime(published_date, '%Y-%m-%d') >= now:
                    break
                all_records.append(record)

            page += 1
            records = requests.get(search_url + '&page={}#{}'.format(str(page), date.today()), throttle=10)
            current_records = len(records.json()['entries'])

        return all_records
<commit_msg>CR response - fix uri processing, a lambda, and check all records for date<commit_after>"""
BioMed Central harvester of public projects for the SHARE Notification Service
Note: At the moment, this harvester only harvests basic data on each article, and does
not make a seperate request for additional metadata for each record.

Example API query: http://www.biomedcentral.com/search/results?terms=*&format=json&drpAddedInLast=1&itemsPerPage=250#2015-04-23
"""

from __future__ import unicode_literals

import json
import logging
from datetime import date, timedelta, datetime

from nameparser import HumanName

from scrapi import requests
from scrapi import settings
from scrapi.base import JSONHarvester
from scrapi.linter.document import RawDocument
from scrapi.base.helpers import build_properties, date_formatter

logger = logging.getLogger(__name__)

def process_contributors(authors):

    authors = authors.strip().replace('', '').replace('', '')
    authors = authors.split(',')

    if ' and ' in authors[-1] or ' et al.' in authors[-1]:
        split_name = authors.pop(-1).replace(' et al.', '').split(' and ')
        authors.extend(split_name)

    contributor_list = []
    for person in authors:
        name = HumanName(person)
        contributor = {
            'name': person,
            'givenName': name.first,
            'additionalName': name.middle,
            'familyName': name.last,
            'sameAs': [],
        }
        contributor_list.append(contributor)

    return contributor_list

class BiomedCentralHarvester(JSONHarvester):
    short_name = 'biomedcentral'
    long_name = 'BioMed Central'
    url = 'http://www.biomedcentral.com/'
    count = 0

    URL = 'http://www.biomedcentral.com/search/results?terms=*&format=json&drpAddedInLast={}&itemsPerPage=250'

    @property
    def schema(self):
        return {
            'contributors': ('/authorNames', process_contributors),
            'uris': {
                'canonicalUri': '/articleFullUrl',
                'objectUris': ('/doi', lambda x: ['http://dx.doi.org/' + x]),
                'providerUris': ('/articleFullUrl', '/abstractPath', lambda x, y: [x] + [y])
            },
            'title': ('/bibliographyTitle', '/blurbTitle', lambda x, y: x or y),
            'providerUpdatedDateTime': ('/published Date', date_formatter),
            'description': '/blurbText',
            'freeToRead': {
                'startDate': ('/is_free', '/published Date', lambda x, y: y if x else None)
            },
            'otherProperties': build_properties(
                ('imageURL', '/imageUrl', {'description': 'a image url'}),
                ('type', '/type'),
                ('isOpenAccess', '/isOpenAccess'),
                ('articleUrl', '/articleUrl'),
                ('articleFullUrl', '/articleFullUrl'),
                ('isFree', '/isFree'),
                ('isHighlyAccessed', '/isHighlyAccessed'),
                ('status', '/status'),
                ('abstractPath', '/abstractPath'),
                ('journal Id', '/journal Id'),
                ('article_host', '/article_host'),
                ('longCitation', '/longCitation'),
                ('is_subscription', '/is_subscription')
            )
        }

    def harvest(self, start_date=None, end_date=None):

        start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)

        # Biomed central can only have a start date
        end_date = date.today()
        date_number = end_date - start_date

        search_url = self.URL.format(date_number.days)
        records = self.get_records(search_url)

        record_list = []
        for record in records:
            doc_id = record['arxId']

            record_list.append(
                RawDocument(
                    {
                        'doc': json.dumps(record),
                        'source': self.short_name,
                        'docID': doc_id,
                        'filetype': 'json'
                    }
                )
            )

        return record_list

    def get_records(self, search_url):
        now = datetime.now()
        records = requests.get(search_url + "#{}".format(date.today()))
        page = 1

        all_records = []
        current_records = len(records.json()['entries'])
        while current_records > 0:
            record_list = records.json()['entries']

            for record in record_list:
                # Check to see if this is from the future, don't grab if so
                published_date = record.get('published Date')
                if datetime.strptime(published_date, '%Y-%m-%d') >= now:
                    continue
                all_records.append(record)

            page += 1
            records = requests.get(search_url + '&page={}#{}'.format(str(page), date.today()), throttle=10)
            current_records = len(records.json()['entries'])

        return all_records
<|endoftext|>


<commit_before>import os
import subprocess
import collections

import yaml
import jinja2
from jinja2.runtime import StrictUndefined

def run_cmd(args, env=None, cwd=None, stdout=None):
    cmd_env = os.environ.copy()
    cmd_env.update(env or {})
    cmd = subprocess.Popen(args, env=cmd_env, cwd=cwd, stdout=stdout, stderr=subprocess.STDOUT)
    return cmd.communicate()[0]

def read_yaml_file(path):
    with open(path) as f:
        return yaml.load(f) or {}

def load_file(path):
    with open(path) as f:
        return f.read()

def dict_from_path(path, value):
    result = {}
    if isinstance(path, basestring):
        path = path.split('.')

    if not path:
        return value

    result[path[0]] = dict_from_path(path[1:], value)

    return result

def merge_dicts(a, b):
    if not (isinstance(a, dict) and isinstance(b, dict)):
        raise ValueError("Error merging variables: '{}' and '{}'".format(
            type(a).__name__, type(b).__name__
        ))

    result = a.copy()
    for key, val in b.iteritems():
        if isinstance(result.get(key), collections.Mapping):
            result[key] = merge_dicts(a[key], b[key])
        else:
            result[key] = val

    return result

def template(item, variables, **kwargs):
    variables = merge_dicts(variables, kwargs)

    if isinstance(item, basestring):
        varname = template_string(item, variables)
        return varname

    elif isinstance(item, collections.Sequence):
        return [template(i, variables) for i in item]

    elif isinstance(item, collections.Mapping):
        result = {}
        for (key, val) in item.iteritems():
            result[key] = template(val, variables)
        return result

    else:
        return item

class LazyTemplateMapping(object):
    def __init__(self, mapping, variables, **kwargs):
        self._mapping = mapping
        self._cache = {}
        self._variables = merge_dicts(variables, kwargs)

    def keys(self):
        return self._mapping.keys()

    def items(self):
        for key in self.keys():
            self.__getitem__(key)
        return self._cache.items()

    def __getitem__(self, key):
        if key not in self._cache:
            self._cache[key] = template(self._mapping[key], self._variables)

        return self._cache[key]

def template_string(string, variables):
    jinja_env = jinja2.Environment(
        trim_blocks=True,
        undefined=StrictUndefined,
    )

    try:
        template = jinja_env.from_string(string)
    except jinja2.exceptions.TemplateSyntaxError, e:
        raise ValueError(u"Template error: {}".format(e))

    try:
        return template.render(variables)
    except jinja2.exceptions.UndefinedError, e:
        raise ValueError(u"Variable {} in '{}'".format(e, string))
<commit_msg>Implement items on LazyTemplateMapping<commit_after>import os
import subprocess
import collections

import yaml
import jinja2
from jinja2.runtime import StrictUndefined

def run_cmd(args, env=None, cwd=None, stdout=None):
    cmd_env = os.environ.copy()
    cmd_env.update(env or {})
    cmd = subprocess.Popen(args, env=cmd_env, cwd=cwd, stdout=stdout, stderr=subprocess.STDOUT)
    return cmd.communicate()[0]

def read_yaml_file(path):
    with open(path) as f:
        return yaml.load(f) or {}

def load_file(path):
    with open(path) as f:
        return f.read()

def dict_from_path(path, value):
    result = {}
    if isinstance(path, basestring):
        path = path.split('.')

    if not path:
        return value

    result[path[0]] = dict_from_path(path[1:], value)

    return result

def merge_dicts(a, b):
    if not (isinstance(a, dict) and isinstance(b, dict)):
        raise ValueError("Error merging variables: '{}' and '{}'".format(
            type(a).__name__, type(b).__name__
        ))

    result = a.copy()
    for key, val in b.iteritems():
        if isinstance(result.get(key), collections.Mapping):
            result[key] = merge_dicts(a[key], b[key])
        else:
            result[key] = val

    return result

def template(item, variables, **kwargs):
    variables = merge_dicts(variables, kwargs)

    if isinstance(item, basestring):
        varname = template_string(item, variables)
        return varname

    elif isinstance(item, collections.Sequence):
        return [template(i, variables) for i in item]

    elif isinstance(item, collections.Mapping):
        result = {}
        for (key, val) in item.iteritems():
            result[key] = template(val, variables)
        return result

    else:
        return item

class LazyTemplateMapping(object):
    def __init__(self, mapping, variables, **kwargs):
        self._mapping = mapping
        self._cache = {}
        self._variables = merge_dicts(variables, kwargs)

    def keys(self):
        return self._mapping.keys()

    def items(self):
        return [(key, self[key]) for key in self.keys()]

    def __getitem__(self, key):
        if key not in self._cache:
            self._cache[key] = template(self._mapping[key], self._variables)

        return self._cache[key]

def template_string(string, variables):
    jinja_env = jinja2.Environment(
        trim_blocks=True,
        undefined=StrictUndefined,
    )

    try:
        template = jinja_env.from_string(string)
    except jinja2.exceptions.TemplateSyntaxError, e:
        raise ValueError(u"Template error: {}".format(e))

    try:
        return template.render(variables)
    except jinja2.exceptions.UndefinedError, e:
        raise ValueError(u"Variable {} in '{}'".format(e, string))
<|endoftext|>


<commit_before>import json
import logging
import uuid
import urllib.parse

from django.db import models
from django.utils import timezone
from django.utils.text import slugify
from django.conf import settings
from channels import Group
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.contrib.auth.models import User
from django.core.mail import send_mail

import radio.choices as choice

from pinax.stripe.models import Plan as pinax_Plan

log = logging.getLogger(__name__)

class Agency(models.Model):
    name = models.CharField(max_length=100)
    short = models.CharField(max_length=5, unique=True)

    def __str__(self):
        return self.name

    def get_short(self):
        return self.short

class System(models.Model):
    name = models.CharField(max_length=100)
    system_id = models.CharField(max_length=20, blank=True, null=True)

    def __str__(self):
        return self.name

class Source(models.Model):
    description = models.CharField(max_length=100)

    class Meta:
        ordering = ["pk"]

    def __str__(self):
        return self.description

class Unit(models.Model):
    dec_id = models.IntegerField()
    description = models.CharField(max_length=100, blank=True, null=True)
    agency = models.ForeignKey(Agency, default=settings.RADIO_DEFAULT_UNIT_AGENCY)
    #agency = models.ForeignKey(Agency, default=2)
    type = models.CharField(max_length=1, choices=choice.RADIO_TYPE_CHOICES, default=choice.RADIO_TYPE_MOBILE)
    number = models.IntegerField(default=1)
    system = models.ForeignKey(System, default=0)
    slug = models.SlugField(null=True, blank=True)

    class Meta:
        unique_together = ('dec_id', 'system',)

    def __str__(self):
        if(self.description):
            return self.description
        else:
            return str(self.dec_id)

    def save(self, *args, **kwargs):
        if self.description:
            self.slug = slugify(self.description)
        super(Unit, self).save(*args, **kwargs)

class RepeaterSite(models.Model):
    name = models.CharField(max_length=100, unique=True)
    description = models.CharField(max_length=500, blank=True, null=True)

    def __str__(self):
        return self.name

class Service(models.Model):
    name = models.CharField(max_length=100, unique=True)
    description = models.CharField(max_length=500, blank=True, null=True)

    def __str__(self):
        return self.name

class TalkGroup(models.Model):
    dec_id = models.IntegerField()
    alpha_tag = models.CharField(max_length=30)
    common_name = models.CharField(max_length=10, blank=True, null=True)
    description = models.CharField(max_length=100, blank=True, null=True)
    slug = models.SlugField(null=True)
    public = models.BooleanField(default=True)
    comments = models.CharField(max_length=100, blank=True, null=True)
    system = models.ForeignKey(System, default=0)
    mode = models.CharField(max_length=1, choices=choice.TG_MODE_CHOICES, default=choice.TG_MODE_DIGITAL, help_text='mode used by trunk-recorder')
    priority = models.IntegerField(default=3, help_text='record priority used by trunk-recorder')
    _home_site = models.ForeignKey(RepeaterSite, blank=True, null=True)
    _service_type = models.ForeignKey(Service, blank=True, null=True)

    class Meta:
        ordering = ["alpha_tag"]
        unique_together = ('dec_id', 'system',)

    def __str__(self):
        return self.alpha_tag

    def save(self, *args, **kwargs):
        self.slug = slugify(self.alpha_tag)
        super(TalkGroup, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return '/tg/{}/'.format(self.slug)

    @property
    def home_site(self):
        return self._home_site

    @home_site.setter
    def home_site(self, value):
        hs, created = RepeaterSite.objects.get_or_create(name=value)
        self._home_site = hs

    @property
    def service_type(self):
        return self._service_type

    @service_type.setter
    def service_type(self, value):
        st, created = Service.objects.get_or_create(name=value)
        self._service_type = st

def AddToDefaultAccessGroup(sender, instance, created, **kwargs):
    if created:
        new_tg = TalkGroupWithSystem.objects.get(pk=instance.pk)
        for access_group in TalkGroupAccess.objects.filter(default_new_talkgroups=True):
            access_group.talkgroups.add(new_tg)
            access_group.save()

post_save.connect(AddToDefaultAccessGroup, sender=TalkGroup)

class TalkGroupWithSystem(TalkGroup):
    class Meta:
        proxy = True

    def __str__(self):
        return "{} ({})".format(self.alpha_tag, self.system)

class Transmission(models.Model):
    slug = models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)
    start_datetime = models.DateTimeField(db_index=True)
    audio_file = models.FileField()
    audio_file_url_path = models.CharField(max_length=100, default='/')
    talkgroup = models.IntegerField()
    talkgroup_info = models.ForeignKey(TalkGroup)
    freq = models.IntegerField()
    emergency = models.BooleanField(default=False)
    units = models.ManyToManyField(Unit, through='TranmissionUnit')
    play_length = models.FloatField(default=0.0)
    source = models.ForeignKey(Source, default=0)
    system = models.ForeignKey(System, default=0)

    def __str__(self):
        return '{} {}'.format(self.talkgroup, self.start_datetime)

    def get_absolute_url(self):
        return '/audio/{}/'.format(self.slug)

    @property
    def local_start_datetime(self):
        return timezone.localtime(self.start_datetime).strftime('%H:%M:%S %m/%d/%Y')

    def as_dict(self):
        return {'start_datetime': str(self.start_datetime), 'audio_file': str(self.audio_file), 'talkgroup_desc': self.talkgroup_info.alpha_tag}

    def print_play_length(self):
        m, s = divmod(int(self.play_length), 60)
        return '{:02d}:{:02d}'.format(m,s)

    def freq_mhz(self):
        return '{0:07.3f}'.format(self.freq / 1000000)

    def tg_name(self):
        """Returns TalkGroup name used for page title
           First tries TalkGroup.common_name
           then TalkGroup.alpha_tag
        """
        if self.talkgroup_info.common_name:
            return self.talkgroup_info.common_name
        else:
            return self.talkgroup_info.alpha_tag

    @property
    def audio_url(self):
        base_path = settings.AUDIO_URL_BASE
        return urllib.parse.urljoin(base_path, self.audio_file_url_path.lstrip('/'))

    class Meta:
        ordering = ["-pk"]

@receiver(post_save, sender=Transmission, dispatch_uid="send_mesg")
def send_mesg(sender, instance, **kwargs):
    #log.debug('Hit post save()')
    #log.debug('DATA %s', json.dumps(instance.as_dict()))
    #log.debug('DATA %s', json.dumps(instance.as_dict()))
    tg = TalkGroup.objects.get(pk=instance.talkgroup_info.pk)
    groups = tg.scanlist_set.all()
    for g in groups:
        Group('livecall-scan-'+g.slug, ).send({'text': json.dumps(instance.as_dict())})
    Group('livecall-tg-' + tg.slug, ).send({'text': json.dumps(instance.as_dict())})

    #def save(self, *args, **kwargs):
    #    try:
    #        self.talkgroup_info = TalkGroup.objects.get(dec_id=self.talkgroup)
    #    except TalkGroup.DoesNotExist:
    #        new_tg = TalkGroup(dec_id=self.talkgroup, alpha_tag='UNK')
    #        new_tg.save
    #        self.talkgroup_info = TalkGroup.objects.get(dec_id=self.talkgroup)
    #    super(Transmission, self).save(*args, **kwargs)

class TranmissionUnit(models.Model):
    transmission = models.ForeignKey(Transmission, on_delete=models.CASCADE)
    unit = models.ForeignKey(Unit, on_delete=models.CASCADE)
    order = models.IntegerField()

    class Meta:
        ordering = ["order"]

    def __str__(self):
        return '{} on {}'.format(self.unit,self.transmission)

class ScanList(models.Model):
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL)
    public = models.BooleanField(default=False)
    name = models.CharField(max_length=30, unique=True)
    description = models.CharField(max_length=100)
    talkgroups = models.ManyToManyField(TalkGroup)
    slug = models.SlugField(null=True, blank=True)

    class Meta:
        ordering = ["name"]

    def __str__(self):
        return '{}'.format(self.name)

    def save(self, *args, **kwargs):
        self.slug = slugify(self.name)
        super(ScanList, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return '/scan/{}/'.format(self.slug)

class MenuList(models.Model):
    order = models.IntegerField(default=1)

    class Meta:
        abstract = True
        ordering = ["order", "name"]

    def __str__(self):
        return '{}'.format(self.name)

    @property
    def scan_name(self):
        return self.name

    @property
    def scan_description(self):
        return self.name.description

    @property
    def scan_slug(self):
        return self.name.slug

class MenuScanList(MenuList):
    name = models.ForeignKey(ScanList)

    @property
    def scan_name(self):
        return self.name.name

    @property
    def scan_description(self):
        return self.name.description

class MenuTalkGroupList(MenuList):
    name = models.ForeignKey(TalkGroupWithSystem)

    @property
    def tg_name(self):
        return self.name.alpha_tag

    @property
    def tg_slug(self):
        return self.name.slug

class TalkGroupAccess(models.Model):
    name = models.CharField(max_length=30, unique=True)
    talkgroups = models.ManyToManyField(TalkGroupWithSystem)
    default_group = models.BooleanField(default=False, help_text='new users get this group by default')
    default_new_talkgroups = models.BooleanField(default=False, help_text='new talkgroups are added to this group')

    class Meta:
        verbose_name_plural = 'talk group access'

    def __str__(self):
        return '{}'.format(self.name)

class Plan(models.Model):
    DEFAULT_PK = 1 # This is added via a migration
    name = models.CharField(max_length=30, unique=True)
    history = models.IntegerField(default=15, help_text='visible history in minutes')

    def __str__(self):
        return '{}'.format(self.name)

class Profile(models.Model):
    user = models.OneToOneField(User, on_delete=models.CASCADE)
    plan = models.ForeignKey(Plan, default=Plan.DEFAULT_PK)
    talkgroup_access = models.ManyToManyField(TalkGroupAccess, blank=True)

class WebHtml(models.Model):
    name = models.CharField(max_length=30, unique=True)
    bodytext = models.TextField()

    def __str__(self):
        return self.name

class StripePlanMatrix(models.Model):
    name = models.CharField(max_length=30, unique=True)
    stripe_plan = models.ForeignKey(pinax_Plan)
    radio_plan = models.ForeignKey(Plan)
    active = models.BooleanField(default=True)
    order = models.IntegerField(default=99)

    class Meta:
        ordering = ["order"]

    def __str__(self):
       return self.name

    def stripe_amount(self):
       return int(self.stripe_plan.amount * 100)

    def history_days(self):
       return int(self.radio_plan.history / 1440)

class SiteOption(models.Model):
    name = models.CharField(max_length=30, unique=True)
    value = models.CharField(max_length=60)
    javascript_visible = models.BooleanField(default=False)
    template_visible = models.BooleanField(default=False)
    description = models.CharField(max_length=100, null=True, blank=True)

    def __str__(self):
        return self.name

    def value_boolean_or_string(self):
        if self.value.lower() == 'true':
            return True
        elif self.value.lower() == 'false':
            return False
        else:
            return self.value

def create_profile(sender, **kwargs):
    user = kwargs["instance"]
    if kwargs["created"]:
        default_plan = Plan.objects.get(pk=Plan.DEFAULT_PK)
        up = Profile(user=user, plan=default_plan)
        up.save()
        for tg in TalkGroupAccess.objects.filter(default_group=True):
            up.talkgroup_access.add(tg)
        try:
            new_user_email = SiteOption.objects.get(name='SEND_ADMIN_EMAIL_ON_NEW_USER')
            if new_user_email.value_boolean_or_string() == True:
                send_mail(
                      'New ScanOC User {}'.format(user.username),
                      'New User {} {} Username {} Email {} just registered'.format(user.first_name, user.last_name, user.username, user.email),
                      settings.SERVER_EMAIL,
                      [ mail for name, mail in settings.ADMINS],
                      fail_silently=False,
                     )
        except SiteOption.DoesNotExist:
            pass

post_save.connect(create_profile, sender=User)
<commit_msg>Remove hardcoded site name<commit_after>import json
import logging
import uuid
import urllib.parse

from django.db import models
from django.utils import timezone
from django.utils.text import slugify
from django.conf import settings
from channels import Group
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.contrib.auth.models import User
from django.core.mail import send_mail

import radio.choices as choice

from pinax.stripe.models import Plan as pinax_Plan

log = logging.getLogger(__name__)

class Agency(models.Model):
    name = models.CharField(max_length=100)
    short = models.CharField(max_length=5, unique=True)

    def __str__(self):
        return self.name

    def get_short(self):
        return self.short

class System(models.Model):
    name = models.CharField(max_length=100)
    system_id = models.CharField(max_length=20, blank=True, null=True)

    def __str__(self):
        return self.name

class Source(models.Model):
    description = models.CharField(max_length=100)

    class Meta:
        ordering = ["pk"]

    def __str__(self):
        return self.description

class Unit(models.Model):
    dec_id = models.IntegerField()
    description = models.CharField(max_length=100, blank=True, null=True)
    agency = models.ForeignKey(Agency, default=settings.RADIO_DEFAULT_UNIT_AGENCY)
    #agency = models.ForeignKey(Agency, default=2)
    type = models.CharField(max_length=1, choices=choice.RADIO_TYPE_CHOICES, default=choice.RADIO_TYPE_MOBILE)
    number = models.IntegerField(default=1)
    system = models.ForeignKey(System, default=0)
    slug = models.SlugField(null=True, blank=True)

    class Meta:
        unique_together = ('dec_id', 'system',)

    def __str__(self):
        if(self.description):
            return self.description
        else:
            return str(self.dec_id)

    def save(self, *args, **kwargs):
        if self.description:
            self.slug = slugify(self.description)
        super(Unit, self).save(*args, **kwargs)

class RepeaterSite(models.Model):
    name = models.CharField(max_length=100, unique=True)
    description = models.CharField(max_length=500, blank=True, null=True)

    def __str__(self):
        return self.name

class Service(models.Model):
    name = models.CharField(max_length=100, unique=True)
    description = models.CharField(max_length=500, blank=True, null=True)

    def __str__(self):
        return self.name

class TalkGroup(models.Model):
    dec_id = models.IntegerField()
    alpha_tag = models.CharField(max_length=30)
    common_name = models.CharField(max_length=10, blank=True, null=True)
    description = models.CharField(max_length=100, blank=True, null=True)
    slug = models.SlugField(null=True)
    public = models.BooleanField(default=True)
    comments = models.CharField(max_length=100, blank=True, null=True)
    system = models.ForeignKey(System, default=0)
    mode = models.CharField(max_length=1, choices=choice.TG_MODE_CHOICES, default=choice.TG_MODE_DIGITAL, help_text='mode used by trunk-recorder')
    priority = models.IntegerField(default=3, help_text='record priority used by trunk-recorder')
    _home_site = models.ForeignKey(RepeaterSite, blank=True, null=True)
    _service_type = models.ForeignKey(Service, blank=True, null=True)

    class Meta:
        ordering = ["alpha_tag"]
        unique_together = ('dec_id', 'system',)

    def __str__(self):
        return self.alpha_tag

    def save(self, *args, **kwargs):
        self.slug = slugify(self.alpha_tag)
        super(TalkGroup, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return '/tg/{}/'.format(self.slug)

    @property
    def home_site(self):
        return self._home_site

    @home_site.setter
    def home_site(self, value):
        hs, created = RepeaterSite.objects.get_or_create(name=value)
        self._home_site = hs

    @property
    def service_type(self):
        return self._service_type

    @service_type.setter
    def service_type(self, value):
        st, created = Service.objects.get_or_create(name=value)
        self._service_type = st

def AddToDefaultAccessGroup(sender, instance, created, **kwargs):
    if created:
        new_tg = TalkGroupWithSystem.objects.get(pk=instance.pk)
        for access_group in TalkGroupAccess.objects.filter(default_new_talkgroups=True):
            access_group.talkgroups.add(new_tg)
            access_group.save()

post_save.connect(AddToDefaultAccessGroup, sender=TalkGroup)

class TalkGroupWithSystem(TalkGroup):
    class Meta:
        proxy = True

    def __str__(self):
        return "{} ({})".format(self.alpha_tag, self.system)

class Transmission(models.Model):
    slug = models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)
    start_datetime = models.DateTimeField(db_index=True)
    audio_file = models.FileField()
    audio_file_url_path = models.CharField(max_length=100, default='/')
    talkgroup = models.IntegerField()
    talkgroup_info = models.ForeignKey(TalkGroup)
    freq = models.IntegerField()
    emergency = models.BooleanField(default=False)
    units = models.ManyToManyField(Unit, through='TranmissionUnit')
    play_length = models.FloatField(default=0.0)
    source = models.ForeignKey(Source, default=0)
    system = models.ForeignKey(System, default=0)

    def __str__(self):
        return '{} {}'.format(self.talkgroup, self.start_datetime)

    def get_absolute_url(self):
        return '/audio/{}/'.format(self.slug)

    @property
    def local_start_datetime(self):
        return timezone.localtime(self.start_datetime).strftime('%H:%M:%S %m/%d/%Y')

    def as_dict(self):
        return {'start_datetime': str(self.start_datetime), 'audio_file': str(self.audio_file), 'talkgroup_desc': self.talkgroup_info.alpha_tag}

    def print_play_length(self):
        m, s = divmod(int(self.play_length), 60)
        return '{:02d}:{:02d}'.format(m,s)

    def freq_mhz(self):
        return '{0:07.3f}'.format(self.freq / 1000000)

    def tg_name(self):
        """Returns TalkGroup name used for page title
           First tries TalkGroup.common_name
           then TalkGroup.alpha_tag
        """
        if self.talkgroup_info.common_name:
            return self.talkgroup_info.common_name
        else:
            return self.talkgroup_info.alpha_tag

    @property
    def audio_url(self):
        base_path = settings.AUDIO_URL_BASE
        return urllib.parse.urljoin(base_path, self.audio_file_url_path.lstrip('/'))

    class Meta:
        ordering = ["-pk"]

@receiver(post_save, sender=Transmission, dispatch_uid="send_mesg")
def send_mesg(sender, instance, **kwargs):
    #log.debug('Hit post save()')
    #log.debug('DATA %s', json.dumps(instance.as_dict()))
    #log.debug('DATA %s', json.dumps(instance.as_dict()))
    tg = TalkGroup.objects.get(pk=instance.talkgroup_info.pk)
    groups = tg.scanlist_set.all()
    for g in groups:
        Group('livecall-scan-'+g.slug, ).send({'text': json.dumps(instance.as_dict())})
    Group('livecall-tg-' + tg.slug, ).send({'text': json.dumps(instance.as_dict())})

    #def save(self, *args, **kwargs):
    #    try:
    #        self.talkgroup_info = TalkGroup.objects.get(dec_id=self.talkgroup)
    #    except TalkGroup.DoesNotExist:
    #        new_tg = TalkGroup(dec_id=self.talkgroup, alpha_tag='UNK')
    #        new_tg.save
    #        self.talkgroup_info = TalkGroup.objects.get(dec_id=self.talkgroup)
    #    super(Transmission, self).save(*args, **kwargs)

class TranmissionUnit(models.Model):
    transmission = models.ForeignKey(Transmission, on_delete=models.CASCADE)
    unit = models.ForeignKey(Unit, on_delete=models.CASCADE)
    order = models.IntegerField()

    class Meta:
        ordering = ["order"]

    def __str__(self):
        return '{} on {}'.format(self.unit,self.transmission)

class ScanList(models.Model):
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL)
    public = models.BooleanField(default=False)
    name = models.CharField(max_length=30, unique=True)
    description = models.CharField(max_length=100)
    talkgroups = models.ManyToManyField(TalkGroup)
    slug = models.SlugField(null=True, blank=True)

    class Meta:
        ordering = ["name"]

    def __str__(self):
        return '{}'.format(self.name)

    def save(self, *args, **kwargs):
        self.slug = slugify(self.name)
        super(ScanList, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return '/scan/{}/'.format(self.slug)

class MenuList(models.Model):
    order = models.IntegerField(default=1)

    class Meta:
        abstract = True
        ordering = ["order", "name"]

    def __str__(self):
        return '{}'.format(self.name)

    @property
    def scan_name(self):
        return self.name

    @property
    def scan_description(self):
        return self.name.description

    @property
    def scan_slug(self):
        return self.name.slug

class MenuScanList(MenuList):
    name = models.ForeignKey(ScanList)

    @property
    def scan_name(self):
        return self.name.name

    @property
    def scan_description(self):
        return self.name.description

class MenuTalkGroupList(MenuList):
    name = models.ForeignKey(TalkGroupWithSystem)

    @property
    def tg_name(self):
        return self.name.alpha_tag

    @property
    def tg_slug(self):
        return self.name.slug

class TalkGroupAccess(models.Model):
    name = models.CharField(max_length=30, unique=True)
    talkgroups = models.ManyToManyField(TalkGroupWithSystem)
    default_group = models.BooleanField(default=False, help_text='new users get this group by default')
    default_new_talkgroups = models.BooleanField(default=False, help_text='new talkgroups are added to this group')

    class Meta:
        verbose_name_plural = 'talk group access'

    def __str__(self):
        return '{}'.format(self.name)

class Plan(models.Model):
    DEFAULT_PK = 1 # This is added via a migration
    name = models.CharField(max_length=30, unique=True)
    history = models.IntegerField(default=15, help_text='visible history in minutes')

    def __str__(self):
        return '{}'.format(self.name)

class Profile(models.Model):
    user = models.OneToOneField(User, on_delete=models.CASCADE)
    plan = models.ForeignKey(Plan, default=Plan.DEFAULT_PK)
    talkgroup_access = models.ManyToManyField(TalkGroupAccess, blank=True)

class WebHtml(models.Model):
    name = models.CharField(max_length=30, unique=True)
    bodytext = models.TextField()

    def __str__(self):
        return self.name

class StripePlanMatrix(models.Model):
    name = models.CharField(max_length=30, unique=True)
    stripe_plan = models.ForeignKey(pinax_Plan)
    radio_plan = models.ForeignKey(Plan)
    active = models.BooleanField(default=True)
    order = models.IntegerField(default=99)

    class Meta:
        ordering = ["order"]

    def __str__(self):
       return self.name

    def stripe_amount(self):
       return int(self.stripe_plan.amount * 100)

    def history_days(self):
       return int(self.radio_plan.history / 1440)

class SiteOption(models.Model):
    name = models.CharField(max_length=30, unique=True)
    value = models.CharField(max_length=60)
    javascript_visible = models.BooleanField(default=False)
    template_visible = models.BooleanField(default=False)
    description = models.CharField(max_length=100, null=True, blank=True)

    def __str__(self):
        return self.name

    def value_boolean_or_string(self):
        if self.value.lower() == 'true':
            return True
        elif self.value.lower() == 'false':
            return False
        else:
            return self.value

def create_profile(sender, **kwargs):
    user = kwargs["instance"]
    if kwargs["created"]:
        default_plan = Plan.objects.get(pk=Plan.DEFAULT_PK)
        up = Profile(user=user, plan=default_plan)
        up.save()
        for tg in TalkGroupAccess.objects.filter(default_group=True):
            up.talkgroup_access.add(tg)
        try:
            new_user_email = SiteOption.objects.get(name='SEND_ADMIN_EMAIL_ON_NEW_USER')
            if new_user_email.value_boolean_or_string() == True:
                send_mail(
                      'New {} User {}'.format(settings.SITE_TITLE, user.username),
                      'New User {} {} Username {} Email {} just registered'.format(user.first_name, user.last_name, user.username, user.email),
                      settings.SERVER_EMAIL,
                      [ mail for name, mail in settings.ADMINS],
                      fail_silently=False,
                     )
        except SiteOption.DoesNotExist:
            pass

post_save.connect(create_profile, sender=User)
<|endoftext|>


<commit_before>##########################################################################
#
#  Copyright (c) 2011-2012, John Haddon. All rights reserved.
#  Copyright (c) 2012-2013, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import os
import functools
import imath

import IECore

import Gaffer
import GafferUI

## Appends items to the IECore.MenuDefinition object passed to build a File menu containing
# standard open/save/revert/etc
def appendDefinitions( menuDefinition, prefix="" ) :

	menuDefinition.append( prefix + "/New", { "command" : new, "shortCut" : "Ctrl+N" } )
	menuDefinition.append( prefix + "/Open...", { "command" : open, "shortCut" : "Ctrl+O" } )
	menuDefinition.append( prefix + "/Open Recent", { "subMenu" : openRecent } )
	menuDefinition.append( prefix + "/OpenDivider", { "divider" : True } )
	menuDefinition.append( prefix + "/Save", { "command" : save, "shortCut" : "Ctrl+S" } )
	menuDefinition.append( prefix + "/Save As...", { "command" : saveAs, "shortCut" : "Shift+Ctrl+S" } )
	menuDefinition.append( prefix + "/Revert To Saved", { "command" : revertToSaved, "active" : __revertToSavedAvailable } )
	menuDefinition.append( prefix + "/SaveDivider", { "divider" : True } )
	menuDefinition.append( prefix + "/Export Selection...", { "command" : exportSelection, "active" : __selectionAvailable } )
	menuDefinition.append( prefix + "/Import...", { "command" : importFile } )
	menuDefinition.append( prefix + "/ImportExportDivider", { "divider" : True } )
	menuDefinition.append( prefix + "/Settings...", { "command" : showSettings } )

## A function suitable as the command for a File/New menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def new( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	application = scriptWindow.scriptNode().ancestor( Gaffer.ApplicationRoot )

	newScript = Gaffer.ScriptNode()
	Gaffer.NodeAlgo.applyUserDefaults( newScript )
	application["scripts"].addChild( newScript )

## A function suitable as the command for a File/Open menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def open( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	path, bookmarks = __pathAndBookmarks( scriptWindow )

	dialogue = GafferUI.PathChooserDialogue( path, title="Open script", confirmLabel="Open", valid=True, leaf=True, bookmarks=bookmarks )
	path = dialogue.waitForPath( parentWindow = scriptWindow )

	if not path :
		return

	__open( scriptWindow.scriptNode(), str( path ) )

## Opens a script and adds it to the application, as if the user
# had done so via the File Open dialogue.
def addScript( application, fileName ) :

	return __addScript( application, fileName )

def __addScript( application, fileName, dialogueParentWindow = None ) :

	recoveryFileName = None
	backups = GafferUI.Backups.acquire( application, createIfNecessary = False )
	if backups is not None :
		recoveryFileName = backups.recoveryFile( fileName )
		if recoveryFileName :
			dialogue = GafferUI.ConfirmationDialogue(
				title = "Backup Available",
				message = "A more recent backup is available. Open backup instead?",
				confirmLabel = "Open Backup",
				cancelLabel = "Open",
			)
			if not dialogue.waitForConfirmation( parentWindow = dialogueParentWindow ) :
				recoveryFileName = None
			dialogue.setVisible( False )

	script = Gaffer.ScriptNode()
	script["fileName"].setValue( recoveryFileName or fileName )

	with GafferUI.ErrorDialogue.ErrorHandler(
		title = "Errors Occurred During Loading",
		closeLabel = "Oy vey",
		parentWindow = dialogueParentWindow
	) :
		script.load( continueOnError = True )

	if recoveryFileName :
		# If we loaded a backup, give the script the original
		# filename so the user can resave and continue as before.
		script["fileName"].setValue( fileName )
		script["unsavedChanges"].setValue( True )

	application["scripts"].addChild( script )

	addRecentFile( application, fileName )

	return script

def __open( currentScript, fileName ) :

	application = currentScript.ancestor( Gaffer.ApplicationRoot )
	currentWindow = GafferUI.ScriptWindow.acquire( currentScript )

	script = __addScript( application, fileName, dialogueParentWindow = currentWindow )

	removeCurrentScript = False
	if not currentScript["fileName"].getValue() and not currentScript["unsavedChanges"].getValue() :
		# the current script is empty - the user will think of the operation as loading
		# the new script into the current window, rather than adding a new window. so make it
		# look like that.
		newWindow = GafferUI.ScriptWindow.acquire( script )
		## \todo We probably want a way of querying and setting geometry in the public API
		newWindow._qtWidget().restoreGeometry( currentWindow._qtWidget().saveGeometry() )
		currentWindow.setVisible( False )
		removeCurrentScript = True

	# We must defer the removal of the old script because otherwise we trigger a crash bug
	# in PySide - I think this is because the menu item that invokes us is a child of
	# currentWindow, and that will get deleted immediately when the script is removed.
	if removeCurrentScript :
		GafferUI.EventLoop.addIdleCallback( functools.partial( __removeScript, application, currentScript ) )

def __removeScript( application, script ) :

	application["scripts"].removeChild( script )
	return False # remove idle callback

## A function suitable as the submenu callable for a File/OpenRecent menu item. It must be invoked
# from a menu which has a ScriptWindow in its ancestry.
def openRecent( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	currentScript = scriptWindow.scriptNode()
	applicationRoot = currentScript.ancestor( Gaffer.ApplicationRoot )

	recentFiles = []
	with IECore.IgnoredExceptions( AttributeError ) :
		recentFiles = applicationRoot.__recentFiles

	result = IECore.MenuDefinition()
	if recentFiles :
		for index, fileName in enumerate( recentFiles ) :
			result.append(
				"/" + str( index ),
				{
					"label": os.path.basename( fileName ),
					"command" : functools.partial( __open, currentScript, fileName ),
					"description" : fileName,
					"active" : os.path.isfile( fileName )
				}
			)
	else :
		result.append( "/None Available", { "active" : False } )

	return result

## This function adds a file to the list shown in the File/OpenRecent menu, and saves a recentFiles.py
# in the application's user startup folder so the settings will persist.
def addRecentFile( application, fileName ) :

	if isinstance( application, Gaffer.Application ) :
		applicationRoot = application.root()
	else :
		applicationRoot = application

	try :
		applicationRoot.__recentFiles
	except AttributeError :
		applicationRoot.__recentFiles = []

	if fileName in applicationRoot.__recentFiles :
		applicationRoot.__recentFiles.remove( fileName )

	applicationRoot.__recentFiles.insert( 0, fileName )
	del applicationRoot.__recentFiles[6:]

	f = file( os.path.join( applicationRoot.preferencesLocation(), "recentFiles.py" ), "w" )
	f.write( "# This file was automatically generated by Gaffer.\n" )
	f.write( "# Do not edit this file - it will be overwritten.\n\n" )

	f.write( "import GafferUI\n" )
	for fileName in reversed( applicationRoot.__recentFiles ) :
		f.write( "GafferUI.FileMenu.addRecentFile( application, \"%s\" )\n" % fileName )

## A function suitable as the command for a File/Save menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def save( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	script = scriptWindow.scriptNode()
	if script["fileName"].getValue() :
		with GafferUI.ErrorDialogue.ErrorHandler( title = "Error Saving File", parentWindow = scriptWindow ) :
			script.save()
	else :
		saveAs( menu )

## A function suitable as the command for a File/Save As menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def saveAs( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	script = scriptWindow.scriptNode()
	path, bookmarks = __pathAndBookmarks( scriptWindow )

	dialogue = GafferUI.PathChooserDialogue( path, title="Save script", confirmLabel="Save", leaf=True, bookmarks=bookmarks )
	path = dialogue.waitForPath( parentWindow = scriptWindow )

	if not path :
		return

	path = str( path )
	if not path.endswith( ".gfr" ) :
		path += ".gfr"

	script["fileName"].setValue( path )
	with GafferUI.ErrorDialogue.ErrorHandler( title = "Error Saving File", parentWindow = scriptWindow ) :
		script.save()

	application = script.ancestor( Gaffer.ApplicationRoot )
	addRecentFile( application, path )

## A function suitable as the command for a File/Revert To Saved menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def revertToSaved( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	script = scriptWindow.scriptNode()

	script.load()

def __revertToSavedAvailable( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	script = scriptWindow.scriptNode()

	if script["fileName"].getValue() and script["unsavedChanges"].getValue() :
		return True

	return False

## A function suitable as the command for a File/Export Selection... menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def exportSelection( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	script = scriptWindow.scriptNode()
	path, bookmarks = __pathAndBookmarks( scriptWindow )

	selection = script.selection()
	parent = selection[0].parent()
	for node in selection :
		if not parent.isAncestorOf( node ) :
			assert( node.parent().isAncestorOf( parent ) )
			parent = node.parent()

	dialogue = GafferUI.PathChooserDialogue( path, title="Export selection", confirmLabel="Export", leaf=True, bookmarks=bookmarks )
	path = dialogue.waitForPath( parentWindow = scriptWindow )

	if not path :
		return

	path = str( path )
	if not path.endswith( ".gfr" ) :
		path += ".gfr"

	script.serialiseToFile( path, parent, script.selection() )

## A function suitable as the command for a File/Import File... menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def importFile( menu ) :

	scope = GafferUI.EditMenu.scope( menu )
	path, bookmarks = __pathAndBookmarks( scope.scriptWindow )

	dialogue = GafferUI.PathChooserDialogue( path, title="Import script", confirmLabel="Import", valid=True, leaf=True, bookmarks=bookmarks )
	path = dialogue.waitForPath( parentWindow = scope.scriptWindow )

	if path is None :
		return

	errorHandler = GafferUI.ErrorDialogue.ErrorHandler(
		title = "Errors Occurred During Loading",
		closeLabel = "Oy vey",
		parentWindow = scope.scriptWindow
	)

	with Gaffer.UndoScope( scope.script ), errorHandler :

		newChildren = []
		c = scope.parent.childAddedSignal().connect( lambda parent, child : newChildren.append( child ) )

		scope.script.importFile( str( path ), parent = scope.parent, continueOnError = True )

		newNodes = [ c for c in newChildren if isinstance( c, Gaffer.Node ) ]
		scope.script.selection().clear()
		scope.script.selection().add( newNodes )

		if scope.graphEditor :

			fallbackPosition = scope.graphEditor.bound().size() / 2
			fallbackPosition = scope.graphEditor.graphGadgetWidget().getViewportGadget().rasterToGadgetSpace(
				imath.V2f( fallbackPosition.x, fallbackPosition.y ),
				gadget = scope.graphEditor.graphGadget()
			).p0
			fallbackPosition = imath.V2f( fallbackPosition.x, fallbackPosition.y )

			scope.graphEditor.graphGadget().getLayout().positionNodes(
				scope.graphEditor.graphGadget(), scope.script.selection(), fallbackPosition
			)

			scope.graphEditor.frame( scope.script.selection(), extend = True )

## A function suitable as the command for a File/Settings... menu item.
def showSettings( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )

	settingsWindow = None
	for window in scriptWindow.childWindows() :
		if hasattr( window, "_settingsEditor" ) :
			settingsWindow = window
			break

	if settingsWindow is None :
		settingsWindow = GafferUI.Window( "Settings", borderWidth=8 )
		settingsWindow._settingsEditor = True
		settingsWindow.setChild( GafferUI.NodeUI.create( scriptWindow.scriptNode() ) )
		# The NodeUI builds lazily, so we force it to build now so we can
		# resize the window to fit.
		settingsWindow.getChild().plugValueWidget( scriptWindow.scriptNode()["fileName"], lazy=False )
		settingsWindow.resizeToFitChild()
		scriptWindow.addChildWindow( settingsWindow )

	settingsWindow.setVisible( True )

def __selectionAvailable( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	return True if scriptWindow.scriptNode().selection().size() else False

def __pathAndBookmarks( scriptWindow ) :

	bookmarks = GafferUI.Bookmarks.acquire(
		scriptWindow,
		pathType = Gaffer.FileSystemPath,
		category = "script",
	)

	currentFileName = scriptWindow.scriptNode()["fileName"].getValue()
	if currentFileName :
		path = Gaffer.FileSystemPath( os.path.dirname( os.path.abspath( currentFileName ) ) )
	else :
		path = Gaffer.FileSystemPath( bookmarks.getDefault( scriptWindow ) )

	path.setFilter( Gaffer.FileSystemPath.createStandardFilter( [ "gfr" ] ) )

	return path, bookmarks
<commit_msg>FileMenu : allow escaping from backup dialog<commit_after>##########################################################################
#
#  Copyright (c) 2011-2012, John Haddon. All rights reserved.
#  Copyright (c) 2012-2013, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import os
import functools
import imath

import IECore

import Gaffer
import GafferUI

## Appends items to the IECore.MenuDefinition object passed to build a File menu containing
# standard open/save/revert/etc
def appendDefinitions( menuDefinition, prefix="" ) :

	menuDefinition.append( prefix + "/New", { "command" : new, "shortCut" : "Ctrl+N" } )
	menuDefinition.append( prefix + "/Open...", { "command" : open, "shortCut" : "Ctrl+O" } )
	menuDefinition.append( prefix + "/Open Recent", { "subMenu" : openRecent } )
	menuDefinition.append( prefix + "/OpenDivider", { "divider" : True } )
	menuDefinition.append( prefix + "/Save", { "command" : save, "shortCut" : "Ctrl+S" } )
	menuDefinition.append( prefix + "/Save As...", { "command" : saveAs, "shortCut" : "Shift+Ctrl+S" } )
	menuDefinition.append( prefix + "/Revert To Saved", { "command" : revertToSaved, "active" : __revertToSavedAvailable } )
	menuDefinition.append( prefix + "/SaveDivider", { "divider" : True } )
	menuDefinition.append( prefix + "/Export Selection...", { "command" : exportSelection, "active" : __selectionAvailable } )
	menuDefinition.append( prefix + "/Import...", { "command" : importFile } )
	menuDefinition.append( prefix + "/ImportExportDivider", { "divider" : True } )
	menuDefinition.append( prefix + "/Settings...", { "command" : showSettings } )

## A function suitable as the command for a File/New menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def new( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	application = scriptWindow.scriptNode().ancestor( Gaffer.ApplicationRoot )

	newScript = Gaffer.ScriptNode()
	Gaffer.NodeAlgo.applyUserDefaults( newScript )
	application["scripts"].addChild( newScript )

## A function suitable as the command for a File/Open menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def open( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	path, bookmarks = __pathAndBookmarks( scriptWindow )

	dialogue = GafferUI.PathChooserDialogue( path, title="Open script", confirmLabel="Open", valid=True, leaf=True, bookmarks=bookmarks )
	path = dialogue.waitForPath( parentWindow = scriptWindow )

	if not path :
		return

	__open( scriptWindow.scriptNode(), str( path ) )

## Opens a script and adds it to the application, as if the user
# had done so via the File Open dialogue.
def addScript( application, fileName ) :

	return __addScript( application, fileName )

def __addScript( application, fileName, dialogueParentWindow = None ) :

	recoveryFileName = None
	backups = GafferUI.Backups.acquire( application, createIfNecessary = False )
	if backups is not None :
		recoveryFileName = backups.recoveryFile( fileName )
		if recoveryFileName :
			dialogue = GafferUI.ConfirmationDialogue(
				title = "Backup Available",
				message = "A more recent backup is available. Open backup instead?",
				confirmLabel = "Open Backup",
				cancelLabel = "Open",
			)
			useBackup = dialogue.waitForConfirmation( parentWindow = dialogueParentWindow )
			if useBackup is None :
				return
			elif not useBackup :
				recoveryFileName = None
			dialogue.setVisible( False )

	script = Gaffer.ScriptNode()
	script["fileName"].setValue( recoveryFileName or fileName )

	with GafferUI.ErrorDialogue.ErrorHandler(
		title = "Errors Occurred During Loading",
		closeLabel = "Oy vey",
		parentWindow = dialogueParentWindow
	) :
		script.load( continueOnError = True )

	if recoveryFileName :
		# If we loaded a backup, give the script the original
		# filename so the user can resave and continue as before.
		script["fileName"].setValue( fileName )
		script["unsavedChanges"].setValue( True )

	application["scripts"].addChild( script )

	addRecentFile( application, fileName )

	return script

def __open( currentScript, fileName ) :

	application = currentScript.ancestor( Gaffer.ApplicationRoot )
	currentWindow = GafferUI.ScriptWindow.acquire( currentScript )

	script = __addScript( application, fileName, dialogueParentWindow = currentWindow )
	if not script :
		return

	removeCurrentScript = False
	if not currentScript["fileName"].getValue() and not currentScript["unsavedChanges"].getValue() :
		# the current script is empty - the user will think of the operation as loading
		# the new script into the current window, rather than adding a new window. so make it
		# look like that.
		newWindow = GafferUI.ScriptWindow.acquire( script )
		## \todo We probably want a way of querying and setting geometry in the public API
		newWindow._qtWidget().restoreGeometry( currentWindow._qtWidget().saveGeometry() )
		currentWindow.setVisible( False )
		removeCurrentScript = True

	# We must defer the removal of the old script because otherwise we trigger a crash bug
	# in PySide - I think this is because the menu item that invokes us is a child of
	# currentWindow, and that will get deleted immediately when the script is removed.
	if removeCurrentScript :
		GafferUI.EventLoop.addIdleCallback( functools.partial( __removeScript, application, currentScript ) )

def __removeScript( application, script ) :

	application["scripts"].removeChild( script )
	return False # remove idle callback

## A function suitable as the submenu callable for a File/OpenRecent menu item. It must be invoked
# from a menu which has a ScriptWindow in its ancestry.
def openRecent( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	currentScript = scriptWindow.scriptNode()
	applicationRoot = currentScript.ancestor( Gaffer.ApplicationRoot )

	recentFiles = []
	with IECore.IgnoredExceptions( AttributeError ) :
		recentFiles = applicationRoot.__recentFiles

	result = IECore.MenuDefinition()
	if recentFiles :
		for index, fileName in enumerate( recentFiles ) :
			result.append(
				"/" + str( index ),
				{
					"label": os.path.basename( fileName ),
					"command" : functools.partial( __open, currentScript, fileName ),
					"description" : fileName,
					"active" : os.path.isfile( fileName )
				}
			)
	else :
		result.append( "/None Available", { "active" : False } )

	return result

## This function adds a file to the list shown in the File/OpenRecent menu, and saves a recentFiles.py
# in the application's user startup folder so the settings will persist.
def addRecentFile( application, fileName ) :

	if isinstance( application, Gaffer.Application ) :
		applicationRoot = application.root()
	else :
		applicationRoot = application

	try :
		applicationRoot.__recentFiles
	except AttributeError :
		applicationRoot.__recentFiles = []

	if fileName in applicationRoot.__recentFiles :
		applicationRoot.__recentFiles.remove( fileName )

	applicationRoot.__recentFiles.insert( 0, fileName )
	del applicationRoot.__recentFiles[6:]

	f = file( os.path.join( applicationRoot.preferencesLocation(), "recentFiles.py" ), "w" )
	f.write( "# This file was automatically generated by Gaffer.\n" )
	f.write( "# Do not edit this file - it will be overwritten.\n\n" )

	f.write( "import GafferUI\n" )
	for fileName in reversed( applicationRoot.__recentFiles ) :
		f.write( "GafferUI.FileMenu.addRecentFile( application, \"%s\" )\n" % fileName )

## A function suitable as the command for a File/Save menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def save( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	script = scriptWindow.scriptNode()
	if script["fileName"].getValue() :
		with GafferUI.ErrorDialogue.ErrorHandler( title = "Error Saving File", parentWindow = scriptWindow ) :
			script.save()
	else :
		saveAs( menu )

## A function suitable as the command for a File/Save As menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def saveAs( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	script = scriptWindow.scriptNode()
	path, bookmarks = __pathAndBookmarks( scriptWindow )

	dialogue = GafferUI.PathChooserDialogue( path, title="Save script", confirmLabel="Save", leaf=True, bookmarks=bookmarks )
	path = dialogue.waitForPath( parentWindow = scriptWindow )

	if not path :
		return

	path = str( path )
	if not path.endswith( ".gfr" ) :
		path += ".gfr"

	script["fileName"].setValue( path )
	with GafferUI.ErrorDialogue.ErrorHandler( title = "Error Saving File", parentWindow = scriptWindow ) :
		script.save()

	application = script.ancestor( Gaffer.ApplicationRoot )
	addRecentFile( application, path )

## A function suitable as the command for a File/Revert To Saved menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def revertToSaved( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	script = scriptWindow.scriptNode()

	script.load()

def __revertToSavedAvailable( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	script = scriptWindow.scriptNode()

	if script["fileName"].getValue() and script["unsavedChanges"].getValue() :
		return True

	return False

## A function suitable as the command for a File/Export Selection... menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def exportSelection( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	script = scriptWindow.scriptNode()
	path, bookmarks = __pathAndBookmarks( scriptWindow )

	selection = script.selection()
	parent = selection[0].parent()
	for node in selection :
		if not parent.isAncestorOf( node ) :
			assert( node.parent().isAncestorOf( parent ) )
			parent = node.parent()

	dialogue = GafferUI.PathChooserDialogue( path, title="Export selection", confirmLabel="Export", leaf=True, bookmarks=bookmarks )
	path = dialogue.waitForPath( parentWindow = scriptWindow )

	if not path :
		return

	path = str( path )
	if not path.endswith( ".gfr" ) :
		path += ".gfr"

	script.serialiseToFile( path, parent, script.selection() )

## A function suitable as the command for a File/Import File... menu item. It must be invoked from a menu which
# has a ScriptWindow in its ancestry.
def importFile( menu ) :

	scope = GafferUI.EditMenu.scope( menu )
	path, bookmarks = __pathAndBookmarks( scope.scriptWindow )

	dialogue = GafferUI.PathChooserDialogue( path, title="Import script", confirmLabel="Import", valid=True, leaf=True, bookmarks=bookmarks )
	path = dialogue.waitForPath( parentWindow = scope.scriptWindow )

	if path is None :
		return

	errorHandler = GafferUI.ErrorDialogue.ErrorHandler(
		title = "Errors Occurred During Loading",
		closeLabel = "Oy vey",
		parentWindow = scope.scriptWindow
	)

	with Gaffer.UndoScope( scope.script ), errorHandler :

		newChildren = []
		c = scope.parent.childAddedSignal().connect( lambda parent, child : newChildren.append( child ) )

		scope.script.importFile( str( path ), parent = scope.parent, continueOnError = True )

		newNodes = [ c for c in newChildren if isinstance( c, Gaffer.Node ) ]
		scope.script.selection().clear()
		scope.script.selection().add( newNodes )

		if scope.graphEditor :

			fallbackPosition = scope.graphEditor.bound().size() / 2
			fallbackPosition = scope.graphEditor.graphGadgetWidget().getViewportGadget().rasterToGadgetSpace(
				imath.V2f( fallbackPosition.x, fallbackPosition.y ),
				gadget = scope.graphEditor.graphGadget()
			).p0
			fallbackPosition = imath.V2f( fallbackPosition.x, fallbackPosition.y )

			scope.graphEditor.graphGadget().getLayout().positionNodes(
				scope.graphEditor.graphGadget(), scope.script.selection(), fallbackPosition
			)

			scope.graphEditor.frame( scope.script.selection(), extend = True )

## A function suitable as the command for a File/Settings... menu item.
def showSettings( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )

	settingsWindow = None
	for window in scriptWindow.childWindows() :
		if hasattr( window, "_settingsEditor" ) :
			settingsWindow = window
			break

	if settingsWindow is None :
		settingsWindow = GafferUI.Window( "Settings", borderWidth=8 )
		settingsWindow._settingsEditor = True
		settingsWindow.setChild( GafferUI.NodeUI.create( scriptWindow.scriptNode() ) )
		# The NodeUI builds lazily, so we force it to build now so we can
		# resize the window to fit.
		settingsWindow.getChild().plugValueWidget( scriptWindow.scriptNode()["fileName"], lazy=False )
		settingsWindow.resizeToFitChild()
		scriptWindow.addChildWindow( settingsWindow )

	settingsWindow.setVisible( True )

def __selectionAvailable( menu ) :

	scriptWindow = menu.ancestor( GafferUI.ScriptWindow )
	return True if scriptWindow.scriptNode().selection().size() else False

def __pathAndBookmarks( scriptWindow ) :

	bookmarks = GafferUI.Bookmarks.acquire(
		scriptWindow,
		pathType = Gaffer.FileSystemPath,
		category = "script",
	)

	currentFileName = scriptWindow.scriptNode()["fileName"].getValue()
	if currentFileName :
		path = Gaffer.FileSystemPath( os.path.dirname( os.path.abspath( currentFileName ) ) )
	else :
		path = Gaffer.FileSystemPath( bookmarks.getDefault( scriptWindow ) )

	path.setFilter( Gaffer.FileSystemPath.createStandardFilter( [ "gfr" ] ) )

	return path, bookmarks
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Copyright (c) 2012 Giacomo Bagnoli <g.bagnoli@asidev.com>

Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
from datetime import datetime
import inspect
import logging
import functools
from .api import logs
from .utils import MultiDict
log = logging.getLogger(__name__)

class VarnishLogs(object):
    default_settings = {
           'process_old_entries': False,
           'process_backend_requests': False,
           'process_client_requests': False,
           'include_tag': None,
           'include_tag_regex': None,
           'exclude_tag': None,
           'exclude_tag_regex': None,
           'stop_after': None,
           'skip_first': None,
           'read_entries_from_file': None,
           'filter_transactions_by_tag_regex': None,
           'ignore_case_in_regex': False
        }

    def __init__(self, varnish, **settings):
        self.settings = dict(self.default_settings)
        self.settings.update(settings)
        self.varnish = varnish
        self.vd = varnish.vd
        logs.init(self.vd, True)
        for st, value in self.settings.items():
            if value and self.default_settings[st] is None:
                getattr(logs, st)(self.vd, value)

            elif value and self.default_settings[st] is False:
                getattr(logs, st)(self.vd)

            if value and st == "filter_transactions_by_tag_regex":
                getattr(logs, st)(self.vd, value[0], value[1])

    def __getattr__(self, attr):
        if attr in self.default_settings:
            return functools.partial(getattr(logs, attr), self.vd)

        raise AttributeError(attr)

    def dispatch_chunks(self, callback, source=None):
        """ Read logs from varnish shared memory logs, then call callback
            for every chunk as returned from the low level api
            `callback` must be a callable that accepts 0 or 1 positional
            parameter (an instance of the varnish.api.logs.LogChunk class)
        """
        if callback:
            args = len(inspect.getargspec(callback).args)

        def wrapper(chunk, priv):
            res = False

            if callback and args == 0:
                res = callback()

            elif callback:
                res = callback(chunk)

            return res

        if source:
            self.read_entries_from_file(source)

        logs.dispatch(self.vd, wrapper)

    def dispatch_requests(self, callback, aggregate=1000, source=None):
        """ Read logs from Varnish shared memory Logs, then call callback
            when a RequestLog is complete (all its chunks have been read).
            `callback` must be a callable that accepts 1 positional parameter
            (an instance of ClientRequestLog or BackendRequestLog, subclasses
             of RequestLog)
            if `aggregate` is true, try to relate backend requests to the
            client request that generated it
        """
        if aggregate:
            # use a multidict because it is ordered
            backend_requests = MultiDict()

        def cb(chunk):
            ev = RequestLog(chunk)
            # discard invalid, incomplete and empty logs
            res = True
            if not ev or not ev.complete or (ev.backend and not ev.chunks):
                return res

            if not aggregate:
                res = callback(ev)

            elif ev.backend:
                id_ = ev.txheaders.getone('x-varnish')
                log.debug("Adding %s to backend_requests", ev)
                backend_requests.overwrite(id_, ev)

            elif ev.client and ev.id in backend_requests:
                ev.backend_request = backend_requests.getone(ev.id)
                res = callback(ev)

            else:
                # backend request was not read, leaving to None
                res = callback(ev)

            if aggregate and len(backend_requests) > aggregate:
                backend_requests.trim(aggregate)

            return res

        self.dispatch_chunks(callback=cb)

    def __str__(self):
        return "<%s [instance: %s]>" % (self.__class__.__name__,
                                        self.varnish.name)

    def __repr__(self):
        return str(self)

class RequestLog(object):
    """ This class is a factory for its subclasses. It keeps returning the
        same objects as long as the chunk belongs to an existing instance.
        It returns None if the chunk belongs to neither a client of backend
        request
    """
    _lines = {}

    def __new__(cls, chunk, active=False):
        if chunk.fd in cls._lines:
            obj = cls._lines[chunk.fd]

        else:
            if chunk.client:
                obj = super(RequestLog, cls).__new__(ClientRequestLog)

            elif chunk.backend:
                obj = super(RequestLog, cls).__new__(BackendRequestLog)

            else:
                return None

            obj.init(chunk, active)
            cls._lines[chunk.fd] = obj

        obj.add_chunk(chunk)
        return obj

    def init(self, chunk, active=False):
        if hasattr(self, "fd"):
            return

        self.fd = chunk.fd
        self.chunks = []
        self.active = active
        self.complete = False
        self.rxheaders = MultiDict()
        self.txheaders = MultiDict()
        self.rxprotocol = None
        self.txprotocol = None
        self.method = None
        self.url = None
        self.status = None
        self.response = None
        self.length = None

    def add_chunk(self, chunk):
        if self.fd == 0:
            return False

        if not self.active and \
            ((chunk.client and chunk.tag.name == 'reqstart') or
             (chunk.backend and chunk.tag.name == 'backendopen')):
            self.active = True

        elif not self.active:
            return False

        if (chunk.client and chunk.tag.name == 'reqend') or \
           (chunk.backend and (chunk.tag.name == 'backendclose' or
                               chunk.tag.name == 'backendreuse')):
            self.complete = True
            self.active = False
            del self.__class__._lines[self.fd]
            if chunk.tag.name == 'backendreuse':
                # backend reuse need a special case to get the next backend
                # request as no backendopen will arrive
                # then we create a new empry object that we now is active
                next_backend = super(RequestLog, self.__class__)\
                                        .__new__(BackendRequestLog)
                next_backend.init(chunk, active=True)
                self.__class__._lines[chunk.fd] = next_backend
                next_backend.on_append_chunk(chunk)

        self.on_append_chunk(chunk)
        return self.complete

    def on_append_chunk(self, chunk):
        self.client = chunk.client
        self.backend = chunk.backend
        self.chunks.append(chunk)
        name = chunk.tag.name
        if name == 'rxheader':
            key, value = chunk.data.split(":", 1)
            self.rxheaders[key.strip().lower()] = value.strip()

        elif name == 'txheader':
            key, value = chunk.data.split(":", 1)
            self.txheaders[key.strip().lower()] = value.strip()

        elif name == 'rxprotocol':
            self.rxprotocol = chunk.data

        elif name == 'txprotocol':
            self.txprotocol = chunk.data

        elif name == 'length':
            self.length = int(chunk.data)

    def __repr__(self):
        res = "<%s %s" % (self.__class__.__name__, self.id)
        for c in self.chunks:
            res += "\n\t%s" % (c)
        res += ">"
        return res

class ClientRequestLog(RequestLog):
    """ Aggragates chunks for a client request """

    def init(self, chunk, active=False):
        super(ClientRequestLog, self).init(chunk, active)
        self.id = None
        self.vcl_calls = MultiDict()
        self.hash_data = []
        self.client_ip = None
        self.client_port = None
        self.started_at = None
        self.completed_at = None
        self.req_start_delay = None
        self.processing_time = None
        self.deliver_time = None
        self.backend_request = None

    def on_append_chunk(self, chunk):
        super(ClientRequestLog, self).on_append_chunk(chunk)
        name = chunk.tag.name

        if name == 'vcl_call':
            self._last_vcl = chunk.data

        elif name == 'vcl_return':
            self.vcl_calls[self._last_vcl] = chunk.data
            del self._last_vcl

        elif name == 'hash':
            self.hash_data.append(chunk.data)

        elif name == 'length':
            self.length = int(chunk.data)

        elif name == 'rxrequest':
            self.method = chunk.data

        elif name == 'rxurl':
            self.url = chunk.data

        elif name == 'reqstart':
            self.client_ip, self.client_port, self.id = \
                chunk.data.split(" ")

        elif name == 'txstatus':
            self.status = int(chunk.data)

        elif name == 'txresponse':
            self.response = chunk.data

        elif name == 'reqend':
            xid, started_at, completed_at, \
                req_start_delay, processing_time, \
                deliver_time = chunk.data.split(" ")
            self.started_at = datetime.fromtimestamp(float(started_at))
            self.completed_at = datetime.fromtimestamp(float(completed_at))
            self.req_start_delay = float(req_start_delay)
            self.processing_time = float(processing_time)
            self.deliver_time = float(deliver_time)

    @property
    def hit(self):
        return "hit" in self.vcl_calls

    @property
    def miss(self):
        return "miss" in self.vcl_calls

    def __repr__(self):
        res = """
<{self.__class__.__name__} XID: {self.id}
    Client: {self.client_ip}:{self.client_port}

    Timing:
        started   : {self.started_at}
        completed : {self.completed_at}
        delay     : {self.req_start_delay} [s]
        processing: {self.processing_time} [s]
        deliver   : {self.deliver_time} [s]

    Request: {self.rxprotocol} {self.method} {self.url}
        headers   : {self.rxheaders}

    Hash: {self.hash_data}
    VCL Calls: {self.vcl_calls}

    Response: {self.txprotocol} {self.status} {self.response} [{self.length}B]
        headers   : {self.txheaders}
>""".format(self=self)
        return res

    def __str__(self):
        return "<{self.__class__.__name__} XID: {self.id}>".format(self=self)

class BackendRequestLog(RequestLog):
    """ Aggragates chunks for a backend request """

    def init(self, chunk, active=False):
        super(BackendRequestLog, self).init(chunk, active)
        self.backend_name = None

    def on_append_chunk(self, chunk):
        super(BackendRequestLog, self).on_append_chunk(chunk)
        name = chunk.tag.name
        if name == 'txrequest':
            self.method = chunk.data

        elif name == 'txurl':
            self.url = chunk.data

        elif name == 'rxstatus':
            self.status = int(chunk.data)

        elif name == 'rxresponse':
            self.response = chunk.data

        elif name in ('backendopen', 'backendreuse'):
            self.backend_name = chunk.data.split(" ")[0]

    def __repr__(self):
        return """
<{self.__class__.__name__} [backend: {self.backend_name}]
    Request: {self.rxprotocol} {self.method} {self.url}
        headers   : {self.rxheaders}

    Response: {self.txprotocol} {self.status} {self.response} [{self.length}B]
        headers   : {self.txheaders}
>""".format(self=self)

    def __str__(self):
        return "<{self.__class__.__name__} ({self.backend_name})>"\
                .format(self=self)
<commit_msg>Propagate source parameter to dispatch_chunks()<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Copyright (c) 2012 Giacomo Bagnoli <g.bagnoli@asidev.com>

Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
from datetime import datetime
import inspect
import logging
import functools
from .api import logs
from .utils import MultiDict
log = logging.getLogger(__name__)

class VarnishLogs(object):
    default_settings = {
           'process_old_entries': False,
           'process_backend_requests': False,
           'process_client_requests': False,
           'include_tag': None,
           'include_tag_regex': None,
           'exclude_tag': None,
           'exclude_tag_regex': None,
           'stop_after': None,
           'skip_first': None,
           'read_entries_from_file': None,
           'filter_transactions_by_tag_regex': None,
           'ignore_case_in_regex': False
        }

    def __init__(self, varnish, **settings):
        self.settings = dict(self.default_settings)
        self.settings.update(settings)
        self.varnish = varnish
        self.vd = varnish.vd
        logs.init(self.vd, True)
        for st, value in self.settings.items():
            if value and self.default_settings[st] is None:
                getattr(logs, st)(self.vd, value)

            elif value and self.default_settings[st] is False:
                getattr(logs, st)(self.vd)

            if value and st == "filter_transactions_by_tag_regex":
                getattr(logs, st)(self.vd, value[0], value[1])

    def __getattr__(self, attr):
        if attr in self.default_settings:
            return functools.partial(getattr(logs, attr), self.vd)

        raise AttributeError(attr)

    def dispatch_chunks(self, callback, source=None):
        """ Read logs from varnish shared memory logs, then call callback
            for every chunk as returned from the low level api
            `callback` must be a callable that accepts 0 or 1 positional
            parameter (an instance of the varnish.api.logs.LogChunk class)
        """
        if callback:
            args = len(inspect.getargspec(callback).args)

        def wrapper(chunk, priv):
            res = False

            if callback and args == 0:
                res = callback()

            elif callback:
                res = callback(chunk)

            return res

        if source:
            self.read_entries_from_file(source)

        logs.dispatch(self.vd, wrapper)

    def dispatch_requests(self, callback, aggregate=1000, source=None):
        """ Read logs from Varnish shared memory Logs, then call callback
            when a RequestLog is complete (all its chunks have been read).
            `callback` must be a callable that accepts 1 positional parameter
            (an instance of ClientRequestLog or BackendRequestLog, subclasses
             of RequestLog)
            if `aggregate` is true, try to relate backend requests to the
            client request that generated it
        """
        if aggregate:
            # use a multidict because it is ordered
            backend_requests = MultiDict()

        def cb(chunk):
            ev = RequestLog(chunk)
            # discard invalid, incomplete and empty logs
            res = True
            if not ev or not ev.complete or (ev.backend and not ev.chunks):
                return res

            if not aggregate:
                res = callback(ev)

            elif ev.backend:
                id_ = ev.txheaders.getone('x-varnish')
                log.debug("Adding %s to backend_requests", ev)
                backend_requests.overwrite(id_, ev)

            elif ev.client and ev.id in backend_requests:
                ev.backend_request = backend_requests.getone(ev.id)
                res = callback(ev)

            else:
                # backend request was not read, leaving to None
                res = callback(ev)

            if aggregate and len(backend_requests) > aggregate:
                backend_requests.trim(aggregate)

            return res

        self.dispatch_chunks(callback=cb, source=source)

    def __str__(self):
        return "<%s [instance: %s]>" % (self.__class__.__name__,
                                        self.varnish.name)

    def __repr__(self):
        return str(self)

class RequestLog(object):
    """ This class is a factory for its subclasses. It keeps returning the
        same objects as long as the chunk belongs to an existing instance.
        It returns None if the chunk belongs to neither a client of backend
        request
    """
    _lines = {}

    def __new__(cls, chunk, active=False):
        if chunk.fd in cls._lines:
            obj = cls._lines[chunk.fd]

        else:
            if chunk.client:
                obj = super(RequestLog, cls).__new__(ClientRequestLog)

            elif chunk.backend:
                obj = super(RequestLog, cls).__new__(BackendRequestLog)

            else:
                return None

            obj.init(chunk, active)
            cls._lines[chunk.fd] = obj

        obj.add_chunk(chunk)
        return obj

    def init(self, chunk, active=False):
        if hasattr(self, "fd"):
            return

        self.fd = chunk.fd
        self.chunks = []
        self.active = active
        self.complete = False
        self.rxheaders = MultiDict()
        self.txheaders = MultiDict()
        self.rxprotocol = None
        self.txprotocol = None
        self.method = None
        self.url = None
        self.status = None
        self.response = None
        self.length = None

    def add_chunk(self, chunk):
        if self.fd == 0:
            return False

        if not self.active and \
            ((chunk.client and chunk.tag.name == 'reqstart') or
             (chunk.backend and chunk.tag.name == 'backendopen')):
            self.active = True

        elif not self.active:
            return False

        if (chunk.client and chunk.tag.name == 'reqend') or \
           (chunk.backend and (chunk.tag.name == 'backendclose' or
                               chunk.tag.name == 'backendreuse')):
            self.complete = True
            self.active = False
            del self.__class__._lines[self.fd]
            if chunk.tag.name == 'backendreuse':
                # backend reuse need a special case to get the next backend
                # request as no backendopen will arrive
                # then we create a new empry object that we now is active
                next_backend = super(RequestLog, self.__class__)\
                                        .__new__(BackendRequestLog)
                next_backend.init(chunk, active=True)
                self.__class__._lines[chunk.fd] = next_backend
                next_backend.on_append_chunk(chunk)

        self.on_append_chunk(chunk)
        return self.complete

    def on_append_chunk(self, chunk):
        self.client = chunk.client
        self.backend = chunk.backend
        self.chunks.append(chunk)
        name = chunk.tag.name
        if name == 'rxheader':
            key, value = chunk.data.split(":", 1)
            self.rxheaders[key.strip().lower()] = value.strip()

        elif name == 'txheader':
            key, value = chunk.data.split(":", 1)
            self.txheaders[key.strip().lower()] = value.strip()

        elif name == 'rxprotocol':
            self.rxprotocol = chunk.data

        elif name == 'txprotocol':
            self.txprotocol = chunk.data

        elif name == 'length':
            self.length = int(chunk.data)

    def __repr__(self):
        res = "<%s %s" % (self.__class__.__name__, self.id)
        for c in self.chunks:
            res += "\n\t%s" % (c)
        res += ">"
        return res

class ClientRequestLog(RequestLog):
    """ Aggragates chunks for a client request """

    def init(self, chunk, active=False):
        super(ClientRequestLog, self).init(chunk, active)
        self.id = None
        self.vcl_calls = MultiDict()
        self.hash_data = []
        self.client_ip = None
        self.client_port = None
        self.started_at = None
        self.completed_at = None
        self.req_start_delay = None
        self.processing_time = None
        self.deliver_time = None
        self.backend_request = None

    def on_append_chunk(self, chunk):
        super(ClientRequestLog, self).on_append_chunk(chunk)
        name = chunk.tag.name

        if name == 'vcl_call':
            self._last_vcl = chunk.data

        elif name == 'vcl_return':
            self.vcl_calls[self._last_vcl] = chunk.data
            del self._last_vcl

        elif name == 'hash':
            self.hash_data.append(chunk.data)

        elif name == 'length':
            self.length = int(chunk.data)

        elif name == 'rxrequest':
            self.method = chunk.data

        elif name == 'rxurl':
            self.url = chunk.data

        elif name == 'reqstart':
            self.client_ip, self.client_port, self.id = \
                chunk.data.split(" ")

        elif name == 'txstatus':
            self.status = int(chunk.data)

        elif name == 'txresponse':
            self.response = chunk.data

        elif name == 'reqend':
            xid, started_at, completed_at, \
                req_start_delay, processing_time, \
                deliver_time = chunk.data.split(" ")
            self.started_at = datetime.fromtimestamp(float(started_at))
            self.completed_at = datetime.fromtimestamp(float(completed_at))
            self.req_start_delay = float(req_start_delay)
            self.processing_time = float(processing_time)
            self.deliver_time = float(deliver_time)

    @property
    def hit(self):
        return "hit" in self.vcl_calls

    @property
    def miss(self):
        return "miss" in self.vcl_calls

    def __repr__(self):
        res = """
<{self.__class__.__name__} XID: {self.id}
    Client: {self.client_ip}:{self.client_port}

    Timing:
        started   : {self.started_at}
        completed : {self.completed_at}
        delay     : {self.req_start_delay} [s]
        processing: {self.processing_time} [s]
        deliver   : {self.deliver_time} [s]

    Request: {self.rxprotocol} {self.method} {self.url}
        headers   : {self.rxheaders}

    Hash: {self.hash_data}
    VCL Calls: {self.vcl_calls}

    Response: {self.txprotocol} {self.status} {self.response} [{self.length}B]
        headers   : {self.txheaders}
>""".format(self=self)
        return res

    def __str__(self):
        return "<{self.__class__.__name__} XID: {self.id}>".format(self=self)

class BackendRequestLog(RequestLog):
    """ Aggragates chunks for a backend request """

    def init(self, chunk, active=False):
        super(BackendRequestLog, self).init(chunk, active)
        self.backend_name = None

    def on_append_chunk(self, chunk):
        super(BackendRequestLog, self).on_append_chunk(chunk)
        name = chunk.tag.name
        if name == 'txrequest':
            self.method = chunk.data

        elif name == 'txurl':
            self.url = chunk.data

        elif name == 'rxstatus':
            self.status = int(chunk.data)

        elif name == 'rxresponse':
            self.response = chunk.data

        elif name in ('backendopen', 'backendreuse'):
            self.backend_name = chunk.data.split(" ")[0]

    def __repr__(self):
        return """
<{self.__class__.__name__} [backend: {self.backend_name}]
    Request: {self.rxprotocol} {self.method} {self.url}
        headers   : {self.rxheaders}

    Response: {self.txprotocol} {self.status} {self.response} [{self.length}B]
        headers   : {self.txheaders}
>""".format(self=self)

    def __str__(self):
        return "<{self.__class__.__name__} ({self.backend_name})>"\
                .format(self=self)
<|endoftext|>