<commit_before>from datetime import timedelta

from django.conf import settings
from django.core.cache import cache
from django.db import models
from django.db.models import Q
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from c3nav.mapdata.models import MapUpdate
from c3nav.mapdata.models.base import SerializableMixin, TitledMixin

class AccessRestriction(TitledMixin, models.Model):
    """
    An access restriction
    """
    users = models.ManyToManyField(settings.AUTH_USER_MODEL, through='AccessPermission')
    open = models.BooleanField(default=False, verbose_name=_('open'))

    class Meta:
        verbose_name = _('Access Restriction')
        verbose_name_plural = _('Access Restrictions')
        default_related_name = 'accessrestrictions'

    @classmethod
    def qs_for_request(cls, request):
        if request.user.is_authenticated and request.user.is_superuser:
            return cls.objects.all()
        return cls.objects.none()

class AccessPermission(models.Model):
    user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
    access_restriction = models.ForeignKey(AccessRestriction, on_delete=models.CASCADE)
    expire_date = models.DateTimeField(null=True, verbose_name=_('expires'))

    class Meta:
        verbose_name = _('Access Permission')
        verbose_name_plural = _('Access Permissions')
        default_related_name = 'accesspermissions'
        unique_together = (('user', 'access_restriction'), )

    @staticmethod
    def user_access_permission_key(user):
        return 'mapdata:user_access_permission:%d' % user.pk

    @classmethod
    def get_for_request(cls, request):
        if not request.user.is_authenticated:
            return set()

        cache_key = cls.user_access_permission_key(request.user)
        access_restriction_ids = cache.get(cache_key, None)
        if access_restriction_ids is None:
            result = tuple(request.user.accesspermissions.filter(
                Q(expire_date__isnull=True) | Q(expire_date__lt=timezone.now())
            ).values_list('access_restriction_id', 'expire_date'))
            if result:
                access_restriction_ids, expire_dates = zip(*result)
            else:
                access_restriction_ids, expire_dates = (), ()

            expire_date = min((e for e in expire_dates if e), default=timezone.now()+timedelta(seconds=120))
            cache.set(cache_key, access_restriction_ids, max(0, (expire_date-timezone.now()).total_seconds()))
        return set(access_restriction_ids)

    @classmethod
    def cache_key_for_request(cls, request, with_update=True):
        return (
            ((MapUpdate.current_cache_key()+':') if with_update else '') +
            ','.join(str(i) for i in sorted(AccessPermission.get_for_request(request)) or '0')
        )

    @classmethod
    def etag_func(cls, request, *args, **kwargs):
        return cls.cache_key_for_request(request)

class AccessRestrictionMixin(SerializableMixin, models.Model):
    access_restriction = models.ForeignKey(AccessRestriction, null=True, blank=True,
                                           verbose_name=_('Access Restriction'))

    class Meta:
        abstract = True

    def _serialize(self, **kwargs):
        result = super()._serialize(**kwargs)
        result['access_restriction'] = self.access_restriction_id
        return result

    def details_display(self):
        result = super().details_display()
        result['display'].extend([
            (str(_('Access Restriction')), self.access_restriction_id and self.access_restriction.title),
        ])
        return result

    @classmethod
    def qs_for_request(cls, request, allow_none=False):
        return cls.objects.filter(cls.q_for_request(request, allow_none=allow_none))

    @classmethod
    def q_for_request(cls, request, prefix='', allow_none=False):
        if request is None and allow_none or request.user.is_superuser:
            return Q()
        return (Q(**{prefix+'access_restriction__isnull': True}) |
                Q(**{prefix+'access_restriction__in': AccessPermission.get_for_request(request)}))
<commit_msg>superusers get their own cache key<commit_after>from datetime import timedelta

from django.conf import settings
from django.core.cache import cache
from django.db import models
from django.db.models import Q
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from c3nav.mapdata.models import MapUpdate
from c3nav.mapdata.models.base import SerializableMixin, TitledMixin

class AccessRestriction(TitledMixin, models.Model):
    """
    An access restriction
    """
    users = models.ManyToManyField(settings.AUTH_USER_MODEL, through='AccessPermission')
    open = models.BooleanField(default=False, verbose_name=_('open'))

    class Meta:
        verbose_name = _('Access Restriction')
        verbose_name_plural = _('Access Restrictions')
        default_related_name = 'accessrestrictions'

    @classmethod
    def qs_for_request(cls, request):
        if request.user.is_authenticated and request.user.is_superuser:
            return cls.objects.all()
        return cls.objects.none()

class AccessPermission(models.Model):
    user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
    access_restriction = models.ForeignKey(AccessRestriction, on_delete=models.CASCADE)
    expire_date = models.DateTimeField(null=True, verbose_name=_('expires'))

    class Meta:
        verbose_name = _('Access Permission')
        verbose_name_plural = _('Access Permissions')
        default_related_name = 'accesspermissions'
        unique_together = (('user', 'access_restriction'), )

    @staticmethod
    def user_access_permission_key(user):
        return 'mapdata:user_access_permission:%d' % user.pk

    @classmethod
    def get_for_request(cls, request):
        if not request.user.is_authenticated:
            return set()

        cache_key = cls.user_access_permission_key(request.user)
        access_restriction_ids = cache.get(cache_key, None)
        if access_restriction_ids is None:
            result = tuple(request.user.accesspermissions.filter(
                Q(expire_date__isnull=True) | Q(expire_date__lt=timezone.now())
            ).values_list('access_restriction_id', 'expire_date'))
            if result:
                access_restriction_ids, expire_dates = zip(*result)
            else:
                access_restriction_ids, expire_dates = (), ()

            expire_date = min((e for e in expire_dates if e), default=timezone.now()+timedelta(seconds=120))
            cache.set(cache_key, access_restriction_ids, max(0, (expire_date-timezone.now()).total_seconds()))
        return set(access_restriction_ids)

    @classmethod
    def cache_key_for_request(cls, request, with_update=True):
        if request.user.is_superuser:
            return (
                ((MapUpdate.current_cache_key() + ':') if with_update else '') +
                'SU'
            )
        return (
            ((MapUpdate.current_cache_key()+':') if with_update else '') +
            ','.join(str(i) for i in sorted(AccessPermission.get_for_request(request)) or '0')
        )

    @classmethod
    def etag_func(cls, request, *args, **kwargs):
        return cls.cache_key_for_request(request)

class AccessRestrictionMixin(SerializableMixin, models.Model):
    access_restriction = models.ForeignKey(AccessRestriction, null=True, blank=True,
                                           verbose_name=_('Access Restriction'))

    class Meta:
        abstract = True

    def _serialize(self, **kwargs):
        result = super()._serialize(**kwargs)
        result['access_restriction'] = self.access_restriction_id
        return result

    def details_display(self):
        result = super().details_display()
        result['display'].extend([
            (str(_('Access Restriction')), self.access_restriction_id and self.access_restriction.title),
        ])
        return result

    @classmethod
    def qs_for_request(cls, request, allow_none=False):
        return cls.objects.filter(cls.q_for_request(request, allow_none=allow_none))

    @classmethod
    def q_for_request(cls, request, prefix='', allow_none=False):
        if request is None and allow_none or request.user.is_superuser:
            return Q()
        return (Q(**{prefix+'access_restriction__isnull': True}) |
                Q(**{prefix+'access_restriction__in': AccessPermission.get_for_request(request)}))
<|endoftext|>


<commit_before>from collections import namedtuple
import datetime
import json
import string
from urllib import quote

import adzerk_api
from pylons import c, g
import requests

from r2.controllers import api, add_controller
from r2.lib import (
    amqp,
    authorize,
    organic,
    promote,
)
from r2.lib.db.sorts import epoch_seconds
from r2.lib.filters import spaceCompress, _force_utf8
from r2.lib.pages.things import default_thing_wrapper
from r2.lib.template_helpers import replace_render
from r2.lib.hooks import HookRegistrar
from r2.models import (
    Account,
    CampaignBuilder,
    Frontpage,
    Link,
    LinkListing,
    PromoCampaign,
    PromotionLog,
    Subreddit,
)

hooks = HookRegistrar()

ADZERK_IMPRESSION_BUMP = 500    # add extra impressions to the number we
                                # request from adzerk in case their count
                                # is lower than our internal traffic tracking

DELCHARS = ''.join(c for c in map(chr, range(256)) if not (c.isalnum() or c.isspace()))

def sanitize_text(text):
    return _force_utf8(text).translate(None, DELCHARS)

def date_to_adzerk(d):
    utc_date = d - promote.timezone_offset
    epoch_milliseconds = int(epoch_seconds(utc_date) * 1000)
    return '/Date(%s)/' % epoch_milliseconds

def srname_to_keyword(srname):
    return srname or Frontpage.name

def render_link(link, campaign):
    author = Account._byID(link.author_id, data=True)
    return json.dumps({
        'link': link._fullname,
        'campaign': campaign._fullname,
        'title': sanitize_text(link.title),
        'author': sanitize_text(author.name),
        'target': campaign.sr_name,
    })

def update_changed(adzerk_object, **d):
    changed = [(attr, val) for attr, val in d.iteritems()
                          if getattr(adzerk_object, attr) != val]
    if changed:
        for (attr, val) in changed:
            setattr(adzerk_object, attr, val)
        adzerk_object._send()
    return changed

def update_campaign(link):
    """Add/update a reddit link as an Adzerk Campaign"""
    if hasattr(link, 'adzerk_campaign_id'):
        az_campaign = adzerk_api.Campaign.get(link.adzerk_campaign_id)
    else:
        az_campaign = None

    d = {
        'AdvertiserId': g.az_selfserve_advertiser_id,
        'IsDeleted': False,
        'IsActive': not link._deleted,
        'Price': 0,
    }

    log_text = None
    if az_campaign:
        changed = update_changed(az_campaign, **d)
    else:
        d.update({
            'Name': link._fullname,
            'Flights': [],
            'StartDate': date_to_adzerk(datetime.datetime.now(g.tz)),
        })
        az_campaign = adzerk_api.Campaign.create(**d)
        link.adzerk_campaign_id = az_campaign.Id
        link._commit()
        log_text = 'created %s' % az_campaign

    if log_text:
        PromotionLog.add(link, log_text)
        g.log.info(log_text)

    return az_campaign

def update_creative(link, campaign):
    """Add/update a reddit link/campaign as an Adzerk Creative"""
    if hasattr(campaign, 'adzerk_creative_id'):
        az_creative = adzerk_api.Creative.get(campaign.adzerk_creative_id)
    else:
        az_creative = None

    title = '-'.join((link._fullname, campaign._fullname))
    d = {
        'Body': title,
        'ScriptBody': render_link(link, campaign),
        'AdvertiserId': g.az_selfserve_advertiser_id,
        'AdTypeId': g.az_selfserve_ad_type,
        'Alt': sanitize_text(link.title),
        'Url': quote(link.url),
        'IsHTMLJS': True,
        'IsSync': False,
        'IsDeleted': False,
        'IsActive': not campaign._deleted,
    }

    log_text = None
    if az_creative:
        changed = update_changed(az_creative, **d)
    else:
        d.update({'Title': title})
        try:
            az_creative = adzerk_api.Creative.create(**d)
        except:
            raise ValueError(d)

        campaign.adzerk_creative_id = az_creative.Id
        campaign._commit()
        log_text = 'created %s' % az_creative

    if log_text:
        PromotionLog.add(link, log_text)
        g.log.info(log_text)

    return az_creative

def update_flight(link, campaign):
    """Add/update a reddit campaign as an Adzerk Flight"""
    if hasattr(campaign, 'adzerk_flight_id'):
        az_flight = adzerk_api.Flight.get(campaign.adzerk_flight_id)
    else:
        az_flight = None

    az_campaign = adzerk_api.Campaign.get(link.adzerk_campaign_id)

    d = {
        'StartDate': date_to_adzerk(campaign.start_date),
        'EndDate': date_to_adzerk(campaign.end_date),
        'OptionType': 1, # 1: CPM, 2: Remainder
        'IsUnlimited': False,
        'IsFullSpeed': False,
        'Keywords': srname_to_keyword(campaign.sr_name),
        'CampaignId': az_campaign.Id,
        'PriorityId': g.az_selfserve_priority_id, # TODO: property of PromoCampaign
        'IsDeleted': False,
        'IsActive': not campaign._deleted,
        'IsFreqCap': None,
    }

    is_cpm = hasattr(campaign, 'cpm')
    if is_cpm:
        d.update({
            'Price': campaign.cpm / 100.,   # convert from cents to dollars
            'Impressions': campaign.impressions + ADZERK_IMPRESSION_BUMP,
            'GoalType': 1, # 1: Impressions
            'RateType': 2, # 2: CPM
        })
    else:
        d.update({
            'Price': campaign.bid,
            'Impressions': int(campaign.bid / campaign.ndays),
            'GoalType': 2, # 2: Percentage
            'RateType': 1, # 1: Flat
        })

    log_text = None
    if az_flight:
        changed = update_changed(az_flight, **d)
    else:
        d.update({'Name': campaign._fullname})
        az_flight = adzerk_api.Flight.create(**d)
        campaign.adzerk_flight_id = az_flight.Id
        campaign._commit()
        log_text = 'created %s' % az_campaign

    if log_text:
        PromotionLog.add(link, log_text)
        g.log.info(log_text)

    return az_flight

def update_cfmap(link, campaign):
    """Add/update a CreativeFlightMap.

    Map the the reddit link (adzerk Creative) and reddit campaign (adzerk
    Flight).

    """

    az_campaign = adzerk_api.Campaign.get(link.adzerk_campaign_id)
    az_creative = adzerk_api.Creative.get(campaign.adzerk_creative_id)
    az_flight = adzerk_api.Flight.get(campaign.adzerk_flight_id)

    if hasattr(campaign, 'adzerk_cfmap_id'):
        az_cfmap = adzerk_api.CreativeFlightMap.get(az_flight.Id,
                                                    campaign.adzerk_cfmap_id)
    else:
        az_cfmap = None

    d = {
        'SizeOverride': False,
        'CampaignId': az_campaign.Id,
        'PublisherAccountId': g.az_selfserve_advertiser_id,
        'Percentage': 100,  # Each flight only has one creative (what about autobalanced)
        'DistributionType': 2, # 2: Percentage, 1: Auto-Balanced, 0: ???
        'Iframe': False,
        'Creative': {'Id': az_creative.Id},
        'FlightId': az_flight.Id,
        'Impressions': 100, # Percentage
        'IsDeleted': False,
        'IsActive': not campaign._deleted,
    }

    log_text = None
    if az_cfmap:
        changed = update_changed(az_cfmap, **d)
    else:
        az_cfmap = adzerk_api.CreativeFlightMap.create(az_flight.Id, **d)
        campaign.adzerk_cfmap_id = az_cfmap.Id
        campaign._commit()
        log_text = 'created %s' % az_cfmap

    if log_text:
        PromotionLog.add(link, log_text)
        g.log.info(log_text)

    return az_cfmap

def update_adzerk(link, campaign):
    g.log.debug('queuing update_adzerk %s %s' % (link, campaign))
    msg = json.dumps({
        'action': 'update_adzerk',
        'link': link._fullname,
        'campaign': campaign._fullname,
    })
    amqp.add_item('adzerk_q', msg)

def _update_adzerk(link, campaign):
    msg = '%s updating/creating adzerk objects for %s - %s'
    g.log.info(msg % (datetime.datetime.now(g.tz), link, campaign))
    az_campaign = update_campaign(link)
    az_creative = update_creative(link, campaign)
    az_flight = update_flight(link, campaign)
    az_cfmap = update_cfmap(link, campaign)

def make_adzerk_promotions(offset=0):
    # campaign goes live if is_charged_transaction and is_accepted
    for link, campaign, weight in promote.accepted_campaigns(offset=offset):
        if (authorize.is_charged_transaction(campaign.trans_id, campaign._id)
            and promote.is_accepted(link)):
            update_adzerk(link, campaign)

@hooks.on('promote.make_daily_promotions')
def adzerk_live_promotions(offset=0):
    make_adzerk_promotions(offset)

@hooks.on('promote.charge_pending')
def adzerk_future_promotions(offset=1):
    make_adzerk_promotions(offset)

@hooks.on('promotion.void')
def deactivate_link(link):
    # deactivating the adzerk campaign will deactivate associated adzerk flights

    if not hasattr(link, 'adzerk_campaign_id'):
        # Link can get voided without having been sent to adzerk if its
        # start date is several days in the future
        return

    g.log.debug('queuing deactivate_link %s' % link)
    msg = json.dumps({
        'action': 'deactivate_link',
        'link': link._fullname,
    })
    amqp.add_item('adzerk_q', msg)

def _deactivate_link(link):
    g.log.debug('running deactivate_link %s' % link)
    az_campaign = update_campaign(link)
    az_campaign.IsActive = False
    az_campaign._send()
    PromotionLog.add(link, 'deactivated %s' % az_campaign)

@hooks.on('campaign.void')
def deactivate_campaign(link, campaign):
    if not (hasattr(link, 'adzerk_campaign_id') and
            hasattr(campaign, 'adzerk_flight_id')):
        # Campaign can get voided without having been sent to adzerk if its
        # start date is several days in the future
        return

    g.log.debug('queuing deactivate_campaign %s' % link)
    msg = json.dumps({
        'action': 'deactivate_campaign',
        'link': link._fullname,
        'campaign': campaign._fullname,
    })
    amqp.add_item('adzerk_q', msg)

def _deactivate_campaign(link, campaign):
    g.log.debug('running deactivate_campaign %s' % link)
    az_flight = update_flight(link, campaign)
    az_flight.IsActive = False
    az_flight._send()
    PromotionLog.add(link, 'deactivated %s' % az_flight)

def process_adzerk():
    @g.stats.amqp_processor('adzerk_q')
    def _handle_adzerk(msg):
        data = json.loads(msg.body)
        g.log.debug('data: %s' % data)
        action = data.get('action')
        if action == 'deactivate_link':
            link = Link._by_fullname(data['link'], data=True)
            _deactivate_link(link)
        elif action == 'deactivate_campaign':
            link = Link._by_fullname(data['link'], data=True)
            campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
            _deactivate_campaign(campaign)
        elif action == 'update_adzerk':
            link = Link._by_fullname(data['link'], data=True)
            campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
            _update_adzerk(link, campaign)
    amqp.consume_items('adzerk_q', _handle_adzerk, verbose=False)

AdzerkResponse = namedtuple('AdzerkResponse',
                    ['link', 'campaign', 'target', 'imp_pixel', 'click_url'])

def adzerk_request(keywords, num_placements=1, timeout=0.1):
    placements = []
    divs = ["div%s" % i for i in xrange(num_placements)]
    for div in divs:
        placement = {
          "divName": div,
          "networkId": g.az_selfserve_network_id,
          "siteId": g.az_selfserve_site_id,
          "adTypes": [g.az_selfserve_ad_type]
        }
        placements.append(placement)

    data = {
        "placements": placements,
        "keywords": [word.lower() for word in keywords],
    }

    url = 'http://engine.adzerk.net/api/v2'
    headers = {'content-type': 'application/json'}

    timer = g.stats.get_timer("adzerk_timer")
    timer.start()

    try:
        r = requests.post(url, data=json.dumps(data), headers=headers,
                          timeout=timeout)
    except requests.exceptions.Timeout:
        g.log.info('adzerk request timeout')
        return None

    timer.stop()

    response = json.loads(r.text)
    decisions = response['decisions']

    if not decisions:
        return None

    res = []
    for div in divs:
        decision = decisions[div]
        if not decision:
            continue

        imp_pixel = decision['impressionUrl']
        click_url = decision['clickUrl']
        body = json.loads(decision['contents'][0]['body'])
        campaign = body['campaign']
        link = body['link']
        target = body['target']
        res.append(AdzerkResponse(link, campaign, target, imp_pixel, click_url))
    return res

@add_controller
class AdzerkApiController(api.ApiController):
    def POST_request_promo(self):
        if not (c.site and c.site.name in g.cpm_beta_srs):
            return super(AdzerkApiController, self).POST_request_promo()

        srids = promote.srids_with_live_promos(c.user, c.site)
        if not srids:
            return

        if '' in srids:
            srnames = [Frontpage.name]
            srids.remove('')
        else:
            srnames = []

        srs = Subreddit._byID(srids, data=True, return_dict=False)
        srnames.extend(sr.name for sr in srs)

        # request multiple ads in case some are hidden by the builder due
        # to the user's hides/preferences
        response = adzerk_request(srnames, num_placements=g.az_selfserve_num_request)

        if not response:
            return

        res_by_campaign = {r.campaign: r for r in response}
        tuples = [promote.PromoTuple(r.link, 1., r.campaign) for r in response]
        builder = CampaignBuilder(tuples, wrap=default_thing_wrapper(),
                                  keep_fn=promote.is_promoted,
                                  num=1,
                                  skip=True)
        listing = LinkListing(builder, nextprev=False).listing()
        if listing.things:
            w = listing.things[0]
            r = res_by_campaign[w.campaign]
            w.adserver_imp_pixel = r.imp_pixel
            w.adserver_click_url = r.click_url
            return spaceCompress(w.render())
<commit_msg>adzerk_request: increase timeout.<commit_after>from collections import namedtuple
import datetime
import json
import string
from urllib import quote

import adzerk_api
from pylons import c, g
import requests

from r2.controllers import api, add_controller
from r2.lib import (
    amqp,
    authorize,
    organic,
    promote,
)
from r2.lib.db.sorts import epoch_seconds
from r2.lib.filters import spaceCompress, _force_utf8
from r2.lib.pages.things import default_thing_wrapper
from r2.lib.template_helpers import replace_render
from r2.lib.hooks import HookRegistrar
from r2.models import (
    Account,
    CampaignBuilder,
    Frontpage,
    Link,
    LinkListing,
    PromoCampaign,
    PromotionLog,
    Subreddit,
)

hooks = HookRegistrar()

ADZERK_IMPRESSION_BUMP = 500    # add extra impressions to the number we
                                # request from adzerk in case their count
                                # is lower than our internal traffic tracking

DELCHARS = ''.join(c for c in map(chr, range(256)) if not (c.isalnum() or c.isspace()))

def sanitize_text(text):
    return _force_utf8(text).translate(None, DELCHARS)

def date_to_adzerk(d):
    utc_date = d - promote.timezone_offset
    epoch_milliseconds = int(epoch_seconds(utc_date) * 1000)
    return '/Date(%s)/' % epoch_milliseconds

def srname_to_keyword(srname):
    return srname or Frontpage.name

def render_link(link, campaign):
    author = Account._byID(link.author_id, data=True)
    return json.dumps({
        'link': link._fullname,
        'campaign': campaign._fullname,
        'title': sanitize_text(link.title),
        'author': sanitize_text(author.name),
        'target': campaign.sr_name,
    })

def update_changed(adzerk_object, **d):
    changed = [(attr, val) for attr, val in d.iteritems()
                          if getattr(adzerk_object, attr) != val]
    if changed:
        for (attr, val) in changed:
            setattr(adzerk_object, attr, val)
        adzerk_object._send()
    return changed

def update_campaign(link):
    """Add/update a reddit link as an Adzerk Campaign"""
    if hasattr(link, 'adzerk_campaign_id'):
        az_campaign = adzerk_api.Campaign.get(link.adzerk_campaign_id)
    else:
        az_campaign = None

    d = {
        'AdvertiserId': g.az_selfserve_advertiser_id,
        'IsDeleted': False,
        'IsActive': not link._deleted,
        'Price': 0,
    }

    log_text = None
    if az_campaign:
        changed = update_changed(az_campaign, **d)
    else:
        d.update({
            'Name': link._fullname,
            'Flights': [],
            'StartDate': date_to_adzerk(datetime.datetime.now(g.tz)),
        })
        az_campaign = adzerk_api.Campaign.create(**d)
        link.adzerk_campaign_id = az_campaign.Id
        link._commit()
        log_text = 'created %s' % az_campaign

    if log_text:
        PromotionLog.add(link, log_text)
        g.log.info(log_text)

    return az_campaign

def update_creative(link, campaign):
    """Add/update a reddit link/campaign as an Adzerk Creative"""
    if hasattr(campaign, 'adzerk_creative_id'):
        az_creative = adzerk_api.Creative.get(campaign.adzerk_creative_id)
    else:
        az_creative = None

    title = '-'.join((link._fullname, campaign._fullname))
    d = {
        'Body': title,
        'ScriptBody': render_link(link, campaign),
        'AdvertiserId': g.az_selfserve_advertiser_id,
        'AdTypeId': g.az_selfserve_ad_type,
        'Alt': sanitize_text(link.title),
        'Url': quote(link.url),
        'IsHTMLJS': True,
        'IsSync': False,
        'IsDeleted': False,
        'IsActive': not campaign._deleted,
    }

    log_text = None
    if az_creative:
        changed = update_changed(az_creative, **d)
    else:
        d.update({'Title': title})
        try:
            az_creative = adzerk_api.Creative.create(**d)
        except:
            raise ValueError(d)

        campaign.adzerk_creative_id = az_creative.Id
        campaign._commit()
        log_text = 'created %s' % az_creative

    if log_text:
        PromotionLog.add(link, log_text)
        g.log.info(log_text)

    return az_creative

def update_flight(link, campaign):
    """Add/update a reddit campaign as an Adzerk Flight"""
    if hasattr(campaign, 'adzerk_flight_id'):
        az_flight = adzerk_api.Flight.get(campaign.adzerk_flight_id)
    else:
        az_flight = None

    az_campaign = adzerk_api.Campaign.get(link.adzerk_campaign_id)

    d = {
        'StartDate': date_to_adzerk(campaign.start_date),
        'EndDate': date_to_adzerk(campaign.end_date),
        'OptionType': 1, # 1: CPM, 2: Remainder
        'IsUnlimited': False,
        'IsFullSpeed': False,
        'Keywords': srname_to_keyword(campaign.sr_name),
        'CampaignId': az_campaign.Id,
        'PriorityId': g.az_selfserve_priority_id, # TODO: property of PromoCampaign
        'IsDeleted': False,
        'IsActive': not campaign._deleted,
        'IsFreqCap': None,
    }

    is_cpm = hasattr(campaign, 'cpm')
    if is_cpm:
        d.update({
            'Price': campaign.cpm / 100.,   # convert from cents to dollars
            'Impressions': campaign.impressions + ADZERK_IMPRESSION_BUMP,
            'GoalType': 1, # 1: Impressions
            'RateType': 2, # 2: CPM
        })
    else:
        d.update({
            'Price': campaign.bid,
            'Impressions': int(campaign.bid / campaign.ndays),
            'GoalType': 2, # 2: Percentage
            'RateType': 1, # 1: Flat
        })

    log_text = None
    if az_flight:
        changed = update_changed(az_flight, **d)
    else:
        d.update({'Name': campaign._fullname})
        az_flight = adzerk_api.Flight.create(**d)
        campaign.adzerk_flight_id = az_flight.Id
        campaign._commit()
        log_text = 'created %s' % az_campaign

    if log_text:
        PromotionLog.add(link, log_text)
        g.log.info(log_text)

    return az_flight

def update_cfmap(link, campaign):
    """Add/update a CreativeFlightMap.

    Map the the reddit link (adzerk Creative) and reddit campaign (adzerk
    Flight).

    """

    az_campaign = adzerk_api.Campaign.get(link.adzerk_campaign_id)
    az_creative = adzerk_api.Creative.get(campaign.adzerk_creative_id)
    az_flight = adzerk_api.Flight.get(campaign.adzerk_flight_id)

    if hasattr(campaign, 'adzerk_cfmap_id'):
        az_cfmap = adzerk_api.CreativeFlightMap.get(az_flight.Id,
                                                    campaign.adzerk_cfmap_id)
    else:
        az_cfmap = None

    d = {
        'SizeOverride': False,
        'CampaignId': az_campaign.Id,
        'PublisherAccountId': g.az_selfserve_advertiser_id,
        'Percentage': 100,  # Each flight only has one creative (what about autobalanced)
        'DistributionType': 2, # 2: Percentage, 1: Auto-Balanced, 0: ???
        'Iframe': False,
        'Creative': {'Id': az_creative.Id},
        'FlightId': az_flight.Id,
        'Impressions': 100, # Percentage
        'IsDeleted': False,
        'IsActive': not campaign._deleted,
    }

    log_text = None
    if az_cfmap:
        changed = update_changed(az_cfmap, **d)
    else:
        az_cfmap = adzerk_api.CreativeFlightMap.create(az_flight.Id, **d)
        campaign.adzerk_cfmap_id = az_cfmap.Id
        campaign._commit()
        log_text = 'created %s' % az_cfmap

    if log_text:
        PromotionLog.add(link, log_text)
        g.log.info(log_text)

    return az_cfmap

def update_adzerk(link, campaign):
    g.log.debug('queuing update_adzerk %s %s' % (link, campaign))
    msg = json.dumps({
        'action': 'update_adzerk',
        'link': link._fullname,
        'campaign': campaign._fullname,
    })
    amqp.add_item('adzerk_q', msg)

def _update_adzerk(link, campaign):
    msg = '%s updating/creating adzerk objects for %s - %s'
    g.log.info(msg % (datetime.datetime.now(g.tz), link, campaign))
    az_campaign = update_campaign(link)
    az_creative = update_creative(link, campaign)
    az_flight = update_flight(link, campaign)
    az_cfmap = update_cfmap(link, campaign)

def make_adzerk_promotions(offset=0):
    # campaign goes live if is_charged_transaction and is_accepted
    for link, campaign, weight in promote.accepted_campaigns(offset=offset):
        if (authorize.is_charged_transaction(campaign.trans_id, campaign._id)
            and promote.is_accepted(link)):
            update_adzerk(link, campaign)

@hooks.on('promote.make_daily_promotions')
def adzerk_live_promotions(offset=0):
    make_adzerk_promotions(offset)

@hooks.on('promote.charge_pending')
def adzerk_future_promotions(offset=1):
    make_adzerk_promotions(offset)

@hooks.on('promotion.void')
def deactivate_link(link):
    # deactivating the adzerk campaign will deactivate associated adzerk flights

    if not hasattr(link, 'adzerk_campaign_id'):
        # Link can get voided without having been sent to adzerk if its
        # start date is several days in the future
        return

    g.log.debug('queuing deactivate_link %s' % link)
    msg = json.dumps({
        'action': 'deactivate_link',
        'link': link._fullname,
    })
    amqp.add_item('adzerk_q', msg)

def _deactivate_link(link):
    g.log.debug('running deactivate_link %s' % link)
    az_campaign = update_campaign(link)
    az_campaign.IsActive = False
    az_campaign._send()
    PromotionLog.add(link, 'deactivated %s' % az_campaign)

@hooks.on('campaign.void')
def deactivate_campaign(link, campaign):
    if not (hasattr(link, 'adzerk_campaign_id') and
            hasattr(campaign, 'adzerk_flight_id')):
        # Campaign can get voided without having been sent to adzerk if its
        # start date is several days in the future
        return

    g.log.debug('queuing deactivate_campaign %s' % link)
    msg = json.dumps({
        'action': 'deactivate_campaign',
        'link': link._fullname,
        'campaign': campaign._fullname,
    })
    amqp.add_item('adzerk_q', msg)

def _deactivate_campaign(link, campaign):
    g.log.debug('running deactivate_campaign %s' % link)
    az_flight = update_flight(link, campaign)
    az_flight.IsActive = False
    az_flight._send()
    PromotionLog.add(link, 'deactivated %s' % az_flight)

def process_adzerk():
    @g.stats.amqp_processor('adzerk_q')
    def _handle_adzerk(msg):
        data = json.loads(msg.body)
        g.log.debug('data: %s' % data)
        action = data.get('action')
        if action == 'deactivate_link':
            link = Link._by_fullname(data['link'], data=True)
            _deactivate_link(link)
        elif action == 'deactivate_campaign':
            link = Link._by_fullname(data['link'], data=True)
            campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
            _deactivate_campaign(campaign)
        elif action == 'update_adzerk':
            link = Link._by_fullname(data['link'], data=True)
            campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
            _update_adzerk(link, campaign)
    amqp.consume_items('adzerk_q', _handle_adzerk, verbose=False)

AdzerkResponse = namedtuple('AdzerkResponse',
                    ['link', 'campaign', 'target', 'imp_pixel', 'click_url'])

def adzerk_request(keywords, num_placements=1, timeout=10):
    placements = []
    divs = ["div%s" % i for i in xrange(num_placements)]
    for div in divs:
        placement = {
          "divName": div,
          "networkId": g.az_selfserve_network_id,
          "siteId": g.az_selfserve_site_id,
          "adTypes": [g.az_selfserve_ad_type]
        }
        placements.append(placement)

    data = {
        "placements": placements,
        "keywords": [word.lower() for word in keywords],
    }

    url = 'http://engine.adzerk.net/api/v2'
    headers = {'content-type': 'application/json'}

    timer = g.stats.get_timer("adzerk_timer")
    timer.start()

    try:
        r = requests.post(url, data=json.dumps(data), headers=headers,
                          timeout=timeout)
    except requests.exceptions.Timeout:
        g.log.info('adzerk request timeout')
        return None

    timer.stop()

    response = json.loads(r.text)
    decisions = response['decisions']

    if not decisions:
        return None

    res = []
    for div in divs:
        decision = decisions[div]
        if not decision:
            continue

        imp_pixel = decision['impressionUrl']
        click_url = decision['clickUrl']
        body = json.loads(decision['contents'][0]['body'])
        campaign = body['campaign']
        link = body['link']
        target = body['target']
        res.append(AdzerkResponse(link, campaign, target, imp_pixel, click_url))
    return res

@add_controller
class AdzerkApiController(api.ApiController):
    def POST_request_promo(self):
        if not (c.site and c.site.name in g.cpm_beta_srs):
            return super(AdzerkApiController, self).POST_request_promo()

        srids = promote.srids_with_live_promos(c.user, c.site)
        if not srids:
            return

        if '' in srids:
            srnames = [Frontpage.name]
            srids.remove('')
        else:
            srnames = []

        srs = Subreddit._byID(srids, data=True, return_dict=False)
        srnames.extend(sr.name for sr in srs)

        # request multiple ads in case some are hidden by the builder due
        # to the user's hides/preferences
        response = adzerk_request(srnames, num_placements=g.az_selfserve_num_request)

        if not response:
            return

        res_by_campaign = {r.campaign: r for r in response}
        tuples = [promote.PromoTuple(r.link, 1., r.campaign) for r in response]
        builder = CampaignBuilder(tuples, wrap=default_thing_wrapper(),
                                  keep_fn=promote.is_promoted,
                                  num=1,
                                  skip=True)
        listing = LinkListing(builder, nextprev=False).listing()
        if listing.things:
            w = listing.things[0]
            r = res_by_campaign[w.campaign]
            w.adserver_imp_pixel = r.imp_pixel
            w.adserver_click_url = r.click_url
            return spaceCompress(w.render())
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2002 Google Inc. All Rights Reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Contains Flag class - information about single command-line flag."""

from functools import total_ordering
from gflags import _helpers
from gflags import argument_parser
from gflags import exceptions

@total_ordering
class Flag(object):
  """Information about a command-line flag.

  'Flag' objects define the following fields:
    .name - the name for this flag;
    .default - the default value for this flag;
    .default_as_str - default value as repr'd string, e.g., "'true'" (or None);
    .value - the most recent parsed value of this flag; set by Parse();
    .help - a help string or None if no help is available;
    .short_name - the single letter alias for this flag (or None);
    .boolean - if 'true', this flag does not accept arguments;
    .present - true if this flag was parsed from command line flags;
    .parser - an ArgumentParser object;
    .serializer - an ArgumentSerializer object;
    .allow_override - the flag may be redefined without raising an error;
    .allow_hide_cpp - the flag may be redefined despite hiding a C++ flag with
                      the same name;
    .using_default_value - the flag value has not been set by user;
    .allow_overwrite - the flag may be parsed more than once without raising
                       an error, the last set value will be used.

  The only public method of a 'Flag' object is Parse(), but it is
  typically only called by a 'FlagValues' object.  The Parse() method is
  a thin wrapper around the 'ArgumentParser' Parse() method.  The parsed
  value is saved in .value, and the .present attribute is updated.  If
  this flag was already present, a FlagsError is raised.

  Parse() is also called during __init__ to parse the default value and
  initialize the .value attribute.  This enables other python modules to
  safely use flags even if the __main__ module neglects to parse the
  command line arguments.  The .present attribute is cleared after
  __init__ parsing.  If the default value is set to None, then the
  __init__ parsing step is skipped and the .value attribute is
  initialized to None.

  Note: The default value is also presented to the user in the help
  string, so it is important that it be a legal value for this flag.
  """

  def __init__(self, parser, serializer, name, default, help_string,
               short_name=None, boolean=False, allow_override=False,
               allow_hide_cpp=False, allow_overwrite=True,
               parse_default=True):
    self.name = name

    if not help_string:
      help_string = '(no help available)'

    self.help = help_string
    self.short_name = short_name
    self.boolean = boolean
    self.present = 0
    self.parser = parser
    self.serializer = serializer
    self.allow_override = allow_override
    self.allow_hide_cpp = allow_hide_cpp
    self.allow_overwrite = allow_overwrite
    self.using_default_value = True
    self.value = None
    self.validators = []

    if parse_default:
      self.SetDefault(default)
    else:
      self.default = default

  def __hash__(self):
    return hash(id(self))

  def __eq__(self, other):
    return self is other

  def __lt__(self, other):
    if isinstance(other, Flag):
      return id(self) < id(other)
    return NotImplemented

  def __GetParsedValueAsString(self, value):
    """Get parsed flag value as string."""
    if value is None:
      return None
    if self.serializer:
      return repr(self.serializer.Serialize(value))
    if self.boolean:
      if value:
        return repr('true')
      else:
        return repr('false')
    return repr(_helpers.StrOrUnicode(value))

  def Parse(self, argument):
    """Parse string and set flag value.

    Args:
      argument: String, value to be parsed for flag.
    """
    if self.present and not self.allow_overwrite:
      raise exceptions.IllegalFlagValue(
          'flag --%s=%s: already defined as %s' % (
              self.name, argument, self.value))
    try:
      self.value = self.parser.Parse(argument)
    except ValueError as e:  # Recast ValueError as IllegalFlagValue.
      raise exceptions.IllegalFlagValue(
          'flag --%s=%s: %s' % (self.name, argument, e))
    self.present += 1

  def Unparse(self):
    if self.default is None:
      self.value = None
    else:
      self.present = 0
      self.Parse(self.default)
    self.using_default_value = True
    self.present = 0

  def Serialize(self):
    if self.value is None:
      return ''
    if self.boolean:
      if self.value:
        return '--%s' % self.name
      else:
        return '--no%s' % self.name
    else:
      if not self.serializer:
        raise exceptions.FlagsError(
            'Serializer not present for flag %s' % self.name)
      return '--%s=%s' % (self.name, self.serializer.Serialize(self.value))

  def SetDefault(self, value):
    """Changes the default value (and current value too) for this Flag."""
    # We can't allow a None override because it may end up not being
    # passed to C++ code when we're overriding C++ flags.  So we
    # cowardly bail out until someone fixes the semantics of trying to
    # pass None to a C++ flag.  See swig_flags.Init() for details on
    # this behavior.
    # TODO(olexiy): Users can directly call this method, bypassing all flags
    # validators (we don't have FlagValues here, so we can not check
    # validators).
    # The simplest solution I see is to make this method private.
    # Another approach would be to store reference to the corresponding
    # FlagValues with each flag, but this seems to be an overkill.
    if value is None and self.allow_override:
      raise exceptions.DuplicateFlagCannotPropagateNoneToSwig(self.name)

    self.default = value
    self.Unparse()
    self.default_as_str = self.__GetParsedValueAsString(self.value)

  def Type(self):
    """Get type of flag.

    NOTE: we use strings, and not the types.*Type constants because
    our flags can have more exotic types, e.g., 'comma separated list
    of strings', 'whitespace separated list of strings', etc.

    Returns:
      a string that describes the type of this Flag.
    """
    return self.parser.Type()

  def WriteInfoInXMLFormat(self, outfile, module_name, is_key=False, indent=''):
    """Writes common info about this flag, in XML format.

    This is information that is relevant to all flags (e.g., name,
    meaning, etc.).  If you defined a flag that has some other pieces of
    info, then please override _WriteCustomInfoInXMLFormat.

    Please do NOT override this method.

    Args:
      outfile: File object we write to.
      module_name: A string, the name of the module that defines this flag.
      is_key: A boolean, True iff this flag is key for main module.
      indent: A string that is prepended to each generated line.
    """
    outfile.write(indent + '<flag>\n')
    inner_indent = indent + '  '
    if is_key:
      _helpers.WriteSimpleXMLElement(outfile, 'key', 'yes', inner_indent)
    _helpers.WriteSimpleXMLElement(outfile, 'file', module_name, inner_indent)
    # Print flag features that are relevant for all flags.
    _helpers.WriteSimpleXMLElement(outfile, 'name', self.name, inner_indent)
    if self.short_name:
      _helpers.WriteSimpleXMLElement(outfile, 'short_name', self.short_name,
                                     inner_indent)
    if self.help:
      _helpers.WriteSimpleXMLElement(outfile, 'meaning', self.help,
                                     inner_indent)
    # The default flag value can either be represented as a string like on the
    # command line, or as a Python object.  We serialize this value in the
    # latter case in order to remain consistent.
    if self.serializer and not isinstance(self.default, str):
      if self.default is not None:
        default_serialized = self.serializer.Serialize(self.default)
      else:
        default_serialized = ''
    else:
      default_serialized = self.default
    _helpers.WriteSimpleXMLElement(outfile, 'default', default_serialized,
                                   inner_indent)
    _helpers.WriteSimpleXMLElement(outfile, 'current', self.value, inner_indent)
    _helpers.WriteSimpleXMLElement(outfile, 'type', self.Type(), inner_indent)
    # Print extra flag features this flag may have.
    self._WriteCustomInfoInXMLFormat(outfile, inner_indent)
    outfile.write(indent + '</flag>\n')

  def _WriteCustomInfoInXMLFormat(self, outfile, indent):
    """Writes extra info about this flag, in XML format.

    "Extra" means "not already printed by WriteInfoInXMLFormat above."

    Args:
      outfile: File object we write to.
      indent: A string that is prepended to each generated line.
    """
    # Usually, the parser knows the extra details about the flag, so
    # we just forward the call to it.
    self.parser.WriteCustomInfoInXMLFormat(outfile, indent)

class BooleanFlag(Flag):
  """Basic boolean flag.

  Boolean flags do not take any arguments, and their value is either
  True (1) or False (0).  The false value is specified on the command
  line by prepending the word 'no' to either the long or the short flag
  name.

  For example, if a Boolean flag was created whose long name was
  'update' and whose short name was 'x', then this flag could be
  explicitly unset through either --noupdate or --nox.
  """

  def __init__(self, name, default, help, short_name=None, **args):  # pylint: disable=redefined-builtin
    p = argument_parser.BooleanParser()
    Flag.__init__(self, p, None, name, default, help, short_name, 1, **args)

class EnumFlag(Flag):
  """Basic enum flag; its value can be any string from list of enum_values."""

  def __init__(self, name, default, help, enum_values=None,  # pylint: disable=redefined-builtin
               short_name=None, case_sensitive=True, **args):
    enum_values = enum_values or []
    p = argument_parser.EnumParser(enum_values, case_sensitive)
    g = argument_parser.ArgumentSerializer()
    Flag.__init__(self, p, g, name, default, help, short_name, **args)
    self.help = '<%s>: %s' % ('|'.join(enum_values), self.help)

  def _WriteCustomInfoInXMLFormat(self, outfile, indent):
    for enum_value in self.parser.enum_values:
      _helpers.WriteSimpleXMLElement(outfile, 'enum_value', enum_value, indent)

class MultiFlag(Flag):
  """A flag that can appear multiple time on the command-line.

  The value of such a flag is a list that contains the individual values
  from all the appearances of that flag on the command-line.

  See the __doc__ for Flag for most behavior of this class.  Only
  differences in behavior are described here:

    * The default value may be either a single value or a list of values.
      A single value is interpreted as the [value] singleton list.

    * The value of the flag is always a list, even if the option was
      only supplied once, and even if the default value is a single
      value
  """

  def __init__(self, *args, **kwargs):
    Flag.__init__(self, *args, **kwargs)
    self.help += ';\n    repeat this option to specify a list of values'

  def Parse(self, arguments):
    """Parses one or more arguments with the installed parser.

    Args:
      arguments: a single argument or a list of arguments (typically a
        list of default values); a single argument is converted
        internally into a list containing one item.
    """
    if not isinstance(arguments, list):
      # Default value may be a list of values.  Most other arguments
      # will not be, so convert them into a single-item list to make
      # processing simpler below.
      arguments = [arguments]

    if self.present:
      # keep a backup reference to list of previously supplied option values
      values = self.value
    else:
      # "erase" the defaults with an empty list
      values = []

    for item in arguments:
      # have Flag superclass parse argument, overwriting self.value reference
      Flag.Parse(self, item)  # also increments self.present
      values.append(self.value)

    # put list of option values back in the 'value' attribute
    self.value = values

  def Serialize(self):
    if not self.serializer:
      raise exceptions.FlagsError(
          'Serializer not present for flag %s' % self.name)
    if self.value is None:
      return ''

    s = ''

    multi_value = self.value

    for self.value in multi_value:
      if s: s += ' '
      s += Flag.Serialize(self)

    self.value = multi_value

    return s

  def Type(self):
    return 'multi ' + self.parser.Type()

  def _WriteCustomInfoInXMLFormat(self, outfile, indent):
    if hasattr(self.parser, 'enum_values'):
      for enum_value in self.parser.enum_values:
        _helpers.WriteSimpleXMLElement(outfile, 'enum_value',
                                       enum_value, indent)
<commit_msg>Allow to create a Python flag that will be taken over by C++ flag at InitGoogle if cpp_flags present.<commit_after>#!/usr/bin/env python
# Copyright 2002 Google Inc. All Rights Reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Contains Flag class - information about single command-line flag."""

from functools import total_ordering
from gflags import _helpers
from gflags import argument_parser
from gflags import exceptions

@total_ordering
class Flag(object):
  """Information about a command-line flag.

  'Flag' objects define the following fields:
    .name - the name for this flag;
    .default - the default value for this flag;
    .default_as_str - default value as repr'd string, e.g., "'true'" (or None);
    .value - the most recent parsed value of this flag; set by Parse();
    .help - a help string or None if no help is available;
    .short_name - the single letter alias for this flag (or None);
    .boolean - if 'true', this flag does not accept arguments;
    .present - true if this flag was parsed from command line flags;
    .parser - an ArgumentParser object;
    .serializer - an ArgumentSerializer object;
    .allow_override - the flag may be redefined without raising an error;
    .allow_override_cpp - the flag may be redefined in C++ without raising an
                          error, value "transfered" to C++;
    .allow_hide_cpp - the flag may be redefined despite hiding a C++ flag with
                      the same name;
    .using_default_value - the flag value has not been set by user;
    .allow_overwrite - the flag may be parsed more than once without raising
                       an error, the last set value will be used.

  The only public method of a 'Flag' object is Parse(), but it is
  typically only called by a 'FlagValues' object.  The Parse() method is
  a thin wrapper around the 'ArgumentParser' Parse() method.  The parsed
  value is saved in .value, and the .present attribute is updated.  If
  this flag was already present, a FlagsError is raised.

  Parse() is also called during __init__ to parse the default value and
  initialize the .value attribute.  This enables other python modules to
  safely use flags even if the __main__ module neglects to parse the
  command line arguments.  The .present attribute is cleared after
  __init__ parsing.  If the default value is set to None, then the
  __init__ parsing step is skipped and the .value attribute is
  initialized to None.

  Note: The default value is also presented to the user in the help
  string, so it is important that it be a legal value for this flag.
  """

  def __init__(self, parser, serializer, name, default, help_string,
               short_name=None, boolean=False, allow_override=False,
               allow_override_cpp=False, allow_hide_cpp=False,
               allow_overwrite=True, parse_default=True):
    self.name = name

    if not help_string:
      help_string = '(no help available)'

    self.help = help_string
    self.short_name = short_name
    self.boolean = boolean
    self.present = 0
    self.parser = parser
    self.serializer = serializer
    self.allow_override = allow_override
    self.allow_override_cpp = allow_override_cpp
    self.allow_hide_cpp = allow_hide_cpp
    self.allow_overwrite = allow_overwrite
    self.using_default_value = True
    self.value = None
    self.validators = []
    if allow_hide_cpp and allow_override_cpp:
      raise exceptions.FlagsError(
          "Can't have both allow_hide_cpp (means use Python flag) and "
          'allow_override_cpp (means use C++ flag after InitGoogle)')

    if parse_default:
      self.SetDefault(default)
    else:
      self.default = default

  def __hash__(self):
    return hash(id(self))

  def __eq__(self, other):
    return self is other

  def __lt__(self, other):
    if isinstance(other, Flag):
      return id(self) < id(other)
    return NotImplemented

  def __GetParsedValueAsString(self, value):
    """Get parsed flag value as string."""
    if value is None:
      return None
    if self.serializer:
      return repr(self.serializer.Serialize(value))
    if self.boolean:
      if value:
        return repr('true')
      else:
        return repr('false')
    return repr(_helpers.StrOrUnicode(value))

  def Parse(self, argument):
    """Parse string and set flag value.

    Args:
      argument: String, value to be parsed for flag.
    """
    if self.present and not self.allow_overwrite:
      raise exceptions.IllegalFlagValue(
          'flag --%s=%s: already defined as %s' % (
              self.name, argument, self.value))
    try:
      self.value = self.parser.Parse(argument)
    except ValueError as e:  # Recast ValueError as IllegalFlagValue.
      raise exceptions.IllegalFlagValue(
          'flag --%s=%s: %s' % (self.name, argument, e))
    self.present += 1

  def Unparse(self):
    if self.default is None:
      self.value = None
    else:
      self.present = 0
      self.Parse(self.default)
    self.using_default_value = True
    self.present = 0

  def Serialize(self):
    if self.value is None:
      return ''
    if self.boolean:
      if self.value:
        return '--%s' % self.name
      else:
        return '--no%s' % self.name
    else:
      if not self.serializer:
        raise exceptions.FlagsError(
            'Serializer not present for flag %s' % self.name)
      return '--%s=%s' % (self.name, self.serializer.Serialize(self.value))

  def SetDefault(self, value):
    """Changes the default value (and current value too) for this Flag."""
    # We can't allow a None override because it may end up not being
    # passed to C++ code when we're overriding C++ flags.  So we
    # cowardly bail out until someone fixes the semantics of trying to
    # pass None to a C++ flag.  See swig_flags.Init() for details on
    # this behavior.
    # TODO(olexiy): Users can directly call this method, bypassing all flags
    # validators (we don't have FlagValues here, so we can not check
    # validators).
    # The simplest solution I see is to make this method private.
    # Another approach would be to store reference to the corresponding
    # FlagValues with each flag, but this seems to be an overkill.
    if value is None and self.allow_override:
      raise exceptions.DuplicateFlagCannotPropagateNoneToSwig(self.name)

    self.default = value
    self.Unparse()
    self.default_as_str = self.__GetParsedValueAsString(self.value)

  def Type(self):
    """Get type of flag.

    NOTE: we use strings, and not the types.*Type constants because
    our flags can have more exotic types, e.g., 'comma separated list
    of strings', 'whitespace separated list of strings', etc.

    Returns:
      a string that describes the type of this Flag.
    """
    return self.parser.Type()

  def WriteInfoInXMLFormat(self, outfile, module_name, is_key=False, indent=''):
    """Writes common info about this flag, in XML format.

    This is information that is relevant to all flags (e.g., name,
    meaning, etc.).  If you defined a flag that has some other pieces of
    info, then please override _WriteCustomInfoInXMLFormat.

    Please do NOT override this method.

    Args:
      outfile: File object we write to.
      module_name: A string, the name of the module that defines this flag.
      is_key: A boolean, True iff this flag is key for main module.
      indent: A string that is prepended to each generated line.
    """
    outfile.write(indent + '<flag>\n')
    inner_indent = indent + '  '
    if is_key:
      _helpers.WriteSimpleXMLElement(outfile, 'key', 'yes', inner_indent)
    _helpers.WriteSimpleXMLElement(outfile, 'file', module_name, inner_indent)
    # Print flag features that are relevant for all flags.
    _helpers.WriteSimpleXMLElement(outfile, 'name', self.name, inner_indent)
    if self.short_name:
      _helpers.WriteSimpleXMLElement(outfile, 'short_name', self.short_name,
                                     inner_indent)
    if self.help:
      _helpers.WriteSimpleXMLElement(outfile, 'meaning', self.help,
                                     inner_indent)
    # The default flag value can either be represented as a string like on the
    # command line, or as a Python object.  We serialize this value in the
    # latter case in order to remain consistent.
    if self.serializer and not isinstance(self.default, str):
      if self.default is not None:
        default_serialized = self.serializer.Serialize(self.default)
      else:
        default_serialized = ''
    else:
      default_serialized = self.default
    _helpers.WriteSimpleXMLElement(outfile, 'default', default_serialized,
                                   inner_indent)
    _helpers.WriteSimpleXMLElement(outfile, 'current', self.value, inner_indent)
    _helpers.WriteSimpleXMLElement(outfile, 'type', self.Type(), inner_indent)
    # Print extra flag features this flag may have.
    self._WriteCustomInfoInXMLFormat(outfile, inner_indent)
    outfile.write(indent + '</flag>\n')

  def _WriteCustomInfoInXMLFormat(self, outfile, indent):
    """Writes extra info about this flag, in XML format.

    "Extra" means "not already printed by WriteInfoInXMLFormat above."

    Args:
      outfile: File object we write to.
      indent: A string that is prepended to each generated line.
    """
    # Usually, the parser knows the extra details about the flag, so
    # we just forward the call to it.
    self.parser.WriteCustomInfoInXMLFormat(outfile, indent)

class BooleanFlag(Flag):
  """Basic boolean flag.

  Boolean flags do not take any arguments, and their value is either
  True (1) or False (0).  The false value is specified on the command
  line by prepending the word 'no' to either the long or the short flag
  name.

  For example, if a Boolean flag was created whose long name was
  'update' and whose short name was 'x', then this flag could be
  explicitly unset through either --noupdate or --nox.
  """

  def __init__(self, name, default, help, short_name=None, **args):  # pylint: disable=redefined-builtin
    p = argument_parser.BooleanParser()
    Flag.__init__(self, p, None, name, default, help, short_name, 1, **args)

class EnumFlag(Flag):
  """Basic enum flag; its value can be any string from list of enum_values."""

  def __init__(self, name, default, help, enum_values=None,  # pylint: disable=redefined-builtin
               short_name=None, case_sensitive=True, **args):
    enum_values = enum_values or []
    p = argument_parser.EnumParser(enum_values, case_sensitive)
    g = argument_parser.ArgumentSerializer()
    Flag.__init__(self, p, g, name, default, help, short_name, **args)
    self.help = '<%s>: %s' % ('|'.join(enum_values), self.help)

  def _WriteCustomInfoInXMLFormat(self, outfile, indent):
    for enum_value in self.parser.enum_values:
      _helpers.WriteSimpleXMLElement(outfile, 'enum_value', enum_value, indent)

class MultiFlag(Flag):
  """A flag that can appear multiple time on the command-line.

  The value of such a flag is a list that contains the individual values
  from all the appearances of that flag on the command-line.

  See the __doc__ for Flag for most behavior of this class.  Only
  differences in behavior are described here:

    * The default value may be either a single value or a list of values.
      A single value is interpreted as the [value] singleton list.

    * The value of the flag is always a list, even if the option was
      only supplied once, and even if the default value is a single
      value
  """

  def __init__(self, *args, **kwargs):
    Flag.__init__(self, *args, **kwargs)
    self.help += ';\n    repeat this option to specify a list of values'

  def Parse(self, arguments):
    """Parses one or more arguments with the installed parser.

    Args:
      arguments: a single argument or a list of arguments (typically a
        list of default values); a single argument is converted
        internally into a list containing one item.
    """
    if not isinstance(arguments, list):
      # Default value may be a list of values.  Most other arguments
      # will not be, so convert them into a single-item list to make
      # processing simpler below.
      arguments = [arguments]

    if self.present:
      # keep a backup reference to list of previously supplied option values
      values = self.value
    else:
      # "erase" the defaults with an empty list
      values = []

    for item in arguments:
      # have Flag superclass parse argument, overwriting self.value reference
      Flag.Parse(self, item)  # also increments self.present
      values.append(self.value)

    # put list of option values back in the 'value' attribute
    self.value = values

  def Serialize(self):
    if not self.serializer:
      raise exceptions.FlagsError(
          'Serializer not present for flag %s' % self.name)
    if self.value is None:
      return ''

    s = ''

    multi_value = self.value

    for self.value in multi_value:
      if s: s += ' '
      s += Flag.Serialize(self)

    self.value = multi_value

    return s

  def Type(self):
    return 'multi ' + self.parser.Type()

  def _WriteCustomInfoInXMLFormat(self, outfile, indent):
    if hasattr(self.parser, 'enum_values'):
      for enum_value in self.parser.enum_values:
        _helpers.WriteSimpleXMLElement(outfile, 'enum_value',
                                       enum_value, indent)
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from django.db import models

# Create your models here.
class Certificate(models.Model):
    id = models.IntegerField(default=0, primary_key=True)
    ssl_certificate = models.CharField(max_length=255)
    ssl_key = models.CharField(max_length=200)
    domain = models.CharField(max_length=200)
    date_registered = models.DateField(auto_now=True)
    expiration_date = models.DateField()
    last_updated = models.DateField(auto_now=True)
<commit_msg>Update apps/certificate/models.py<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from flask.ext.restful import reqparse, Resource
import flask

from server import api
from server import repository as repo
from server import authentication as auth

parser = reqparse.RequestParser()
parser.add_argument('name',
    type=unicode,
    location=["json", "values"],
    required=True)

parser.add_argument('brewery',
    type=unicode,
    location=["json", "values"],
    required=True)

parser.add_argument('count',
    type=int,
    location=["json", "values"],
    required=False,
    default=1)

parser.add_argument('vintage',
    type=int,
    location=["json", "values"],
    required=True)

class Beer(Resource):

    @auth.requires_auth
    def get(self, beer_id):
        user = flask.g.user["username"]
        return repo.get_beer(user, beer_id)

    @auth.requires_auth
    def delete(self, beer_id):
        user = flask.g.user["username"]
        repo.delete_beer(user, beer_id)
        return '', 204

    @auth.requires_auth
    def put(self, beer_id):
        user = flask.g.user["username"]
        beer = repo.update_beer(user, beer_id, parser.parse_args())
        return beer, 201

class BeerList(Resource):

    @auth.requires_auth
    def get(self):
        user = flask.g.user["username"]
        return repo.list_beers(user)

    @auth.requires_auth
    def post(self):
        beer = parser.parse_args()
        user = flask.g.user["username"]
        saved_beer = repo.add_beer(user, beer)
        return saved_beer, 201

api.add_resource(BeerList, '/api/beers')
api.add_resource(Beer,     '/api/beers/<string:beer_id>')
<commit_msg>Explicit return status on GET calls<commit_after># -*- coding: utf-8 -*-

from flask.ext.restful import reqparse, Resource
import flask

from server import api
from server import repository as repo
from server import authentication as auth

parser = reqparse.RequestParser()
parser.add_argument('name',
    type=unicode,
    location=["json", "values"],
    required=True)

parser.add_argument('brewery',
    type=unicode,
    location=["json", "values"],
    required=True)

parser.add_argument('count',
    type=int,
    location=["json", "values"],
    required=False,
    default=1)

parser.add_argument('vintage',
    type=int,
    location=["json", "values"],
    required=True)

class Beer(Resource):

    @auth.requires_auth
    def get(self, beer_id):
        user = flask.g.user["username"]
        return repo.get_beer(user, beer_id), 200

    @auth.requires_auth
    def delete(self, beer_id):
        user = flask.g.user["username"]
        repo.delete_beer(user, beer_id)
        return '', 204

    @auth.requires_auth
    def put(self, beer_id):
        user = flask.g.user["username"]
        beer = repo.update_beer(user, beer_id, parser.parse_args())
        return beer, 201

class BeerList(Resource):

    @auth.requires_auth
    def get(self):
        user = flask.g.user["username"]
        return repo.list_beers(user), 200

    @auth.requires_auth
    def post(self):
        beer = parser.parse_args()
        user = flask.g.user["username"]
        saved_beer = repo.add_beer(user, beer)
        return saved_beer, 201

api.add_resource(BeerList, '/api/beers')
api.add_resource(Beer,     '/api/beers/<string:beer_id>')
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os
import shutil
import sys
import re
from time import sleep
from bs4 import BeautifulSoup as bs
#import psutil
import requests
import subprocess
import signal

__doc__ = 'Script has to be run relative to the Plex directory and as running user or sudo'
LINK = 'https://aur.archlinux.org/packages/plex-media-server-plexpass/'
SLEEP = 10

def download_file(url):
    local_filename = '/tmp/' + url.split('/')[-1]
    if os.path.exists(local_filename):
        print(local_filename + "Already exists")
        return local_filename
    # NOTE the stream=True parameter
    r = requests.get(url, stream=True)
    with open(local_filename, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
                # f.flush() commented by recommendation from J.F.Sebastian
    return local_filename

def extract(f):
    try:
        command = 'rpm2cpio /tmp/{} | cpio -idmv'.format(f)
        print("Running " + command)
        subprocess.check_output(command, shell=True)
    except subprocess.CalledProcessError as err:
        print(str(err))
        sys.exit(1)

def kill_plex(process_gid):
    """ Retrieves the process group of Plex and sends TERM signal """
    os.killpg(int(process_gid), signal.SIGTERM)
    print("SIGTERM sent to Plex process group ID")
    print("Waiting {} seconds".format(SLEEP))

def rename(ff):
    f = '.'.join(ff.split('.')[:-2])
    print("Renaming relative dir usr/lib/plexmediaserver to " + f)
    try:
        os.rename('usr/lib/plexmediaserver', f)
    except OSError as err:
        print(str(err))
        sys.exit(2)

def symlink(f):
    """ symlinks new version to Plex """
    file_name = '.'.join(f.split('.')[:-2])
    link_name = 'Plex'
    try:
        os.remove(link_name)
    except FileNotFoundError as err:
        print(str(err))
    print("Symlinking {} to {}".format(file_name, link_name))
    os.symlink(file_name, link_name)

def remove():
    """ Removes relative files extracted by cpio"""
    print('removing usr')
    shutil.rmtree('usr')
    print('removing etc')
    shutil.rmtree('etc')
    print('removing lib')
    shutil.rmtree('lib')

def extract_link():
    """ extracts link from AUR page """
    html = requests.get(LINK).content
    soup = bs(html, 'html.parser')
    for link in soup.find_all('a'):
        match = re.search(r'^http.*x86_64.*rpm$', link.get('href'))
        if match:
            url = match.group()
            file_name = url.split('/')[-1]
            if check_exists(file_name):
                print("Downloading to /tmp/" + file_name)
                download_file(url)
    return file_name

def check_exists(f):
    """ check if plexmediaserver-1.4.3.3433-03e4cfa35 exists """
    _f = '.'.join(f.split('.')[:-2])
    print("Checking if {} exists".format(_f))
    if os.path.exists(_f):
        print("Exiting dir already exists")
        sys.exit(3)
    else:
        return True

def get_plex_pgid():
    # psutil has to many issues depending on version used switch to pgrep
    #	and has C based extensions which is a pain sometimes
    #"""
    # try:
    #     for proc in psutil.process_iter():
    #         if proc.name == "Plex Media Server":
    #             process_gid = os.getpgid(int(proc.pid))
    #             return process_gid
    # except psutil.ZombieProcess:
    #     pass
    try:
        pid = subprocess.check_output("pgrep -f 'Plex Media Server$'", shell=True)
        process_gid = os.getpgid(int(pid))
        return process_gid
    except subprocess.CalledProcessError as err:
        print(str(err))

def main():
    file_name = extract_link()
    extract(file_name)
    rename(file_name)
    remove()

    while True:
        process_gid = get_plex_pgid()
        if process_gid:
            kill_plex(process_gid)
            sleep(SLEEP)
        else:
            symlink(file_name)
            print("No Plex pid quitting")
            break

if __name__ == '__main__':
    main()
<commit_msg>remove rpm file when done<commit_after>#!/usr/bin/env python3

import os
import shutil
import sys
import re
from time import sleep
from bs4 import BeautifulSoup as bs
#import psutil
import requests
import subprocess
import signal

__doc__ = 'Script has to be run relative to the Plex directory and as running user or sudo'
LINK = 'https://aur.archlinux.org/packages/plex-media-server-plexpass/'
SLEEP = 10
DDIR = '/tmp'

def download_file(url):
    local_filename = DDIR + '/' + url.split('/')[-1]
    if os.path.exists(local_filename):
        print(local_filename + "Already exists")
        return local_filename
    # NOTE the stream=True parameter
    r = requests.get(url, stream=True)
    with open(local_filename, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
                # f.flush() commented by recommendation from J.F.Sebastian
    return local_filename

def extract(f):
    try:
        command = 'rpm2cpio /tmp/{} | cpio -idmv'.format(f)
        print("Running " + command)
        subprocess.check_output(command, shell=True)
    except subprocess.CalledProcessError as err:
        print(str(err))
        sys.exit(1)

def kill_plex(process_gid):
    """ Retrieves the process group of Plex and sends TERM signal """
    os.killpg(int(process_gid), signal.SIGTERM)
    print("SIGTERM sent to Plex process group ID")
    print("Waiting {} seconds".format(SLEEP))

def rename(ff):
    f = '.'.join(ff.split('.')[:-2])
    print("Renaming relative dir usr/lib/plexmediaserver to " + f)
    try:
        os.rename('usr/lib/plexmediaserver', f)
    except OSError as err:
        print(str(err))
        sys.exit(2)

def symlink(f):
    """ symlinks new version to Plex """
    file_name = '.'.join(f.split('.')[:-2])
    link_name = 'Plex'
    try:
        os.remove(link_name)
    except FileNotFoundError as err:
        print(str(err))
    print("Symlinking {} to {}".format(file_name, link_name))
    os.symlink(file_name, link_name)

def remove(f=None):
    """ Removes relative files extracted by cpio"""
    if f:
        os.remove(DDIR + '/' + f)
    else:
        print('removing usr')
        shutil.rmtree('usr')
        print('removing etc')
        shutil.rmtree('etc')
        print('removing lib')
        shutil.rmtree('lib')

def extract_link():
    """ extracts link from AUR page """
    html = requests.get(LINK).content
    soup = bs(html, 'html.parser')
    for link in soup.find_all('a'):
        match = re.search(r'^http.*x86_64.*rpm$', link.get('href'))
        if match:
            url = match.group()
            file_name = url.split('/')[-1]
            if check_exists(file_name):
                print("Downloading to /tmp/" + file_name)
                download_file(url)
    return file_name

def check_exists(f):
    """ check if plexmediaserver-1.4.3.3433-03e4cfa35 exists """
    _f = '.'.join(f.split('.')[:-2])
    print("Checking if {} exists".format(_f))
    if os.path.exists(_f):
        print("Exiting dir already exists")
        sys.exit(3)
    else:
        return True

def get_plex_pgid():
    # psutil has to many issues depending on version used switch to pgrep
    #	and has C based extensions which is a pain sometimes
    #"""
    # try:
    #     for proc in psutil.process_iter():
    #         if proc.name == "Plex Media Server":
    #             process_gid = os.getpgid(int(proc.pid))
    #             return process_gid
    # except psutil.ZombieProcess:
    #     pass
    try:
        pid = subprocess.check_output("pgrep -f 'Plex Media Server$'", shell=True)
        process_gid = os.getpgid(int(pid))
        return process_gid
    except subprocess.CalledProcessError as err:
        print(str(err))

def main():
    file_name = extract_link()
    extract(file_name)
    rename(file_name)
    remove()

    while True:
        process_gid = get_plex_pgid()
        if process_gid:
            kill_plex(process_gid)
            sleep(SLEEP)
        else:
            symlink(file_name)
            print("No Plex pid quitting")
            print("Removing %s in 10 seconds" % file_name)
            #remove rpm file
            sleep(SLEEP)
            remove(file_name)
            break

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import json

import requests
from django.utils import timezone

from apps.contribution.models import Repository, RepositoryLanguage
from apps.mommy import schedule
from apps.mommy.registry import Task

git_domain = "https://api.github.com"

class UpdateRepositories(Task):

    @staticmethod
    def run():
        # Load new data
        fresh = UpdateRepositories.get_git_repositories()
        for repo in fresh:
            fresh_repo = Repository(
                id=int(repo['id']),
                name=repo['name'],
                description=repo['description'],
                updated_at=timezone.datetime.strptime(repo['updated_at'], "%Y-%m-%dT%H:%M:%SZ"),
                url=repo['url']
            )

            # If repository exists, only update data
            if Repository.objects.filter(id=fresh_repo.id).exists():
                stored_repo = Repository.objects.get(id=fresh_repo.id)
                repo_languages = UpdateRepositories.get_repository_languages(stored_repo.url)
                UpdateRepositories.update_repository(stored_repo, fresh_repo, repo_languages)

            # else: repository does not exist
            else:
                repo_languages = UpdateRepositories.get_repository_languages(fresh_repo.url)
                UpdateRepositories.new_repository(fresh_repo, repo_languages)

    @staticmethod
    def update_repository(stored_repo, fresh_repo, repo_languages):
        stored_repo.name = fresh_repo.name
        stored_repo.description = fresh_repo.description
        stored_repo.updated_at = fresh_repo.updated_at
        stored_repo.url = fresh_repo.url
        stored_repo.save()

        # Update languages if they exist, and add if not
        for language in repo_languages:
            if RepositoryLanguage.objects.filter(type=language, repository=stored_repo).exists():
                stored_language = RepositoryLanguage.objects.get(type=language, repository=stored_repo)
                stored_language.size = repo_languages[language]
                stored_language.save()
            else:
                new_language = RepositoryLanguage(
                    type=language,
                    size=(int(repo_languages[language])),
                    repository=stored_repo
                )
                new_language.save()

    @staticmethod
    def new_repository(new_repo, new_languages):
        # Filter out repositories with inactivity past 2 years (365 days * 2)
        if new_repo.updated_at > (timezone.now() - timezone.timedelta(days=730)).replace(tzinfo=None):
            new_repo = Repository(
                id=new_repo.id,
                name=new_repo.name,
                description=new_repo.description,
                updated_at=new_repo.updated_at,
                url=new_repo.url
            )
            new_repo.save()

            # Add repository languages
            for language in new_languages:
                new_language = RepositoryLanguage(
                    type=language,
                    size=int(new_languages[language]),
                    repository=new_repo
                )
                new_language.save()

    @staticmethod
    def get_git_repositories():
        url = git_domain + "/users/dotkom/repos"
        r = requests.get(url)
        data = json.loads(r.text)
        return data

    @staticmethod
    def get_repository_languages(url):
        r = requests.get(url + "/languages")
        data = json.loads(r.text)
        return data

schedule.register(UpdateRepositories, day_of_week="mon-sun", hour=3, minute=0)
<commit_msg>make the intial object timezone aware as its parsed from string<commit_after>import json

import requests
from django.utils import timezone
from pytz import timezone as tz

from apps.contribution.models import Repository, RepositoryLanguage
from apps.mommy import schedule
from apps.mommy.registry import Task

git_domain = "https://api.github.com"

class UpdateRepositories(Task):

    @staticmethod
    def run():
        # Load new data
        fresh = UpdateRepositories.get_git_repositories()
        localtz = tz('Europe/Oslo')
        for repo in fresh:
            fresh_repo = Repository(
                id=int(repo['id']),
                name=repo['name'],
                description=repo['description'],
                updated_at=localtz.localize(timezone.datetime.strptime(repo['updated_at'], "%Y-%m-%dT%H:%M:%SZ")),
                url=repo['url']
            )

            # If repository exists, only update data
            if Repository.objects.filter(id=fresh_repo.id).exists():
                stored_repo = Repository.objects.get(id=fresh_repo.id)
                repo_languages = UpdateRepositories.get_repository_languages(stored_repo.url)
                UpdateRepositories.update_repository(stored_repo, fresh_repo, repo_languages)

            # else: repository does not exist
            else:
                repo_languages = UpdateRepositories.get_repository_languages(fresh_repo.url)
                UpdateRepositories.new_repository(fresh_repo, repo_languages)

    @staticmethod
    def update_repository(stored_repo, fresh_repo, repo_languages):
        stored_repo.name = fresh_repo.name
        stored_repo.description = fresh_repo.description
        stored_repo.updated_at = fresh_repo.updated_at
        stored_repo.url = fresh_repo.url
        stored_repo.save()

        # Update languages if they exist, and add if not
        for language in repo_languages:
            if RepositoryLanguage.objects.filter(type=language, repository=stored_repo).exists():
                stored_language = RepositoryLanguage.objects.get(type=language, repository=stored_repo)
                stored_language.size = repo_languages[language]
                stored_language.save()
            else:
                new_language = RepositoryLanguage(
                    type=language,
                    size=(int(repo_languages[language])),
                    repository=stored_repo
                )
                new_language.save()

    @staticmethod
    def new_repository(new_repo, new_languages):
        # Filter out repositories with inactivity past 2 years (365 days * 2)
        if new_repo.updated_at > timezone.now() - timezone.timedelta(days=730):
            new_repo = Repository(
                id=new_repo.id,
                name=new_repo.name,
                description=new_repo.description,
                updated_at=new_repo.updated_at,
                url=new_repo.url
            )
            new_repo.save()

            # Add repository languages
            for language in new_languages:
                new_language = RepositoryLanguage(
                    type=language,
                    size=int(new_languages[language]),
                    repository=new_repo
                )
                new_language.save()

    @staticmethod
    def get_git_repositories():
        url = git_domain + "/users/dotkom/repos"
        r = requests.get(url)
        data = json.loads(r.text)
        return data

    @staticmethod
    def get_repository_languages(url):
        r = requests.get(url + "/languages")
        data = json.loads(r.text)
        return data

schedule.register(UpdateRepositories, day_of_week="mon-sun", hour=16, minute=36)
<|endoftext|>


<commit_before>from pliers.extractors.base import Extractor, merge_results
from pliers.transformers import get_transformer
from itertools import chain
from pliers.utils import listify, flatten
from six import string_types
from collections import OrderedDict

class Node(object):

    ''' A graph node/vertex. Represents a single transformer, optionally with
    references to children.
    Args:
        name (str): Name of the node
        transformer (Transformer): the Transformer instance at this node
    '''

    def __init__(self, name, transformer):
        self.name = name
        self.children = []
        if isinstance(transformer, string_types):
            transformer = get_transformer(transformer)
        self.transformer = transformer

    def collect(self, stim):
        if hasattr(self, 'transformer') and self.transformer is not None:
            if isinstance(self.transformer, Extractor):
                return listify(self.transformer.transform(stim))
            stim = self.transformer.transform(stim)
        return list(chain(*[c.collect(stim) for c in self.children]))

    def add_child(self, node):
        ''' Append a child to the list of children. '''
        self.children.append(node)

    def is_leaf(self):
        return len(self.children) > 0

class Graph(Node):

    def __init__(self, nodes=None):

        self.nodes = OrderedDict()
        self.children = []
        if nodes is not None:
            self.add_children(nodes)

    def add_branch(self, nodes, parent=None):
        for n in nodes:
            node_args = self._parse_node_args(n)
            node = self.add_node(parent=parent, return_node=True, **node_args)
            parent = node

    def add_children(self, nodes, parent=None):
        for n in nodes:
            node_args = self._parse_node_args(n)
            self.add_node(parent=parent, **node_args)

    @staticmethod
    def _parse_node_args(node):

        if isinstance(node, dict):
            return node

        kwargs = {}

        if isinstance(node, (list, tuple)):
            kwargs['transformer'] = node[0]
            if len(node) > 1:
                kwargs['children'] = node[1]
            if len(node) > 2:
                kwargs['name'] = node[2]
        else:
            kwargs['transformer'] = node

        return kwargs

    def add_node(self, transformer, name=None, children=None, parent=None,
                 return_node=False):

        if name is None:
            name = id(transformer)

        node = Node(name, transformer)
        self.nodes[name] = node

        parent = self if parent is None else self.nodes[parent.name]
        parent.add_child(node)

        if children is not None:
            self.add_children(children, parent=node)

        if return_node:
            return node

    def extract(self, stims, merge=True):
        results = list(flatten(self.collect(stims)))
        return merge_results(results) if merge else results
<commit_msg>graph caches intermediate results when they're generators<commit_after>from pliers.extractors.base import Extractor, merge_results
from pliers.transformers import get_transformer
from itertools import chain
from pliers.utils import listify, flatten
from six import string_types
from collections import OrderedDict
from types import GeneratorType

class Node(object):

    ''' A graph node/vertex. Represents a single transformer, optionally with
    references to children.
    Args:
        name (str): Name of the node
        transformer (Transformer): the Transformer instance at this node
    '''

    def __init__(self, name, transformer):
        self.name = name
        self.children = []
        if isinstance(transformer, string_types):
            transformer = get_transformer(transformer)
        self.transformer = transformer

    def collect(self, stim):
        if hasattr(self, 'transformer') and self.transformer is not None:
            result = listify(self.transformer.transform(stim))
            if isinstance(self.transformer, Extractor):
                return result
            stim = result
            # If result is a generator, the first child will destroy the
            # iterable, so cache via list conversion
            if len(self.children) > 1 and isinstance(stim, GeneratorType):
                stim = list(stim)
        return list(chain(*[c.collect(stim) for c in self.children]))

    def add_child(self, node):
        ''' Append a child to the list of children. '''
        self.children.append(node)

    def is_leaf(self):
        return len(self.children)

class Graph(Node):

    def __init__(self, nodes=None):

        self.nodes = OrderedDict()
        self.children = []
        if nodes is not None:
            self.add_children(nodes)

    def add_branch(self, nodes, parent=None):
        for n in nodes:
            node_args = self._parse_node_args(n)
            node = self.add_node(parent=parent, return_node=True, **node_args)
            parent = node

    def add_children(self, nodes, parent=None):
        for n in nodes:
            node_args = self._parse_node_args(n)
            self.add_node(parent=parent, **node_args)

    @staticmethod
    def _parse_node_args(node):

        if isinstance(node, dict):
            return node

        kwargs = {}

        if isinstance(node, (list, tuple)):
            kwargs['transformer'] = node[0]
            if len(node) > 1:
                kwargs['children'] = node[1]
            if len(node) > 2:
                kwargs['name'] = node[2]
        else:
            kwargs['transformer'] = node

        return kwargs

    def add_node(self, transformer, name=None, children=None, parent=None,
                 return_node=False):

        if name is None:
            name = id(transformer)

        node = Node(name, transformer)
        self.nodes[name] = node

        parent = self if parent is None else self.nodes[parent.name]
        parent.add_child(node)

        if children is not None:
            self.add_children(children, parent=node)

        if return_node:
            return node

    def extract(self, stims, merge=True):
        results = list(flatten(self.collect(stims)))
        return merge_results(results) if merge else results
<|endoftext|>