<commit_before>from django.db import models
from socket import gethostname
from django.contrib.auth.models import User
from django.utils.safestring import mark_safe
import os, sys

host = gethostname()

stage = {0: 'failed (check settings and restart)',1:'preprocessing', 1001:'preprocessing image stack', 2:'initial alignment', 1002:'calculating initial alignment', 3:'affine alignment', 1003:'processing affine alignment', 4:'final warp alignment', 1004:'processing warp alignment', 5:'checking alignment', 1005:'aligning BG channel', 6: 'aligning other channels', 1006:'aligning all channel using BG warp', 7: 'alignment done', 10:'request merged tif',  1010:'creating merged tif', 20:'merged tif created'}
comp = {0: 'awaiting processing',1:'convertion complete', 2:'preprocessing complete', 3:'initial alignment complete', 4:'affine alignment complete', 5:'final warp complete', 6: 'background alignment complete', 7: 'all channels aligned', 10: 'merged tif requested', 20: 'merged tif created'}
chan = {0: 'to be calculated', 1:'Channel 1', 2:'Channel 2', 3:'Channel 3'}
ori = ['LPS','RPI','RAS','LAI','PLI','PRS','ALS','ARI'] #X(>),Y(\/),Z(X).
orien = [str(x).replace('R','right-').replace('L','left-').replace('P','posterior-').replace('A','anterior-').replace('S','superior').replace('I','inferior') for x in ori]
comp_orien = dict(zip(ori,orien))
orien = zip(orien,orien)
conv_orien = dict(zip(comp_orien.values(),comp_orien.keys()))
# Create your models here.

class Alignment(models.Model):
    import system.models
    #from users.models import User
    #from system.models import Setting
    name = models.CharField(max_length=500)
    settings = models.ForeignKey('system.Setting', default=1)
    max_stage = models.IntegerField(choices=comp.items(), default=0)
    last_host = models.CharField(max_length=100, blank=True, default=host)
    aligned_score = models.CharField(max_length=20, blank=True)
    alignment_stage = models.IntegerField(choices=stage.items(), default=1)
    orig_orientation = models.CharField(max_length=50, choices=orien, default='left-posterior-superior', blank=True)
    loading_host = models.CharField(max_length=100, blank=True)
    original_ext = models.CharField(max_length=10)
    original_path = models.TextField(max_length=1000)
    crop_xyz = models.CharField(max_length=100, blank=True)
    temp_initial_nrrd = models.TextField(max_length=1000, blank=True)
    temp_initial_score = models.CharField(max_length=20, blank=True)
    background_channel = models.IntegerField(choices=chan.items(), default=0)
    signal_channel = models.IntegerField(choices=chan.items(), default=0)
    ac1_channel = models.IntegerField(choices=chan.items(), default=0)
    aligned_bg = models.TextField(max_length=1000, blank=True)
    aligned_sg = models.TextField(max_length=1000, blank=True)
    aligned_ac1 = models.TextField(max_length=1000, blank=True)
    aligned_slice_score = models.CharField(max_length=20, blank=True)
    aligned_avgslice_score = models.CharField(max_length=20, blank=True)
    notes = models.TextField(max_length=10000, blank=True)
    reference = models.TextField(max_length=5000, blank=True)
    user = models.ForeignKey(User, blank=False, default=0)
    def __str__(self):
        return self.name
    def complete(self):
        return 1000 > self.alignment_stage > 6
    complete.admin_order_field = 'name'
    complete.boolean = True
    complete.short_description = 'Alignment complete?'
    def curStage(self):
        return stage[self.alignment_stage]
    curStage.short_description = 'Current stage?'
    def ch1_image(self):
      if self.max_stage > 1:
        return '' % str(self.id)
      return ''
    ch1_image.short_description = 'original image ch1'
    ch1_image.allow_tags = True
    def ch2_image(self):
      if self.max_stage > 1:
        return '' % str(self.id)
      return ''
    ch2_image.short_description = 'original image ch2'
    ch2_image.allow_tags = True
    def ch3_image(self):
      if self.max_stage > 1:
        return '' % str(self.id)
      return ''
    ch3_image.short_description = 'original image ch3'
    ch3_image.allow_tags = True
    def bg_image(self):
      if self.max_stage > 5:
        return '' % str(self.id)
      return ''
    bg_image.short_description = 'aligned BG image'
    bg_image.allow_tags = True
    def sg_image(self):
      if self.max_stage > 6:
        return '' % str(self.id)
      return ''
    sg_image.short_description = 'aligned SG image'
    sg_image.allow_tags = True
    def ac1_image(self):
      if self.max_stage > 6:
        return '' % str(self.id)
      return ''
    ac1_image.short_description = 'aligned AC1 image'
    ac1_image.allow_tags = True
    def hist_image(self):
      if self.max_stage > 1:
        return '' % str(self.id)
      return ''
    hist_image.short_description = 'original image histogram (new contrast range maked in cyan)'
    hist_image.allow_tags = True
    def init_image(self):
      if self.max_stage > 2:
        return '' % str(self.id)
      return ''
    init_image.short_description = 'initial BG alignment'
    init_image.allow_tags = True
    def temp_image(self):
        return '' % str(self.settings.template.image)
    temp_image.short_description = 'template image for reference'
    temp_image.allow_tags = True
    # def available_files(self):
    #     created = {}
    #     if 'nrrd' in self.aligned_bg:
    #       created.append({'Aligned BG':str(self.aligned_bg)})
    #     if 'nrrd' in self.aligned_sg:
    #       created.append({'Aligned SG':str(self.aligned_sg)})
    #     if 'nrrd' in self.aligned_ac1:
    #       created.append({'Aligned AC1':str(self.aligned_ac1)})
    #     return created

class Original_nrrd(models.Model):
    image = models.ForeignKey(Alignment)
    channel = models.IntegerField(default=0)
    new_min = models.IntegerField(default=0)
    new_max = models.IntegerField(default=255)
    file = models.TextField(max_length=1000, blank=True)
    is_index = models.BooleanField(default=False)
    pre_hist = models.CommaSeparatedIntegerField(max_length=255, default=range(0,255))
    def __str__(self):
        return str(self.image) + ' channel ' + str(self.channel)
    def owner(self):
        im_id = self.image
        return str(im_id.user)
    owner.admin_order_field = 'image'
    owner.short_description = 'User'
    # def available_files(self):
    #   created = {}
    #   if 'nrrd' in str(self.file):
    #     created.append({('Original Ch' + str(self.channel)):str(self.file)})
    #   return created
    def chan_image(self):
      return '' % (str(self.channel), str(self.image.id))
    chan_image.short_description = 'channel image'
    chan_image.allow_tags = True
    def chan_ident(self):
      ident = '...'
      if str(self.channel) == str(self.image.background_channel):
        ident = 'background'
      if str(self.channel) == str(self.image.signal_channel):
        ident = 'signal'
      if str(self.channel) == str(self.image.ac1_channel):
        ident = 'additional channel 1'
      return ident
    chan_ident.short_description = 'channel identified as'
    def parent(self):
      return '%s' % (str(self.image.id), str(self.image))
    parent.short_description = 'parent details'
    parent.allow_tags = True

class Upload(models.Model):
    import system.models
    #from users.models import User
    #from system.models import Server, Setting, tempfolder
    file = models.FileField(upload_to='web' + os.path.sep)
    settings = models.ForeignKey('system.Setting', default=1)
    orientation = models.CharField(max_length=50, choices=orien, default='left-posterior-superior', blank=True)
    def curStage(self):
        return self.file

class Mask_original(models.Model):
    image = models.ForeignKey(Original_nrrd)
    intensity_threshold = models.IntegerField(default=35)
    min_object_size = models.IntegerField(default=10000)
    complete = models.BooleanField(default=False)
    detected_objects = models.CommaSeparatedIntegerField(max_length=255, blank=True, default={})
    cut_objects = models.CommaSeparatedIntegerField(max_length=255, blank=True, default={})
    cut_complete = models.BooleanField(default=False)
    crop_objects = models.CommaSeparatedIntegerField(max_length=255, blank=True, default={})
    crop_complete = models.BooleanField(default=False)
    auto_restart_alignment = models.BooleanField(default=False)
    def __str__(self):
        return 'Mask for ' + str(self.image)
    def mask_image(self):
      if self.complete:
        return '' % str(self.id)
      return ''
    mask_image.short_description = 'detected objects'
    mask_image.allow_tags = True
    def orig_image(self):
      ch_id = self.image
      im_id = ch_id.image
      try:
        return '' % (str(ch_id.channel), str(im_id.id))
      except:
        return ''
    orig_image.short_description = 'original image'
    orig_image.allow_tags = True
    def owner(self):
      ch_id = self.image
      im_id = ch_id.image
      return str(im_id.user)
    owner.admin_order_field = 'image'
    owner.short_description = 'User'
    def chan_ident(self):
      ident = '...'
      if str(self.image.channel) == str(self.image.image.background_channel):
        ident = 'background'
      if str(self.image.channel) == str(self.image.image.signal_channel):
        ident = 'signal'
      if str(self.image.channel) == str(self.image.image.ac1_channel):
        ident = 'additional channel 1'
      return ident
    chan_ident.short_description = 'channel identified as'
    def parent(self):
      return '%s' % (str(self.image.image.id), str(self.image.image))
    parent.short_description = 'parent details'
    parent.allow_tags = True
    def available(self):
      try:
        if self.image.image.alignment_stage > 0:
          return False
        else:
          return True
      except:
        return False
    available.boolean = True
    available.short_description = 'available for manual processing'

class Mask_aligned(models.Model):
    image = models.ForeignKey(Alignment)
    channel = models.CharField(max_length=3, choices=(('bg', 'Background'),('sg', 'Signal'),('ac1', 'Additional')), default='sg')
    intensity_threshold = models.IntegerField(default=20)
    min_object_size = models.IntegerField(default=1000)
    complete = models.BooleanField(default=False)
    detected_objects = models.CommaSeparatedIntegerField(max_length=255, blank=True, default={})
    cut_objects = models.CommaSeparatedIntegerField(max_length=255, blank=True, default={})
    cut_complete = models.BooleanField(default=False)
    crop_objects = models.CommaSeparatedIntegerField(max_length=255, blank=True, default={})
    crop_complete = models.BooleanField(default=False)
    def __str__(self):
        return 'Mask for aligned ' + str(self.image) + ' ' + str(self.channel).upper() + ' channel'
    def mask_image(self):
      # if self.complete:
      #   return '' % str(self.channel), str(self.id)
      return ''
    mask_image.short_description = 'detected objects'
    mask_image.allow_tags = True
    def orig_image(self):
      try:
        return '' % (str(self.channel), str(self.image.id))
      except:
        return "Unexpected error:" % sys.exc_info()[0]
    orig_image.short_description = 'original image'
    orig_image.allow_tags = True
    def owner(self):
      im_id = self.image
      return str(im_id.user)
    owner.admin_order_field = 'image'
    owner.short_description = 'User'
    def parent(self):
      return '%s' % (str(self.image.id), str(self.image))
    parent.short_description = 'parent details'
    parent.allow_tags = True
    def available(self):
      try:
        if self.image.alignment_stage < 1:
          return True
        if self.image.alignment_stage < 7:
          return False
        else:
          return True
      except:
        return False
    available.boolean = True
    available.short_description = 'available for manual processing'
<commit_msg>Revert "test"<commit_after>from django.db import models
from socket import gethostname
from django.contrib.auth.models import User
from django.utils.safestring import mark_safe
import os

host = gethostname()

stage = {0: 'failed (check settings and restart)',1:'preprocessing', 1001:'preprocessing image stack', 2:'initial alignment', 1002:'calculating initial alignment', 3:'affine alignment', 1003:'processing affine alignment', 4:'final warp alignment', 1004:'processing warp alignment', 5:'checking alignment', 1005:'aligning BG channel', 6: 'aligning other channels', 1006:'aligning all channel using BG warp', 7: 'alignment done', 10:'request merged tif',  1010:'creating merged tif', 20:'merged tif created'}
comp = {0: 'awaiting processing',1:'convertion complete', 2:'preprocessing complete', 3:'initial alignment complete', 4:'affine alignment complete', 5:'final warp complete', 6: 'background alignment complete', 7: 'all channels aligned', 10: 'merged tif requested', 20: 'merged tif created'}
chan = {0: 'to be calculated', 1:'Channel 1', 2:'Channel 2', 3:'Channel 3'}
ori = ['LPS','RPI','RAS','LAI','PLI','PRS','ALS','ARI'] #X(>),Y(\/),Z(X).
orien = [str(x).replace('R','right-').replace('L','left-').replace('P','posterior-').replace('A','anterior-').replace('S','superior').replace('I','inferior') for x in ori]
comp_orien = dict(zip(ori,orien))
orien = zip(orien,orien)
conv_orien = dict(zip(comp_orien.values(),comp_orien.keys()))
# Create your models here.

class Alignment(models.Model):
    import system.models
    #from users.models import User
    #from system.models import Setting
    name = models.CharField(max_length=500)
    settings = models.ForeignKey('system.Setting', default=1)
    max_stage = models.IntegerField(choices=comp.items(), default=0)
    last_host = models.CharField(max_length=100, blank=True, default=host)
    aligned_score = models.CharField(max_length=20, blank=True)
    alignment_stage = models.IntegerField(choices=stage.items(), default=1)
    orig_orientation = models.CharField(max_length=50, choices=orien, default='left-posterior-superior', blank=True)
    loading_host = models.CharField(max_length=100, blank=True)
    original_ext = models.CharField(max_length=10)
    original_path = models.TextField(max_length=1000)
    crop_xyz = models.CharField(max_length=100, blank=True)
    temp_initial_nrrd = models.TextField(max_length=1000, blank=True)
    temp_initial_score = models.CharField(max_length=20, blank=True)
    background_channel = models.IntegerField(choices=chan.items(), default=0)
    signal_channel = models.IntegerField(choices=chan.items(), default=0)
    ac1_channel = models.IntegerField(choices=chan.items(), default=0)
    aligned_bg = models.TextField(max_length=1000, blank=True)
    aligned_sg = models.TextField(max_length=1000, blank=True)
    aligned_ac1 = models.TextField(max_length=1000, blank=True)
    aligned_slice_score = models.CharField(max_length=20, blank=True)
    aligned_avgslice_score = models.CharField(max_length=20, blank=True)
    notes = models.TextField(max_length=10000, blank=True)
    reference = models.TextField(max_length=5000, blank=True)
    user = models.ForeignKey(User, blank=False, default=0)
    def __str__(self):
        return self.name
    def complete(self):
        return 1000 > self.alignment_stage > 6
    complete.admin_order_field = 'name'
    complete.boolean = True
    complete.short_description = 'Alignment complete?'
    def curStage(self):
        return stage[self.alignment_stage]
    curStage.short_description = 'Current stage?'
    def ch1_image(self):
      if self.max_stage > 1:
        return '' % str(self.id)
      return ''
    ch1_image.short_description = 'original image ch1'
    ch1_image.allow_tags = True
    def ch2_image(self):
      if self.max_stage > 1:
        return '' % str(self.id)
      return ''
    ch2_image.short_description = 'original image ch2'
    ch2_image.allow_tags = True
    def ch3_image(self):
      if self.max_stage > 1:
        return '' % str(self.id)
      return ''
    ch3_image.short_description = 'original image ch3'
    ch3_image.allow_tags = True
    def bg_image(self):
      if self.max_stage > 5:
        return '' % str(self.id)
      return ''
    bg_image.short_description = 'aligned BG image'
    bg_image.allow_tags = True
    def sg_image(self):
      if self.max_stage > 6:
        return '' % str(self.id)
      return ''
    sg_image.short_description = 'aligned SG image'
    sg_image.allow_tags = True
    def ac1_image(self):
      if self.max_stage > 6:
        return '' % str(self.id)
      return ''
    ac1_image.short_description = 'aligned AC1 image'
    ac1_image.allow_tags = True
    def hist_image(self):
      if self.max_stage > 1:
        return '' % str(self.id)
      return ''
    hist_image.short_description = 'original image histogram (new contrast range maked in cyan)'
    hist_image.allow_tags = True
    def init_image(self):
      if self.max_stage > 2:
        return '' % str(self.id)
      return ''
    init_image.short_description = 'initial BG alignment'
    init_image.allow_tags = True
    def temp_image(self):
        return '' % str(self.settings.template.image)
    temp_image.short_description = 'template image for reference'
    temp_image.allow_tags = True
    # def available_files(self):
    #     created = {}
    #     if 'nrrd' in self.aligned_bg:
    #       created.append({'Aligned BG':str(self.aligned_bg)})
    #     if 'nrrd' in self.aligned_sg:
    #       created.append({'Aligned SG':str(self.aligned_sg)})
    #     if 'nrrd' in self.aligned_ac1:
    #       created.append({'Aligned AC1':str(self.aligned_ac1)})
    #     return created

class Original_nrrd(models.Model):
    image = models.ForeignKey(Alignment)
    channel = models.IntegerField(default=0)
    new_min = models.IntegerField(default=0)
    new_max = models.IntegerField(default=255)
    file = models.TextField(max_length=1000, blank=True)
    is_index = models.BooleanField(default=False)
    pre_hist = models.CommaSeparatedIntegerField(max_length=255, default=range(0,255))
    def __str__(self):
        return str(self.image) + ' channel ' + str(self.channel)
    def owner(self):
        im_id = self.image
        return str(im_id.user)
    owner.admin_order_field = 'image'
    owner.short_description = 'User'
    # def available_files(self):
    #   created = {}
    #   if 'nrrd' in str(self.file):
    #     created.append({('Original Ch' + str(self.channel)):str(self.file)})
    #   return created
    def chan_image(self):
      return '' % (str(self.channel), str(self.image.id))
    chan_image.short_description = 'channel image'
    chan_image.allow_tags = True
    def chan_ident(self):
      ident = '...'
      if str(self.channel) == str(self.image.background_channel):
        ident = 'background'
      if str(self.channel) == str(self.image.signal_channel):
        ident = 'signal'
      if str(self.channel) == str(self.image.ac1_channel):
        ident = 'additional channel 1'
      return ident
    chan_ident.short_description = 'channel identified as'
    def parent(self):
      return '%s' % (str(self.image.id), str(self.image))
    parent.short_description = 'parent details'
    parent.allow_tags = True

class Upload(models.Model):
    import system.models
    #from users.models import User
    #from system.models import Server, Setting, tempfolder
    file = models.FileField(upload_to='web' + os.path.sep)
    settings = models.ForeignKey('system.Setting', default=1)
    orientation = models.CharField(max_length=50, choices=orien, default='left-posterior-superior', blank=True)
    def curStage(self):
        return self.file

class Mask_original(models.Model):
    image = models.ForeignKey(Original_nrrd)
    intensity_threshold = models.IntegerField(default=35)
    min_object_size = models.IntegerField(default=10000)
    complete = models.BooleanField(default=False)
    detected_objects = models.CommaSeparatedIntegerField(max_length=255, blank=True, default={})
    cut_objects = models.CommaSeparatedIntegerField(max_length=255, blank=True, default={})
    cut_complete = models.BooleanField(default=False)
    crop_objects = models.CommaSeparatedIntegerField(max_length=255, blank=True, default={})
    crop_complete = models.BooleanField(default=False)
    auto_restart_alignment = models.BooleanField(default=False)
    def __str__(self):
        return 'Mask for ' + str(self.image)
    def mask_image(self):
      if self.complete:
        return '' % str(self.id)
      return ''
    mask_image.short_description = 'detected objects'
    mask_image.allow_tags = True
    def orig_image(self):
      ch_id = self.image
      im_id = ch_id.image
      try:
        return '' % (str(ch_id.channel), str(im_id.id))
      except:
        return ''
    orig_image.short_description = 'original image'
    orig_image.allow_tags = True
    def owner(self):
      ch_id = self.image
      im_id = ch_id.image
      return str(im_id.user)
    owner.admin_order_field = 'image'
    owner.short_description = 'User'
    def chan_ident(self):
      ident = '...'
      if str(self.image.channel) == str(self.image.image.background_channel):
        ident = 'background'
      if str(self.image.channel) == str(self.image.image.signal_channel):
        ident = 'signal'
      if str(self.image.channel) == str(self.image.image.ac1_channel):
        ident = 'additional channel 1'
      return ident
    chan_ident.short_description = 'channel identified as'
    def parent(self):
      return '%s' % (str(self.image.image.id), str(self.image.image))
    parent.short_description = 'parent details'
    parent.allow_tags = True
    def available(self):
      try:
        if self.image.image.alignment_stage > 0:
          return False
        else:
          return True
      except:
        return False
    available.boolean = True
    available.short_description = 'available for manual processing'

class Mask_aligned(models.Model):
    image = models.ForeignKey(Alignment)
    channel = models.CharField(max_length=3, choices=(('bg', 'Background'),('sg', 'Signal'),('ac1', 'Additional')), default='sg')
    intensity_threshold = models.IntegerField(default=20)
    min_object_size = models.IntegerField(default=1000)
    complete = models.BooleanField(default=False)
    detected_objects = models.CommaSeparatedIntegerField(max_length=255, blank=True, default={})
    cut_objects = models.CommaSeparatedIntegerField(max_length=255, blank=True, default={})
    cut_complete = models.BooleanField(default=False)
    crop_objects = models.CommaSeparatedIntegerField(max_length=255, blank=True, default={})
    crop_complete = models.BooleanField(default=False)
    def __str__(self):
        return 'Mask for aligned ' + str(self.image) + ' ' + str(self.channel).upper() + ' channel'
    def mask_image(self):
      # if self.complete:
      #   return '' % str(self.channel), str(self.id)
      return ''
    mask_image.short_description = 'detected objects'
    mask_image.allow_tags = True
    def orig_image(self):
      im_id = self.image
      try:
        return '' % (str(self.channel), str(im_id.id))
      except:
        return ''
    orig_image.short_description = 'original image'
    orig_image.allow_tags = True
    def owner(self):
      im_id = self.image
      return str(im_id.user)
    owner.admin_order_field = 'image'
    owner.short_description = 'User'
    def parent(self):
      return '%s' % (str(self.image.id), str(self.image))
    parent.short_description = 'parent details'
    parent.allow_tags = True
    def available(self):
      try:
        if self.image.alignment_stage < 1:
          return True
        if self.image.alignment_stage < 7:
          return False
        else:
          return True
      except:
        return False
    available.boolean = True
    available.short_description = 'available for manual processing'
<|endoftext|>


<commit_before>from gpfit.fit import fit
import gpkitmodels.GP.aircraft.wing.gustloading as GustL
import os
import matplotlib.pyplot as plt
import numpy as np
import sys
plt.rcParams.update({'font.size':15})
GENERATE = True

def arctanfit():
    u = np.linspace(1e-15, 0.7, 100)
    w = np.arctan(u)

    x = np.log(u)
    y = np.log(w)

    cn, err = fit(x, y, 1, "MA")
    rm = err
    print "RMS error: %.4f" % rm

    yfit = cn.evaluate(x)
    df = cn.get_dataframe()
    fig, ax = plt.subplots()
    ax.plot(u, w, lw=2)
    ax.plot(u, np.exp(yfit), "--", lw=2)
    ax.set_xlim([0, 0.7])
    ax.grid()
    ax.set_xlabel("$V_{\\mathrm{gust}}/V$")
    ax.set_ylabel("$\\alpha_{\\mathrm{gust}}$")
    ax.legend(["$\\arctan{(V_{\\mathrm{gust}}/V)}$",
               "$0.905 (V_{\\mathrm{gust}}/V)^{0.961}$"], loc=2, fontsize=15)
    return df, fig, ax

if __name__ == "__main__":
    df, fig, ax = arctanfit()
    if GENERATE:
        path = os.path.dirname(GustL.__file__)
        df.to_csv(path + os.sep + "arctan_fit.csv", index=False)
    else:
        df.to_csv("arctan_fit.csv")
    if len(sys.argv) > 1:
        path = sys.argv[1]
        fig.savefig(path + "arctanfit.pdf", bbox_inches="tight")
    else:
        fig.savefig("arctanfit.pdf", bbox_inches="tight")
<commit_msg>modify arctan for changed scenario<commit_after>from gpfit.fit import fit
import os
import matplotlib.pyplot as plt
import numpy as np
import sys
plt.rcParams.update({'font.size':15})
GENERATE = True

def arctanfit():
    u = np.linspace(1e-15, 0.7, 100)
    w = np.arctan(u)

    x = np.log(u)
    y = np.log(w)

    cn, err = fit(x, y, 1, "MA")
    rm = err
    print "RMS error: %.4f" % rm

    yfit = cn.evaluate(x)
    df = cn.get_dataframe()
    fig, ax = plt.subplots()
    ax.plot(u, w, lw=2)
    ax.plot(u, np.exp(yfit), "--", lw=2)
    ax.set_xlim([0, 0.7])
    ax.grid()
    ax.set_xlabel("$V_{\\mathrm{gust}}/V$")
    ax.set_ylabel("$\\alpha_{\\mathrm{gust}}$")
    ax.legend(["$\\arctan{(V_{\\mathrm{gust}}/V)}$",
               "$0.905 (V_{\\mathrm{gust}}/V)^{0.961}$"], loc=2, fontsize=15)
    return df, fig, ax

if __name__ == "__main__":
    df, fig, ax = arctanfit()
    df.to_csv("arctan_fit.csv")
    fig.savefig("arctanfit.pdf", bbox_inches="tight")
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
import sys
import threading

import mesos.interface
from mesos.interface import mesos_pb2
import mesos.native
from subprocess import call
import pickle

log = logging.getLogger( __name__ )

class JobTreeMesosExecutor(mesos.interface.Executor):
    """Part of mesos framework, runs on mesos slave. A jobTree job is passed to it via the task.data field,
     and launched via call(jobTree.command). Uses the ExecutorDriver to communicate.
    """

    # TODO: rename internal methods, i.e. methods *not* invoked by the driver to start with two
    # underscores. FOr each callback, briefly document in a docstring when it is called.

    def registered(self, driver, executorInfo, frameworkInfo, slaveInfo):
        """
        Invoked once the executor driver has been able to successfully connect with Mesos.
        :param driver:
        :param executorInfo:
        :param frameworkInfo:
        :param slaveInfo:
        :return:
        """
        log.debug("Registered with framework")

    def reregistered(self, driver, slaveInfo):
        """
        Invoked when the executor re-registers with a restarted slave.
        :param driver:
        :param slaveInfo:
        :return:
        """
        log.debug("Re-Registered")

    def disconnected(self, driver):
        """
        Invoked when the executor becomes "disconnected" from the slave (e.g., the slave is being restarted due to an upgrade).
        :param driver:
        :return:
        """
        print "disconnected from slave"

    def error(self, driver, message):
        """
        Invoked when a fatal error has occurred with the executor and/or executor driver.
        :param driver:
        :param message:
        :return:
        """
        log.error(message)
        self.frameworkMessage(driver, message)

    def launchTask(self, driver, task):
        """
        Invoked by SchedulerDriver when a task has been launched on this executor
        :param driver:
        :param task:
        :return:
        """
        def __run_task():
            log.debug("Running task %s" % task.task_id.value)
            self.sendUpdate(driver, task, mesos_pb2.TASK_RUNNING)

            jobTreeJob = pickle.loads( task.data )
            os.chdir( jobTreeJob.cwd )

            result = call(jobTreeJob.command, shell=True)

            if result != 0:
                self.sendUpdate(driver, task, mesos_pb2.TASK_FAILED)
            else:
                self.sendUpdate(driver, task, mesos_pb2.TASK_FINISHED)

        # TODO: I think there needs to be a thread.join() somewhere for each thread. Come talk to me about this.
        thread = threading.Thread(target=__run_task)
        thread.start()

    # TODO: why is TASK_STATE upper case?
    def sendUpdate(self, driver, task, TASK_STATE):
        log.debug("Sending status update...")
        update = mesos_pb2.TaskStatus()
        update.task_id.value = task.task_id.value
        update.state = TASK_STATE
        # TODO: What's this for?
        update.data = 'data with a \0 byte'
        driver.sendStatusUpdate(update)
        log.debug("Sent status update")

    def frameworkMessage(self, driver, message):
        """
        Invoked when a framework message has arrived for this executor.
        :param driver:
        :param message:
        :return:
        """
        # Send it back to the scheduler.
        driver.sendFrameworkMessage(message)

if __name__ == "__main__":
    log.debug("Starting executor")
    driver = mesos.native.MesosExecutorDriver(JobTreeMesosExecutor())
    sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
<commit_msg>Refactor method signatures<commit_after>#!/usr/bin/env python

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
import sys
import threading

import mesos.interface
from mesos.interface import mesos_pb2
import mesos.native
from subprocess import call
import pickle

log = logging.getLogger( __name__ )

class JobTreeMesosExecutor(mesos.interface.Executor):
    """Part of mesos framework, runs on mesos slave. A jobTree job is passed to it via the task.data field,
     and launched via call(jobTree.command). Uses the ExecutorDriver to communicate.
    """

    # TODO: rename internal methods, i.e. methods *not* invoked by the driver to start with two
    # underscores. FOr each callback, briefly document in a docstring when it is called.

    def registered(self, driver, executorInfo, frameworkInfo, slaveInfo):
        """
        Invoked once the executor driver has been able to successfully connect with Mesos.
        :param driver:
        :param executorInfo:
        :param frameworkInfo:
        :param slaveInfo:
        :return:
        """
        log.debug("Registered with framework")

    def reregistered(self, driver, slaveInfo):
        """
        Invoked when the executor re-registers with a restarted slave.
        :param driver:
        :param slaveInfo:
        :return:
        """
        log.debug("Re-Registered")

    def disconnected(self, driver):
        """
        Invoked when the executor becomes "disconnected" from the slave (e.g., the slave is being restarted due to an upgrade).
        :param driver:
        :return:
        """
        print "disconnected from slave"

    def error(self, driver, message):
        """
        Invoked when a fatal error has occurred with the executor and/or executor driver.
        :param driver:
        :param message:
        :return:
        """
        log.error(message)
        self.frameworkMessage(driver, message)

    def launchTask(self, driver, task):
        """
        Invoked by SchedulerDriver when a task has been launched on this executor
        :param driver:
        :param task:
        :return:
        """
        def __run_task():
            log.debug("Running task %s" % task.task_id.value)
            self.__sendUpdate(driver, task, mesos_pb2.TASK_RUNNING)

            jobTreeJob = pickle.loads( task.data )
            os.chdir( jobTreeJob.cwd )

            result = call(jobTreeJob.command, shell=True)

            if result != 0:
                self.__sendUpdate(driver, task, mesos_pb2.TASK_FAILED)
            else:
                self.__sendUpdate(driver, task, mesos_pb2.TASK_FINISHED)

        # TODO: I think there needs to be a thread.join() somewhere for each thread. Come talk to me about this.
        thread = threading.Thread(target=__run_task)
        thread.start()

    # TODO: why is TASK_STATE upper case?
    def __sendUpdate(self, driver, task, TASK_STATE):
        log.debug("Sending status update...")
        update = mesos_pb2.TaskStatus()
        update.task_id.value = task.task_id.value
        update.state = TASK_STATE
        # TODO: What's this for?
        update.data = 'data with a \0 byte'
        driver.sendStatusUpdate(update)
        log.debug("Sent status update")

    def frameworkMessage(self, driver, message):
        """
        Invoked when a framework message has arrived for this executor.
        :param driver:
        :param message:
        :return:
        """
        # Send it back to the scheduler.
        driver.sendFrameworkMessage(message)

if __name__ == "__main__":
    log.debug("Starting executor")
    driver = mesos.native.MesosExecutorDriver(JobTreeMesosExecutor())
    sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
<|endoftext|>


<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyCvxpy(PythonPackage):
    """Convex optimization, for everyone."""

    homepage = "https://www.cvxpy.org/index.html"
    pypi = "cvxpy/cvxpy-1.0.25.tar.gz"

    version('1.0.25', sha256='8535529ddb807067b0d59661dce1d9a6ddb2a218398a38ea7772328ad8a6ea13')

    depends_on('py-setuptools', type='build')
    depends_on('py-numpy@1.15:',        type=('build', 'run'))
    depends_on('py-scipy@1.1.0:',        type=('build', 'run'))
    depends_on('py-ecos@2:',        type=('build', 'run'))
    depends_on('py-scs@1.1.3:',        type=('build', 'run'))
    depends_on('py-osqp@0.4.1',        type=('build', 'run'))
    depends_on('py-multiprocess', type=('build', 'run'))
    depends_on('py-six', type=('build', 'run'))
<commit_msg>[py-cvxpy]: Add new version, fix depends_on versions (#24147)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyCvxpy(PythonPackage):
    """Convex optimization, for everyone."""

    homepage = "https://www.cvxpy.org/index.html"
    pypi = "cvxpy/cvxpy-1.0.25.tar.gz"

    version('1.1.13', sha256='a9c781e74ad76097b47b86456cb3a943898f7ec9ac8f47bcefc922051cdc4a04')
    version('1.0.25', sha256='8535529ddb807067b0d59661dce1d9a6ddb2a218398a38ea7772328ad8a6ea13')

    # Dependency versions based on README.md in python packages
    depends_on('python@3.4:', type=('build', 'run'), when='@1.1:')
    depends_on('python@3.6:', type=('build', 'run'), when='@1.1.13:')
    depends_on('py-setuptools', type='build')
    depends_on('py-numpy@1.15:',        type=('build', 'run'))
    depends_on('py-scipy@1.1.0:',        type=('build', 'run'))
    depends_on('py-ecos@2:',        type=('build', 'run'))
    depends_on('py-scs@1.1.3:',        type=('build', 'run'))
    depends_on('py-scs@1.1.6:',        type=('build', 'run'), when='@1.1.13:')
    depends_on('py-osqp@0.4.1:',        type=('build', 'run'))
    depends_on('py-multiprocess', type=('build', 'run'))
    depends_on('py-six', type=('build', 'run'), when='@:1.0.99')
<|endoftext|>


<commit_before>#!/usr/bin/python

# Owner of the setup
USERNAME = 'gokmen'

# GIT repository to deploy
GIT_BRANCH = 'dev-gokmen'
GIT_REVISION = 'HEAD'

# Configuration to deploy (main.<name>.coffee)
CONFIG = 'gokmen'

# Network topology
NETWORK = [
    {'roles': ['rabbitmq_server', 'broker'], 'instance_type': 'm1.xlarge'},
    {'roles': ['web_server', 'cacheworker', 'emailworker', 'guestcleanup'], 'instance_type': 'm1.xlarge'},
    {'roles': ['authworker', 'socialworker'], 'autoscale': (4, 5), 'instance_type': 'm1.large'},
]

#
# You shouldn't mess with the rest.
#

# Domain name populated from USERNAME
DOMAIN = '%s.dev.service.aws.koding.com' % USERNAME

# AWS DNS Zone ID for 'dev.service.aws.koding.com'
ZONE_ID = 'Z3OWM4DB88IDTJ'

ATTRIBUTES = {
    'kd_deploy': {'revision_tag': GIT_REVISION, 'git_branch': GIT_BRANCH},
    'nginx': {'server_name': DOMAIN},
    'launch': {'config': CONFIG},
}

NAME_PATTERN = '%(username)s-%(roles)s'

# Instance defaults
DEFAULTS = {
    'ami': 'ami-3d4ff254',
    'instance_type': 'm1.small',
    'ssh_key_name': 'koding',
    'sec_groups': ['sg-26167d4f'],
    'subnet_id': '',
    'roles': [],
    'tags': {},
}

# Expanded roles, don't edit if you are not familiar with Chef roles/cookbooks
ROLES = {
    'authworker': ['role[base_server]', 'role[authworker]', 'recipe[kd_deploy]'],
    'broker': ['role[base_server]', 'role[broker]', 'recipe[kd_deploy]'],
    'cacheworker': ['role[base_server]', 'role[cacheworker]', 'recipe[kd_deploy]'],
    'guestcleanup': ['role[base_server]', 'role[guestcleanup]', 'recipe[kd_deploy]'],
    'rabbitmq_server': ['role[base_server]', 'role[rabbitmq_server]'],
    'socialworker': ['role[base_server]', 'role[socialworker]', 'recipe[kd_deploy]'],
    'web_server': ['role[base_server]', 'role[web_server]', 'recipe[kd_deploy]'],
    'emailworker': ['role[base_server]', 'role[emailworker]', 'recipe[kd_deploy]'],
}

TEMPLATE = 'userdata.txt.template'

import boto
import boto.ec2
import boto.ec2.autoscale

conn_ec2 = boto.connect_ec2()
conn_as = boto.connect_autoscale()
conn_r53 = boto.connect_route53()

import copy
import time

class DNSRecord:
    def __init__(self, zone, name, value, ttl=600):
        self.zone = zone
        self.name = name
        self.value = value
        self.ttl = ttl

def get_file_content(filename):
    return file(filename).read()

def save_file_content(filename, data):
    file(filename, 'w').write(data)

def get_user_data(roles, attributes):
    tmp = get_file_content(TEMPLATE)

    # Create list of required recipes
    run_list = []
    for role in roles:
        run_list.extend(ROLES[role])
    run_list = set(run_list)
    run_list = '- ' + '\n        - '.join(run_list)
    tmp = tmp.replace('{{CHEF_RUN_LIST}}', run_list)

    # Set attributes
    attrs = []
    for recipe, settings in attributes.iteritems():
        attrs.append('%s:' % recipe)
        for key, value in settings.iteritems():
            attrs.append('    %s: "%s"' % (key, value))
    attrs = '\n        '.join(attrs)
    tmp = tmp.replace('{{CHEF_ATTRIBUTES}}', attrs)

    return tmp

def aws_run_interface(**kwargs):
    if 'autoscale' in kwargs:
        as_min, as_max = kwargs['autoscale']
        as_group = '%s-group' % kwargs['name']
        as_config = '%s-config' % kwargs['name']
        print 'Creating autoscaling instance group: %s (%s)' % (as_group, kwargs['instance_type'])

        lc = boto.ec2.autoscale.LaunchConfiguration(name=as_config,
                                                    image_id=kwargs['ami'],
                                                    key_name=kwargs['ssh_key_name'],
                                                    security_groups=kwargs['sec_groups'],
                                                    user_data=kwargs['user_data'])
        conn_as.create_launch_configuration(lc)

        ag = boto.ec2.autoscale.AutoScalingGroup(group_name=as_group,
                                                 launch_config=lc,
                                                 availability_zones=['us-east-1a'],
                                                 min_size=as_min,
                                                 max_size=as_max,
                                                 connection=conn_as)
        conn_as.create_auto_scaling_group(ag)

        return [ag, lc]
    else:
        print 'Creating instance: %s (%s)' % (kwargs['name'], kwargs['instance_type'])
        # Create instance
        reservation = conn_ec2.run_instances(image_id=kwargs['ami'],
                                             key_name=kwargs['ssh_key_name'],
                                             security_group_ids=kwargs['sec_groups'],
                                             user_data=kwargs['user_data'],
                                             instance_type=kwargs['instance_type'],
                                             subnet_id=kwargs['subnet_id'])
        instance = reservation.instances[0]

        # Wait for instance to get ready
        status = instance.update()
        while status == 'pending':
            time.sleep(3)
            status = instance.update()
        if status == 'running':
            print '  ID: %s' % instance.id
            print '  Private IP: %s' % instance.private_ip_address
            if instance.public_dns_name:
                print '  Public IP: %s' % instance.public_dns_name
        else:
            print '  Error: %s' % (status)
            return []

        # Set instance name
        instance.add_tag('Name', kwargs['name'])
        # Set instance tags
        for key, value in kwargs['tags'].iteritems():
            instance.add_tag(key, value)

        domains = []
        if 'broker' in kwargs['roles']:
            domains.append('broker.%s' % DOMAIN)
        if 'web_server' in kwargs['roles']:
            domains.append(DOMAIN)
        if 'rabbitmq_server' in kwargs['roles']:
            domains.append('mq.%s' % DOMAIN)

        # Add/update Route 53 entry
        dns_records = []
        if instance.public_dns_name:
            for domain in domains:
                record_sets = conn_r53.get_all_rrsets(ZONE_ID)
                change = record_sets.add_change('CREATE', domain, 'CNAME', 300)
                change.add_value(instance.public_dns_name)
                record_sets.commit()
                print '  Domain: %s' % domain
                dns_records.append(DNSRecord(ZONE_ID, domain, instance.public_dns_name, 300))

        return [instance] + dns_records

def render():
    instance_no = 0

    # List of AWS objects
    aws_objects = []

    template = get_file_content(TEMPLATE)
    for instance in NETWORK:
        tmp = template

        spec = copy.deepcopy(DEFAULTS)
        spec.update(copy.deepcopy(instance))

        # Roles
        if not isinstance(spec['roles'], list):
            spec['roles'] = [spec['roles']]

        # User data
        spec['user_data'] = get_user_data(spec['roles'], ATTRIBUTES)

        # Name
        spec['name'] = NAME_PATTERN % {'username': USERNAME,
                                       'roles': '_'.join(spec['roles']),
                                       'domain': DOMAIN}

        # Tags
        tags = {'init-username': USERNAME,
                'init-roles': ', '.join(spec['roles'])}
        spec.setdefault('tags', {}).update(tags)

        # Create instance
        items = aws_run_interface(**spec)
        aws_objects.extend(items)

    print 'Created objects:'
    for item in aws_objects:
        if isinstance(item, boto.ec2.autoscale.group.AutoScalingGroup):
            print '  AS Group  : %s' % item.name
        elif isinstance(item, boto.ec2.autoscale.launchconfig.LaunchConfiguration):
            print '  AS Config : %s' % item.name
        elif isinstance(item, boto.ec2.instance.Instance):
            print '  Instance  : %s' % item.id
        elif isinstance(item, DNSRecord):
            print '  R53 Entry : %s' % item.name

    print 'Press ENTER to delete everything.'
    raw_input()
    for item in aws_objects:
        if isinstance(item, boto.ec2.autoscale.group.AutoScalingGroup):
            print 'Shutting down AS group %s' % item.name
            item.shutdown_instances()
            conn_as.delete_auto_scaling_group(item.name, force_delete=True)
        elif isinstance(item, boto.ec2.autoscale.launchconfig.LaunchConfiguration):
            print 'Deleting AS config %s' % item.name
            conn_as.delete_launch_configuration(item.name)
        elif isinstance(item, boto.ec2.instance.Instance):
            print 'Terminating %s' % item.id
            conn_ec2.terminate_instances([item.id])
        elif isinstance(item, DNSRecord):
            print 'Removing %s' % item.name
            record_sets = conn_r53.get_all_rrsets(ZONE_ID)
            change = record_sets.add_change('DELETE', item.name, 'CNAME', item.ttl)
            change.add_value(item.value)
            record_sets.commit()

def main():
    render()

if __name__ == '__main__':
    main()
<commit_msg>AWS init script: Policies to scale up/down AS network<commit_after>#!/usr/bin/python

# Owner of the setup
USERNAME = 'bahadir'

# GIT repository to deploy
GIT_BRANCH = 'dev-bahadir'
GIT_REVISION = 'HEAD'

# Configuration to deploy (main.<name>.coffee)
CONFIG = 'bahadir'

# Network topology
NETWORK = [
    {'roles': ['rabbitmq_server', 'broker'], 'instance_type': 'm1.xlarge'},
    {'roles': ['web_server', 'cacheworker', 'emailworker', 'guestcleanup'], 'instance_type': 'm1.xlarge'},
    {'roles': ['authworker', 'socialworker'], 'autoscale': (1, 2), 'instance_type': 'm1.large'},
]

#
# You shouldn't mess with the rest.
#

# Domain name populated from USERNAME
DOMAIN = '%s.dev.service.aws.koding.com' % USERNAME

# AWS DNS Zone ID for 'dev.service.aws.koding.com'
ZONE_ID = 'Z3OWM4DB88IDTJ'

ATTRIBUTES = {
    'kd_deploy': {'revision_tag': GIT_REVISION, 'git_branch': GIT_BRANCH},
    'nginx': {'server_name': DOMAIN},
    'launch': {'config': CONFIG},
}

NAME_PATTERN = '%(username)s-%(roles)s'

# Instance defaults
DEFAULTS = {
    'ami': 'ami-3d4ff254',
    'instance_type': 'm1.small',
    'ssh_key_name': 'koding',
    'sec_groups': ['sg-26167d4f'],
    'subnet_id': '',
    'roles': [],
    'tags': {},
}

# Expanded roles, don't edit if you are not familiar with Chef roles/cookbooks
ROLES = {
    'authworker': ['role[base_server]', 'role[authworker]', 'recipe[kd_deploy]'],
    'broker': ['role[base_server]', 'role[broker]', 'recipe[kd_deploy]'],
    'cacheworker': ['role[base_server]', 'role[cacheworker]', 'recipe[kd_deploy]'],
    'guestcleanup': ['role[base_server]', 'role[guestcleanup]', 'recipe[kd_deploy]'],
    'rabbitmq_server': ['role[base_server]', 'role[rabbitmq_server]'],
    'socialworker': ['role[base_server]', 'role[socialworker]', 'recipe[kd_deploy]'],
    'web_server': ['role[base_server]', 'role[web_server]', 'recipe[kd_deploy]'],
    'emailworker': ['role[base_server]', 'role[emailworker]', 'recipe[kd_deploy]'],
}

TEMPLATE = 'userdata.txt.template'

import boto
import boto.ec2
import boto.ec2.autoscale
import boto.ec2.cloudwatch

conn_ec2 = boto.connect_ec2()
conn_as = boto.connect_autoscale()
conn_r53 = boto.connect_route53()
conn_cw = boto.connect_cloudwatch()

import copy
import time

class DNSRecord:
    def __init__(self, zone, name, value, ttl=600):
        self.zone = zone
        self.name = name
        self.value = value
        self.ttl = ttl

def get_file_content(filename):
    return file(filename).read()

def save_file_content(filename, data):
    file(filename, 'w').write(data)

def get_user_data(roles, attributes):
    tmp = get_file_content(TEMPLATE)

    # Create list of required recipes
    run_list = []
    for role in roles:
        run_list.extend(ROLES[role])
    run_list = set(run_list)
    run_list = '- ' + '\n        - '.join(run_list)
    tmp = tmp.replace('{{CHEF_RUN_LIST}}', run_list)

    # Set attributes
    attrs = []
    for recipe, settings in attributes.iteritems():
        attrs.append('%s:' % recipe)
        for key, value in settings.iteritems():
            attrs.append('    %s: "%s"' % (key, value))
    attrs = '\n        '.join(attrs)
    tmp = tmp.replace('{{CHEF_ATTRIBUTES}}', attrs)

    return tmp

def create_alarm(as_group):
    scale_up_policy = boto.ec2.autoscale.ScalingPolicy(name='scale_up',
                                                       adjustment_type='ChangeInCapacity',
                                                       as_name=as_group,
                                                       scaling_adjustment=1,
                                                       cooldown=180)
    scale_down_policy = boto.ec2.autoscale.ScalingPolicy(name='scale_down',
                                                         adjustment_type='ChangeInCapacity',
                                                         as_name=as_group,
                                                         scaling_adjustment=-1,
                                                         cooldown=180)

    conn_as.create_scaling_policy(scale_up_policy)
    conn_as.create_scaling_policy(scale_down_policy)

    scale_up_policy = conn_as.get_all_policies(as_group=as_group, policy_names=['scale_up'])[0]
    scale_down_policy = conn_as.get_all_policies(as_group=as_group, policy_names=['scale_down'])[0]

    alarm_dimensions = {"AutoScalingGroupName": as_group}

    scale_up_alarm = boto.ec2.cloudwatch.MetricAlarm(name='%s-scale_up_on_cpu' % as_group,
                                                     namespace='AWS/EC2',
                                                     metric='CPUUtilization',
                                                     statistic='Average',
                                                     comparison='>',
                                                     threshold='50',
                                                     period='120',
                                                     evaluation_periods=2,
                                                     alarm_actions=[scale_up_policy.policy_arn],
                                                     dimensions=alarm_dimensions)
    conn_cw.create_alarm(scale_up_alarm)
    scale_down_alarm = boto.ec2.cloudwatch.MetricAlarm(name='%s-scale_down_on_cpu' % as_group,
                                                       namespace='AWS/EC2',
                                                       metric='CPUUtilization',
                                                       statistic='Average',
                                                       comparison='<',
                                                       threshold='40',
                                                       period='240',
                                                       evaluation_periods=2,
                                                       alarm_actions=[scale_down_policy.policy_arn],
                                                       dimensions=alarm_dimensions)
    conn_cw.create_alarm(scale_down_alarm)

def aws_create_autoscale(**kwargs):
    as_min, as_max = kwargs['autoscale']
    as_group = '%s-group' % kwargs['name']
    as_config = '%s-config' % kwargs['name']
    lc = boto.ec2.autoscale.LaunchConfiguration(name=as_config,
                                                image_id=kwargs['ami'],
                                                key_name=kwargs['ssh_key_name'],
                                                security_groups=kwargs['sec_groups'],
                                                user_data=kwargs['user_data'])
    conn_as.create_launch_configuration(lc)

    ag = boto.ec2.autoscale.AutoScalingGroup(group_name=as_group,
                                             launch_config=lc,
                                             availability_zones=['us-east-1a'],
                                             min_size=as_min,
                                             max_size=as_max,
                                             connection=conn_as)
    conn_as.create_auto_scaling_group(ag)

    create_alarm(as_group)

    return [ag, lc]

def aws_run_instance(**kwargs):
    # Create instance
    reservation = conn_ec2.run_instances(image_id=kwargs['ami'],
                                         key_name=kwargs['ssh_key_name'],
                                         security_group_ids=kwargs['sec_groups'],
                                         user_data=kwargs['user_data'],
                                         instance_type=kwargs['instance_type'],
                                         subnet_id=kwargs['subnet_id'])
    instance = reservation.instances[0]

    # Wait for instance to get ready
    status = instance.update()
    while status == 'pending':
        time.sleep(3)
        status = instance.update()
    if status == 'running':
        print '  ID: %s' % instance.id
        print '  Private IP: %s' % instance.private_ip_address
        if instance.public_dns_name:
            print '  Public IP: %s' % instance.public_dns_name
    else:
        print '  Error: %s' % (status)
        return []

    # Set instance name
    instance.add_tag('Name', kwargs['name'])
    # Set instance tags
    for key, value in kwargs['tags'].iteritems():
        instance.add_tag(key, value)

    domains = []
    if 'broker' in kwargs['roles']:
        domains.append('broker.%s' % DOMAIN)
    if 'web_server' in kwargs['roles']:
        domains.append(DOMAIN)
    if 'rabbitmq_server' in kwargs['roles']:
        domains.append('mq.%s' % DOMAIN)

    # Add/update Route 53 entry
    dns_records = []
    if instance.public_dns_name:
        for domain in domains:
            record_sets = conn_r53.get_all_rrsets(ZONE_ID)
            change = record_sets.add_change('CREATE', domain, 'CNAME', 300)
            change.add_value(instance.public_dns_name)
            record_sets.commit()
            print '  Domain: %s' % domain
            dns_records.append(DNSRecord(ZONE_ID, domain, instance.public_dns_name, 300))

    return [instance] + dns_records

def main():
    instance_no = 0

    # List of AWS objects
    aws_objects = []

    template = get_file_content(TEMPLATE)
    for instance in NETWORK:
        tmp = template

        spec = copy.deepcopy(DEFAULTS)
        spec.update(copy.deepcopy(instance))

        # Roles
        if not isinstance(spec['roles'], list):
            spec['roles'] = [spec['roles']]

        # User data
        spec['user_data'] = get_user_data(spec['roles'], ATTRIBUTES)

        # Name
        spec['name'] = NAME_PATTERN % {'username': USERNAME,
                                       'roles': '_'.join(spec['roles']),
                                       'domain': DOMAIN}

        # Tags
        tags = {'init-username': USERNAME,
                'init-roles': ', '.join(spec['roles'])}
        spec.setdefault('tags', {}).update(tags)

        # Create instance
        if 'autoscale' in spec:
            print 'Creating autoscaling instance group: %s-group (%s)' % (spec['name'], spec['instance_type'])
            items = aws_create_autoscale(**spec)
        else:
            print 'Creating instance: %s (%s)' % (spec['name'], spec['instance_type'])
            items = aws_run_instance(**spec)

        aws_objects.extend(items)

    aws_data = []
    print 'Created objects:'
    for item in aws_objects:
        if isinstance(item, boto.ec2.autoscale.group.AutoScalingGroup):
            aws_data.append(('as_group', item.name))
            print '  AS Group  : %s' % item.name
        elif isinstance(item, boto.ec2.autoscale.launchconfig.LaunchConfiguration):
            aws_data.append(('as_config', item.name))
            print '  AS Config : %s' % item.name
        elif isinstance(item, boto.ec2.instance.Instance):
            aws_data.append(('instance', item.id))
            print '  Instance  : %s' % item.id
        elif isinstance(item, DNSRecord):
            record = '%s|%s|%s|%s' % (item.zone, item.name, item.value, item.ttl)
            aws_data.append(('domain', record))
            print '  R53 Entry : %s' % item.name

    # aws_data = '\n'.join(['%s %s' % (x, y) for x, y in aws_data])
    # save_file_content('.', aws_data)

    print 'Press ENTER to delete everything.'
    raw_input()
    for item in aws_objects:
        if isinstance(item, boto.ec2.autoscale.group.AutoScalingGroup):
            print 'Shutting down AS group %s' % item.name
            item.shutdown_instances()
            conn_as.delete_auto_scaling_group(item.name, force_delete=True)
        elif isinstance(item, boto.ec2.autoscale.launchconfig.LaunchConfiguration):
            print 'Deleting AS config %s' % item.name
            conn_as.delete_launch_configuration(item.name)
        elif isinstance(item, boto.ec2.instance.Instance):
            print 'Terminating %s' % item.id
            conn_ec2.terminate_instances([item.id])
        elif isinstance(item, DNSRecord):
            print 'Removing %s' % item.name
            record_sets = conn_r53.get_all_rrsets(ZONE_ID)
            change = record_sets.add_change('DELETE', item.name, 'CNAME', item.ttl)
            change.add_value(item.value)
            record_sets.commit()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import asyncio
import base64
import logging
import os
import sys

from cryptography.hazmat.bindings.openssl.binding import Binding
from pylibsrtp import Policy, Session

from .rtp import is_rtcp
from .utils import first_completed

binding = Binding()
binding.init_static_locks()
ffi = binding.ffi
lib = binding.lib

SRTP_KEY_LEN = 16
SRTP_SALT_LEN = 14

CERT_PATH = os.path.join(os.path.dirname(__file__), 'dtls.crt')
KEY_PATH = os.path.join(os.path.dirname(__file__), 'dtls.key')

logger = logging.getLogger('dtls')

def certificate_digest(x509):
    digest = lib.EVP_get_digestbyname(b'SHA256')
    if digest == ffi.NULL:
        raise ValueError("No such digest method")

    result_buffer = ffi.new('unsigned char[]', lib.EVP_MAX_MD_SIZE)
    result_length = ffi.new('unsigned int[]', 1)
    result_length[0] = len(result_buffer)

    digest_result = lib.X509_digest(x509, digest, result_buffer, result_length)
    assert digest_result == 1

    return b":".join([
        base64.b16encode(ch).upper() for ch
        in ffi.buffer(result_buffer, result_length[0])]).decode('ascii')

def get_srtp_key_salt(src, idx):
    key_start = idx * SRTP_KEY_LEN
    salt_start = 2 * SRTP_KEY_LEN + idx * SRTP_SALT_LEN
    return (
        src[key_start:key_start + SRTP_KEY_LEN] +
        src[salt_start:salt_start + SRTP_SALT_LEN]
    )

@ffi.callback('int(int, X509_STORE_CTX *)')
def verify_callback(x, y):
    return 1

class DtlsSrtpContext:
    def __init__(self):
        ctx = lib.SSL_CTX_new(lib.DTLSv1_method())
        self.ctx = ffi.gc(ctx, lib.SSL_CTX_free)

        lib.SSL_CTX_set_verify(self.ctx, lib.SSL_VERIFY_PEER | lib.SSL_VERIFY_FAIL_IF_NO_PEER_CERT,
                               verify_callback)
        if not lib.SSL_CTX_use_certificate_file(self.ctx,
                                                CERT_PATH.encode(sys.getfilesystemencoding()),
                                                lib.SSL_FILETYPE_PEM):
            print("SSL could not use certificate")
        if not lib.SSL_CTX_use_PrivateKey_file(self.ctx,
                                               KEY_PATH.encode(sys.getfilesystemencoding()),
                                               lib.SSL_FILETYPE_PEM):
            print("SSL could not use private key")
        if not lib.SSL_CTX_set_cipher_list(self.ctx, b'HIGH:!CAMELLIA:!aNULL'):
            print("SSL could not set cipher list")
        if lib.SSL_CTX_set_tlsext_use_srtp(self.ctx, b'SRTP_AES128_CM_SHA1_80'):
            print("SSL could not enable SRTP extension")
        if lib.SSL_CTX_set_read_ahead(self.ctx, 1):
            print("SSL could not enable read ahead")

class Channel:
    def __init__(self, closed, queue, send):
        self.closed = closed
        self.queue = queue
        self.send = send

    async def recv(self):
        data = await first_completed(self.queue.get(), self.closed.wait())
        if data is True:
            raise ConnectionError
        return data

class DtlsSrtpSession:
    def __init__(self, context, is_server, transport):
        self.closed = asyncio.Event()
        self.encrypted = False
        self.is_server = is_server
        self.remote_fingerprint = None
        self.transport = transport

        self.data_cdata = ffi.new('char[]', 1500)
        self.data_queue = asyncio.Queue()
        self.data = Channel(
            closed=self.closed,
            queue=self.data_queue,
            send=self._send_data)

        self.rtp_queue = asyncio.Queue()
        self.rtp = Channel(
            closed=self.closed,
            queue=self.rtp_queue,
            send=self._send_rtp)

        ssl = lib.SSL_new(context.ctx)
        self.ssl = ffi.gc(ssl, lib.SSL_free)

        self.read_bio = lib.BIO_new(lib.BIO_s_mem())
        self.write_bio = lib.BIO_new(lib.BIO_s_mem())
        lib.SSL_set_bio(self.ssl, self.read_bio, self.write_bio)

        if self.is_server:
            lib.SSL_set_accept_state(self.ssl)
        else:
            lib.SSL_set_connect_state(self.ssl)

        # local fingerprint
        x509 = lib.SSL_get_certificate(self.ssl)
        self.local_fingerprint = certificate_digest(x509)

    async def close(self):
        lib.SSL_shutdown(self.ssl)
        await self._write_ssl()
        logger.info('DTLS shutdown complete')

    async def connect(self):
        while not self.encrypted:
            result = lib.SSL_do_handshake(self.ssl)
            await self._write_ssl()

            if result > 0:
                self.encrypted = True
                break

            error = lib.SSL_get_error(self.ssl, result)
            if error == lib.SSL_ERROR_WANT_READ:
                data = await self.transport.recv()
                lib.BIO_write(self.read_bio, data, len(data))
            else:
                raise Exception('DTLS handshake failed (error %d)' % error)

        # check remote fingerprint
        x509 = lib.SSL_get_peer_certificate(self.ssl)
        remote_fingerprint = certificate_digest(x509)
        if remote_fingerprint != self.remote_fingerprint.upper():
            raise Exception('DTLS fingerprint does not match')

        # generate keying material
        buf = ffi.new('unsigned char[]', 2 * (SRTP_KEY_LEN + SRTP_SALT_LEN))
        extractor = b'EXTRACTOR-dtls_srtp'
        if not lib.SSL_export_keying_material(self.ssl, buf, len(buf),
                                              extractor, len(extractor),
                                              ffi.NULL, 0, 0):
            raise Exception('DTLS could not extract SRTP keying material')

        view = ffi.buffer(buf)
        if self.is_server:
            srtp_tx_key = get_srtp_key_salt(view, 1)
            srtp_rx_key = get_srtp_key_salt(view, 0)
        else:
            srtp_tx_key = get_srtp_key_salt(view, 0)
            srtp_rx_key = get_srtp_key_salt(view, 1)

        logger.info('DTLS handshake complete')
        rx_policy = Policy(key=srtp_rx_key, ssrc_type=Policy.SSRC_ANY_INBOUND)
        self._rx_srtp = Session(rx_policy)
        tx_policy = Policy(key=srtp_tx_key, ssrc_type=Policy.SSRC_ANY_OUTBOUND)
        self._tx_srtp = Session(tx_policy)

        # start data pump
        asyncio.ensure_future(self.__run())

    async def __run(self):
        while True:
            try:
                data = await first_completed(self.transport.recv(), self.closed.wait())
            except Exception:
                # underlying transport was closed
                break

            if data is True:
                # session was closed
                break

            first_byte = data[0]
            if first_byte > 19 and first_byte < 64:
                # DTLS
                lib.BIO_write(self.read_bio, data, len(data))
                result = lib.SSL_read(self.ssl, self.data_cdata, len(self.data_cdata))
                if result > 0:
                    await self.data_queue.put(ffi.buffer(self.data_cdata)[0:result])
            elif first_byte > 127 and first_byte < 192:
                # SRTP / SRTCP
                if is_rtcp(data):
                    data = self._rx_srtp.unprotect_rtcp(data)
                else:
                    data = self._rx_srtp.unprotect(data)
                await self.rtp_queue.put(data)

        # mark session as closed
        self.closed.set()

    async def _send_data(self, data):
        lib.SSL_write(self.ssl, data, len(data))
        await self._write_ssl()

    async def _send_rtp(self, data):
        if is_rtcp(data):
            data = self._tx_srtp.protect_rtcp(data)
        else:
            data = self._tx_srtp.protect(data)
        await self.transport.send(data)

    async def _write_ssl(self):
        """
        Flush outgoing data which OpenSSL put in our BIO to the transport.
        """
        pending = lib.BIO_ctrl_pending(self.write_bio)
        if pending > 0:
            buf = ffi.new('char[]', pending)
            lib.BIO_read(self.write_bio, buf, len(buf))
            data = b''.join(buf)
            await self.transport.send(data)
<commit_msg>[dtls] move all data reception to _recv_next<commit_after>import asyncio
import base64
import logging
import os
import sys

from cryptography.hazmat.bindings.openssl.binding import Binding
from pylibsrtp import Policy, Session

from .rtp import is_rtcp
from .utils import first_completed

binding = Binding()
binding.init_static_locks()
ffi = binding.ffi
lib = binding.lib

SRTP_KEY_LEN = 16
SRTP_SALT_LEN = 14

CERT_PATH = os.path.join(os.path.dirname(__file__), 'dtls.crt')
KEY_PATH = os.path.join(os.path.dirname(__file__), 'dtls.key')

logger = logging.getLogger('dtls')

def certificate_digest(x509):
    digest = lib.EVP_get_digestbyname(b'SHA256')
    if digest == ffi.NULL:
        raise ValueError("No such digest method")

    result_buffer = ffi.new('unsigned char[]', lib.EVP_MAX_MD_SIZE)
    result_length = ffi.new('unsigned int[]', 1)
    result_length[0] = len(result_buffer)

    digest_result = lib.X509_digest(x509, digest, result_buffer, result_length)
    assert digest_result == 1

    return b":".join([
        base64.b16encode(ch).upper() for ch
        in ffi.buffer(result_buffer, result_length[0])]).decode('ascii')

def get_srtp_key_salt(src, idx):
    key_start = idx * SRTP_KEY_LEN
    salt_start = 2 * SRTP_KEY_LEN + idx * SRTP_SALT_LEN
    return (
        src[key_start:key_start + SRTP_KEY_LEN] +
        src[salt_start:salt_start + SRTP_SALT_LEN]
    )

@ffi.callback('int(int, X509_STORE_CTX *)')
def verify_callback(x, y):
    return 1

class DtlsSrtpContext:
    def __init__(self):
        ctx = lib.SSL_CTX_new(lib.DTLSv1_method())
        self.ctx = ffi.gc(ctx, lib.SSL_CTX_free)

        lib.SSL_CTX_set_verify(self.ctx, lib.SSL_VERIFY_PEER | lib.SSL_VERIFY_FAIL_IF_NO_PEER_CERT,
                               verify_callback)
        if not lib.SSL_CTX_use_certificate_file(self.ctx,
                                                CERT_PATH.encode(sys.getfilesystemencoding()),
                                                lib.SSL_FILETYPE_PEM):
            print("SSL could not use certificate")
        if not lib.SSL_CTX_use_PrivateKey_file(self.ctx,
                                               KEY_PATH.encode(sys.getfilesystemencoding()),
                                               lib.SSL_FILETYPE_PEM):
            print("SSL could not use private key")
        if not lib.SSL_CTX_set_cipher_list(self.ctx, b'HIGH:!CAMELLIA:!aNULL'):
            print("SSL could not set cipher list")
        if lib.SSL_CTX_set_tlsext_use_srtp(self.ctx, b'SRTP_AES128_CM_SHA1_80'):
            print("SSL could not enable SRTP extension")
        if lib.SSL_CTX_set_read_ahead(self.ctx, 1):
            print("SSL could not enable read ahead")

class Channel:
    def __init__(self, closed, queue, send):
        self.closed = closed
        self.queue = queue
        self.send = send

    async def recv(self):
        data = await first_completed(self.queue.get(), self.closed.wait())
        if data is True:
            raise ConnectionError
        return data

class DtlsSrtpSession:
    def __init__(self, context, is_server, transport):
        self.closed = asyncio.Event()
        self.encrypted = False
        self.is_server = is_server
        self.remote_fingerprint = None
        self.transport = transport

        self.data_cdata = ffi.new('char[]', 1500)
        self.data_queue = asyncio.Queue()
        self.data = Channel(
            closed=self.closed,
            queue=self.data_queue,
            send=self._send_data)

        self.rtp_queue = asyncio.Queue()
        self.rtp = Channel(
            closed=self.closed,
            queue=self.rtp_queue,
            send=self._send_rtp)

        ssl = lib.SSL_new(context.ctx)
        self.ssl = ffi.gc(ssl, lib.SSL_free)

        self.read_bio = lib.BIO_new(lib.BIO_s_mem())
        self.write_bio = lib.BIO_new(lib.BIO_s_mem())
        lib.SSL_set_bio(self.ssl, self.read_bio, self.write_bio)

        if self.is_server:
            lib.SSL_set_accept_state(self.ssl)
        else:
            lib.SSL_set_connect_state(self.ssl)

        # local fingerprint
        x509 = lib.SSL_get_certificate(self.ssl)
        self.local_fingerprint = certificate_digest(x509)

    async def close(self):
        lib.SSL_shutdown(self.ssl)
        await self._write_ssl()
        logger.info('DTLS shutdown complete')

    async def connect(self):
        while not self.encrypted:
            result = lib.SSL_do_handshake(self.ssl)
            await self._write_ssl()

            if result > 0:
                self.encrypted = True
                break

            error = lib.SSL_get_error(self.ssl, result)
            if error == lib.SSL_ERROR_WANT_READ:
                await self._recv_next()
            else:
                raise Exception('DTLS handshake failed (error %d)' % error)

        # check remote fingerprint
        x509 = lib.SSL_get_peer_certificate(self.ssl)
        remote_fingerprint = certificate_digest(x509)
        if remote_fingerprint != self.remote_fingerprint.upper():
            raise Exception('DTLS fingerprint does not match')

        # generate keying material
        buf = ffi.new('unsigned char[]', 2 * (SRTP_KEY_LEN + SRTP_SALT_LEN))
        extractor = b'EXTRACTOR-dtls_srtp'
        if not lib.SSL_export_keying_material(self.ssl, buf, len(buf),
                                              extractor, len(extractor),
                                              ffi.NULL, 0, 0):
            raise Exception('DTLS could not extract SRTP keying material')

        view = ffi.buffer(buf)
        if self.is_server:
            srtp_tx_key = get_srtp_key_salt(view, 1)
            srtp_rx_key = get_srtp_key_salt(view, 0)
        else:
            srtp_tx_key = get_srtp_key_salt(view, 0)
            srtp_rx_key = get_srtp_key_salt(view, 1)

        logger.info('DTLS handshake complete')
        rx_policy = Policy(key=srtp_rx_key, ssrc_type=Policy.SSRC_ANY_INBOUND)
        self._rx_srtp = Session(rx_policy)
        tx_policy = Policy(key=srtp_tx_key, ssrc_type=Policy.SSRC_ANY_OUTBOUND)
        self._tx_srtp = Session(tx_policy)

        # start data pump
        asyncio.ensure_future(self.__run())

    async def __run(self):
        try:
            while True:
                await self._recv_next()
        except ConnectionError:
            pass
        finally:
            self.closed.set()

    async def _recv_next(self):
        try:
            data = await first_completed(self.transport.recv(), self.closed.wait())
        except Exception:
            # underlying transport was closed
            raise ConnectionError

        if data is True:
            # session was closed
            raise ConnectionError

        first_byte = data[0]
        if first_byte > 19 and first_byte < 64:
            # DTLS
            lib.BIO_write(self.read_bio, data, len(data))
            result = lib.SSL_read(self.ssl, self.data_cdata, len(self.data_cdata))
            if result > 0:
                await self.data_queue.put(ffi.buffer(self.data_cdata)[0:result])
        elif first_byte > 127 and first_byte < 192:
            # SRTP / SRTCP
            if is_rtcp(data):
                data = self._rx_srtp.unprotect_rtcp(data)
            else:
                data = self._rx_srtp.unprotect(data)
            await self.rtp_queue.put(data)

    async def _send_data(self, data):
        lib.SSL_write(self.ssl, data, len(data))
        await self._write_ssl()

    async def _send_rtp(self, data):
        if is_rtcp(data):
            data = self._tx_srtp.protect_rtcp(data)
        else:
            data = self._tx_srtp.protect(data)
        await self.transport.send(data)

    async def _write_ssl(self):
        """
        Flush outgoing data which OpenSSL put in our BIO to the transport.
        """
        pending = lib.BIO_ctrl_pending(self.write_bio)
        if pending > 0:
            buf = ffi.new('char[]', pending)
            lib.BIO_read(self.write_bio, buf, len(buf))
            data = b''.join(buf)
            await self.transport.send(data)
<|endoftext|>


<commit_before>from twisted.python import log
from twisted.python.log import logging
from twisted.internet.defer import inlineCallbacks, returnValue
from django.contrib.auth.models import User

from vumi.service import Worker, Consumer, Publisher
from vumi.webapp.api import utils

import json
import time

from vumi.webapp.api import models

class SMSKeywordConsumer(Consumer):
    exchange_name = "vumi"
    exchange_type = "direct"
    durable = True
    delivery_mode = 2
    queue_name = "" # overwritten by subclass
    routing_key = "" # overwritten by subclass

    def consume_json(self, dictionary):
        message = dictionary.get('short_message')
        head = message.split(' ')[0]
        try:
            user = User.objects.get(username=head)
            profile = user.get_profile()
            urlcallback_set = profile.urlcallback_set.filter(name='sms_received')
            for urlcallback in urlcallback_set:
                try:
                    url = urlcallback.url
                    log.msg('URL: %s' % urlcallback.url)
                    params = [
                            ("callback_name", "sms_received"),
                            ("to_msisdn", str(dictionary.get('destination_addr'))),
                            ("from_msisdn", str(dictionary.get('source_addr'))),
                            ("message", str(dictionary.get('short_message')))
                            ]
                    url, resp = utils.callback(url, params)
                    log.msg('RESP: %s' % resp)
                except Exception, e:
                    log.err(e)

        except User.DoesNotExist:
            log.msg("Couldn't find user for message: %s" % message)
        log.msg("DELIVER SM %s consumed by %s" % (json.dumps(dictionary),self.__class__.__name__))

class FallbackSMSKeywordConsumer(SMSKeywordConsumer):
    routing_key = 'sms.fallback'

def dynamically_create_keyword_consumer(name,**kwargs):
    return type("%s_SMSKeywordConsumer" % name, (SMSKeywordConsumer,), kwargs)

class SMSKeywordWorker(Worker):
    """
    A worker that fires off URLCallback's for incoming SMSs
    with keywords
    """

    @inlineCallbacks
    def startWorker(self):
        log.msg("Starting the SMSKeywordWorkers for: %s" % self.config.get('OPERATOR_NUMBER'))
        for network,msisdn in self.config.get('OPERATOR_NUMBER').items():
            if len(msisdn):
                yield self.start_consumer(dynamically_create_keyword_consumer(network,
                    routing_key='sms.%s' % msisdn,
                    queue_name='sms.keywords.%s' % network.lower()
                ))
        yield self.start_consumer(FallbackSMSKeywordConsumer)

    def stopWorker(self):
        log.msg("Stopping the SMSKeywordWorker")

#==================================================================================================

class SMSReceiptConsumer(Consumer):
    exchange_name = "vumi"
    exchange_type = "direct"
    durable = True
    delivery_mode = 2
    queue_name = "" # overwritten by subclass
    routing_key = "" # overwritten by subclass

    def consume_json(self, dictionary):
        _id = dictionary['delivery_report']['id']
        if len(_id):
            resp = models.SMPPResp.objects.get(message_id=_id)
            sent = resp.sent_sms
            user = sent.user
            profile = user.get_profile()
            urlcallback_set = profile.urlcallback_set.filter(name='sms_receipt')
            for urlcallback in urlcallback_set:
                try:
                    url = urlcallback.url
                    log.msg('URL: %s' % urlcallback.url)
                    params = [
                            ("id", sent.id),
                            ("transport_status", dictionary['delivery_report']['stat']),
                            ("transport_status_display", dictionary['delivery_report']['stat']),
                            ("created_at", sent.created_at),
                            ("updated_at", sent.updated_at),
                            ("delivered_at", time.strftime(
                                    "%Y-%m-%d %H:%M:%S",
                                    time.strptime(
                                        "20"+dictionary['delivery_report']['done_date'],
                                        "%Y%m%d%H%M%S"
                                        )
                                    )),
                            ("from_msisdn", dictionary['destination_addr']),
                            ("to_msisdn", sent.to_msisdn),
                            ("message", sent.message),
                            ]
                    url, resp = utils.callback(url, params)
                    log.msg('RESP: %s' % resp)
                except Exception, e:
                    log.err(e)
        log.msg("RECEIPT SM %s consumed by %s" % (json.dumps(dictionary),self.__class__.__name__))

class FallbackSMSReceiptConsumer(SMSReceiptConsumer):
    routing_key = 'receipt.fallback'

def dynamically_create_receipt_consumer(name,**kwargs):
    return type("%s_SMSReceiptConsumer" % name, (SMSReceiptConsumer,), kwargs)

class SMSReceiptWorker(Worker):
    """
    A worker that fires off URLCallback's for incoming Receipts
    """

    @inlineCallbacks
    def startWorker(self):
        log.msg("Starting the SMSReceiptWorkers for: %s" % self.config.get('OPERATOR_NUMBER'))
        for network,msisdn in self.config.get('OPERATOR_NUMBER').items():
            if len(msisdn):
                yield self.start_consumer(dynamically_create_receipt_consumer(network,
                    routing_key='receipt.%s' % msisdn,
                    queue_name='receipt.%s' % network.lower()
                ))
        yield self.start_consumer(FallbackSMSReceiptConsumer)

    def stopWorker(self):
        log.msg("Stopping the SMSReceiptWorker")

#==================================================================================================

class SMSBatchConsumer(Consumer):
    exchange_name = "vumi"
    exchange_type = "direct"
    durable = True
    delivery_mode = 2
    queue_name = "batch"
    routing_key = "batch"

    def __init__(self, publisher):
        self.publisher = publisher

    def consume_json(self, dictionary):
        log.msg("SM BATCH %s consumed by %s" % (json.dumps(dictionary),self.__class__.__name__))
        payload = []
        kwargs = dictionary.get('kwargs')
        if kwargs:
            pk = kwargs.get('pk')
            for o in models.SentSMS.objects.filter(send_group=pk):
                mess = {
                        'transport_name':o.transport_name,
                        'send_group':o.send_group_id,
                        'from_msisdn':o.from_msisdn,
                        'user':o.user_id,
                        'to_msisdn':o.to_msisdn,
                        'message':o.message,
                        'id':o.id
                        }
                print ">>>>", json.dumps(mess)
                self.publisher.publish_json(mess)
                #reactor.callLater(0, self.publisher.publish_json, mess)
        return True

    def consume(self, message):
        if self.consume_json(json.loads(message.content.body)):
            self.ack(message)

class FallbackSMSBatchConsumer(SMSBatchConsumer):
    routing_key = 'batch.fallback'

class IndivPublisher(Publisher):
    """
    This publisher publishes all incoming SMPP messages to the
    `vumi.smpp` exchange, its default routing key is `smpp.fallback`
    """
    exchange_name = "vumi"
    exchange_type = "direct"
    routing_key = "sms.indiv"
    durable = True
    auto_delete = False
    delivery_mode = 2

    def publish_json(self, dictionary, **kwargs):
        log.msg("Publishing JSON %s with extra args: %s" % (dictionary, kwargs))
        super(IndivPublisher, self).publish_json(dictionary, **kwargs)

class SMSBatchWorker(Worker):
    """
    A worker that breaks up batches of sms's into individual sms's
    """

    @inlineCallbacks
    def startWorker(self):
        log.msg("Starting the SMSBatchWorker")
        self.publisher = yield self.start_publisher(IndivPublisher)
        yield self.start_consumer(SMSBatchConsumer, self.publisher)
        yield self.start_consumer(FallbackSMSBatchConsumer)

    def stopWorker(self):
        log.msg("Stopping the SMSBatchWorker")

<commit_msg>bloody pycurl & unicode<commit_after>from twisted.python import log
from twisted.python.log import logging
from twisted.internet.defer import inlineCallbacks, returnValue
from django.contrib.auth.models import User

from vumi.service import Worker, Consumer, Publisher
from vumi.webapp.api import utils

import json
import time

from vumi.webapp.api import models

class SMSKeywordConsumer(Consumer):
    exchange_name = "vumi"
    exchange_type = "direct"
    durable = True
    delivery_mode = 2
    queue_name = "" # overwritten by subclass
    routing_key = "" # overwritten by subclass

    def consume_json(self, dictionary):
        message = dictionary.get('short_message')
        head = message.split(' ')[0]
        try:
            user = User.objects.get(username=head)
            profile = user.get_profile()
            urlcallback_set = profile.urlcallback_set.filter(name='sms_received')
            for urlcallback in urlcallback_set:
                try:
                    url = urlcallback.url
                    log.msg('URL: %s' % urlcallback.url)
                    params = [
                            ("callback_name", "sms_received"),
                            ("to_msisdn", str(dictionary.get('destination_addr'))),
                            ("from_msisdn", str(dictionary.get('source_addr'))),
                            ("message", str(dictionary.get('short_message')))
                            ]
                    url, resp = utils.callback(url, params)
                    log.msg('RESP: %s' % resp)
                except Exception, e:
                    log.err(e)

        except User.DoesNotExist:
            log.msg("Couldn't find user for message: %s" % message)
        log.msg("DELIVER SM %s consumed by %s" % (json.dumps(dictionary),self.__class__.__name__))

class FallbackSMSKeywordConsumer(SMSKeywordConsumer):
    routing_key = 'sms.fallback'

def dynamically_create_keyword_consumer(name,**kwargs):
    return type("%s_SMSKeywordConsumer" % name, (SMSKeywordConsumer,), kwargs)

class SMSKeywordWorker(Worker):
    """
    A worker that fires off URLCallback's for incoming SMSs
    with keywords
    """

    @inlineCallbacks
    def startWorker(self):
        log.msg("Starting the SMSKeywordWorkers for: %s" % self.config.get('OPERATOR_NUMBER'))
        for network,msisdn in self.config.get('OPERATOR_NUMBER').items():
            if len(msisdn):
                yield self.start_consumer(dynamically_create_keyword_consumer(network,
                    routing_key='sms.%s' % msisdn,
                    queue_name='sms.keywords.%s' % network.lower()
                ))
        yield self.start_consumer(FallbackSMSKeywordConsumer)

    def stopWorker(self):
        log.msg("Stopping the SMSKeywordWorker")

#==================================================================================================

class SMSReceiptConsumer(Consumer):
    exchange_name = "vumi"
    exchange_type = "direct"
    durable = True
    delivery_mode = 2
    queue_name = "" # overwritten by subclass
    routing_key = "" # overwritten by subclass

    def consume_json(self, dictionary):
        _id = dictionary['delivery_report']['id']
        if len(_id):
            resp = models.SMPPResp.objects.get(message_id=_id)
            sent = resp.sent_sms
            user = sent.user
            profile = user.get_profile()
            urlcallback_set = profile.urlcallback_set.filter(name='sms_receipt')
            for urlcallback in urlcallback_set:
                try:
                    url = urlcallback.url
                    log.msg('URL: %s' % urlcallback.url)
                    params = [
                            ("id", sent.id),
                            ("transport_status", dictionary['delivery_report']['stat']),
                            ("transport_status_display", dictionary['delivery_report']['stat']),
                            ("created_at", sent.created_at),
                            ("updated_at", sent.updated_at),
                            ("delivered_at", time.strftime(
                                    "%Y-%m-%d %H:%M:%S",
                                    time.strptime(
                                        "20"+dictionary['delivery_report']['done_date'],
                                        "%Y%m%d%H%M%S"
                                        )
                                    )),
                            ("from_msisdn", dictionary['destination_addr']),
                            ("to_msisdn", sent.to_msisdn),
                            ("message", sent.message),
                            ]
                    print repr(params)
                    url, resp = utils.callback(url, params)
                    log.msg('RESP: %s' % resp)
                except Exception, e:
                    log.err(e)
        log.msg("RECEIPT SM %s consumed by %s" % (json.dumps(dictionary),self.__class__.__name__))

class FallbackSMSReceiptConsumer(SMSReceiptConsumer):
    routing_key = 'receipt.fallback'

def dynamically_create_receipt_consumer(name,**kwargs):
    return type("%s_SMSReceiptConsumer" % name, (SMSReceiptConsumer,), kwargs)

class SMSReceiptWorker(Worker):
    """
    A worker that fires off URLCallback's for incoming Receipts
    """

    @inlineCallbacks
    def startWorker(self):
        log.msg("Starting the SMSReceiptWorkers for: %s" % self.config.get('OPERATOR_NUMBER'))
        for network,msisdn in self.config.get('OPERATOR_NUMBER').items():
            if len(msisdn):
                yield self.start_consumer(dynamically_create_receipt_consumer(network,
                    routing_key='receipt.%s' % msisdn,
                    queue_name='receipt.%s' % network.lower()
                ))
        yield self.start_consumer(FallbackSMSReceiptConsumer)

    def stopWorker(self):
        log.msg("Stopping the SMSReceiptWorker")

#==================================================================================================

class SMSBatchConsumer(Consumer):
    exchange_name = "vumi"
    exchange_type = "direct"
    durable = True
    delivery_mode = 2
    queue_name = "batch"
    routing_key = "batch"

    def __init__(self, publisher):
        self.publisher = publisher

    def consume_json(self, dictionary):
        log.msg("SM BATCH %s consumed by %s" % (json.dumps(dictionary),self.__class__.__name__))
        payload = []
        kwargs = dictionary.get('kwargs')
        if kwargs:
            pk = kwargs.get('pk')
            for o in models.SentSMS.objects.filter(send_group=pk):
                mess = {
                        'transport_name':o.transport_name,
                        'send_group':o.send_group_id,
                        'from_msisdn':o.from_msisdn,
                        'user':o.user_id,
                        'to_msisdn':o.to_msisdn,
                        'message':o.message,
                        'id':o.id
                        }
                print ">>>>", json.dumps(mess)
                self.publisher.publish_json(mess)
                #reactor.callLater(0, self.publisher.publish_json, mess)
        return True

    def consume(self, message):
        if self.consume_json(json.loads(message.content.body)):
            self.ack(message)

class FallbackSMSBatchConsumer(SMSBatchConsumer):
    routing_key = 'batch.fallback'

class IndivPublisher(Publisher):
    """
    This publisher publishes all incoming SMPP messages to the
    `vumi.smpp` exchange, its default routing key is `smpp.fallback`
    """
    exchange_name = "vumi"
    exchange_type = "direct"
    routing_key = "sms.indiv"
    durable = True
    auto_delete = False
    delivery_mode = 2

    def publish_json(self, dictionary, **kwargs):
        log.msg("Publishing JSON %s with extra args: %s" % (dictionary, kwargs))
        super(IndivPublisher, self).publish_json(dictionary, **kwargs)

class SMSBatchWorker(Worker):
    """
    A worker that breaks up batches of sms's into individual sms's
    """

    @inlineCallbacks
    def startWorker(self):
        log.msg("Starting the SMSBatchWorker")
        self.publisher = yield self.start_publisher(IndivPublisher)
        yield self.start_consumer(SMSBatchConsumer, self.publisher)
        yield self.start_consumer(FallbackSMSBatchConsumer)

    def stopWorker(self):
        log.msg("Stopping the SMSBatchWorker")

<|endoftext|>


<commit_before># coding: utf-8

from django.db import models
from django.utils.translation import ugettext_lazy as _

class Subdomain(models.Model):
    '''One atlas page'''
    SUBDOMAIN_STATUS = (
        (0, _('inactive')),
        (1, _('active')))

    class Meta:
        ordering = ['order']

    url = models.CharField(max_length=50, blank=True, null=True)
    title = models.CharField(max_length=50)
    isCurrent = models.BooleanField(db_column='is_current')
    status = models.SmallIntegerField(choices=SUBDOMAIN_STATUS)
    order = models.IntegerField()
    disclaimer = models.TextField(blank=True)

    def __unicode__(self):
        return self.title

    def name(self):
        return self.title

    def full_url(self):
        if self.url:
            return "%s.newrynda.org" % self.url
        else:
            return "newrynda.org"

class Category(models.Model):
    ''' Категория сообщения '''
    class Meta:
        ordering = ['order']

    name = models.CharField(
        max_length=200, db_column='name',
        verbose_name=_('name'))
    description = models.TextField(
        blank=True, db_column='description',
        verbose_name=_('description'))
    color = models.CharField(
        max_length=7, db_column='color',
        default='#000000', verbose_name=_('color'))
    slug = models.SlugField(
        max_length=255, db_column='slug',
        verbose_name=_('slug'), blank=True)
    icon = models.CharField(
        max_length=255, null=True, blank=True,
        db_column='icon', verbose_name=_('icon'))
    order = models.SmallIntegerField(db_column='order')
    subdomain = models.ForeignKey(
        Subdomain, null=True, blank=True,
        db_column='subdomain_id', verbose_name=_('subdomain'))
    group = models.ForeignKey("CategoryGroup", null=True, blank=True)

    def __unicode__(self):
        return self.name

    def unlink(self):
        '''Remove link to category group'''
        self.group = None
        self.save()

class CategoryGroup(models.Model):
    '''Grouping categories. One category must be in one group.'''
    class Meta:
        ordering = ['order']

    name = models.CharField(
        max_length=200,
        verbose_name=_('category group name'))
    order = models.IntegerField()

    def __unicode__(self):
        return _("Category group %s") % self.name

    def add_category(self, category):
        '''Add category to group '''
        category.group = self
        category.save()

class Infopage(models.Model):
    class Meta:
        pass

    title = models.CharField(
        max_length=255, db_column='title',
        verbose_name=_('title'))
    text = models.TextField(db_column='text', verbose_name=_('text'))
    active = models.BooleanField(default=False, verbose_name=_('is active'))
    slug = models.SlugField(max_length=255, verbose_name=_('slug'))
    default = models.BooleanField(
        default=False, verbose_name=_('default page'))

    def __unicode__(self):
        return self.title
<commit_msg>Added i18n for core models<commit_after># coding: utf-8

from django.db import models
from django.utils.translation import ugettext_lazy as _

class Subdomain(models.Model):
    '''One atlas page'''
    SUBDOMAIN_STATUS = (
        (0, _('inactive')),
        (1, _('active')))

    class Meta:
        ordering = ['order']
        verbose_name = _('map')
        verbose_name_plural = _('maps')

    url = models.CharField(max_length=50, blank=True, null=True)
    title = models.CharField(max_length=50)
    isCurrent = models.BooleanField(db_column='is_current')
    status = models.SmallIntegerField(choices=SUBDOMAIN_STATUS)
    order = models.IntegerField()
    disclaimer = models.TextField(blank=True)

    def __unicode__(self):
        return self.title

    def name(self):
        return self.title

    def full_url(self):
        if self.url:
            return "%s.newrynda.org" % self.url
        else:
            return "newrynda.org"

class Category(models.Model):
    ''' Категория сообщения '''
    class Meta:
        ordering = ['order']
        verbose_name = _('category')
        verbose_name_plural = _('categories')

    name = models.CharField(
        max_length=200, db_column='name',
        verbose_name=_('name'))
    description = models.TextField(
        blank=True, db_column='description',
        verbose_name=_('description'))
    color = models.CharField(
        max_length=7, db_column='color',
        default='#000000', verbose_name=_('color'))
    slug = models.SlugField(
        max_length=255, db_column='slug',
        verbose_name=_('slug'), blank=True)
    icon = models.CharField(
        max_length=255, null=True, blank=True,
        db_column='icon', verbose_name=_('icon'))
    order = models.SmallIntegerField(db_column='order')
    subdomain = models.ForeignKey(
        Subdomain, null=True, blank=True,
        db_column='subdomain_id', verbose_name=_('subdomain'))
    group = models.ForeignKey("CategoryGroup", null=True, blank=True)

    def __unicode__(self):
        return self.name

    def unlink(self):
        '''Remove link to category group'''
        self.group = None
        self.save()

class CategoryGroup(models.Model):
    '''Grouping categories. One category must be in one group.'''
    class Meta:
        ordering = ['order']
        verbose_name = _('category group')
        verbose_name_plural = _('category groups')

    name = models.CharField(
        max_length=200,
        verbose_name=_('category group name'))
    order = models.IntegerField()

    def __unicode__(self):
        return self.name

    def add_category(self, category):
        '''Add category to group '''
        category.group = self
        category.save()

class Infopage(models.Model):
    class Meta:
        pass

    title = models.CharField(
        max_length=255, db_column='title',
        verbose_name=_('title'))
    text = models.TextField(db_column='text', verbose_name=_('text'))
    active = models.BooleanField(default=False, verbose_name=_('is active'))
    slug = models.SlugField(max_length=255, verbose_name=_('slug'))
    default = models.BooleanField(
        default=False, verbose_name=_('default page'))

    def __unicode__(self):
        return self.title
<|endoftext|>