<commit_before>import hashlib, base64
import urlparse, urllib

TITLE = 'MYTF1'
ART = 'art-default.jpg'
ICON = 'icon-default.png'
PREFIX = '/video/mytf1'

BASE_URL = 'http://www.tf1.fr'
PROGRAMS = '%s/programmes-tv' % BASE_URL
VIDEOS = '%s/%s/videos'

RE_MEDIA_ID = Regex("c_idwat = '(?P<media_id>[^']+)'")

####################################################################################################
def Start():

    ObjectContainer.title1 = TITLE
    ObjectContainer.art = R(ART)

    DirectoryObject.thumb = R(ICON)
    DirectoryObject.art = R(ART)
    VideoClipObject.thumb = R(ICON)
    VideoClipObject.art = R(ART)

    HTTP.CacheTime = CACHE_1HOUR
    HTTP.Headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'

####################################################################################################
@handler(PREFIX, TITLE)
def MainMenu():

    oc = ObjectContainer(
        title2="Program Categories"
    )

    html = HTML.ElementFromURL(PROGRAMS)

    for category in html.xpath('//ul[contains(@class, "filters_2") and contains(@class, "contentopen")]/li/a'):
        oc.add(DirectoryObject(
            key=Callback(Programs, prog_cat=category.xpath('./@data-target')[0]),
            title=category.xpath('./text()')[0]
        ))

    return oc

@route(PREFIX + '/programs/{prog_cat}')
def Programs(prog_cat):

    oc = ObjectContainer(
        title2="Programs"
    )

    html = HTML.ElementFromURL(PROGRAMS)

    for program in html.xpath('//ul[contains(@id, "js_filter_el_container")]/li'):
        if program.xpath('./@data-type')[0] != prog_cat and prog_cat != 'all':
            continue

        imgs = program.xpath('./div/a/div/picture/source/@data-srcset')
        if (len(imgs) != 0):
            img = imgs[0]
        else:
            img = program.xpath('./div/a/div/picture/source/@srcset')[0]

        oc.add(DirectoryObject(
            key=Callback(VideoCategories, prog_url=program.xpath('./div/a/@href')[0]),
            title=program.xpath('./div/div/a/div/p/text()')[0],
            thumb='http:' + img.split(',')[-1].split(' ')[0]
        ))

    return oc

@route(PREFIX + '/videos')
def VideoCategories(prog_url):

    oc = ObjectContainer(
        title2="Video Categories"
    )

    html = HTML.ElementFromURL(VIDEOS % (BASE_URL, prog_url))

    #TODO: it's possible to retrieve video filters, but I can't find a way to filter the videos, idk how they are doing it
    for category in html.xpath('//ul[contains(@class, "filters_1") and contains(@class, "contentopen")]/li/a'):
        oc.add(DirectoryObject(
            key=Callback(Videos, video_cat=category.xpath('./@data-filter')[0], prog_url=prog_url),
            title=category.xpath('./text()')[0]
        ))

    return oc

@route(PREFIX + '/videos/{video_cat}')
def Videos(video_cat, prog_url):

    oc = ObjectContainer(
        title2="Videos"
    )

    html = HTML.ElementFromURL(VIDEOS % (BASE_URL, prog_url))

    for video in html.xpath('//ul[contains(@class, "grid")]/li'):
        #if video.xpath('./@data-filter')[0] != video_cat and video_cat != 'all':
        #    continue

        imgs = video.xpath('./div/a/div/picture/source/@data-srcset')
        if (len(imgs) != 0):
            img = imgs[0]
        else:
            img = video.xpath('./div/a/div/picture/source/@srcset')[0]

        url = video.xpath('./div/div/a/@href')[0]
        title = video.xpath('./div/div/a/div/p[contains(@class, "title")]/text()')[0]
        summary = video.xpath('./div/div/a/div/p[contains(@class, "stitle")]/text()')[0]
        thumb = 'http:' + img.split(',')[-1].split(' ')[0]
        duration = video.xpath('./div/div/a/div/p[contains(@class, "uptitle")]/span/text()')[0]
        originally_available_at = video.xpath('./div/div/a/div/p[contains(@class, "uptitle")]/span/text()')[2]

        oc.add(VideoClipObject(
            key=Callback(VideoDetails, title=title, summary=summary, thumb=thumb, duration=duration, originally_available_at=originally_available_at, rating_key=title, url=url),
            title=title,
            summary=summary,
            thumb=thumb,
            rating_key=title
        ))

    return oc

@route(PREFIX + '/video/details')
def VideoDetails(title, summary, thumb, duration, originally_available_at, rating_key, url):

    oc = ObjectContainer()

    playlist_url = GetVideoPlaylistURL(url)
    #streams = GetHLSStreams(playlist_url)

    oc.add(VideoClipObject(
        key=Callback(VideoDetails, title=title, summary=summary, thumb=thumb, duration=duration, originally_available_at=originally_available_at, rating_key=title, url=url),
        title=title,
        summary=summary,
        thumb=thumb,
        duration=int(duration) * 1000,
        originally_available_at=Datetime.ParseDate(originally_available_at).date(),
        rating_key=rating_key,
        items=[
            MediaObject(
                bitrate=bitrate,
                audio_channels=2,
                audio_codec=AudioCodec.AAC,
                video_codec=VideoCodec.H264,
                video_resolution=video_resolution,
                container=Container.MP4,
                video_frame_rate=25,
                optimized_for_streaming=True,
                parts=[
                    PartObject(
                        key=HTTPLiveStreamURL(Callback(PlayVideo, url=playlist_url))
                    )
                ]
            ) for video_resolution, bitrate in [(720, 2719), (540, 1977), (360, 1340), (360, 704)]
        ]
    ))

    return oc

####################################################################################################
@indirect
@route(PREFIX + '/video/play.m3u8')
def PlayVideo(url):
    return IndirectResponse(VideoClipObject, key=url)

####################################################################################################
def GetVideoPlaylistURL(prog_url):

    page = HTTP.Request(BASE_URL + prog_url).content
    media_id = RE_MEDIA_ID.search(page).group('media_id')

    def GetAuthKey(app_name, media_id):
        secret = base64.b64decode('VzNtMCMxbUZJ')
        timestamp = HTTP.Request('http://www.wat.tv/servertime2', cacheTime=60).content

        string = '%s-%s-%s-%s-%s' % (media_id, secret, app_name, secret, timestamp)

        auth_key = hashlib.md5(string).hexdigest() + '/' + timestamp
        return auth_key

    user_agent = 'MyTF1/16090602 CFNetwork/808.1.4 Darwin/16.1.0'
    app_name = 'sdk/Iphone/1.0'
    method = 'getUrl'
    auth_key = GetAuthKey(app_name, media_id)
    version = '1.8.14'
    hosting_application_name = 'com.tf1.applitf1'
    hosting_application_version = '6.3'

    data = ('appName=%s&method=%s&mediaId=%s&authKey=%s&version=%s&hostingApplicationName=%s&hostingApplicationVersion=%s') % (app_name, method, media_id, auth_key, version, hosting_application_name, hosting_application_version)
    data = HTTP.Request('http://api.wat.tv/services/Delivery', headers={'User-Agent': user_agent}, cacheTime=60, data=data).content
    data = JSON.ObjectFromString(data)

    # remove bandwidth limit to get a playlist with hd stream
    (scheme,
     netloc,
     path,
     query_string,
     fragment) = urlparse.urlsplit(data['message'])
    query_params = urlparse.parse_qs(query_string)
    query_params.pop('bwmax', None)
    new_query_string = urllib.urlencode(query_params, doseq=True)
    m3u8_url = urlparse.urlunsplit((scheme,
                                    netloc,
                                    path,
                                    new_query_string,
                                    fragment))
    return m3u8_url

####################################################################################################
def GetHLSStreams(url):

    playlist = HTTP.Request(url).content
    streams = []

    for line in playlist.splitlines():
        if "BANDWIDTH" in line:
            stream            = {}
            stream["bitrate"] = int(Regex('(?<=BANDWIDTH=)[0-9]+').search(line).group(0))

            if "RESOLUTION" in line:
                stream["resolution"] = int(Regex('(?<=RESOLUTION=)[0-9]+x[0-9]+').search(line).group(0).split("x")[1])
            else:
                stream["resolution"] = 0

            if "FRAME-RATE" in line:
                stream["frame-rate"] = int(Regex('(?<=FRAME-RATE=)[0-9]+').search(line).group(0))
            else:
                stream["frame-rate"] = 0

        elif ".m3u8" in line:
            path = ''

            if not "audio=" in line or not "video=" in line:
                break

            if not line.startswith("http"):
                path = url[ : url.rfind('/') + 1]

            stream["url"] = path + line

            streams.append(stream)

    return streams
<commit_msg>Cleanup code, it's working but not with Chrome...<commit_after>import hashlib, base64
import urlparse, urllib

TITLE = 'MYTF1'
ART = 'art-default.jpg'
ICON = 'icon-default.png'
PREFIX = '/video/mytf1'

BASE_URL = 'http://www.tf1.fr'
PROGRAMS = '%s/programmes-tv' % BASE_URL
VIDEOS = '%s/%s/videos'

RE_MEDIA_ID = Regex("c_idwat = '(?P<media_id>[^']+)'")

####################################################################################################
def Start():

    ObjectContainer.title1 = TITLE
    ObjectContainer.art = R(ART)

    DirectoryObject.thumb = R(ICON)
    DirectoryObject.art = R(ART)
    VideoClipObject.thumb = R(ICON)
    VideoClipObject.art = R(ART)

    HTTP.CacheTime = CACHE_1HOUR
    HTTP.Headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'

####################################################################################################
@handler(PREFIX, TITLE)
def MainMenu():

    oc = ObjectContainer(
        title2="Program Categories"
    )

    html = HTML.ElementFromURL(PROGRAMS)

    for category in html.xpath('//ul[contains(@class, "filters_2") and contains(@class, "contentopen")]/li/a'):
        oc.add(DirectoryObject(
            key=Callback(Programs, prog_cat=category.xpath('./@data-target')[0]),
            title=category.xpath('./text()')[0]
        ))

    return oc

@route(PREFIX + '/programs/{prog_cat}')
def Programs(prog_cat):

    oc = ObjectContainer(
        title2="Programs"
    )

    html = HTML.ElementFromURL(PROGRAMS)

    for program in html.xpath('//ul[contains(@id, "js_filter_el_container")]/li'):
        if program.xpath('./@data-type')[0] != prog_cat and prog_cat != 'all':
            continue

        imgs = program.xpath('./div/a/div/picture/source/@data-srcset')
        if (len(imgs) != 0):
            img = imgs[0]
        else:
            img = program.xpath('./div/a/div/picture/source/@srcset')[0]

        oc.add(DirectoryObject(
            key=Callback(VideoCategories, prog_url=program.xpath('./div/a/@href')[0]),
            title=program.xpath('./div/div/a/div/p/text()')[0],
            thumb='http:' + img.split(',')[-1].split(' ')[0]
        ))

    return oc

@route(PREFIX + '/videos')
def VideoCategories(prog_url):

    oc = ObjectContainer(
        title2="Video Categories"
    )

    html = HTML.ElementFromURL(VIDEOS % (BASE_URL, prog_url))

    #TODO: it's possible to retrieve video filters, but I can't find a way to filter the videos, idk how they are doing it
    for category in html.xpath('//ul[contains(@class, "filters_1") and contains(@class, "contentopen")]/li/a'):
        oc.add(DirectoryObject(
            key=Callback(Videos, video_cat=category.xpath('./@data-filter')[0], prog_url=prog_url),
            title=category.xpath('./text()')[0]
        ))

    return oc

@route(PREFIX + '/videos/{video_cat}')
def Videos(video_cat, prog_url):

    oc = ObjectContainer(
        title2="Videos"
    )

    html = HTML.ElementFromURL(VIDEOS % (BASE_URL, prog_url))

    for video in html.xpath('//ul[contains(@class, "grid")]/li'):
        #if video.xpath('./@data-filter')[0] != video_cat and video_cat != 'all':
        #    continue

        imgs = video.xpath('./div/a/div/picture/source/@data-srcset')
        if (len(imgs) != 0):
            img = imgs[0]
        else:
            img = video.xpath('./div/a/div/picture/source/@srcset')[0]

        url = video.xpath('./div/div/a/@href')[0]
        title = video.xpath('./div/div/a/div/p[contains(@class, "title")]/text()')[0]
        summary = video.xpath('./div/div/a/div/p[contains(@class, "stitle")]/text()')[0]
        thumb = 'http:' + img.split(',')[-1].split(' ')[0]
        duration = video.xpath('./div/div/a/div/p[contains(@class, "uptitle")]/span/text()')[0]
        originally_available_at = video.xpath('./div/div/a/div/p[contains(@class, "uptitle")]/span/text()')[2]

        oc.add(VideoClipObject(
            key=Callback(VideoDetails, title=title, summary=summary, thumb=thumb, duration=duration, originally_available_at=originally_available_at, rating_key=title, url=url),
            title=title,
            summary=summary,
            thumb=thumb,
            rating_key=title
        ))

    return oc

@route(PREFIX + '/video/details')
def VideoDetails(title, summary, thumb, duration, originally_available_at, rating_key, url):

    oc = ObjectContainer()

    oc.add(VideoClipObject(
        key=Callback(VideoDetails, title=title, summary=summary, thumb=thumb, duration=duration, originally_available_at=originally_available_at, rating_key=title, url=url),
        title=title,
        summary=summary,
        thumb=thumb,
        duration=int(duration) * 1000,
        originally_available_at=Datetime.ParseDate(originally_available_at).date(),
        rating_key=rating_key,
        items=[
            MediaObject(
                bitrate=bitrate,
                audio_channels=2,
                audio_codec=AudioCodec.AAC,
                video_codec=VideoCodec.H264,
                video_resolution=video_resolution,
                container=Container.MP4,
                video_frame_rate=25,
                optimized_for_streaming=True,
                parts=[
                    PartObject(
                        key=HTTPLiveStreamURL(Callback(PlayVideo, url=url))
                    )
                ]
            ) for video_resolution, bitrate in [(720, 2719), (540, 1977), (360, 1340), (360, 704)]
        ]
    ))

    return oc

####################################################################################################
@indirect
@route(PREFIX + '/video/play.m3u8')
def PlayVideo(url):

    playlist_url = GetVideoPlaylistURL(url)

    return IndirectResponse(VideoClipObject, key=playlist_url)

####################################################################################################
def GetVideoPlaylistURL(prog_url):

    page = HTTP.Request(BASE_URL + prog_url).content
    media_id = RE_MEDIA_ID.search(page).group('media_id')

    def GetAuthKey(app_name, media_id):

        secret = base64.b64decode('VzNtMCMxbUZJ')
        timestamp = HTTP.Request('http://www.wat.tv/servertime2', cacheTime=60).content

        string = '%s-%s-%s-%s-%s' % (media_id, secret, app_name, secret, timestamp)

        auth_key = hashlib.md5(string).hexdigest() + '/' + timestamp
        return auth_key

    user_agent = 'MyTF1/16090602 CFNetwork/808.1.4 Darwin/16.1.0'
    app_name = 'sdk/Iphone/1.0'
    method = 'getUrl'
    auth_key = GetAuthKey(app_name, media_id)
    version = '1.8.14'
    hosting_application_name = 'com.tf1.applitf1'
    hosting_application_version = '6.3'

    data = ('appName=%s&method=%s&mediaId=%s&authKey=%s&version=%s&hostingApplicationName=%s&hostingApplicationVersion=%s') % (app_name, method, media_id, auth_key, version, hosting_application_name, hosting_application_version)
    data = HTTP.Request('http://api.wat.tv/services/Delivery', headers={'User-Agent': user_agent}, cacheTime=60, data=data).content
    data = JSON.ObjectFromString(data)

    # remove bandwidth limit to get a playlist with hd stream
    (scheme,
     netloc,
     path,
     query_string,
     fragment) = urlparse.urlsplit(data['message'])
    query_params = urlparse.parse_qs(query_string)
    query_params.pop('bwmax', None)
    new_query_string = urllib.urlencode(query_params, doseq=True)
    m3u8_url = urlparse.urlunsplit((scheme,
                                    netloc,
                                    path,
                                    new_query_string,
                                    fragment))
    return m3u8_url
<|endoftext|>


<commit_before>from django.http import HttpResponse
from django.shortcuts import render
from django.views.decorators.csrf import csrf_exempt

import json
import string
import random
from time import time
from DBNlogic.nets import DBN, RBM
from DBNlogic.sets import DataSet
from DBNlogic.util import Configuration

training_jobs = {}

def index(request):
    """Return the main page."""
    ordered_datasets = sorted(DataSet.allSets(), key = str.lower)
    context = {
        'config': Configuration(),   # default configuration
        'datasets': ordered_datasets # available datasets
    }
    return render(request, 'DBNtrain/index.html', context)

@csrf_exempt
def train(request):
    """Set up a network to be trained according to the
    parameters in the request."""
    trainset_name = request.POST['dataset']
    trainset = DataSet.fromWhatever(trainset_name)

    try:
        num_layers = int(request.POST['num_layers'])
    except ValueError:
        num_layers = 1
        # return HttpResponse({'error': 'you haven\'t specified [...]'})

    net = DBN(name = trainset_name)
    vis_size = int(request.POST['vis_sz'])
    for layer in range(1, num_layers):
        hid_size = int(request.POST['hid_sz_' + str(layer)])
        print('creating a', vis_size, 'x', hid_size, 'RBM...')
        net.append(RBM(vis_size, hid_size))
        vis_size = hid_size # for the next RBM

    config = {
        'max_epochs' : int(request.POST['epochs']),
        'batch_size' : int(request.POST['batch_size']),
        'learn_rate' : float(request.POST['learn_rate']),
        'momentum'   : float(request.POST['momentum'])
    }

    random_id = ''.join(random.choices(string.ascii_uppercase + string.digits, k = 10))
    print(random_id)
    training_jobs[random_id] = {
        'birthday': time(),
        'generator': net.learn(trainset, Configuration(**config))
    }

    # delete a random job older than one hour:
    random_old_job = random.choice(list(training_jobs.keys()))
    if time() - training_jobs[random_old_job]['birthday'] > 60 * 60:
        print('deleting old job n.', random_old_job)
        del training_jobs[random_old_job]

    return HttpResponse(random_id)

@csrf_exempt
def getError(request):
    """Run one training iteration for a particular network
    and return the reconstruction error."""
    body = json.loads(request.body.decode())
    job = body['job_id']
    train_gen = training_jobs[job]['generator']
    stop = False

    try:
        next_err = next(train_gen)
    except StopIteration:
        next_err = None
        stop = True

    response = {
        'error': next_err,
        'stop': stop
    }
    json_response = json.dumps(response)
    return HttpResponse(json_response, content_type = 'application/json')
<commit_msg>Python 3.5 compatibility<commit_after>from django.http import HttpResponse
from django.shortcuts import render
from django.views.decorators.csrf import csrf_exempt

import json
import string
import random
from time import time
from DBNlogic.nets import DBN, RBM
from DBNlogic.sets import DataSet
from DBNlogic.util import Configuration

training_jobs = {}

def index(request):
    """Return the main page."""
    ordered_datasets = sorted(DataSet.allSets(), key = str.lower)
    context = {
        'config': Configuration(),   # default configuration
        'datasets': ordered_datasets # available datasets
    }
    return render(request, 'DBNtrain/index.html', context)

@csrf_exempt
def train(request):
    """Set up a network to be trained according to the
    parameters in the request."""
    trainset_name = request.POST['dataset']
    trainset = DataSet.fromWhatever(trainset_name)

    try:
        num_layers = int(request.POST['num_layers'])
    except ValueError:
        num_layers = 1
        # return HttpResponse({'error': 'you haven\'t specified [...]'})

    net = DBN(name = trainset_name)
    vis_size = int(request.POST['vis_sz'])
    for layer in range(1, num_layers):
        hid_size = int(request.POST['hid_sz_' + str(layer)])
        print('creating a', vis_size, 'x', hid_size, 'RBM...')
        net.append(RBM(vis_size, hid_size))
        vis_size = hid_size # for the next RBM

    config = {
        'max_epochs' : int(request.POST['epochs']),
        'batch_size' : int(request.POST['batch_size']),
        'learn_rate' : float(request.POST['learn_rate']),
        'momentum'   : float(request.POST['momentum'])
    }

    random_id = ''.join(random.choice(string.ascii_uppercase + string.digits) for i in range(10))
    print(random_id)
    training_jobs[random_id] = {
        'birthday': time(),
        'generator': net.learn(trainset, Configuration(**config))
    }

    # delete a random job older than one hour:
    random_old_job = random.choice(list(training_jobs.keys()))
    if time() - training_jobs[random_old_job]['birthday'] > 60 * 60:
        print('deleting old job n.', random_old_job)
        del training_jobs[random_old_job]

    return HttpResponse(random_id)

@csrf_exempt
def getError(request):
    """Run one training iteration for a particular network
    and return the reconstruction error."""
    body = json.loads(request.body.decode())
    job = body['job_id']
    train_gen = training_jobs[job]['generator']
    stop = False

    try:
        next_err = next(train_gen)
    except StopIteration:
        next_err = None
        stop = True

    response = {
        'error': next_err,
        'stop': stop
    }
    json_response = json.dumps(response)
    return HttpResponse(json_response, content_type = 'application/json')
<|endoftext|>


<commit_before>from apiclient.discovery import build
from oauth2client.service_account import ServiceAccountCredentials
from httplib2 import Http
import os
import logging
from pandas_bigquery.exceptions import *
from pandas_bigquery.datasets import Datasets
from pandas_bigquery.tables import Tables
from pandas_bigquery.tabledata import Tabledata
from pandas_bigquery.jobs import Jobs
from pandas import DataFrame, concat
from pandas.compat import lzip
from datetime import datetime
from random import randint
import numpy as np
from time import sleep

try:
    from googleapiclient.errors import HttpError
except:
    from apiclient.errors import HttpError

log = logging.getLogger()

class Bigquery:
    def __init__(self, project_id=os.getenv('BIGQUERY_PROJECT'), private_key_path=os.getenv('BIGQUERY_KEY_PATH')):

        if private_key_path is None:
            raise RuntimeError('Invalid bigquery key path')

        self.project_id = project_id
        self.private_key_path = private_key_path

        credentials = ServiceAccountCredentials.from_json_keyfile_name(
            private_key_path, ['https://www.googleapis.com/auth/bigquery'])

        http_auth = credentials.authorize(Http())
        self._service = build('bigquery', 'v2', http=http_auth)

        with open(private_key_path) as data_file:
            self.private_key = data_file.read()

        self._tables = Tables(self.project_id, private_key=self.private_key_path)
        self._datasets = Datasets(self.project_id, private_key=self.private_key_path)
        self._jobs = Jobs(self.project_id, private_key=self.private_key_path)
        self._tabledata = Tabledata(self.project_id, private_key=self.private_key_path)

    @staticmethod
    def _parse_data(schema, rows):
        # see:
        # http://pandas.pydata.org/pandas-docs/dev/missing_data.html
        # #missing-data-casting-rules-and-indexing
        dtype_map = {'FLOAT': np.dtype(float),
                     'TIMESTAMP': 'M8[ns]'}

        fields = schema['fields']
        col_types = [field['type'] for field in fields]
        col_names = [str(field['name']) for field in fields]
        col_dtypes = [dtype_map.get(field['type'], object) for field in fields]
        page_array = np.zeros((len(rows),), dtype=lzip(col_names, col_dtypes))
        for row_num, raw_row in enumerate(rows):
            entries = raw_row.get('f', [])
            for col_num, field_type in enumerate(col_types):
                field_value = Bigquery._parse_entry(entries[col_num].get('v', ''),
                                                    field_type)
                page_array[row_num][col_num] = field_value

        return DataFrame(page_array, columns=col_names)

    @staticmethod
    def _parse_entry(field_value, field_type):
        if field_value is None or field_value == 'null':
            return None
        if field_type == 'INTEGER':
            return int(field_value)
        elif field_type == 'FLOAT':
            return float(field_value)
        elif field_type == 'TIMESTAMP':
            timestamp = datetime.utcfromtimestamp(float(field_value))
            return np.datetime64(timestamp)
        elif field_type == 'BOOLEAN':
            return field_value == 'true'
        return field_value

    @property
    def datasets(self):
        return self._datasets

    @property
    def tables(self):
        return self._tables

    @property
    def jobs(self):
        return self._jobs

    @property
    def tabledata(self):
        return self._tabledata

    def query_async(self, query, dialect='standard', priority='BATCH', strict=True, **kwargs):

        if Bigquery._check_strict(query, strict):
            raise Exception('Strict mode error',
                            "partition reference not found in query, "
                            "please add a partitiondate, _partitiontime or _table_suffix restriction "
                            "in the where-clause or set strict = False if you are confident in what you're doing.")

        config = kwargs.get('configuration')
        if config is not None and 'query' in config:
            config['query']['priority'] = priority
        else:
            config = {
                'query': {
                    'priority': priority
                }
            }
        config['query']['useLegacySql'] = dialect == 'legacy'

        return self.jobs.query_async(query, configuration=config)

    def query(self, query, dialect='standard', priority='BATCH', strict=True, **kwargs):

        if Bigquery._check_strict(query, strict):
            raise Exception('Strict mode error',
                            "partition reference not found in query, "
                            "please add a partitiondate, _partitiontime or _table_suffix restriction "
                            "in the where-clause or set strict = False if you are confident in what you're doing.")

        if dialect not in ('legacy', 'standard'):
            raise ValueError("'{0}' is not valid for dialect".format(dialect))

        if priority not in ('BATCH', 'INTERACTIVE'):
            raise ValueError("'{0}' is not valid for priority".format(dialect))

        config = kwargs.get('configuration')
        if config is not None and 'query' in config:
            config['query']['priority'] = priority
        else:
            config = {
                'query': {
                    'priority': priority
                }
            }
        config['query']['useLegacySql'] = dialect == 'legacy'

        schema, pages = self.jobs.query(query, configuration=config)
        dataframe_list = []
        while len(pages) > 0:
            page = pages.pop()
            dataframe_list.append(Bigquery._parse_data(schema, page))

        if len(dataframe_list) > 0:
            final_df = concat(dataframe_list, ignore_index=True)
        else:
            final_df = Bigquery._parse_data(schema, [])

        # cast BOOLEAN and INTEGER columns from object to bool/int
        # if they dont have any nulls
        type_map = {'BOOLEAN': bool, 'INTEGER': int}
        for field in schema['fields']:
            if field['type'] in type_map and \
                    final_df[field['name']].notnull().all():
                final_df[field['name']] = \
                    final_df[field['name']].astype(type_map[field['type']])

        self.jobs.print_elapsed_seconds(
            'Total time taken',
            datetime.now().strftime('s.\nFinished at %Y-%m-%d %H:%M:%S.'),
            0
        )

        return final_df

    def upload(self, dataframe, destination_table, if_exists='fail', chunksize=10000):
        if if_exists not in ('fail', 'replace', 'append'):
            raise ValueError("'{0}' is not valid for if_exists".format(if_exists))

        if '.' not in destination_table:
            raise NotFoundException(
                "Invalid Table Name. Should be of the form 'datasetId.tableId' ")

        dataset_id, table_id = destination_table.rsplit('.', 1)

        table_schema = Tables.generate_schema_from_dataframe(dataframe)

        if Tables.contains_partition_decorator(table_id):
            root_table_id, partition_id = table_id.rsplit('$', 1)

            if not self.tables.exists(dataset_id, root_table_id):
                raise NotFoundException("Could not write to the partition because "
                                        "the table does not exist.")

            table_resource = self.tables.get(dataset_id, root_table_id)

            if 'timePartitioning' not in table_resource:
                raise InvalidSchema("Could not write to the partition because "
                                    "the table is not partitioned.")

            partition_exists = self.query("SELECT COUNT(*) AS num_rows FROM {0}"
                                          .format(destination_table),
                                          strict=False,
                                          priority='INTERACTIVE',
                                          dialect='legacy')['num_rows'][0] > 0

            if partition_exists:
                if if_exists == 'fail':
                    raise TableCreationError("Could not create the partition "
                                             "because it already exists. "
                                             "Change the if_exists parameter to "
                                             "append or replace data.")
                elif if_exists == 'append':
                    if not self.tables.schema_is_subset(dataset_id,
                                                        root_table_id,
                                                        table_schema):
                        raise InvalidSchema("Please verify that the structure and "
                                            "data types in the DataFrame match "
                                            "the schema of the destination table.")
                    self.tabledata.insert_all(dataframe, dataset_id, table_id, chunksize)

                elif if_exists == 'replace':
                    if not self.tables.schema_is_subset(dataset_id,
                                                        root_table_id,
                                                        table_schema):
                        raise InvalidSchema("Please verify that the structure and "
                                            "data types in the DataFrame match "
                                            "the schema of the destination table.")

                    temporary_table_id = '_'.join(
                        [root_table_id + '_' + partition_id,
                         str(randint(1, 100000))])
                    self.tables.insert(dataset_id, temporary_table_id, table_schema)
                    self.tabledata.insert_all(dataframe, dataset_id, temporary_table_id,
                                              chunksize)
                    sleep(30)  # <- Curses Google!!!
                    self.jobs.query('select * from {0}.{1}'
                                    .format(dataset_id, temporary_table_id),
                                    configuration={
                                        'query': {
                                            'destinationTable': {
                                                'projectId': self.project_id,
                                                'datasetId': dataset_id,
                                                'tableId': table_id
                                            },
                                            'createDisposition':
                                                'CREATE_IF_NEEDED',
                                            'writeDisposition':
                                                'WRITE_TRUNCATE',
                                            'allowLargeResults': True
                                        }
                                    })
                    self.tables.delete(dataset_id, temporary_table_id)

            else:
                self.tabledata.insert_all(dataframe, dataset_id, table_id, chunksize)

        else:
            if self.tables.exists(dataset_id, table_id):
                if if_exists == 'fail':
                    raise TableCreationError(
                        "Could not create the table because it "
                        "already exists. "
                        "Change the if_exists parameter to "
                        "append or replace data.")
                elif if_exists == 'replace':
                    self.tables.delete_and_recreate_table(
                        dataset_id, table_id, table_schema)
                elif if_exists == 'append':
                    if not self.tables.schema_is_subset(dataset_id,
                                                        table_id,
                                                        table_schema):
                        raise InvalidSchema("Please verify that the structure and "
                                            "data types in the DataFrame match "
                                            "the schema of the destination table.")
            else:
                self.tables.insert(dataset_id, table_id, table_schema)

            self.tabledata.insert_all(dataframe, dataset_id, table_id, chunksize)

    @staticmethod
    def _check_strict(query, strict):
        return strict and \
               not any([x in query.lower() for x in ('partitiondate', '_partitiontime', '_table_suffix', '$')])

    @staticmethod
    def run_with_retry(func, max_tries=10, **kwargs):
        for i in range(0, max_tries):
            try:
                return func(**kwargs), i + 1
            except Exception as err:
                log.warning("run_with_retry error, trying again {0}/{1}".format(i + 1, max_tries))
                if i == max_tries - 1:
                    raise err

    # Legacy methods

    def copy(self, source_dataset, source_table, destination_dataset, destination_table):
        return self.jobs.copy(source_dataset, source_table, destination_dataset, destination_table)

    def generate_schema(self, df, default_type='STRING'):
        return Tables.generate_schema_from_dataframe(df, default_type)

    def schema(self, dataset_id, table_id):
        return self.tables.get_schema(dataset_id, table_id)

    def resource(self, dataset_id, table_id):
        return self.tables.get(dataset_id, table_id)

    def verify_schema(self, dataset_id, table_id, schema):
        return self.tables.schema_matches(dataset_id, table_id, schema)

    def table_copy(self, source_dataset_id, source_table_id, destination_dataset_id, destination_table_id):
        return self.jobs.copy(source_dataset_id, source_table_id, destination_dataset_id, destination_table_id)

    def table_create(self, dataset_id, table_id, schema, **kwargs):
        return self.tables.insert(dataset_id, table_id, schema, **kwargs)

    def table_delete(self, dataset_id, table_id):
        return self.tables.delete(dataset_id, table_id)

    @property
    def dataset_list(self):
        return self.datasets.list()

    def dataset_delete(self, dataset_id, delete_contents=False):
        return self.datasets.delete(dataset_id, delete_contents)

    def query_batch(self, query, dialect='standard', strict=True, **kwargs):
        return self.query(query=query, dialect=dialect, priority='BATCH', strict=strict, **kwargs)
<commit_msg>Changed default query behaviour from BATCH to INTERACTIVE<commit_after>from apiclient.discovery import build
from oauth2client.service_account import ServiceAccountCredentials
from httplib2 import Http
import os
import logging
from pandas_bigquery.exceptions import *
from pandas_bigquery.datasets import Datasets
from pandas_bigquery.tables import Tables
from pandas_bigquery.tabledata import Tabledata
from pandas_bigquery.jobs import Jobs
from pandas import DataFrame, concat
from pandas.compat import lzip
from datetime import datetime
from random import randint
import numpy as np
from time import sleep

try:
    from googleapiclient.errors import HttpError
except:
    from apiclient.errors import HttpError

log = logging.getLogger()

class Bigquery:
    def __init__(self, project_id=os.getenv('BIGQUERY_PROJECT'), private_key_path=os.getenv('BIGQUERY_KEY_PATH')):

        if private_key_path is None:
            raise RuntimeError('Invalid bigquery key path')

        self.project_id = project_id
        self.private_key_path = private_key_path

        credentials = ServiceAccountCredentials.from_json_keyfile_name(
            private_key_path, ['https://www.googleapis.com/auth/bigquery'])

        http_auth = credentials.authorize(Http())
        self._service = build('bigquery', 'v2', http=http_auth)

        with open(private_key_path) as data_file:
            self.private_key = data_file.read()

        self._tables = Tables(self.project_id, private_key=self.private_key_path)
        self._datasets = Datasets(self.project_id, private_key=self.private_key_path)
        self._jobs = Jobs(self.project_id, private_key=self.private_key_path)
        self._tabledata = Tabledata(self.project_id, private_key=self.private_key_path)

    @staticmethod
    def _parse_data(schema, rows):
        # see:
        # http://pandas.pydata.org/pandas-docs/dev/missing_data.html
        # #missing-data-casting-rules-and-indexing
        dtype_map = {'FLOAT': np.dtype(float),
                     'TIMESTAMP': 'M8[ns]'}

        fields = schema['fields']
        col_types = [field['type'] for field in fields]
        col_names = [str(field['name']) for field in fields]
        col_dtypes = [dtype_map.get(field['type'], object) for field in fields]
        page_array = np.zeros((len(rows),), dtype=lzip(col_names, col_dtypes))
        for row_num, raw_row in enumerate(rows):
            entries = raw_row.get('f', [])
            for col_num, field_type in enumerate(col_types):
                field_value = Bigquery._parse_entry(entries[col_num].get('v', ''),
                                                    field_type)
                page_array[row_num][col_num] = field_value

        return DataFrame(page_array, columns=col_names)

    @staticmethod
    def _parse_entry(field_value, field_type):
        if field_value is None or field_value == 'null':
            return None
        if field_type == 'INTEGER':
            return int(field_value)
        elif field_type == 'FLOAT':
            return float(field_value)
        elif field_type == 'TIMESTAMP':
            timestamp = datetime.utcfromtimestamp(float(field_value))
            return np.datetime64(timestamp)
        elif field_type == 'BOOLEAN':
            return field_value == 'true'
        return field_value

    @property
    def datasets(self):
        return self._datasets

    @property
    def tables(self):
        return self._tables

    @property
    def jobs(self):
        return self._jobs

    @property
    def tabledata(self):
        return self._tabledata

    def query_async(self, query, dialect='standard', priority='BATCH', strict=True, **kwargs):

        if Bigquery._check_strict(query, strict):
            raise Exception('Strict mode error',
                            "partition reference not found in query, "
                            "please add a partitiondate, _partitiontime or _table_suffix restriction "
                            "in the where-clause or set strict = False if you are confident in what you're doing.")

        config = kwargs.get('configuration')
        if config is not None and 'query' in config:
            config['query']['priority'] = priority
        else:
            config = {
                'query': {
                    'priority': priority
                }
            }
        config['query']['useLegacySql'] = dialect == 'legacy'

        return self.jobs.query_async(query, configuration=config)

    def query(self, query, dialect='standard', priority='INTERACTIVE', strict=True, **kwargs):

        if Bigquery._check_strict(query, strict):
            raise Exception('Strict mode error',
                            "partition reference not found in query, "
                            "please add a partitiondate, _partitiontime or _table_suffix restriction "
                            "in the where-clause or set strict = False if you are confident in what you're doing.")

        if dialect not in ('legacy', 'standard'):
            raise ValueError("'{0}' is not valid for dialect".format(dialect))

        if priority not in ('BATCH', 'INTERACTIVE'):
            raise ValueError("'{0}' is not valid for priority".format(dialect))

        config = kwargs.get('configuration')
        if config is not None and 'query' in config:
            config['query']['priority'] = priority
        else:
            config = {
                'query': {
                    'priority': priority
                }
            }
        config['query']['useLegacySql'] = dialect == 'legacy'

        schema, pages = self.jobs.query(query, configuration=config)
        dataframe_list = []
        while len(pages) > 0:
            page = pages.pop()
            dataframe_list.append(Bigquery._parse_data(schema, page))

        if len(dataframe_list) > 0:
            final_df = concat(dataframe_list, ignore_index=True)
        else:
            final_df = Bigquery._parse_data(schema, [])

        # cast BOOLEAN and INTEGER columns from object to bool/int
        # if they dont have any nulls
        type_map = {'BOOLEAN': bool, 'INTEGER': int}
        for field in schema['fields']:
            if field['type'] in type_map and \
                    final_df[field['name']].notnull().all():
                final_df[field['name']] = \
                    final_df[field['name']].astype(type_map[field['type']])

        self.jobs.print_elapsed_seconds(
            'Total time taken',
            datetime.now().strftime('s.\nFinished at %Y-%m-%d %H:%M:%S.'),
            0
        )

        return final_df

    def upload(self, dataframe, destination_table, if_exists='fail', chunksize=10000):
        if if_exists not in ('fail', 'replace', 'append'):
            raise ValueError("'{0}' is not valid for if_exists".format(if_exists))

        if '.' not in destination_table:
            raise NotFoundException(
                "Invalid Table Name. Should be of the form 'datasetId.tableId' ")

        dataset_id, table_id = destination_table.rsplit('.', 1)

        table_schema = Tables.generate_schema_from_dataframe(dataframe)

        if Tables.contains_partition_decorator(table_id):
            root_table_id, partition_id = table_id.rsplit('$', 1)

            if not self.tables.exists(dataset_id, root_table_id):
                raise NotFoundException("Could not write to the partition because "
                                        "the table does not exist.")

            table_resource = self.tables.get(dataset_id, root_table_id)

            if 'timePartitioning' not in table_resource:
                raise InvalidSchema("Could not write to the partition because "
                                    "the table is not partitioned.")

            partition_exists = self.query("SELECT COUNT(*) AS num_rows FROM {0}"
                                          .format(destination_table),
                                          strict=False,
                                          priority='INTERACTIVE',
                                          dialect='legacy')['num_rows'][0] > 0

            if partition_exists:
                if if_exists == 'fail':
                    raise TableCreationError("Could not create the partition "
                                             "because it already exists. "
                                             "Change the if_exists parameter to "
                                             "append or replace data.")
                elif if_exists == 'append':
                    if not self.tables.schema_is_subset(dataset_id,
                                                        root_table_id,
                                                        table_schema):
                        raise InvalidSchema("Please verify that the structure and "
                                            "data types in the DataFrame match "
                                            "the schema of the destination table.")
                    self.tabledata.insert_all(dataframe, dataset_id, table_id, chunksize)

                elif if_exists == 'replace':
                    if not self.tables.schema_is_subset(dataset_id,
                                                        root_table_id,
                                                        table_schema):
                        raise InvalidSchema("Please verify that the structure and "
                                            "data types in the DataFrame match "
                                            "the schema of the destination table.")

                    temporary_table_id = '_'.join(
                        [root_table_id + '_' + partition_id,
                         str(randint(1, 100000))])
                    self.tables.insert(dataset_id, temporary_table_id, table_schema)
                    self.tabledata.insert_all(dataframe, dataset_id, temporary_table_id,
                                              chunksize)
                    sleep(30)  # <- Curses Google!!!
                    self.jobs.query('select * from {0}.{1}'
                                    .format(dataset_id, temporary_table_id),
                                    configuration={
                                        'query': {
                                            'destinationTable': {
                                                'projectId': self.project_id,
                                                'datasetId': dataset_id,
                                                'tableId': table_id
                                            },
                                            'createDisposition':
                                                'CREATE_IF_NEEDED',
                                            'writeDisposition':
                                                'WRITE_TRUNCATE',
                                            'allowLargeResults': True
                                        }
                                    })
                    self.tables.delete(dataset_id, temporary_table_id)

            else:
                self.tabledata.insert_all(dataframe, dataset_id, table_id, chunksize)

        else:
            if self.tables.exists(dataset_id, table_id):
                if if_exists == 'fail':
                    raise TableCreationError(
                        "Could not create the table because it "
                        "already exists. "
                        "Change the if_exists parameter to "
                        "append or replace data.")
                elif if_exists == 'replace':
                    self.tables.delete_and_recreate_table(
                        dataset_id, table_id, table_schema)
                elif if_exists == 'append':
                    if not self.tables.schema_is_subset(dataset_id,
                                                        table_id,
                                                        table_schema):
                        raise InvalidSchema("Please verify that the structure and "
                                            "data types in the DataFrame match "
                                            "the schema of the destination table.")
            else:
                self.tables.insert(dataset_id, table_id, table_schema)

            self.tabledata.insert_all(dataframe, dataset_id, table_id, chunksize)

    @staticmethod
    def _check_strict(query, strict):
        return strict and \
               not any([x in query.lower() for x in ('partitiondate', '_partitiontime', '_table_suffix', '$')])

    @staticmethod
    def run_with_retry(func, max_tries=10, **kwargs):
        for i in range(0, max_tries):
            try:
                return func(**kwargs), i + 1
            except Exception as err:
                log.warning("run_with_retry error, trying again {0}/{1}".format(i + 1, max_tries))
                if i == max_tries - 1:
                    raise err

    # Legacy methods

    def copy(self, source_dataset, source_table, destination_dataset, destination_table):
        return self.jobs.copy(source_dataset, source_table, destination_dataset, destination_table)

    def generate_schema(self, df, default_type='STRING'):
        return Tables.generate_schema_from_dataframe(df, default_type)

    def schema(self, dataset_id, table_id):
        return self.tables.get_schema(dataset_id, table_id)

    def resource(self, dataset_id, table_id):
        return self.tables.get(dataset_id, table_id)

    def verify_schema(self, dataset_id, table_id, schema):
        return self.tables.schema_matches(dataset_id, table_id, schema)

    def table_copy(self, source_dataset_id, source_table_id, destination_dataset_id, destination_table_id):
        return self.jobs.copy(source_dataset_id, source_table_id, destination_dataset_id, destination_table_id)

    def table_create(self, dataset_id, table_id, schema, **kwargs):
        return self.tables.insert(dataset_id, table_id, schema, **kwargs)

    def table_delete(self, dataset_id, table_id):
        return self.tables.delete(dataset_id, table_id)

    @property
    def dataset_list(self):
        return self.datasets.list()

    def dataset_delete(self, dataset_id, delete_contents=False):
        return self.datasets.delete(dataset_id, delete_contents)

    def query_batch(self, query, dialect='standard', strict=True, **kwargs):
        return self.query(query=query, dialect=dialect, priority='BATCH', strict=strict, **kwargs)
<|endoftext|>


<commit_before>import json

def get_columns(parameters, columns):
    cols = []

    for col in parameters.get('fields', "").split(','):
        if col in columns:
            cols.append(columns[col])

    return cols if len(cols) > 0 else columns

def filter_columns(parameters, query, supported_columns):

    for c in supported_columns:
        if c.name in parameters:
            # try to parse json
            value = parameters[c.name]
            try:
                j = json.loads(value)
                if "gt" in j:
                    query = query.where(c > j['gt'])
                elif "gte" in j:
                    query = query.where(c >= j['gte'])

                if "lt" in j:
                    query = query.where(c < j['lt'])
                elif "lte" in j:
                    query = query.where(c <= j['lte'])

            except ValueError:
                query = query.where(c == value)

    return query
<commit_msg>Add some comments<commit_after>import json

def get_columns(parameters, columns):
    cols = []

    for col in parameters.get('fields', "").split(','):
        if col in columns:
            cols.append(columns[col])

    return cols if len(cols) > 0 else columns

def filter_columns(parameters, query, supported_columns):

    for c in supported_columns:
        if c.name in parameters:
            # try to parse json
            # this is to be able to support filtering like
            # url?parameter={"gt":45} for e.g. greater than
            value = parameters[c.name]
            try:
                j = json.loads(value)
                if "gt" in j:
                    query = query.where(c > j['gt'])
                elif "gte" in j:
                    query = query.where(c >= j['gte'])

                if "lt" in j:
                    query = query.where(c < j['lt'])
                elif "lte" in j:
                    query = query.where(c <= j['lte'])

            except ValueError:
                query = query.where(c == value)

    return query
<|endoftext|>


<commit_before># encoding: utf-8

from datetime import datetime

__author__ = "Tsuyoshi Hombashi"
__copyright__ = "Copyright 2016-{}, {}".format(datetime.now().year, __author__)
__license__ = "MIT License"
__version__ = "0.17.2"
__maintainer__ = __author__
__email__ = "tsuyoshi.hombashi@gmail.com"
<commit_msg>Bump version to 0.17.3<commit_after># encoding: utf-8

from datetime import datetime

__author__ = "Tsuyoshi Hombashi"
__copyright__ = "Copyright 2016-{}, {}".format(datetime.now().year, __author__)
__license__ = "MIT License"
__version__ = "0.17.3"
__maintainer__ = __author__
__email__ = "tsuyoshi.hombashi@gmail.com"
<|endoftext|>


<commit_before>#!/usr/bin/env python
#  Author:
#  Florian Kaufmann (DE-CIX)

import argparse
import atexit
import json
from multiprocessing.connection import Listener, Client
import os
from signal import signal, SIGTERM
from sys import exit
from threading import RLock, Thread
from lib import PConfig
import util.log

''' client for participants to send policy updates '''
class ParticipantClient(object):

    def __init__(self, id, config_file, logger):
        # participant id
        self.id = id

        # used to signal termination
        self.run = True
        #self.prefix_lock = {}

        # Initialize participant params and logger
        self.cfg = PConfig(config_file, self.id)
        self.logger = logger

    def xstart(self, policy_file, action):

        # Initalize participant client
        valid_actions = {"remove", "insert"}

        # Check if action is valid
        if action in valid_actions:
            self.client = self.cfg.get_participant_client(self.id, self.logger)

            # Open File and Parse
            with open(policy_file, 'r') as f:
                raw_data=f.read()
                data = json.loads('{ "policy": [ { "%s": [ %s ] } ] }' % (action, raw_data))

            # Send data
            self.logger.debug("participant_client(%s): send: %s" % (self.id, data))
            self.client.send(data)
        else:
            # Not a valid action
            self.logger.debug("participant_client(%s): action (%s) not found" % (self.id, action))
            self.prtclnt.stop()

    def stop(self):

        # Stop participant client
        self.logger.debug("participant_client(%s): close connection"  % self.id)
        self.run = False

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('policy_file', help='the policy change file')
    parser.add_argument('id', type=int,
                   help='participant id (integer)')
    parser.add_argument('action', type=str, choices={"remove", "insert"}, help='use remove or insert')
    args = parser.parse_args()

    # locate policy changefile
    # TODO: atm same path as this program
    base_path = os.path.abspath(os.path.join(os.path.realpath(__file__),
                                ".."))
    policy_change_file = os.path.join(base_path, args.policy_file)

    # locate config file
    # TODO: hard coded destination to global config file
    base_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                "..","..","endeavour","examples","test-mh","config"))
    config_file = os.path.join(base_path, "sdx_global.cfg")

    logger = util.log.getLogger("P_" + str(args.id))
    logger.debug ("Starting participant_client (%s) with config file: %s" % (self.id, config_file))

    # start controller
    prtclnt = ParticipantClient(args.id, config_file, logger)
    prtclnt_thread = Thread(target=prtclnt.xstart(policy_change_file, args.action))
    prtclnt_thread.daemon = True
    prtclnt_thread.start()

    atexit.register(prtclnt.stop)
    signal(SIGTERM, lambda signum, stack_frame: exit(1))

    while prtclnt_thread.is_alive():
        try:
            prtclnt_thread.join(1)
        except KeyboardInterrupt:
            prtclnt.stop()

    logger.debug ("participant_client extiting")
    print ("done")

if __name__ == '__main__':
    main()<commit_msg>fix syntax error<commit_after>#!/usr/bin/env python
#  Author:
#  Florian Kaufmann (DE-CIX)

import argparse
import atexit
import json
from multiprocessing.connection import Listener, Client
import os
from signal import signal, SIGTERM
from sys import exit
from threading import RLock, Thread
from lib import PConfig
import util.log

''' client for participants to send policy updates '''
class ParticipantClient(object):

    def __init__(self, id, config_file, logger):
        # participant id
        self.id = id

        # used to signal termination
        self.run = True
        #self.prefix_lock = {}

        # Initialize participant params and logger
        self.cfg = PConfig(config_file, self.id)
        self.logger = logger

    def xstart(self, policy_file, action):

        # Initalize participant client
        valid_actions = {"remove", "insert"}

        # Check if action is valid
        if action in valid_actions:
            self.client = self.cfg.get_participant_client(self.id, self.logger)

            # Open File and Parse
            with open(policy_file, 'r') as f:
                raw_data=f.read()
                data = json.loads('{ "policy": [ { "%s": [ %s ] } ] }' % (action, raw_data))

            # Send data
            self.logger.debug("participant_client(%s): send: %s" % (self.id, data))
            self.client.send(data)
        else:
            # Not a valid action
            self.logger.debug("participant_client(%s): action (%s) not found" % (self.id, action))
            self.prtclnt.stop()

    def stop(self):

        # Stop participant client
        self.logger.debug("participant_client(%s): close connection"  % self.id)
        self.run = False

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('policy_file', help='the policy change file')
    parser.add_argument('id', type=int,
                   help='participant id (integer)')
    parser.add_argument('action', type=str, choices={"remove", "insert"}, help='use remove or insert')
    args = parser.parse_args()

    # locate policy changefile
    # TODO: atm same path as this program
    base_path = os.path.abspath(os.path.join(os.path.realpath(__file__),
                                ".."))
    policy_change_file = os.path.join(base_path, args.policy_file)

    # locate config file
    # TODO: hard coded destination to global config file
    base_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                "..","..","endeavour","examples","test-mh","config"))
    config_file = os.path.join(base_path, "sdx_global.cfg")

    logger = util.log.getLogger("P_" + str(args.id))
    logger.debug ("Starting participant_client (%s) with config file: %s" % (args.id, config_file))

    # start controller
    prtclnt = ParticipantClient(args.id, config_file, logger)
    prtclnt_thread = Thread(target=prtclnt.xstart(policy_change_file, args.action))
    prtclnt_thread.daemon = True
    prtclnt_thread.start()

    atexit.register(prtclnt.stop)
    signal(SIGTERM, lambda signum, stack_frame: exit(1))

    while prtclnt_thread.is_alive():
        try:
            prtclnt_thread.join(1)
        except KeyboardInterrupt:
            prtclnt.stop()

    logger.debug ("participant_client(%s): extiting" % args.id)
    print ("done")

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>#
# Copyright (c) 2015 Red Hat
# Licensed under The MIT License (MIT)
# http://opensource.org/licenses/MIT
#
from django.conf import settings

import logging

logger = logging.getLogger(__name__)

class DummyMessenger(object):
    def __init__(self):
        pass

    def send_message(self, topic, msg):
        pass

class KombuMessenger(object):
    def __init__(self):
        from kombu import Connection, Exchange
        self.conn = Connection(settings.MESSAGE_BUS['URL'],
                               **settings.MESSAGE_BUS.get('OPTIONS', {}))
        self.exchange = Exchange(**settings.MESSAGE_BUS['EXCHANGE'])
        self.messenger = self.conn.Producer()

    def send_message(self, topic, msg):
        self.messenger.publish(msg, exchange=self.exchange, routing_key=topic)

class FedmsgMessenger(object):
    def __init__(self):
        import fedmsg
        self.messenger = fedmsg

    def send_message(self, topic, msg):
        self.messenger.publish(topic=topic.strip('.'), msg=msg)

class ProtonMessenger(object):
    def __init__(self):
        import proton
        self.messenger = proton.Messenger()
        self.messenger.certificate = settings.MESSAGE_BUS['CERT_FILE']
        self.messenger.private_key = settings.MESSAGE_BUS['KEY_FILE']
        self.messenger.start()
        self.message = proton.Message()

    def send_message(self, topic, msg):
        self.message.address = settings.MESSAGE_BUS['URL'] + topic
        self.message.body = msg
        self.messenger.put(self.message)
        self.messenger.send()

class StompMessenger(object):
    def __init__(self):
        import stomp
        self.connection = stomp.Connection(
            host_and_ports=settings.MESSAGE_BUS['HOST_AND_PORTS'],
            use_ssl=True,
            ssl_key_file=settings.MESSAGE_BUS['KEY_FILE'],
            ssl_cert_file=settings.MESSAGE_BUS['CERT_FILE'],
        )
        self.connected = self.do_connect()

    def do_connect(self):
        try:
            self.connection.start()
            self.connection.connect()
        except Exception, e:
            logger.warn("StompMessenger connection exception(%s): %s." % (type(e), e))
            return False
        return True

    def send_message(self, topic, msg):
        address = '/topic/' + settings.MESSAGE_BUS['TOPIC'] + str(topic)
        if self.connected or self.do_connect():
            try:
                self.connection.send(body=msg, destination=address,
                                     headers={'persistent': 'true'},
                                     auto='true')
            except Exception, e:
                logger.warn("Send Message exception(%s): %s." % (type(e), e))
        else:
            logger.warn("Send Message exception: Failed to Connect to Messaging Server.")
<commit_msg>Avoid double-encoding the fedmsg json payload.<commit_after>#
# Copyright (c) 2015 Red Hat
# Licensed under The MIT License (MIT)
# http://opensource.org/licenses/MIT
#
import json

from django.conf import settings
from django.utils import six

import logging

logger = logging.getLogger(__name__)

class DummyMessenger(object):
    def __init__(self):
        pass

    def send_message(self, topic, msg):
        pass

class KombuMessenger(object):
    def __init__(self):
        from kombu import Connection, Exchange
        self.conn = Connection(settings.MESSAGE_BUS['URL'],
                               **settings.MESSAGE_BUS.get('OPTIONS', {}))
        self.exchange = Exchange(**settings.MESSAGE_BUS['EXCHANGE'])
        self.messenger = self.conn.Producer()

    def send_message(self, topic, msg):
        self.messenger.publish(msg, exchange=self.exchange, routing_key=topic)

class FedmsgMessenger(object):
    def __init__(self):
        import fedmsg
        self.messenger = fedmsg

    def send_message(self, topic, msg):
        topic = topic.strip('.')
        if isinstance(msg, six.string_types):
            msg = json.loads(msg)
        self.messenger.publish(topic=topic, msg=msg)

class ProtonMessenger(object):
    def __init__(self):
        import proton
        self.messenger = proton.Messenger()
        self.messenger.certificate = settings.MESSAGE_BUS['CERT_FILE']
        self.messenger.private_key = settings.MESSAGE_BUS['KEY_FILE']
        self.messenger.start()
        self.message = proton.Message()

    def send_message(self, topic, msg):
        self.message.address = settings.MESSAGE_BUS['URL'] + topic
        self.message.body = msg
        self.messenger.put(self.message)
        self.messenger.send()

class StompMessenger(object):
    def __init__(self):
        import stomp
        self.connection = stomp.Connection(
            host_and_ports=settings.MESSAGE_BUS['HOST_AND_PORTS'],
            use_ssl=True,
            ssl_key_file=settings.MESSAGE_BUS['KEY_FILE'],
            ssl_cert_file=settings.MESSAGE_BUS['CERT_FILE'],
        )
        self.connected = self.do_connect()

    def do_connect(self):
        try:
            self.connection.start()
            self.connection.connect()
        except Exception, e:
            logger.warn("StompMessenger connection exception(%s): %s." % (type(e), e))
            return False
        return True

    def send_message(self, topic, msg):
        address = '/topic/' + settings.MESSAGE_BUS['TOPIC'] + str(topic)
        if self.connected or self.do_connect():
            try:
                self.connection.send(body=msg, destination=address,
                                     headers={'persistent': 'true'},
                                     auto='true')
            except Exception, e:
                logger.warn("Send Message exception(%s): %s." % (type(e), e))
        else:
            logger.warn("Send Message exception: Failed to Connect to Messaging Server.")
<|endoftext|>


<commit_before>import os
import string
import uuid

from PblCommand import PblCommand

class PblProjectCreator(PblCommand):
    name = 'new-project'
    help = 'Create a new Pebble project'

    def configure_subparser(self, parser):
        parser.add_argument("name", help = "Name of the project you want to create")
        parser.add_argument("--javascript", action="store_true", help = "Generate javascript related files")

    def run(self, args):
        print "Creating new project {}".format(args.name)

        # User can give a path to a new project dir
        project_path = args.name
        project_name = os.path.split(project_path)[1]
        project_root = os.path.join(os.getcwd(), project_path)

        project_src = os.path.join(project_root, "src")

        # Create directories
        os.makedirs(project_root)
        os.makedirs(os.path.join(project_root, "resources"))
        os.makedirs(project_src)

        # Create main .c file
        with open(os.path.join(project_src, "%s.c" % (project_name)), "w") as f:
            f.write(FILE_DUMMY_MAIN)

        # Add wscript file
        with open(os.path.join(project_root, "wscript"), "w") as f:
            f.write(FILE_WSCRIPT)

        # Add appinfo.json file
        appinfo_dummy = DICT_DUMMY_APPINFO.copy()
        appinfo_dummy['uuid'] = str(uuid.uuid4())
        with open(os.path.join(project_root, "appinfo.json"), "w") as f:
            f.write(FILE_DUMMY_APPINFO.substitute(**appinfo_dummy))

        # Add .gitignore file
        with open(os.path.join(project_root, ".gitignore"), "w") as f:
            f.write(FILE_GITIGNORE)

        if args.javascript:
            project_js_src = os.path.join(project_src, "js")
            os.makedirs(project_js_src)

            with open(os.path.join(project_js_src, "pebble-js-app.js"), "w") as f:
                f.write(FILE_DUMMY_JAVASCRIPT_SRC)

FILE_GITIGNORE = """
# Ignore build generated files
build
"""

FILE_WSCRIPT = """
#
# This file is the default set of rules to compile a Pebble project.
#
# Feel free to customize this to your needs.
#

top = '.'
out = 'build'

def options(ctx):
    ctx.load('pebble_sdk')

def configure(ctx):
    ctx.load('pebble_sdk')

def build(ctx):
    ctx.load('pebble_sdk')

    ctx.pbl_program(source=ctx.path.ant_glob('src/**/*.c'),
                    target='pebble-app.elf')

    ctx.pbl_bundle(elf='pebble-app.elf',
                   js=ctx.path.ant_glob('src/js/**/*.js'))
"""

FILE_DUMMY_MAIN = """#include <pebble.h>

static Window *window;
static TextLayer *text_layer;

static void select_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Select");
}

static void up_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Up");
}

static void down_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Down");
}

static void click_config_provider(void *context) {
  window_single_click_subscribe(BUTTON_ID_SELECT, select_click_handler);
  window_single_click_subscribe(BUTTON_ID_UP, up_click_handler);
  window_single_click_subscribe(BUTTON_ID_DOWN, down_click_handler);
}

static void window_load(Window *window) {
  Layer *window_layer = window_get_root_layer(window);
  GRect bounds = layer_get_bounds(window_layer);

  text_layer = text_layer_create((GRect) { .origin = { 0, 72 }, .size = { bounds.size.w, 20 } });
  text_layer_set_text(text_layer, "Press a button");
  text_layer_set_text_alignment(text_layer, GTextAlignmentCenter);
  layer_add_child(window_layer, text_layer_get_layer(text_layer));
}

static void window_unload(Window *window) {
  text_layer_destroy(text_layer);
}

static void init(void) {
  window = window_create();
  window_set_click_config_provider(window, click_config_provider);
  window_set_window_handlers(window, (WindowHandlers) {
    .load = window_load,
    .unload = window_unload,
  });
  const bool animated = true;
  window_stack_push(window, animated);
}

static void deinit(void) {
  window_destroy(window);
}

int main(void) {
  init();

  APP_LOG(APP_LOG_LEVEL_DEBUG, "Done initializing, pushed window: %p", window);

  app_event_loop();
  deinit();
}
"""

DICT_DUMMY_APPINFO = {
    'short_name': 'Template App',
    'long_name': 'Pebble Template App',
    'company_name': 'Your Company',
    'version_code': 1,
    'version_label': '1.0.0',
    'is_watchface': 'false',
    'app_keys': """{
    "dummy": 0
  }""",
    'resources_media': '[]'
}

FILE_DUMMY_APPINFO = string.Template("""{
  "uuid": "${uuid}",
  "shortName": "${short_name}",
  "longName": "${long_name}",
  "companyName": "${company_name}",
  "versionCode": ${version_code},
  "versionLabel": "${version_label}",
  "watchapp": {
    "watchface": ${is_watchface}
  },
  "appKeys": ${app_keys},
  "resources": {
    "media": ${resources_media}
  }
}
""")

FILE_DUMMY_JAVASCRIPT_SRC = """\
Pebble.addEventListener("ready",
    function(e) {
        console.log("Hello world!", "Sent from your javascript application.")
    }
};
"""

class InvalidProjectException(Exception):
    pass

class OutdatedProjectException(Exception):
    pass

def check_project_directory():
    """Check to see if the current directly matches what is created by PblProjectCreator.run.

    Raises an InvalidProjectException or an OutdatedProjectException if everything isn't quite right.
    """

    if not os.path.isdir('src') or not os.path.exists('wscript'):
        raise InvalidProjectException

    if os.path.islink('pebble_app.ld') or os.path.exists('resources/src/resource_map.json'):
        raise OutdatedProjectException

def requires_project_dir(func):
    def wrapper(self, args):
        check_project_directory()
        return func(self, args)
    return wrapper

<commit_msg>Name new projects after the project name / folder<commit_after>import os
import string
import uuid

from PblCommand import PblCommand

class PblProjectCreator(PblCommand):
    name = 'new-project'
    help = 'Create a new Pebble project'

    def configure_subparser(self, parser):
        parser.add_argument("name", help = "Name of the project you want to create")
        parser.add_argument("--javascript", action="store_true", help = "Generate javascript related files")

    def run(self, args):
        print "Creating new project {}".format(args.name)

        # User can give a path to a new project dir
        project_path = args.name
        project_name = os.path.split(project_path)[1]
        project_root = os.path.join(os.getcwd(), project_path)

        project_src = os.path.join(project_root, "src")

        # Create directories
        os.makedirs(project_root)
        os.makedirs(os.path.join(project_root, "resources"))
        os.makedirs(project_src)

        # Create main .c file
        with open(os.path.join(project_src, "%s.c" % (project_name)), "w") as f:
            f.write(FILE_DUMMY_MAIN)

        # Add wscript file
        with open(os.path.join(project_root, "wscript"), "w") as f:
            f.write(FILE_WSCRIPT)

        # Add appinfo.json file
        appinfo_dummy = DICT_DUMMY_APPINFO.copy()
        appinfo_dummy['uuid'] = str(uuid.uuid4())
        appinfo_dummy['project_name'] = project_name
        with open(os.path.join(project_root, "appinfo.json"), "w") as f:
            f.write(FILE_DUMMY_APPINFO.substitute(**appinfo_dummy))

        # Add .gitignore file
        with open(os.path.join(project_root, ".gitignore"), "w") as f:
            f.write(FILE_GITIGNORE)

        if args.javascript:
            project_js_src = os.path.join(project_src, "js")
            os.makedirs(project_js_src)

            with open(os.path.join(project_js_src, "pebble-js-app.js"), "w") as f:
                f.write(FILE_DUMMY_JAVASCRIPT_SRC)

FILE_GITIGNORE = """
# Ignore build generated files
build
"""

FILE_WSCRIPT = """
#
# This file is the default set of rules to compile a Pebble project.
#
# Feel free to customize this to your needs.
#

top = '.'
out = 'build'

def options(ctx):
    ctx.load('pebble_sdk')

def configure(ctx):
    ctx.load('pebble_sdk')

def build(ctx):
    ctx.load('pebble_sdk')

    ctx.pbl_program(source=ctx.path.ant_glob('src/**/*.c'),
                    target='pebble-app.elf')

    ctx.pbl_bundle(elf='pebble-app.elf',
                   js=ctx.path.ant_glob('src/js/**/*.js'))
"""

FILE_DUMMY_MAIN = """#include <pebble.h>

static Window *window;
static TextLayer *text_layer;

static void select_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Select");
}

static void up_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Up");
}

static void down_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Down");
}

static void click_config_provider(void *context) {
  window_single_click_subscribe(BUTTON_ID_SELECT, select_click_handler);
  window_single_click_subscribe(BUTTON_ID_UP, up_click_handler);
  window_single_click_subscribe(BUTTON_ID_DOWN, down_click_handler);
}

static void window_load(Window *window) {
  Layer *window_layer = window_get_root_layer(window);
  GRect bounds = layer_get_bounds(window_layer);

  text_layer = text_layer_create((GRect) { .origin = { 0, 72 }, .size = { bounds.size.w, 20 } });
  text_layer_set_text(text_layer, "Press a button");
  text_layer_set_text_alignment(text_layer, GTextAlignmentCenter);
  layer_add_child(window_layer, text_layer_get_layer(text_layer));
}

static void window_unload(Window *window) {
  text_layer_destroy(text_layer);
}

static void init(void) {
  window = window_create();
  window_set_click_config_provider(window, click_config_provider);
  window_set_window_handlers(window, (WindowHandlers) {
    .load = window_load,
    .unload = window_unload,
  });
  const bool animated = true;
  window_stack_push(window, animated);
}

static void deinit(void) {
  window_destroy(window);
}

int main(void) {
  init();

  APP_LOG(APP_LOG_LEVEL_DEBUG, "Done initializing, pushed window: %p", window);

  app_event_loop();
  deinit();
}
"""

# @ROBERT TODO: how I make subst here?
DICT_DUMMY_APPINFO = {
    'company_name': 'MakeAwesomeHappen',
    'version_code': 1,
    'version_label': '1.0.0',
    'is_watchface': 'false',
    'app_keys': """{
    "dummy": 0
  }""",
    'resources_media': '[]'
}

FILE_DUMMY_APPINFO = string.Template("""{
  "uuid": "${uuid}",
  "shortName": "${project_name}",
  "longName": "${project_name}",
  "companyName": "${company_name}",
  "versionCode": ${version_code},
  "versionLabel": "${version_label}",
  "watchapp": {
    "watchface": ${is_watchface}
  },
  "appKeys": ${app_keys},
  "resources": {
    "media": ${resources_media}
  }
}
""")

FILE_DUMMY_JAVASCRIPT_SRC = """\
Pebble.addEventListener("ready",
    function(e) {
        console.log("Hello world!", "Sent from your javascript application.")
    }
};
"""

class InvalidProjectException(Exception):
    pass

class OutdatedProjectException(Exception):
    pass

def check_project_directory():
    """Check to see if the current directly matches what is created by PblProjectCreator.run.

    Raises an InvalidProjectException or an OutdatedProjectException if everything isn't quite right.
    """

    if not os.path.isdir('src') or not os.path.exists('wscript'):
        raise InvalidProjectException

    if os.path.islink('pebble_app.ld') or os.path.exists('resources/src/resource_map.json'):
        raise OutdatedProjectException

def requires_project_dir(func):
    def wrapper(self, args):
        check_project_directory()
        return func(self, args)
    return wrapper

<|endoftext|>


<commit_before>from webob.exc import WSGIHTTPException
from webob.response import Response

class JSONValidationException(WSGIHTTPException):
    """
    WebOb doesn't allow setting the explicit content type
    when raising an HTTP exception. It forces the server to use plain text
    or HTML. We require a JSON response because we are validating JSON.

    We subclass form the base HTTP WebOb exception and force the response
    to be JSON.
    """

    code = 400
    title = 'Failed Validation'
    explanation = 'Request failed validation schema'

    def generate_response(self, environ, start_response):
        if self.content_length is not None:
            del self.content_length
        headerlist = list(self.headerlist)
        content_type = 'application/json'
        body = '{"error": "%s"}' % self.detail
        resp = Response(
            body,
            status=self.status,
            headerlist=headerlist,
            content_type=content_type
        )
        resp.content_type = content_type
        return resp(environ, start_response)

<commit_msg>exceptions: webob 1.7 compat<commit_after>from webob.exc import WSGIHTTPException
from webob.response import Response

class JSONValidationException(WSGIHTTPException):
    """
    WebOb doesn't allow setting the explicit content type
    when raising an HTTP exception. It forces the server to use plain text
    or HTML. We require a JSON response because we are validating JSON.

    We subclass form the base HTTP WebOb exception and force the response
    to be JSON.
    """

    code = 400
    title = 'Failed Validation'
    explanation = 'Request failed validation schema'

    def generate_response(self, environ, start_response):
        if self.content_length is not None:
            del self.content_length
        headerlist = list(self.headerlist)
        charset = 'utf-8'
        content_type = 'application/json'
        body = '{"error": "%s"}' % self.detail
        resp = Response(
            body,
            charset=charset,
            status=self.status,
            headerlist=headerlist,
            content_type=content_type
        )
        resp.content_type = content_type
        return resp(environ, start_response)

<|endoftext|>


<commit_before>from collections import namedtuple
from optparse import OptionParser

import requests
from logger import logger

from perfrunner.helpers.remote import RemoteHelper
from perfrunner.settings import ClusterSpec

Build = namedtuple('Build', ['arch', 'pkg', 'version', 'openssl', 'toy'])

class CouchbaseInstaller(object):

    CBFS = 'http://cbfs-ext.hq.couchbase.com/builds/'
    LATEST_BUILDS = 'http://builds.hq.northscale.net/latestbuilds/'

    def __init__(self, cluster_spec, options):
        self.remote_helper = RemoteHelper(cluster_spec)
        self.cluster_spec = cluster_spec

        arch = self.remote_helper.detect_arch()
        pkg = self.remote_helper.detect_pkg()
        openssl = self.remote_helper.detect_openssl(pkg)

        self.build = Build(arch, pkg, options.version, openssl, options.toy)
        logger.info('Target build info: {}'.format(self.build))

    def get_expected_filenames(self):
        if self.build.toy:
            patterns = (
                'couchbase-server-community_toy-{toy}-{arch}_{version}-toy.{pkg}',
                'couchbase-server-community_toy-{toy}-{version}-toy_{arch}.{pkg}',
                'couchbase-server-community_cent58-3.0.0-toy-{toy}-{arch}_{version}-toy.{pkg}',
                'couchbase-server-community_cent58-master-toy-{toy}-{arch}_{version}-toy.{pkg}',
                'couchbase-server-community_cent54-master-toy-{toy}-{arch}_{version}-toy.{pkg}',
            )
        elif self.build.openssl == '0.9.8e':
            patterns = (
                'couchbase-server-enterprise_{arch}_{version}-rel.{pkg}',
                'couchbase-server-enterprise_{version}-rel_{arch}_openssl098.{pkg}',
            )
        else:
            patterns = (
                'couchbase-server-enterprise_centos6_{arch}_{version}-rel.{pkg}',
                'couchbase-server-enterprise_{arch}_{version}-rel.{pkg}',
                'couchbase-server-enterprise_{version}-rel_{arch}.{pkg}',
            )
        for pattern in patterns:
            yield pattern.format(**self.build._asdict())

    def find_package(self):
        for filename in self.get_expected_filenames():
            for base in (self.LATEST_BUILDS, self.CBFS):
                url = '{}{}'.format(base, filename)
                try:
                    status_code = requests.head(url).status_code
                except requests.exceptions.ConnectionError:
                    continue
                else:
                    if status_code == 200:
                        logger.info('Found "{}"'.format(url))
                        return filename, url
        logger.interrupt('Target build not found')

    def kill_processes(self):
        self.remote_helper.kill_processes()

    def uninstall_package(self):
        self.remote_helper.uninstall_package(self.build.pkg)

    def clean_data(self):
        self.remote_helper.clean_data()

    def install_package(self):
        filename, url = self.find_package()
        version = self.build.version.split('-')[0]
        self.remote_helper.install_package(self.build.pkg, url, filename,
                                           version)

    def install(self):
        self.kill_processes()
        self.uninstall_package()
        self.clean_data()
        self.install_package()

def main():
    usage = '%prog -c cluster -v version [-t toy]'

    parser = OptionParser(usage)

    parser.add_option('-c', dest='cluster_spec_fname',
                      help='path to cluster specification file',
                      metavar='cluster.spec')
    parser.add_option('-v', dest='version',
                      help='build version', metavar='2.0.0-1976')
    parser.add_option('-t', dest='toy',
                      help='optional toy build ID', metavar='couchstore')

    options, _ = parser.parse_args()
    if not options.cluster_spec_fname or not options.version:
        parser.error('Missing mandatory parameter')

    cluster_spec = ClusterSpec()
    cluster_spec.parse(options.cluster_spec_fname)

    installer = CouchbaseInstaller(cluster_spec, options)
    installer.install()

if __name__ == '__main__':
    main()
<commit_msg>extend list of toy builds<commit_after>from collections import namedtuple
from optparse import OptionParser

import requests
from logger import logger

from perfrunner.helpers.remote import RemoteHelper
from perfrunner.settings import ClusterSpec

Build = namedtuple('Build', ['arch', 'pkg', 'version', 'openssl', 'toy'])

class CouchbaseInstaller(object):

    CBFS = 'http://cbfs-ext.hq.couchbase.com/builds/'
    LATEST_BUILDS = 'http://builds.hq.northscale.net/latestbuilds/'

    def __init__(self, cluster_spec, options):
        self.remote_helper = RemoteHelper(cluster_spec)
        self.cluster_spec = cluster_spec

        arch = self.remote_helper.detect_arch()
        pkg = self.remote_helper.detect_pkg()
        openssl = self.remote_helper.detect_openssl(pkg)

        self.build = Build(arch, pkg, options.version, openssl, options.toy)
        logger.info('Target build info: {}'.format(self.build))

    def get_expected_filenames(self):
        if self.build.toy:
            patterns = (
                'couchbase-server-community_toy-{toy}-{arch}_{version}-toy.{pkg}',
                'couchbase-server-community_toy-{toy}-{version}-toy_{arch}.{pkg}',
                'couchbase-server-community_cent58-2.5.2-toy-{toy}-{arch}_{version}-toy.{pkg}',
                'couchbase-server-community_cent58-3.0.0-toy-{toy}-{arch}_{version}-toy.{pkg}',
                'couchbase-server-community_cent58-master-toy-{toy}-{arch}_{version}-toy.{pkg}',
                'couchbase-server-community_cent54-master-toy-{toy}-{arch}_{version}-toy.{pkg}',
            )
        elif self.build.openssl == '0.9.8e':
            patterns = (
                'couchbase-server-enterprise_{arch}_{version}-rel.{pkg}',
                'couchbase-server-enterprise_{version}-rel_{arch}_openssl098.{pkg}',
            )
        else:
            patterns = (
                'couchbase-server-enterprise_centos6_{arch}_{version}-rel.{pkg}',
                'couchbase-server-enterprise_{arch}_{version}-rel.{pkg}',
                'couchbase-server-enterprise_{version}-rel_{arch}.{pkg}',
            )
        for pattern in patterns:
            yield pattern.format(**self.build._asdict())

    def find_package(self):
        for filename in self.get_expected_filenames():
            for base in (self.LATEST_BUILDS, self.CBFS):
                url = '{}{}'.format(base, filename)
                try:
                    status_code = requests.head(url).status_code
                except requests.exceptions.ConnectionError:
                    continue
                else:
                    if status_code == 200:
                        logger.info('Found "{}"'.format(url))
                        return filename, url
        logger.interrupt('Target build not found')

    def kill_processes(self):
        self.remote_helper.kill_processes()

    def uninstall_package(self):
        self.remote_helper.uninstall_package(self.build.pkg)

    def clean_data(self):
        self.remote_helper.clean_data()

    def install_package(self):
        filename, url = self.find_package()
        version = self.build.version.split('-')[0]
        self.remote_helper.install_package(self.build.pkg, url, filename,
                                           version)

    def install(self):
        self.kill_processes()
        self.uninstall_package()
        self.clean_data()
        self.install_package()

def main():
    usage = '%prog -c cluster -v version [-t toy]'

    parser = OptionParser(usage)

    parser.add_option('-c', dest='cluster_spec_fname',
                      help='path to cluster specification file',
                      metavar='cluster.spec')
    parser.add_option('-v', dest='version',
                      help='build version', metavar='2.0.0-1976')
    parser.add_option('-t', dest='toy',
                      help='optional toy build ID', metavar='couchstore')

    options, _ = parser.parse_args()
    if not options.cluster_spec_fname or not options.version:
        parser.error('Missing mandatory parameter')

    cluster_spec = ClusterSpec()
    cluster_spec.parse(options.cluster_spec_fname)

    installer = CouchbaseInstaller(cluster_spec, options)
    installer.install()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>
class Event:
    """Unit class for storing events inside data."""

    title = ""
    url = ""
    start_rate = 0.0
    end_rate = 0.0
    complete = False

    def __init__(self, start_rate, end_rate, title, url, complete):
        self.start_rate = start_rate
        self.end_rate = end_rate
        self.title = title
        self.url = url
        self.complete = complete

    def get_title(self):
        return self.title

    def print_event(self):
        """Print event for test. """
        print(" - " + self.title)

    def html_event(self):
        html =  "
"
        html += "" + self.title + ""
        html += "
"
        return html
<commit_msg>link button<commit_after>
class Event:
    """Unit class for storing events inside data."""

    title = ""
    url = ""
    start_rate = 0.0
    end_rate = 0.0
    complete = False

    def __init__(self, start_rate, end_rate, title, url, complete):
        self.start_rate = start_rate
        self.end_rate = end_rate
        self.title = title
        self.url = url
        self.complete = complete

    def get_title(self):
        return self.title

    def print_event(self):
        """Print event for test. """
        print(" - " + self.title)

    def html_event(self):
        html =  "
"
        html += self.title
        html += "
"
        return html
<|endoftext|>