<commit_before>#!/bin/python
import os
import optparse
import json

import livetrader.exchangeMonitor
import livetrader.gekkoTrigger
import livetrader.gekkoChecker
import livetrader.strategyRanker

parser = optparse.OptionParser()

parser.add_option('-b', '--balance',
                  dest='balanceChecker', action='store_true', default=False)

parser.add_option('-t', '--trigger <strategy>',
                  dest='botTrigger', type='str', default='')

parser.add_option('-c', dest='runningBotChecker',
                  action='store_true', default=False)

parser.add_option('-l', dest='tradingBot', action='store_true',
                  default=False)

parser.add_option('--candleSize <cs>',
                  dest='candleSize', type='int', default=5)

parser.add_option('--strat <strategy>', dest='strategy',
                  type='str', default='')

parser.add_option('--param <parameters>', dest='alternativeParameters',
                  type='str', default=None)

options, args = parser.parse_args()

if __name__ == '__main__':
    os.chdir(os.path.dirname(os.path.realpath(__file__)))
    exchange = livetrader.exchangeMonitor.Exchange('binance')

    if options.balanceChecker:
        totalUSD = exchange.getUserBalance()

        print("net weight at %s: US$T%.2f" % (
            exchange.name,
            totalUSD)
        )

    if options.botTrigger:
        allPairs = exchange.getAssets()
        assetCurrencyPairs = exchange.parseAssets(allPairs)
        Stratlist = [options.botTrigger]

        exchangeConfPath =\
            exchange.conf.binanceAssetCurrencyTargetFilePath
        if exchangeConfPath:
            exchangeMarketData = exchange.generateMarketsJson(
                assetCurrencyPairs)
            exchangeConfPath = os.path.join(exchangeConfPath,
                                            'binance-markets.json')

            with open(exchangeConfPath, 'w') as F:
                json.dump(exchangeMarketData, F, indent=2)

        livetrader.gekkoTrigger.launchBatchTradingBots(
            assetCurrencyPairs,
            Stratlist,
            options
        )

    if options.runningBotChecker:
        ranker = livetrader.strategyRanker.strategyRanker()
        ranker.loadStrategyRankings()
        userOrderHistory = exchange.getRecentOrders()
        for M in userOrderHistory.keys():
            marketOrderHistory = userOrderHistory[M]
            if marketOrderHistory:
                information = json.dumps(marketOrderHistory, indent=2)
                print(information)

        livetrader.gekkoChecker.checkGekkoRunningBots(exchange, ranker)
<commit_msg>Added.<commit_after>#!/bin/python
import os
import optparse
import json

import livetrader.exchangeMonitor
import livetrader.gekkoTrigger
import livetrader.gekkoChecker
import livetrader.strategyRanker

parser = optparse.OptionParser()

parser.add_option('-b', '--balance',
                  dest='balanceChecker', action='store_true', default=False)

parser.add_option('-t', '--trigger <strategy>',
                  dest='botTrigger', type='str', default='')

parser.add_option('-c', dest='runningBotChecker',
                  action='store_true', default=False)

parser.add_option('-l', dest='tradingBot', action='store_true',
                  default=False)

parser.add_option('--candleSize <cs>',
                  dest='candleSize', type='int', default=5)

parser.add_option('--strat <strategy>', dest='strategy',
                  type='str', default='')

parser.add_option('--param <parameters>', dest='alternativeParameters',
                  type='str', default=None)

parser.add_option('-k', dest='killGekkoBots', action='store_true',
                  default=False,
                  help='Destroy all running gekko bot instances.')

parser.add_option('-s', dest='viewLastTrades', action='store_true',
                  default=False,
                  help='Show last trades done by bots.')

options, args = parser.parse_args()

if __name__ == '__main__':
    os.chdir(os.path.dirname(os.path.realpath(__file__)))
    exchange = livetrader.exchangeMonitor.Exchange('binance')

    if options.balanceChecker:
        totalUSD = exchange.getUserBalance()

        print("net weight at %s: US$T%.2f" % (
            exchange.name,
            totalUSD)
        )

    if options.botTrigger:
        allPairs = exchange.getAssets()
        assetCurrencyPairs = exchange.parseAssets(allPairs)
        Stratlist = [options.botTrigger]

        exchangeConfPath =\
            exchange.conf.binanceAssetCurrencyTargetFilePath
        if exchangeConfPath:
            exchangeMarketData = exchange.generateMarketsJson(
                assetCurrencyPairs)
            exchangeConfPath = os.path.join(exchangeConfPath,
                                            'binance-markets.json')

            with open(exchangeConfPath, 'w') as F:
                json.dump(exchangeMarketData, F, indent=2)

        livetrader.gekkoTrigger.launchBatchTradingBots(
            assetCurrencyPairs,
            Stratlist,
            options
        )

    if options.runningBotChecker:
        ranker = livetrader.strategyRanker.strategyRanker()
        ranker.loadStrategyRankings()
        userOrderHistory = exchange.getRecentOrders()
        for M in userOrderHistory.keys():
            marketOrderHistory = userOrderHistory[M]
            if marketOrderHistory:
                information = json.dumps(marketOrderHistory, indent=2)
                print(information)

        livetrader.gekkoChecker.checkGekkoRunningBots(exchange,
                                                      ranker, options)
    if options.killGekkoBots:
        livetrader.gekkoChecker.stopGekkoBots()

    if options.viewLastTrades:
        Orders = exchange.getRecentOrders()
        print(json.dumps(Orders, indent=2))

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    livereload.watcher
    ~~~~~~~~~~~~~~~~~~

    A file watch management for LiveReload Server.

    :copyright: (c) 2013 - 2015 by Hsiaoming Yang
"""

import os
import glob
import time
try:
    import pyinotify
except ImportError:
    pyinotify = None

class Watcher(object):
    """A file watcher registery."""
    def __init__(self):
        self._tasks = {}
        self._mtimes = {}

        # setting changes
        self._changes = []

        # filepath that is changed
        self.filepath = None
        self._start = time.time()

    def ignore(self, filename):
        """Ignore a given filename or not."""
        _, ext = os.path.splitext(filename)
        return ext in ['.pyc', '.pyo', '.o', '.swp']

    def watch(self, path, func=None, delay=0, ignore=None):
        """Add a task to watcher.

        :param path: a filepath or directory path or glob pattern
        :param func: the function to be executed when file changed
        :param delay: Delay sending the reload message. Use 'forever' to
                      not send it. This is useful to compile sass files to
                      css, but reload on changed css files then only.
        :param ignore: A function return True to ignore a certain pattern of
                       filepath.
        """
        self._tasks[path] = {
            'func': func,
            'delay': delay,
            'ignore': ignore,
        }

    def start(self, callback):
        """Start the watcher running, calling callback when changes are
        observed. If this returns False, regular polling will be used."""
        return False

    def examine(self):
        """Check if there are changes, if true, run the given task."""
        if self._changes:
            return self._changes.pop()

        # clean filepath
        self.filepath = None
        delays = set()
        for path in self._tasks:
            item = self._tasks[path]
            if self.is_changed(path, item['ignore']):
                func = item['func']
                func and func()
                delays.add(item['delay'])

        if delays:
            delay = max(delays)
        else:
            delay = None
        return self.filepath, delay

    def is_changed(self, path, ignore=None):
        if os.path.isfile(path):
            return self.is_file_changed(path, ignore)
        elif os.path.isdir(path):
            return self.is_folder_changed(path, ignore)
        return self.is_glob_changed(path, ignore)

    def is_file_changed(self, path, ignore=None):
        if not os.path.isfile(path):
            return False

        if self.ignore(path):
            return False

        if ignore and ignore(path):
            return False

        mtime = os.path.getmtime(path)

        if path not in self._mtimes:
            self._mtimes[path] = mtime
            self.filepath = path
            return mtime > self._start

        if self._mtimes[path] != mtime:
            self._mtimes[path] = mtime
            self.filepath = path
            return True

        self._mtimes[path] = mtime
        return False

    def is_folder_changed(self, path, ignore=None):
        for root, dirs, files in os.walk(path, followlinks=True):
            if '.git' in dirs:
                dirs.remove('.git')
            if '.hg' in dirs:
                dirs.remove('.hg')
            if '.svn' in dirs:
                dirs.remove('.svn')
            if '.cvs' in dirs:
                dirs.remove('.cvs')

            for f in files:
                if self.is_file_changed(os.path.join(root, f), ignore):
                    return True
        return False

    def is_glob_changed(self, path, ignore=None):
        for f in glob.glob(path):
            if self.is_file_changed(f, ignore):
                return True
        return False

class INotifyWatcher(Watcher):
    def __init__(self):
        Watcher.__init__(self)

        self.wm = pyinotify.WatchManager()
        self.notifier = None
        self.callback = None

    def watch(self, path, func=None, delay=None, ignore=None):
        flag = pyinotify.IN_CREATE | pyinotify.IN_DELETE | pyinotify.IN_MODIFY
        self.wm.add_watch(path, flag, rec=True, do_glob=True, auto_add=True)
        Watcher.watch(self, path, func, delay, ignore)

    def inotify_event(self, event):
        self.callback()

    def start(self, callback):
        if not self.notifier:
            self.callback = callback

            from tornado import ioloop
            self.notifier = pyinotify.TornadoAsyncNotifier(
                self.wm, ioloop.IOLoop.instance(),
                default_proc_fun=self.inotify_event
            )
            callback()
        return True

def get_watcher_class():
    if pyinotify is None:
        return Watcher
    return INotifyWatcher
<commit_msg>Only add int delay to delays. fix #96<commit_after># -*- coding: utf-8 -*-
"""
    livereload.watcher
    ~~~~~~~~~~~~~~~~~~

    A file watch management for LiveReload Server.

    :copyright: (c) 2013 - 2015 by Hsiaoming Yang
"""

import os
import glob
import time
try:
    import pyinotify
except ImportError:
    pyinotify = None

class Watcher(object):
    """A file watcher registery."""
    def __init__(self):
        self._tasks = {}
        self._mtimes = {}

        # setting changes
        self._changes = []

        # filepath that is changed
        self.filepath = None
        self._start = time.time()

    def ignore(self, filename):
        """Ignore a given filename or not."""
        _, ext = os.path.splitext(filename)
        return ext in ['.pyc', '.pyo', '.o', '.swp']

    def watch(self, path, func=None, delay=0, ignore=None):
        """Add a task to watcher.

        :param path: a filepath or directory path or glob pattern
        :param func: the function to be executed when file changed
        :param delay: Delay sending the reload message. Use 'forever' to
                      not send it. This is useful to compile sass files to
                      css, but reload on changed css files then only.
        :param ignore: A function return True to ignore a certain pattern of
                       filepath.
        """
        self._tasks[path] = {
            'func': func,
            'delay': delay,
            'ignore': ignore,
        }

    def start(self, callback):
        """Start the watcher running, calling callback when changes are
        observed. If this returns False, regular polling will be used."""
        return False

    def examine(self):
        """Check if there are changes, if true, run the given task."""
        if self._changes:
            return self._changes.pop()

        # clean filepath
        self.filepath = None
        delays = set()
        for path in self._tasks:
            item = self._tasks[path]
            if self.is_changed(path, item['ignore']):
                func = item['func']
                func and func()
                delay = item['delay']
                if delay and isinstance(delay, int):
                    delays.add(delay)

        if delays:
            delay = max(delays)
        else:
            delay = None
        return self.filepath, delay

    def is_changed(self, path, ignore=None):
        if os.path.isfile(path):
            return self.is_file_changed(path, ignore)
        elif os.path.isdir(path):
            return self.is_folder_changed(path, ignore)
        return self.is_glob_changed(path, ignore)

    def is_file_changed(self, path, ignore=None):
        if not os.path.isfile(path):
            return False

        if self.ignore(path):
            return False

        if ignore and ignore(path):
            return False

        mtime = os.path.getmtime(path)

        if path not in self._mtimes:
            self._mtimes[path] = mtime
            self.filepath = path
            return mtime > self._start

        if self._mtimes[path] != mtime:
            self._mtimes[path] = mtime
            self.filepath = path
            return True

        self._mtimes[path] = mtime
        return False

    def is_folder_changed(self, path, ignore=None):
        for root, dirs, files in os.walk(path, followlinks=True):
            if '.git' in dirs:
                dirs.remove('.git')
            if '.hg' in dirs:
                dirs.remove('.hg')
            if '.svn' in dirs:
                dirs.remove('.svn')
            if '.cvs' in dirs:
                dirs.remove('.cvs')

            for f in files:
                if self.is_file_changed(os.path.join(root, f), ignore):
                    return True
        return False

    def is_glob_changed(self, path, ignore=None):
        for f in glob.glob(path):
            if self.is_file_changed(f, ignore):
                return True
        return False

class INotifyWatcher(Watcher):
    def __init__(self):
        Watcher.__init__(self)

        self.wm = pyinotify.WatchManager()
        self.notifier = None
        self.callback = None

    def watch(self, path, func=None, delay=None, ignore=None):
        flag = pyinotify.IN_CREATE | pyinotify.IN_DELETE | pyinotify.IN_MODIFY
        self.wm.add_watch(path, flag, rec=True, do_glob=True, auto_add=True)
        Watcher.watch(self, path, func, delay, ignore)

    def inotify_event(self, event):
        self.callback()

    def start(self, callback):
        if not self.notifier:
            self.callback = callback

            from tornado import ioloop
            self.notifier = pyinotify.TornadoAsyncNotifier(
                self.wm, ioloop.IOLoop.instance(),
                default_proc_fun=self.inotify_event
            )
            callback()
        return True

def get_watcher_class():
    if pyinotify is None:
        return Watcher
    return INotifyWatcher
<|endoftext|>


<commit_before>import datetime
import json
import unittest2

from consts.event_type import EventType
from consts.district_type import DistrictType

from datafeeds.parsers.fms_api.fms_api_awards_parser import FMSAPIAwardsParser
from datafeeds.parsers.fms_api.fms_api_event_alliances_parser import FMSAPIEventAlliancesParser
from datafeeds.parsers.fms_api.fms_api_event_rankings_parser import FMSAPIEventRankingsParser
from datafeeds.parsers.fms_api.fms_api_match_parser import FMSAPIHybridScheduleParser

from google.appengine.ext import testbed

from models.event import Event

class TestFMSAPIEventParser(unittest2.TestCase):
    def setUp(self):
        self.testbed = testbed.Testbed()
        self.testbed.activate()
        self.testbed.init_datastore_v3_stub()
        self.testbed.init_memcache_stub()

        self.event = Event(
            id="2015waamv",
            end_date=datetime.datetime(2015, 4, 2, 0, 0),
            event_short="waamv",
            event_type_enum=EventType.REGIONAL,
            event_district_enum=DistrictType.NO_DISTRICT,
            first_eid="13467",
            name="PNW District - Auburn Mountainview Event",
            start_date=datetime.datetime(2015, 3, 31, 0, 0),
            year=2015,
            timezone_id='America/Los_Angeles'
        )
        self.event.put()

    def tearDown(self):
        self.testbed.deactivate()

    def test_parseAwards(self):
        with open('test_data/fms_api/2015waamv_staging_awards.json', 'r') as f:
            awards = FMSAPIAwardsParser(self.event).parse(json.loads(f.read()))

        self.assertEqual(len(awards), 5)

        for award in awards:
            if award.key.id() == '2015waamv_3':
                self.assertEqual(award.name_str, 'Woodie Flowers Award')
                self.assertEqual(award.award_type_enum, 3)
                self.assertTrue({'team_number': None, 'awardee': 'Bob'} in award.recipient_list)
            elif award.key.id() == '2015waamv_17':
                self.assertEqual(award.name_str, 'Quality Award sponsored by Motorola')
                self.assertEqual(award.award_type_enum, 17)
                self.assertTrue({'team_number': 1318, 'awardee': None} in award.recipient_list)
            elif award.key.id() == '2015waamv_4':
                self.assertEqual(award.name_str, 'FIRST Dean\'s List Award')
                self.assertEqual(award.award_type_enum, 4)
                self.assertTrue({'team_number': 123, 'awardee': 'Person Name 1'} in award.recipient_list)
                self.assertTrue({'team_number': 321, 'awardee': 'Person Name 2'} in award.recipient_list)

    def test_parseMatches(self):
        with open('test_data/fms_api/2015waamv_staging_matches.json', 'r') as f:
            matches = FMSAPIHybridScheduleParser(2015, 'waamv').parse(json.loads(f.read()))

        self.assertEqual(len(matches), 64)

        matches = sorted(matches, key=lambda m: m.play_order)

        # Test played match
        match = matches[0]
        self.assertEqual(match.comp_level, "qm")
        self.assertEqual(match.set_number, 1)
        self.assertEqual(match.match_number, 1)
        self.assertEqual(match.team_key_names, [u'frc4131', u'frc4469', u'frc3663', u'frc3684', u'frc5295', u'frc2976'])
        self.assertEqual(match.alliances_json, """{"blue": {"score": 30, "teams": ["frc4131", "frc4469", "frc3663"]}, "red": {"score": 18, "teams": ["frc3684", "frc5295", "frc2976"]}}""")
        self.assertEqual(match.time, datetime.datetime(2015, 2, 27, 0, 0))
        self.assertEqual(match.actual_time, datetime.datetime(2015, 2, 27, 0, 0))
        self.assertEqual(match.score_breakdown['red']['foul_points'], 0)
        self.assertEqual(match.score_breakdown['red']['auto_points'], 8)
        self.assertEqual(match.score_breakdown['blue']['foul_points'], 0)
        self.assertEqual(match.score_breakdown['blue']['auto_points'], 14)

        # Test unplayed match
        match = matches[11]
        self.assertEqual(match.comp_level, "qm")
        self.assertEqual(match.set_number, 1)
        self.assertEqual(match.match_number, 12)
        self.assertEqual(match.team_key_names, [u'frc3663', u'frc5295', u'frc2907', u'frc2046', u'frc3218', u'frc2412'])
        self.assertEqual(match.alliances_json, """{"blue": {"score": null, "teams": ["frc3663", "frc5295", "frc2907"]}, "red": {"score": null, "teams": ["frc2046", "frc3218", "frc2412"]}}""")
        self.assertEqual(match.time, datetime.datetime(2015, 2, 27, 2, 17))
        self.assertEqual(match.actual_time, None)
        self.assertEqual(match.score_breakdown['red']['foul_points'], None)
        self.assertEqual(match.score_breakdown['red']['auto_points'], None)
        self.assertEqual(match.score_breakdown['blue']['foul_points'], None)
        self.assertEqual(match.score_breakdown['blue']['auto_points'], None)

    def test_parseEventAlliances(self):
        with open('test_data/fms_api/2015waamv_staging_alliances.json', 'r') as f:
            alliances = FMSAPIEventAlliancesParser().parse(json.loads(f.read()))

        self.assertEqual(
            alliances,
            [{'declines': [], 'picks': ['frc1', 'frc2', 'frc3']},
            {'declines': [], 'picks': ['frc5', 'frc6', 'frc7', 'frc8']},
            {'declines': [], 'picks': ['frc9', 'frc10', 'frc11', 'frc12']},
            {'declines': [], 'picks': ['frc13', 'frc14', 'frc15', 'frc16' ]},
            {'declines': [], 'picks': ['frc17', 'frc18', 'frc19', 'frc20']},
            {'declines': [], 'picks': ['frc21', 'frc22', 'frc23', 'frc24']},
            {'declines': [], 'picks': ['frc25', 'frc26', 'frc27', 'frc28']},
            {'declines': [], 'picks': ['frc29', 'frc30', 'frc31', 'frc31']}])

    def test_parseEventRankings(self):
        with open('test_data/fms_api/2015waamv_staging_rankings.json', 'r') as f:
            rankings = FMSAPIEventRankingsParser(2015).parse(json.loads(f.read()))

        self.assertEqual(
            rankings,
            [['Rank', 'Team', 'Qual Avg', 'Auto', 'Container', 'Coopertition', 'Litter', 'Tote', 'Played'],
            [1, 2906, 76, 6, 48, 80, 6, 12, 2],
            [2, 4726, 74, 6, 48, 80, 6, 8, 2],
            [3, 2929, 70, 6, 48, 80, 6, 0, 2],
            [4, 2907, 59, 14, 0, 40, 0, 64, 2],
            [5, 2046, 58, 14, 24, 20, 4, 60, 2],
            [6, 1294, 54, 14, 0, 40, 0, 60, 2],
            [7, 360, 51, 0, 48, 20, 12, 22, 2],
            [8, 3218, 50, 18, 12, 40, 4, 26, 2],
            [9, 3237, 48, 18, 12, 0, 4, 14, 1],
            [10, 3781, 46, 46, 20, 20, 6, 0, 1],
            [11, 3393, 46, 46, 20, 20, 6, 0, 1],
            [12, 1983, 46, 14, 0, 0, 4, 80, 2],
            [13, 4579, 44, 0, 24, 60, 4, 0, 2],
            [14, 3220, 44, 18, 12, 40, 4, 14, 2],
            [15, 3049, 44, 0, 0, 40, 4, 0, 1],
            [16, 5295, 43, 22, 0, 0, 0, 70, 2],
            [17, 4131, 43, 14, 28, 0, 6, 38, 2],
            [18, 4911, 38, 0, 0, 40, 0, 36, 2],
            [19, 2605, 36, 0, 28, 0, 10, 34, 2],
            [20, 3586, 34, 20, 0, 0, 0, 60, 2],
            [21, 2976, 33, 8, 0, 40, 0, 18, 2],
            [22, 3684, 29, 8, 0, 40, 0, 10, 2],
            [23, 3223, 28, 0, 0, 40, 4, 12, 2],
            [24, 3588, 27, 0, 0, 0, 4, 56, 2],
            [25, 4450, 24, 0, 24, 20, 4, 0, 2],
            [26, 2927, 22, 0, 0, 40, 4, 0, 2],
            [27, 2412, 22, 0, 0, 40, 4, 0, 2],
            [28, 3221, 21, 0, 0, 0, 8, 40, 2],
            [29, 3663, 15, 14, 0, 0, 0, 16, 2],
            [30, 4469, 15, 14, 0, 0, 0, 16, 2],
            [31, 2557, 15, 6, 0, 0,  0, 36, 2],
            [32, 1318, 1, 6, 0, 0, 0, 8, 2]])
<commit_msg>Remove irrelevant tests<commit_after>import datetime
import json
import unittest2

from consts.event_type import EventType
from consts.district_type import DistrictType

from datafeeds.parsers.fms_api.fms_api_awards_parser import FMSAPIAwardsParser
from datafeeds.parsers.fms_api.fms_api_event_alliances_parser import FMSAPIEventAlliancesParser
from datafeeds.parsers.fms_api.fms_api_event_rankings_parser import FMSAPIEventRankingsParser
from datafeeds.parsers.fms_api.fms_api_match_parser import FMSAPIHybridScheduleParser

from google.appengine.ext import testbed

from models.event import Event

class TestFMSAPIEventParser(unittest2.TestCase):
    def setUp(self):
        self.testbed = testbed.Testbed()
        self.testbed.activate()
        self.testbed.init_datastore_v3_stub()
        self.testbed.init_memcache_stub()

        self.event = Event(
            id="2015waamv",
            end_date=datetime.datetime(2015, 4, 2, 0, 0),
            event_short="waamv",
            event_type_enum=EventType.REGIONAL,
            event_district_enum=DistrictType.NO_DISTRICT,
            first_eid="13467",
            name="PNW District - Auburn Mountainview Event",
            start_date=datetime.datetime(2015, 3, 31, 0, 0),
            year=2015,
            timezone_id='America/Los_Angeles'
        )
        self.event.put()

    def tearDown(self):
        self.testbed.deactivate()

    def test_parseAwards(self):
        with open('test_data/fms_api/2015waamv_staging_awards.json', 'r') as f:
            awards = FMSAPIAwardsParser(self.event).parse(json.loads(f.read()))

        self.assertEqual(len(awards), 5)

        for award in awards:
            if award.key.id() == '2015waamv_3':
                self.assertEqual(award.name_str, 'Woodie Flowers Award')
                self.assertEqual(award.award_type_enum, 3)
                self.assertTrue({'team_number': None, 'awardee': 'Bob'} in award.recipient_list)
            elif award.key.id() == '2015waamv_17':
                self.assertEqual(award.name_str, 'Quality Award sponsored by Motorola')
                self.assertEqual(award.award_type_enum, 17)
                self.assertTrue({'team_number': 1318, 'awardee': None} in award.recipient_list)
            elif award.key.id() == '2015waamv_4':
                self.assertEqual(award.name_str, 'FIRST Dean\'s List Award')
                self.assertEqual(award.award_type_enum, 4)
                self.assertTrue({'team_number': 123, 'awardee': 'Person Name 1'} in award.recipient_list)
                self.assertTrue({'team_number': 321, 'awardee': 'Person Name 2'} in award.recipient_list)

    def test_parseMatches(self):
        with open('test_data/fms_api/2015waamv_staging_matches.json', 'r') as f:
            matches = FMSAPIHybridScheduleParser(2015, 'waamv').parse(json.loads(f.read()))

        self.assertEqual(len(matches), 64)

        matches = sorted(matches, key=lambda m: m.play_order)

        # Test played match
        match = matches[0]
        self.assertEqual(match.comp_level, "qm")
        self.assertEqual(match.set_number, 1)
        self.assertEqual(match.match_number, 1)
        self.assertEqual(match.team_key_names, [u'frc4131', u'frc4469', u'frc3663', u'frc3684', u'frc5295', u'frc2976'])
        self.assertEqual(match.alliances_json, """{"blue": {"score": 30, "teams": ["frc4131", "frc4469", "frc3663"]}, "red": {"score": 18, "teams": ["frc3684", "frc5295", "frc2976"]}}""")
        self.assertEqual(match.time, datetime.datetime(2015, 2, 27, 0, 0))
        self.assertEqual(match.actual_time, datetime.datetime(2015, 2, 27, 0, 0))

        # Test unplayed match
        match = matches[11]
        self.assertEqual(match.comp_level, "qm")
        self.assertEqual(match.set_number, 1)
        self.assertEqual(match.match_number, 12)
        self.assertEqual(match.team_key_names, [u'frc3663', u'frc5295', u'frc2907', u'frc2046', u'frc3218', u'frc2412'])
        self.assertEqual(match.alliances_json, """{"blue": {"score": null, "teams": ["frc3663", "frc5295", "frc2907"]}, "red": {"score": null, "teams": ["frc2046", "frc3218", "frc2412"]}}""")
        self.assertEqual(match.time, datetime.datetime(2015, 2, 27, 2, 17))
        self.assertEqual(match.actual_time, None)

    def test_parseEventAlliances(self):
        with open('test_data/fms_api/2015waamv_staging_alliances.json', 'r') as f:
            alliances = FMSAPIEventAlliancesParser().parse(json.loads(f.read()))

        self.assertEqual(
            alliances,
            [{'declines': [], 'picks': ['frc1', 'frc2', 'frc3']},
            {'declines': [], 'picks': ['frc5', 'frc6', 'frc7', 'frc8']},
            {'declines': [], 'picks': ['frc9', 'frc10', 'frc11', 'frc12']},
            {'declines': [], 'picks': ['frc13', 'frc14', 'frc15', 'frc16' ]},
            {'declines': [], 'picks': ['frc17', 'frc18', 'frc19', 'frc20']},
            {'declines': [], 'picks': ['frc21', 'frc22', 'frc23', 'frc24']},
            {'declines': [], 'picks': ['frc25', 'frc26', 'frc27', 'frc28']},
            {'declines': [], 'picks': ['frc29', 'frc30', 'frc31', 'frc31']}])

    def test_parseEventRankings(self):
        with open('test_data/fms_api/2015waamv_staging_rankings.json', 'r') as f:
            rankings = FMSAPIEventRankingsParser(2015).parse(json.loads(f.read()))

        self.assertEqual(
            rankings,
            [['Rank', 'Team', 'Qual Avg', 'Auto', 'Container', 'Coopertition', 'Litter', 'Tote', 'Played'],
            [1, 2906, 76, 6, 48, 80, 6, 12, 2],
            [2, 4726, 74, 6, 48, 80, 6, 8, 2],
            [3, 2929, 70, 6, 48, 80, 6, 0, 2],
            [4, 2907, 59, 14, 0, 40, 0, 64, 2],
            [5, 2046, 58, 14, 24, 20, 4, 60, 2],
            [6, 1294, 54, 14, 0, 40, 0, 60, 2],
            [7, 360, 51, 0, 48, 20, 12, 22, 2],
            [8, 3218, 50, 18, 12, 40, 4, 26, 2],
            [9, 3237, 48, 18, 12, 0, 4, 14, 1],
            [10, 3781, 46, 46, 20, 20, 6, 0, 1],
            [11, 3393, 46, 46, 20, 20, 6, 0, 1],
            [12, 1983, 46, 14, 0, 0, 4, 80, 2],
            [13, 4579, 44, 0, 24, 60, 4, 0, 2],
            [14, 3220, 44, 18, 12, 40, 4, 14, 2],
            [15, 3049, 44, 0, 0, 40, 4, 0, 1],
            [16, 5295, 43, 22, 0, 0, 0, 70, 2],
            [17, 4131, 43, 14, 28, 0, 6, 38, 2],
            [18, 4911, 38, 0, 0, 40, 0, 36, 2],
            [19, 2605, 36, 0, 28, 0, 10, 34, 2],
            [20, 3586, 34, 20, 0, 0, 0, 60, 2],
            [21, 2976, 33, 8, 0, 40, 0, 18, 2],
            [22, 3684, 29, 8, 0, 40, 0, 10, 2],
            [23, 3223, 28, 0, 0, 40, 4, 12, 2],
            [24, 3588, 27, 0, 0, 0, 4, 56, 2],
            [25, 4450, 24, 0, 24, 20, 4, 0, 2],
            [26, 2927, 22, 0, 0, 40, 4, 0, 2],
            [27, 2412, 22, 0, 0, 40, 4, 0, 2],
            [28, 3221, 21, 0, 0, 0, 8, 40, 2],
            [29, 3663, 15, 14, 0, 0, 0, 16, 2],
            [30, 4469, 15, 14, 0, 0, 0, 16, 2],
            [31, 2557, 15, 6, 0, 0,  0, 36, 2],
            [32, 1318, 1, 6, 0, 0, 0, 8, 2]])
<|endoftext|>


<commit_before>"""DIV2K dataset: DIVerse 2K resolution high quality images as used for the challenges @ NTIRE (CVPR 2017 and CVPR 2018) and @ PIRM (ECCV 2018)"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os.path
import re

import tensorflow.compat.v2 as tf
import tensorflow_datasets.public_api as tfds

_CITATION = """@InProceedings{Ignatov_2018_ECCV_Workshops,
author = {Ignatov, Andrey and Timofte, Radu and others},
title = {PIRM challenge on perceptual image enhancement on smartphones: report},
booktitle = {European Conference on Computer Vision (ECCV) Workshops},
url = "http://www.vision.ee.ethz.ch/~timofter/publications/Agustsson-CVPRW-2017.pdf",
month = {January},
year = {2019}
}
"""

_DESCRIPTION = """
DIV2K dataset: DIVerse 2K resolution high quality images as used for the challenges @ NTIRE (CVPR 2017 and CVPR 2018) and @ PIRM (ECCV 2018)
"""

_DL_URL = "https://data.vision.ee.ethz.ch/cvl/DIV2K/"

_DL_URLS = {
    "train_hr": _DL_URL + "DIV2K_train_HR.zip",
    "valid_hr": _DL_URL + "DIV2K_valid_HR.zip",
    "train_bicubic_x2": _DL_URL + "DIV2K_train_LR_bicubic_X2.zip",
    "train_unknown_x2": _DL_URL + "DIV2K_train_LR_unknown_X2.zip",
    "valid_bicubic_x2": _DL_URL + "DIV2K_valid_LR_bicubic_X2.zip",
    "valid_unknown_x2": _DL_URL + "DIV2K_valid_LR_unknown_X2.zip",
    "train_bicubic_x3": _DL_URL + "DIV2K_train_LR_bicubic_X3.zip",
    "train_unknown_x3": _DL_URL + "DIV2K_train_LR_unknown_X3.zip",
    "valid_bicubic_x3": _DL_URL + "DIV2K_valid_LR_bicubic_X3.zip",
    "valid_unknown_x3": _DL_URL + "DIV2K_valid_LR_unknown_X3.zip",
    "train_bicubic_x4": _DL_URL + "DIV2K_train_LR_bicubic_X4.zip",
    "train_unknown_x4": _DL_URL + "DIV2K_train_LR_unknown_X4.zip",
    "valid_bicubic_x4": _DL_URL + "DIV2K_valid_LR_bicubic_X4.zip",
    "valid_unknown_x4": _DL_URL + "DIV2K_valid_LR_unknown_X4.zip",
    "train_bicubic_x8": _DL_URL + "DIV2K_train_LR_x8.zip",
    "valid_bicubic_x8": _DL_URL + "DIV2K_valid_LR_x8.zip",
    "train_realistic_mild_x4": _DL_URL + "DIV2K_train_LR_mild.zip",
    "valid_realistic_mild_x4": _DL_URL + "DIV2K_valid_LR_mild.zip",
    "train_realistic_difficult_x4": _DL_URL + "DIV2K_train_LR_difficult.zip",
    "valid_realistic_difficult_x4": _DL_URL + "DIV2K_valid_LR_difficult.zip",
    "train_realistic_wild_x4": _DL_URL + "DIV2K_train_LR_wild.zip",
    "valid_realistic_wild_x4": _DL_URL + "DIV2K_valid_LR_wild.zip",
}

_DATA_OPTIONS = ["bicubic_x2", "bicubic_x3", "bicubic_x4", "bicubic_x8",
                 "unknown_x2", "unknown_x3", "unknown_x4",
                 "realistic_mild_x4", "realistic_difficult_x4",
                 "realistic_wild_x4"]

class Div2kConfig(tfds.core.BuilderConfig):
  """BuilderConfig for Div2k."""

  def __init__(self, name, **kwargs):
    """Constructs a Div2kConfig."""
    if name not in _DATA_OPTIONS:
      raise ValueError("data must be one of %s" % _DATA_OPTIONS)

    description = kwargs.get("description", "Uses %s data." % name)
    kwargs["description"] = description

    super(Div2kConfig, self).__init__(name=name, **kwargs)
    self.data = name
    self.download_urls = {
        "train_lr_url": _DL_URLS["train_"+self.data],
        "valid_lr_url": _DL_URLS["valid_"+self.data],
        "train_hr_url": _DL_URLS["train_hr"],
        "valid_hr_url": _DL_URLS["valid_hr"],
    }

def _make_builder_configs():
  configs = []
  for data in _DATA_OPTIONS:
    configs.append(Div2kConfig(
        version=tfds.core.Version("2.0.0"),
        name=data))
  return configs

class Div2k(tfds.core.GeneratorBasedBuilder):
  """DIV2K dataset: DIVerse 2K resolution high quality images"""

  BUILDER_CONFIGS = _make_builder_configs()

  def _info(self):
    return tfds.core.DatasetInfo(
        builder=self,
        description=_DESCRIPTION,
        features=tfds.features.FeaturesDict({
            "lr": tfds.features.Image(),
            "hr": tfds.features.Image(),
        }),
        homepage=_DL_URL,
        citation=_CITATION,
    )

  def _split_generators(self, dl_manager):
    """Returns SplitGenerators."""
    extracted_paths = dl_manager.download_and_extract(
        self.builder_config.download_urls)

    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            gen_kwargs={
                "lr_path": extracted_paths["train_lr_url"],
                "hr_path": extracted_paths["train_hr_url"],
            }
        ),
        tfds.core.SplitGenerator(
            name=tfds.Split.VALIDATION,
            gen_kwargs={
                "lr_path": extracted_paths["valid_lr_url"],
                "hr_path": extracted_paths["valid_hr_url"],
            }
        ),
    ]

  def _generate_examples(self, lr_path, hr_path):
    """Yields examples."""
    if not tf.io.gfile.listdir(hr_path)[0].endswith(".png"):
      hr_path = os.path.join(hr_path, tf.io.gfile.listdir(hr_path)[0])

    for root, _, files in tf.io.gfile.walk(lr_path):
      if len(files):
        for file_path in files:
          yield root + file_path, {
              "lr": os.path.join(root, file_path),
              #extract for corresponding file with matching 4 digit id
              "hr": os.path.join(hr_path,
                                 re.search(r'\d{4}',
                                           str(file_path)).group(0) + ".png")
          }
<commit_msg>address requested changes<commit_after>"""DIV2K dataset: DIVerse 2K resolution high quality images as used for the challenges @ NTIRE (CVPR 2017 and CVPR 2018) and @ PIRM (ECCV 2018)"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os.path
import re

import tensorflow.compat.v2 as tf
import tensorflow_datasets.public_api as tfds

_CITATION = """@InProceedings{Ignatov_2018_ECCV_Workshops,
author = {Ignatov, Andrey and Timofte, Radu and others},
title = {PIRM challenge on perceptual image enhancement on smartphones: report},
booktitle = {European Conference on Computer Vision (ECCV) Workshops},
url = "http://www.vision.ee.ethz.ch/~timofter/publications/Agustsson-CVPRW-2017.pdf",
month = {January},
year = {2019}
}
"""

_DESCRIPTION = """
DIV2K dataset: DIVerse 2K resolution high quality images as used for the challenges @ NTIRE (CVPR 2017 and CVPR 2018) and @ PIRM (ECCV 2018)
"""

_DL_URL = "https://data.vision.ee.ethz.ch/cvl/DIV2K/"

_DL_URLS = {
    "train_hr": _DL_URL + "DIV2K_train_HR.zip",
    "valid_hr": _DL_URL + "DIV2K_valid_HR.zip",
    "train_bicubic_x2": _DL_URL + "DIV2K_train_LR_bicubic_X2.zip",
    "train_unknown_x2": _DL_URL + "DIV2K_train_LR_unknown_X2.zip",
    "valid_bicubic_x2": _DL_URL + "DIV2K_valid_LR_bicubic_X2.zip",
    "valid_unknown_x2": _DL_URL + "DIV2K_valid_LR_unknown_X2.zip",
    "train_bicubic_x3": _DL_URL + "DIV2K_train_LR_bicubic_X3.zip",
    "train_unknown_x3": _DL_URL + "DIV2K_train_LR_unknown_X3.zip",
    "valid_bicubic_x3": _DL_URL + "DIV2K_valid_LR_bicubic_X3.zip",
    "valid_unknown_x3": _DL_URL + "DIV2K_valid_LR_unknown_X3.zip",
    "train_bicubic_x4": _DL_URL + "DIV2K_train_LR_bicubic_X4.zip",
    "train_unknown_x4": _DL_URL + "DIV2K_train_LR_unknown_X4.zip",
    "valid_bicubic_x4": _DL_URL + "DIV2K_valid_LR_bicubic_X4.zip",
    "valid_unknown_x4": _DL_URL + "DIV2K_valid_LR_unknown_X4.zip",
    "train_bicubic_x8": _DL_URL + "DIV2K_train_LR_x8.zip",
    "valid_bicubic_x8": _DL_URL + "DIV2K_valid_LR_x8.zip",
    "train_realistic_mild_x4": _DL_URL + "DIV2K_train_LR_mild.zip",
    "valid_realistic_mild_x4": _DL_URL + "DIV2K_valid_LR_mild.zip",
    "train_realistic_difficult_x4": _DL_URL + "DIV2K_train_LR_difficult.zip",
    "valid_realistic_difficult_x4": _DL_URL + "DIV2K_valid_LR_difficult.zip",
    "train_realistic_wild_x4": _DL_URL + "DIV2K_train_LR_wild.zip",
    "valid_realistic_wild_x4": _DL_URL + "DIV2K_valid_LR_wild.zip",
}

_DATA_OPTIONS = ["bicubic_x2", "bicubic_x3", "bicubic_x4", "bicubic_x8",
                 "unknown_x2", "unknown_x3", "unknown_x4",
                 "realistic_mild_x4", "realistic_difficult_x4",
                 "realistic_wild_x4"]

class Div2kConfig(tfds.core.BuilderConfig):
  """BuilderConfig for Div2k."""

  def __init__(self, name, **kwargs):
    """Constructs a Div2kConfig."""
    if name not in _DATA_OPTIONS:
      raise ValueError("data must be one of %s" % _DATA_OPTIONS)

    description = kwargs.get("description", "Uses %s data." % name)
    kwargs["description"] = description

    super(Div2kConfig, self).__init__(name=name, **kwargs)
    self.data = name
    self.download_urls = {
        "train_lr_url": _DL_URLS["train_"+self.data],
        "valid_lr_url": _DL_URLS["valid_"+self.data],
        "train_hr_url": _DL_URLS["train_hr"],
        "valid_hr_url": _DL_URLS["valid_hr"],
    }

def _make_builder_configs():
  configs = []
  for data in _DATA_OPTIONS:
    configs.append(Div2kConfig(
        version=tfds.core.Version("2.0.0"),
        name=data))
  return configs

class Div2k(tfds.core.GeneratorBasedBuilder):
  """DIV2K dataset: DIVerse 2K resolution high quality images"""

  BUILDER_CONFIGS = _make_builder_configs()

  def _info(self):
    return tfds.core.DatasetInfo(
        builder=self,
        description=_DESCRIPTION,
        features=tfds.features.FeaturesDict({
            "lr": tfds.features.Image(),
            "hr": tfds.features.Image(),
        }),
        supervised_keys=("lr", "hr"),
        homepage=_DL_URL,
        citation=_CITATION,
    )

  def _split_generators(self, dl_manager):
    """Returns SplitGenerators."""
    print("EXTRACTING", self.builder_config.download_urls)
    extracted_paths = dl_manager.download_and_extract(
        self.builder_config.download_urls)

    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            gen_kwargs={
                "lr_path": extracted_paths["train_lr_url"],
                "hr_path": extracted_paths["train_hr_url"],
            }
        ),
        tfds.core.SplitGenerator(
            name=tfds.Split.VALIDATION,
            gen_kwargs={
                "lr_path": extracted_paths["valid_lr_url"],
                "hr_path": extracted_paths["valid_hr_url"],
            }
        ),
    ]

  def _generate_examples(self, lr_path, hr_path):
    """Yields examples."""
    for root, _, files in tf.io.gfile.walk(lr_path):
      for file_path in files:
        yield file_path, {
            "lr": os.path.join(root, file_path),
            #Extract the image id from the filename: "0001x2.png"
            "hr": os.path.join(hr_path, file_path[:4]+".png")
        }
<|endoftext|>


<commit_before>""" This class represents a Queue Node to store values and also
    links others Nodes with values."""
class Node:

    """ It starts with a value at all times. A note can not be
        created without a value associated. """
    def __init__(self, value):
        self.value = value
        self.next = None

""" This class represents a Queue to store values. The Queue starts
    with a node called head. Every single element is going to be added
    after the last node entered."""
class Queue:

    """ The Queue is created with it's size zero and the head element
        head is None (undefined)."""
    def __init__(self):
        self.head = None
        self.size = 0

    """ It adds a new value. The value is going to be added always after the
        last value added. If the Queue has no elements, the value added is
        going to be the head/head and also the last/tail value."""
    def enqueue(self, value):
        if (self.head is None):
            self.head = Node(value)
            self.size += 1
        else:
            pointer = self.head
            while(pointer.next is not None):
                pointer = pointer.next
            pointer.next = Node(value)

    """ This routine removes and also returns the first element. After the
        remotion of the element, the head is updated and it turns to be the next
        element of the queue (it's next element). If there are no more elements
        other than the head, the Queue turns to be empty. If there are no elements
        at all, there will be no remotion or return."""
    def dequeue(self):
        if (self.head is not None):
            removed = self.head.value
            self.head = self.head.next
            self.size -= 1
            return removed

    """ It shows all the Queue elements one by one in a correct
        order. """
    def display(self):
        pointer = self.head
        while (pointer is not None):
            print pointer.value
            pointer = pointer.next

    """ It returns the head node value, but it doesn't remove the
        node. """
    def head(self):
        return self.head.value

    """ It verifies whether or not the Queue has elements. If the Queue
        doesn't have any elements, the head or head element is going to
        be None. """
    def is_empty(self):
        return self.head is None
<commit_msg>ADT Queue and it's Node structure<commit_after>""" This class represents a Queue Node to store values and also
    links others Nodes with values."""
class Node:

    """ It starts with a value at all times. A note can not be
        created without a value associated. """
    def __init__(self, value):
        self.value = value
        self.next = None

""" This class represents a Queue to store values. The Queue starts
    with a node called head. Every single element is going to be added
    after the last node entered."""
class Queue:

    """ The Queue is created with it's size zero and the head element
        head is None (undefined)."""
    def __init__(self):
        self.head = None
        self.size = 0

    """ It adds a new value. The value is going to be added always after the
        last value added. If the Queue has no elements, the value added is
        going to be the head/head and also the last/tail value."""
    def enqueue(self, value):
        if (self.head is None):
            self.head = Node(value)
            self.size += 1
        else:
            pointer = self.head
            while(pointer.next is not None):
                pointer = pointer.next
            pointer.next = Node(value)

    """ This routine removes and also returns the first element. After the
        remotion of the element, the head is updated and it turns to be the next
        element of the queue (it's next element). If there are no more elements
        other than the head, the Queue turns to be empty. If there are no elements
        at all, there will be no remotion or return."""
    def dequeue(self):
        if (self.head is not None):
            removed = self.head.value
            self.head = self.head.next
            self.size -= 1
            return removed

    """ It shows all the Queue elements one by one in a correct
        order. """
    def display(self):
        pointer = self.head
        while (pointer is not None):
            print pointer.value
            pointer = pointer.next

    """ It returns the head node value, but it doesn't remove the
        node. """
    def head(self):
        return self.head.value

    """ It verifies whether or not the Queue has elements. If the Queue
        doesn't have any elements, the head or head element is going to
        be None. """
    def is_empty(self):
        return self.head is None

<|endoftext|>


<commit_before>def matchParenthesis(string):
    parenthesis = 0
    brackets = 0
    curlyBrackets = 0
    for i in range(len(string)):
        l = string[i]
        if l == '(':
            parenthesis += 1
        elif l == '[':
            brackets += 1
        elif l == '{':
            curlyBrackets += 1
        elif l == ')':
            if i > 0:
                if not (string[i-1] == ':' or string[i-1] == ';'):
                    if i > 1:
                        if not string[i-2] == '-':
                            parenthesis -= 1
        elif l == '[':
            brackets -= 1
        elif l == '{':
            curlyBrackets -= 1
    fix = ""
    for i in range(parenthesis):
        fix += ")"
    for i in range(brackets):
        fix += "]"
    for i in range(curlyBrackets):
        fix += "}"

    print(parenthesis)
    print(brackets)
    print(curlyBrackets)

    return fix

print(matchParenthesis("hei[{]{{[}((()() :)"))
<commit_msg>Remove debug prints<commit_after>def matchParenthesis(string):
    parenthesis = 0
    brackets = 0
    curlyBrackets = 0
    for i in range(len(string)):
        l = string[i]
        if l == '(':
            parenthesis += 1
        elif l == '[':
            brackets += 1
        elif l == '{':
            curlyBrackets += 1
        elif l == ')':
            if i > 0:
                if not (string[i-1] == ':' or string[i-1] == ';'):
                    if i > 1:
                        if not string[i-2] == '-':
                            parenthesis -= 1
        elif l == '[':
            brackets -= 1
        elif l == '{':
            curlyBrackets -= 1
    fix = ""
    for i in range(parenthesis):
        fix += ")"
    for i in range(brackets):
        fix += "]"
    for i in range(curlyBrackets):
        fix += "}"

    return fix
<|endoftext|>


<commit_before>import hashlib
import hmac
import json
import math
import os
import random
import time
from urlparse import urlparse

import mohawk
from requests.auth import AuthBase

from loads.case import TestCase

OMXEN_URL = os.getenv("OMXEN_URL", "http://omxen.dev.mozaws.net")

print "USING %s OMXEN endpoint" % OMXEN_URL

PERCENTAGE_OF_MT_FLOW = 50  # Remining are MOMT flows
PERCENTAGE_OF_WRONG_CODES = 34  # Remining are valid ones.
PERCENTAGE_OF_SHORT_CODES = 50  # Remining are right ones.
MAX_OMXEN_TIMEOUT = 2  # Seconds to poll from omxen.

PUBLIC_KEY = '{"algorithm":"DS","y":"2bb69e89652d97315f156852c5d1a36c7ebf0fb991a3f812e55c804ca8fa8a09e6f1880de99fcaf4f5417a78a88591679219acda438f9ef4d5afb9d0ae20216414822c6187ec4f180eae6f1f2d2a0f55095e24141c8a5f12344ce4d6319042c168c962de1029a9ac1a528da2d27e4f9550b7c203572a034f077480d9f6e3ef3968902f7547a7374c5e7475dcc81002af3308d09ace22124719a83451cbd872403d6f72cf0b13b5ae0c092a3bf0daf8a26df7d618bc731306ccfbe431c9a30206327bedd0adac35dd5623d017c07aa7fbb991c9364335278b80f1a979d5141fb6cf6cc92849bc6ca5e3c24d25e75f828c0efd57214e37be92d32f2c2208527f20","p":"d6c4e5045697756c7a312d02c2289c25d40f9954261f7b5876214b6df109c738b76226b199bb7e33f8fc7ac1dcc316e1e7c78973951bfc6ff2e00cc987cd76fcfb0b8c0096b0b460fffac960ca4136c28f4bfb580de47cf7e7934c3985e3b3d943b77f06ef2af3ac3494fc3c6fc49810a63853862a02bb1c824a01b7fc688e4028527a58ad58c9d512922660db5d505bc263af293bc93bcd6d885a157579d7f52952236dd9d06a4fc3bc2247d21f1a70f5848eb0176513537c983f5a36737f01f82b44546e8e7f0fabc457e3de1d9c5dba96965b10a2a0580b0ad0f88179e10066107fb74314a07e6745863bc797b7002ebec0b000a98eb697414709ac17b401","q":"b1e370f6472c8754ccd75e99666ec8ef1fd748b748bbbc08503d82ce8055ab3b","g":"9a8269ab2e3b733a5242179d8f8ddb17ff93297d9eab00376db211a22b19c854dfa80166df2132cbc51fb224b0904abb22da2c7b7850f782124cb575b116f41ea7c4fc75b1d77525204cd7c23a15999004c23cdeb72359ee74e886a1dde7855ae05fe847447d0a68059002c3819a75dc7dcbb30e39efac36e07e2c404b7ca98b263b25fa314ba93c0625718bd489cea6d04ba4b0b7f156eeb4c56c44b50e4fb5bce9d7ae0d55b379225feb0214a04bed72f33e0664d290e7c840df3e2abb5e48189fa4e90646f1867db289c6560476799f7be8420a6dc01d078de437f280fff2d7ddf1248d56e1a54b933a41629d6c252983c58795105802d30d7bcd819cf6ef"}'  # NOQA

class TestMSISDN(TestCase):
    omxen_url = OMXEN_URL

    def test_all(self):
        # Discover
        self.discover()

        # Create a token
        self.register()

        # Use the MO flow 50% of the time and the MT flow the remaining
        if random.randint(0, 100) < PERCENTAGE_OF_MT_FLOW:
            # 1. Ask MSISDN validation
            resp = self.start_mt_flow()
            self.assertEqual(resp.status_code, 200,
                             "Start MT Flow failed: %s" % resp.content)
        else:
            # 2. Send SMS /sms/momt/verify hawkId
            resp = self.start_momt_flow()
            self.assertEqual(resp.status_code, 200,
                             "Start MOMT Flow failed: %s" % resp.content)

        # Poll omxen for the message
        message = self.read_message()

        # Get the message code
        if random.randint(0, 100) > PERCENTAGE_OF_WRONG_CODES:
            # 1. Try to validate a valid code
            self.incr_counter("try-right-code")
            resp = self.verify_code(message)
            if resp.status_code == 200:
                # If it was a valid code generate a certificate
                self.incr_counter("ask-for-certificate")
                self.sign_certificate()
            else:
                # If we didn't validate the code from the omxen
                # message it is probably because two test where using
                # the same MSISDN at the same time
                self.assertEquals(resp.status_code, 400,
                                  "Omxen collision failed: %s" % resp.content)

                self.incr_counter("omxen-message-collision")
        else:
            # 2. Try to validate a wrong code
            self.incr_counter("try-wrong-code")
            resp = self.verify_code()
            self.assertEquals(resp.status_code, 400,
                              "Try wrong code failed: %s" % resp.content)

        # Unregister
        self.unregister()

    def __get_random_msisdn(self):
        code = "%d" % random.randint(0, 999999999)
        code = code.zfill(9)
        return "+33%s" % code

    def discover(self):
        resp = self.session.post(
            self.server_url + '/discover',
            data=json.dumps({"mcc": "204"}),
            headers={'Content-type': 'application/json'}
        )
        self.assertEquals(resp.status_code, 200,
                          "Discover endpoint failed: %s" % resp.content)

    def register(self):
        resp = self.session.post(self.server_url + '/register')
        self.incr_counter("register")
        try:
            sessionToken = resp.json()['msisdnSessionToken']
        except ValueError:
            self.fail("No JSON has been returned: %s" % resp.content)
        except KeyError:
            self.fail("msisdnSessionToken not found in response: %s" %
                      resp.content)

        self.hawk_auth = HawkAuth(self.server_url, sessionToken)

    def start_mt_flow(self):
        self.msisdn = self.__get_random_msisdn()
        self.shortVerificationCode = \
            random.randint(0, 100) < PERCENTAGE_OF_SHORT_CODES

        self.incr_counter("mt-flow")

        return self.session.post(
            self.server_url + '/sms/mt/verify',
            data=json.dumps({
                "msisdn": self.msisdn,
                "shortVerification": self.shortVerificationCode
            }),
            headers={'Content-type': 'application/json'},
            auth=self.hawk_auth)

    def start_momt_flow(self):
        self.msisdn = self.__get_random_msisdn()
        self.shortVerificationCode = False

        self.incr_counter("momt-flow")

        # You don't need to be authenticated to revoke a token.
        return self.session.get(
            self.server_url + '/sms/momt/nexmo_callback',
            params={"msisdn": self.msisdn.lstrip("+"),
                    "text": "/sms/momt/verify %s" % self.hawk_auth.hawk_id})

    def read_message(self):
        resp = self.session.get(self.omxen_url + '/receive',
                                params={"to": self.msisdn.lstrip("+")})
        try:
            messages = resp.json()
            self.assertIsInstance(messages, list,
                                  "Wrong JSON from OMXEN: %s" % messages)
        except ValueError:
            print resp.content
            raise

        start_time = time.time()

        #  Poll on the omxen message list for this number
        while len(messages) < 1 and \
                time.time() - start_time < MAX_OMXEN_TIMEOUT:
            time.sleep(1)
            resp = self.session.get(self.omxen_url + '/receive',
                                    params={"to": self.msisdn.lstrip("+")})
            try:
                messages = resp.json()
                self.assertIsInstance(messages, list,
                                      "Wrong JSON from OMXEN: %s" % messages)
            except ValueError:
                print resp.content
                raise

        if len(messages) >= 1:
            return messages[0]["text"]

    def verify_code(self, message=None):
        # Extract the code from the message
        if message is None:
            # No message, build a fake code
            code = "%d" % random.randint(0, 999999)
            code = code.zfill(6)
        else:
            if len(message) == 64:
                # Long Verification code
                # The message is the code
                code = message
            else:
                # Short verification code, the code is the last word.
                code = message.split()[-1]
        return self.session.post(self.server_url + "/sms/verify_code",
                                 data=json.dumps({"code": code}),
                                 headers={'Content-type': 'application/json'},
                                 auth=self.hawk_auth)

    def sign_certificate(self):
        data = {
            'duration': 10,
            'publicKey': PUBLIC_KEY
        }
        resp = self.session.post(self.server_url + '/certificate/sign',
                                 data=json.dumps(data),
                                 headers={'Content-type': 'application/json'},
                                 auth=self.hawk_auth)
        self.assertEqual(resp.status_code, 200,
                         "Sign certificate failed: %s" % resp.content)

    def unregister(self):
        resp = self.session.post(self.server_url + '/unregister',
                                 headers={'Content-type': 'application/json'},
                                 auth=self.hawk_auth)
        self.incr_counter("unregister")
        self.assertEqual(resp.status_code, 200,
                         "Unregister failed: %s" % resp.content)

def HKDF_extract(salt, IKM, hashmod=hashlib.sha256):
    """HKDF-Extract; see RFC-5869 for the details."""
    if salt is None:
        salt = b"\x00" * hashmod().digest_size
    return hmac.new(salt, IKM, hashmod).digest()

def HKDF_expand(PRK, info, L, hashmod=hashlib.sha256):
    """HKDF-Expand; see RFC-5869 for the details."""
    digest_size = hashmod().digest_size
    N = int(math.ceil(L * 1.0 / digest_size))
    assert N <= 255
    T = b""
    output = []
    for i in xrange(1, N + 1):
        data = T + info + chr(i)
        T = hmac.new(PRK, data, hashmod).digest()
        output.append(T)
    return b"".join(output)[:L]

def HKDF(secret, salt, info, size, hashmod=hashlib.sha256):
    """HKDF-extract-and-expand as a single function."""
    PRK = HKDF_extract(salt, secret, hashmod)
    return HKDF_expand(PRK, info, size, hashmod)

class HawkAuth(AuthBase):
    def __init__(self, server_url, tokendata):
        hawk_session = tokendata.decode('hex')
        self.server_url = server_url
        keyInfo = 'identity.mozilla.com/picl/v1/sessionToken'
        keyMaterial = HKDF(hawk_session, "", keyInfo, 32*3)
        self.hawk_id = keyMaterial[:32].encode("hex")
        self.auth_key = keyMaterial[32:64].encode("hex")
        self.credentials = {
            'id': self.hawk_id,
            'key': self.auth_key,
            'algorithm': 'sha256'
        }

    def __call__(self, r):
        r.headers['Host'] = urlparse(self.server_url).netloc
        sender = mohawk.Sender(
            self.credentials,
            r.url,
            r.method,
            content=r.body or '',
            content_type=r.headers.get('Content-Type', '')
        )

        r.headers['Authorization'] = sender.request_header
        return r
<commit_msg>make it 2.6 compliant<commit_after>import hashlib
import hmac
import json
import math
import os
import random
import time
from urlparse import urlparse

import mohawk
from requests.auth import AuthBase

from loads.case import TestCase

OMXEN_URL = os.getenv("OMXEN_URL", "http://omxen.dev.mozaws.net")

print "USING %s OMXEN endpoint" % OMXEN_URL

PERCENTAGE_OF_MT_FLOW = 50  # Remining are MOMT flows
PERCENTAGE_OF_WRONG_CODES = 34  # Remining are valid ones.
PERCENTAGE_OF_SHORT_CODES = 50  # Remining are right ones.
MAX_OMXEN_TIMEOUT = 2  # Seconds to poll from omxen.

PUBLIC_KEY = '{"algorithm":"DS","y":"2bb69e89652d97315f156852c5d1a36c7ebf0fb991a3f812e55c804ca8fa8a09e6f1880de99fcaf4f5417a78a88591679219acda438f9ef4d5afb9d0ae20216414822c6187ec4f180eae6f1f2d2a0f55095e24141c8a5f12344ce4d6319042c168c962de1029a9ac1a528da2d27e4f9550b7c203572a034f077480d9f6e3ef3968902f7547a7374c5e7475dcc81002af3308d09ace22124719a83451cbd872403d6f72cf0b13b5ae0c092a3bf0daf8a26df7d618bc731306ccfbe431c9a30206327bedd0adac35dd5623d017c07aa7fbb991c9364335278b80f1a979d5141fb6cf6cc92849bc6ca5e3c24d25e75f828c0efd57214e37be92d32f2c2208527f20","p":"d6c4e5045697756c7a312d02c2289c25d40f9954261f7b5876214b6df109c738b76226b199bb7e33f8fc7ac1dcc316e1e7c78973951bfc6ff2e00cc987cd76fcfb0b8c0096b0b460fffac960ca4136c28f4bfb580de47cf7e7934c3985e3b3d943b77f06ef2af3ac3494fc3c6fc49810a63853862a02bb1c824a01b7fc688e4028527a58ad58c9d512922660db5d505bc263af293bc93bcd6d885a157579d7f52952236dd9d06a4fc3bc2247d21f1a70f5848eb0176513537c983f5a36737f01f82b44546e8e7f0fabc457e3de1d9c5dba96965b10a2a0580b0ad0f88179e10066107fb74314a07e6745863bc797b7002ebec0b000a98eb697414709ac17b401","q":"b1e370f6472c8754ccd75e99666ec8ef1fd748b748bbbc08503d82ce8055ab3b","g":"9a8269ab2e3b733a5242179d8f8ddb17ff93297d9eab00376db211a22b19c854dfa80166df2132cbc51fb224b0904abb22da2c7b7850f782124cb575b116f41ea7c4fc75b1d77525204cd7c23a15999004c23cdeb72359ee74e886a1dde7855ae05fe847447d0a68059002c3819a75dc7dcbb30e39efac36e07e2c404b7ca98b263b25fa314ba93c0625718bd489cea6d04ba4b0b7f156eeb4c56c44b50e4fb5bce9d7ae0d55b379225feb0214a04bed72f33e0664d290e7c840df3e2abb5e48189fa4e90646f1867db289c6560476799f7be8420a6dc01d078de437f280fff2d7ddf1248d56e1a54b933a41629d6c252983c58795105802d30d7bcd819cf6ef"}'  # NOQA

class TestMSISDN(TestCase):
    omxen_url = OMXEN_URL

    def test_all(self):
        # Discover
        self.discover()

        # Create a token
        self.register()

        # Use the MO flow 50% of the time and the MT flow the remaining
        if random.randint(0, 100) < PERCENTAGE_OF_MT_FLOW:
            # 1. Ask MSISDN validation
            resp = self.start_mt_flow()
            self.assertEqual(resp.status_code, 200,
                             "Start MT Flow failed: %s" % resp.content)
        else:
            # 2. Send SMS /sms/momt/verify hawkId
            resp = self.start_momt_flow()
            self.assertEqual(resp.status_code, 200,
                             "Start MOMT Flow failed: %s" % resp.content)

        # Poll omxen for the message
        message = self.read_message()

        # Get the message code
        if random.randint(0, 100) > PERCENTAGE_OF_WRONG_CODES:
            # 1. Try to validate a valid code
            self.incr_counter("try-right-code")
            resp = self.verify_code(message)
            if resp.status_code == 200:
                # If it was a valid code generate a certificate
                self.incr_counter("ask-for-certificate")
                self.sign_certificate()
            else:
                # If we didn't validate the code from the omxen
                # message it is probably because two test where using
                # the same MSISDN at the same time
                self.assertEquals(resp.status_code, 400,
                                  "Omxen collision failed: %s" % resp.content)

                self.incr_counter("omxen-message-collision")
        else:
            # 2. Try to validate a wrong code
            self.incr_counter("try-wrong-code")
            resp = self.verify_code()
            self.assertEquals(resp.status_code, 400,
                              "Try wrong code failed: %s" % resp.content)

        # Unregister
        self.unregister()

    def __get_random_msisdn(self):
        code = "%d" % random.randint(0, 999999999)
        code = code.zfill(9)
        return "+33%s" % code

    def discover(self):
        resp = self.session.post(
            self.server_url + '/discover',
            data=json.dumps({"mcc": "204"}),
            headers={'Content-type': 'application/json'}
        )
        self.assertEquals(resp.status_code, 200,
                          "Discover endpoint failed: %s" % resp.content)

    def register(self):
        resp = self.session.post(self.server_url + '/register')
        self.incr_counter("register")
        try:
            sessionToken = resp.json()['msisdnSessionToken']
        except ValueError:
            self.fail("No JSON has been returned: %s" % resp.content)
        except KeyError:
            self.fail("msisdnSessionToken not found in response: %s" %
                      resp.content)

        self.hawk_auth = HawkAuth(self.server_url, sessionToken)

    def start_mt_flow(self):
        self.msisdn = self.__get_random_msisdn()
        self.shortVerificationCode = \
            random.randint(0, 100) < PERCENTAGE_OF_SHORT_CODES

        self.incr_counter("mt-flow")

        return self.session.post(
            self.server_url + '/sms/mt/verify',
            data=json.dumps({
                "msisdn": self.msisdn,
                "shortVerification": self.shortVerificationCode
            }),
            headers={'Content-type': 'application/json'},
            auth=self.hawk_auth)

    def start_momt_flow(self):
        self.msisdn = self.__get_random_msisdn()
        self.shortVerificationCode = False

        self.incr_counter("momt-flow")

        # You don't need to be authenticated to revoke a token.
        return self.session.get(
            self.server_url + '/sms/momt/nexmo_callback',
            params={"msisdn": self.msisdn.lstrip("+"),
                    "text": "/sms/momt/verify %s" % self.hawk_auth.hawk_id})

    def read_message(self):
        resp = self.session.get(self.omxen_url + '/receive',
                                params={"to": self.msisdn.lstrip("+")})
        try:
            messages = resp.json()
            self.assertTrue(isinstance(messages, list),
                            "Wrong JSON from OMXEN: %s" % messages)
        except ValueError:
            print resp.content
            raise

        start_time = time.time()

        #  Poll on the omxen message list for this number
        while len(messages) < 1 and \
                time.time() - start_time < MAX_OMXEN_TIMEOUT:
            time.sleep(1)
            resp = self.session.get(self.omxen_url + '/receive',
                                    params={"to": self.msisdn.lstrip("+")})
            try:
                messages = resp.json()
                self.assertTrue(isinstance(messages, list),
                                "Wrong JSON from OMXEN: %s" % messages)
            except ValueError:
                print resp.content
                raise

        if len(messages) >= 1:
            return messages[0]["text"]

    def verify_code(self, message=None):
        # Extract the code from the message
        if message is None:
            # No message, build a fake code
            code = "%d" % random.randint(0, 999999)
            code = code.zfill(6)
        else:
            if len(message) == 64:
                # Long Verification code
                # The message is the code
                code = message
            else:
                # Short verification code, the code is the last word.
                code = message.split()[-1]
        return self.session.post(self.server_url + "/sms/verify_code",
                                 data=json.dumps({"code": code}),
                                 headers={'Content-type': 'application/json'},
                                 auth=self.hawk_auth)

    def sign_certificate(self):
        data = {
            'duration': 10,
            'publicKey': PUBLIC_KEY
        }
        resp = self.session.post(self.server_url + '/certificate/sign',
                                 data=json.dumps(data),
                                 headers={'Content-type': 'application/json'},
                                 auth=self.hawk_auth)
        self.assertEqual(resp.status_code, 200,
                         "Sign certificate failed: %s" % resp.content)

    def unregister(self):
        resp = self.session.post(self.server_url + '/unregister',
                                 headers={'Content-type': 'application/json'},
                                 auth=self.hawk_auth)
        self.incr_counter("unregister")
        self.assertEqual(resp.status_code, 200,
                         "Unregister failed: %s" % resp.content)

def HKDF_extract(salt, IKM, hashmod=hashlib.sha256):
    """HKDF-Extract; see RFC-5869 for the details."""
    if salt is None:
        salt = b"\x00" * hashmod().digest_size
    return hmac.new(salt, IKM, hashmod).digest()

def HKDF_expand(PRK, info, L, hashmod=hashlib.sha256):
    """HKDF-Expand; see RFC-5869 for the details."""
    digest_size = hashmod().digest_size
    N = int(math.ceil(L * 1.0 / digest_size))
    assert N <= 255
    T = b""
    output = []
    for i in xrange(1, N + 1):
        data = T + info + chr(i)
        T = hmac.new(PRK, data, hashmod).digest()
        output.append(T)
    return b"".join(output)[:L]

def HKDF(secret, salt, info, size, hashmod=hashlib.sha256):
    """HKDF-extract-and-expand as a single function."""
    PRK = HKDF_extract(salt, secret, hashmod)
    return HKDF_expand(PRK, info, size, hashmod)

class HawkAuth(AuthBase):
    def __init__(self, server_url, tokendata):
        hawk_session = tokendata.decode('hex')
        self.server_url = server_url
        keyInfo = 'identity.mozilla.com/picl/v1/sessionToken'
        keyMaterial = HKDF(hawk_session, "", keyInfo, 32*3)
        self.hawk_id = keyMaterial[:32].encode("hex")
        self.auth_key = keyMaterial[32:64].encode("hex")
        self.credentials = {
            'id': self.hawk_id,
            'key': self.auth_key,
            'algorithm': 'sha256'
        }

    def __call__(self, r):
        r.headers['Host'] = urlparse(self.server_url).netloc
        sender = mohawk.Sender(
            self.credentials,
            r.url,
            r.method,
            content=r.body or '',
            content_type=r.headers.get('Content-Type', '')
        )

        r.headers['Authorization'] = sender.request_header
        return r
<|endoftext|>


<commit_before>import re

from collections import defaultdict
from pprint import pprint

from mongoengine import *

from flask import Flask, render_template, redirect, url_for, flash
from flask.ext.mongoengine import MongoEngine
from flask_bootstrap import Bootstrap
from flask_debugtoolbar import DebugToolbarExtension

from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import HtmlFormatter

from docdef import *
import settings

app = Flask(__name__)
app.config['MONGODB_DB'] = settings.MONGODB
app.config['SECRET_KEY'] = "b\xac\xea&\x9d\x86\x98Da?\xcaL\x146\x13\x83\x82.$\x91\xee\x03\xe3\x95"
app.debug = True

app.config['DEBUG_TB_PANELS'] = [
    'flask_debugtoolbar.panels.versions.VersionDebugPanel',
    'flask_debugtoolbar.panels.timer.TimerDebugPanel',
    'flask_debugtoolbar.panels.headers.HeaderDebugPanel',
    'flask_debugtoolbar.panels.request_vars.RequestVarsDebugPanel',
    'flask_debugtoolbar.panels.template.TemplateDebugPanel',
    #'flask_debugtoolbar.panels.sqlalchemy.SQLAlchemyDebugPanel',
    'flask_debugtoolbar.panels.logger.LoggingPanel',
    'flask_debugtoolbar.panels.profiler.ProfilerDebugPanel',
    # Add the MongoDB panel
    'flask_debugtoolbar_mongo.panel.MongoDebugPanel',
]

toolbar = DebugToolbarExtension(app)

db = MongoEngine(app)
Bootstrap(app)

@app.route("/")
def index():
    return render_template('index.html', files=File.objects.exclude("frames", "content").order_by('-timestamp'))

@app.route("/view_files/<files_id>")
def view_file(files_id):

    series = {}
    files_object = File.objects(id__in=re.split('&', files_id))

    '''for file in files_object:
        serie = defaultdict(list)
        for line in file.lines:
            for k,listv in line.data.items():
                for v in listv:
                    if type(v) is (int or float):
                        serie[k].append(v)
        series[file.id] = serie'''

    for file in files_object:
        file.content = highlight(file.content, PythonLexer(), HtmlFormatter(linenos=True))

    return render_template('view_files.html', files=files_object, series=series)

@app.route("/remove_file/<file_id>")
def remove_file(file_id):
    File.objects(id=file_id).delete()
    flash('The entry %s was successfully deleted' % file_id)
    return redirect(url_for('index'))

if __name__ == "__main__":
    app.run()<commit_msg>Correction of data for highcharts<commit_after>import re

from collections import defaultdict
from pprint import pprint

from mongoengine import *

from flask import Flask, render_template, redirect, url_for, flash
from flask.ext.mongoengine import MongoEngine
from flask_bootstrap import Bootstrap
from flask_debugtoolbar import DebugToolbarExtension

from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import HtmlFormatter

from docdef import *
import settings

app = Flask(__name__)
app.config['MONGODB_DB'] = settings.MONGODB
app.config['SECRET_KEY'] = "b\xac\xea&\x9d\x86\x98Da?\xcaL\x146\x13\x83\x82.$\x91\xee\x03\xe3\x95"
app.debug = True

app.config['DEBUG_TB_PANELS'] = [
    'flask_debugtoolbar.panels.versions.VersionDebugPanel',
    'flask_debugtoolbar.panels.timer.TimerDebugPanel',
    'flask_debugtoolbar.panels.headers.HeaderDebugPanel',
    'flask_debugtoolbar.panels.request_vars.RequestVarsDebugPanel',
    'flask_debugtoolbar.panels.template.TemplateDebugPanel',
    #'flask_debugtoolbar.panels.sqlalchemy.SQLAlchemyDebugPanel',
    'flask_debugtoolbar.panels.logger.LoggingPanel',
    'flask_debugtoolbar.panels.profiler.ProfilerDebugPanel',
    # Add the MongoDB panel
    'flask_debugtoolbar_mongo.panel.MongoDebugPanel',
]

toolbar = DebugToolbarExtension(app)

db = MongoEngine(app)
Bootstrap(app)

@app.route("/")
def index():
    return render_template('index.html', files=File.objects.exclude("frames", "content").order_by('-timestamp'))

@app.route("/view_files/<files_id>")
def view_file(files_id):

    series = {}
    files_object = File.objects(id__in=re.split('&', files_id))

    for file in files_object:
        serie = defaultdict(list)
        for frame in file.frames:
            for line in frame.lines:
                for k,listv in line.data.items():
                    for v in listv:
                        if type(v) is (int or float):
                            serie[k].append(v)
        series[file.id] = serie

    for file in files_object:
        file.content = highlight(file.content, PythonLexer(), HtmlFormatter(linenos=True))

    return render_template('view_files.html', files=files_object, series=series)

@app.route("/remove_file/<file_id>")
def remove_file(file_id):
    File.objects(id=file_id).delete()
    flash('The entry %s was successfully deleted' % file_id)
    return redirect(url_for('index'))

if __name__ == "__main__":
    app.run()<|endoftext|>


<commit_before># vim: fileencoding=utf-8 :
"""
Test gabble trying alternative nicknames when the nick you wanted is already in
use in a MUC you try to join.
"""

import dbus

from gabbletest import (
    exec_test, make_muc_presence, sync_stream, elem,
    )
from servicetest import (
    call_async, unwrap, sync_dbus, assertEquals, assertSameSets, wrap_channel,
    EventPattern,
    )
import constants as cs
import ns

def test(q, bus, conn, stream):
    test_join(q, bus, conn, stream, 'chat@conf.localhost', False)
    test_join(q, bus, conn, stream, 'chien@conf.localhost', True)

    test_gtalk_weirdness(q, bus, conn, stream,
        'private-chat-massive-uuid@groupchat.google.com')

def test_join(q, bus, conn, stream, room_jid, transient_conflict):
    """
    Tells Gabble to join a MUC, but make the first nick it tries conflict with
    an existing member of the MUC.  If transient_conflict is True, then when
    Gabble successfully joins with a different nick the originally conflicting
    user turns out not actually to be in the room (they left while we were
    retrying).
    """
    # Implementation detail: Gabble uses the first part of your jid (if you
    # don't have an alias) as your room nickname, and appends an underscore a
    # few times before giving up.
    member, member_ = [room_jid + '/' + x for x in ['test', 'test_']]

    call_async(q, conn.Requests, 'CreateChannel',
        dbus.Dictionary({ cs.CHANNEL_TYPE: cs.CHANNEL_TYPE_TEXT,
                          cs.TARGET_HANDLE_TYPE: cs.HT_ROOM,
                          cs.TARGET_ID: room_jid,
                        }, signature='sv'))

    # Gabble first tries to join as test
    q.expect('stream-presence', to=member)

    # MUC says no: there's already someone called test in room_jid
    presence = elem('presence', from_=member, type='error')(
        elem(ns.MUC, 'x'),
        elem('error', type='cancel')(
          elem(ns.STANZA, 'conflict'),
        ))
    stream.send(presence)

    # Gabble tries again as test_
    q.expect('stream-presence', to=member_)

    # MUC says yes!

    if not transient_conflict:
        # Send the other member of the room's presence. This is the nick we
        # originally wanted.
        stream.send(make_muc_presence('owner', 'moderator', room_jid, 'test'))

    # If gabble erroneously thinks the other user's presence is our own, it'll
    # think that it's got the whole userlist now. If so, syncing here will make
    # CreateChannel incorrectly return here.
    sync_stream(q, stream)
    sync_dbus(bus, q, conn)

    # Send presence for own membership of room.
    stream.send(make_muc_presence('none', 'participant', room_jid, 'test_'))

    # Only now should we have finished joining the room.
    event = q.expect('dbus-return', method='CreateChannel')
    path, props = event.value
    text_chan = wrap_channel(bus.get_object(conn.bus_name, path), 'Text')
    group_props = unwrap(text_chan.Properties.GetAll(cs.CHANNEL_IFACE_GROUP))

    t, t_ = conn.RequestHandles(cs.HT_CONTACT, [member, member_])

    # Check that Gabble think our nickname in the room is test_, not test
    muc_self_handle = group_props['SelfHandle']
    assert muc_self_handle == t_, (muc_self_handle, t_, t)

    members = group_props['Members']

    if transient_conflict:
        # The user we originally conflicted with isn't actually here; check
        # there's exactly one member (test_).
        assert members == [t_], (members, t_, t)
    else:
        # Check there are exactly two members (test and test_)
        assertSameSets([t, t_], members)

    # In either case, there should be no pending members.
    assert len(group_props['LocalPendingMembers']) == 0, group_props
    assert len(group_props['RemotePendingMembers']) == 0, group_props

    # Check that test_'s handle owner is us, and that test (if it's there) has
    # no owner.
    handle_owners = group_props['HandleOwners']
    assertEquals (conn.GetSelfHandle(), handle_owners[t_])
    if not transient_conflict:
        assertEquals (0, handle_owners[t])

    # test that closing the channel results in an unavailable message to the
    # right jid
    text_chan.Close()

    event = q.expect('stream-presence', to=member_)
    assertEquals('unavailable', event.stanza['type'])

def test_gtalk_weirdness(q, bus, conn, stream, room_jid):
    """
    There's a strange bug in the Google Talk MUC server where it sends the
    <conflict/> stanza twice. This has been reported to their server team; but
    in any case it triggered a crazy bug in Gabble, so here's a regression test.
    """

    # Implementation detail: Gabble uses the first part of your jid (if you
    # don't have an alias) as your room nickname, and appends an underscore a
    # few times before giving up.
    jids = ['%s/test%s' % (room_jid, x) for x in ['', '_', '__']]
    member, member_, member__ = jids

    # Gabble should never get as far as trying to join as 'test__' since
    # joining as 'test_' will succeed.
    q.forbid_events([ EventPattern('stream-presence', to=member__) ])

    call_async(q, conn.Requests, 'CreateChannel',
        dbus.Dictionary({ cs.CHANNEL_TYPE: cs.CHANNEL_TYPE_TEXT,
                          cs.TARGET_HANDLE_TYPE: cs.HT_ROOM,
                          cs.TARGET_ID: room_jid,
                        }, signature='sv'))

    # Gabble first tries to join as test
    q.expect('stream-presence', to=member)

    # Google Talk says no from 'test', twice.
    presence = elem('presence', from_=member, type='error')(
        elem(ns.MUC, 'x'),
        elem('error', type='cancel')(
          elem(ns.STANZA, 'conflict'),
        ))
    stream.send(presence)
    stream.send(presence)

    # Gabble should try to join again as test_
    q.expect('stream-presence', to=member_)

    # Since 'test_' is not in use in the MUC, joining should succeed. According
    # to XEP-0045 §7.1.3 <http://xmpp.org/extensions/xep-0045.html#enter-pres>:
    #  The service MUST first send the complete list of the existing occupants
    #  to the new occupant and only then send the new occupant's own presence
    #  to the new occupant
    # but groupchat.google.com cheerfully violates this.
    stream.send(make_muc_presence('none', 'participant', room_jid, 'test_'))

    # Here's some other random person, who owns the MUC.
    stream.send(make_muc_presence('owner', 'moderator', room_jid, 'foobar_gmail.com'))
    # And here's our hypothetical other self.
    stream.send(make_muc_presence('none', 'participant', room_jid, 'test'))

    # The Gabble bug makes this time out: because Gabble thinks it's joining as
    # test__ it ignores the presence for test_, since it's not flagged with
    # code='210' to say “this is you”. (This is acceptable behaviour by the
    # server: it only needs to include code='210' if it's assigned the client a
    # name other than the one it asked for.
    #
    # The forbidden stream-presence event above doesn't blow up here because
    # servicetest doesn't process events on the 'stream-*' queue at all when
    # we're not waiting for one. But during disconnection in the test clean-up,
    # the forbidden event is encountered and correctly flagged up.
    event = q.expect('dbus-return', method='CreateChannel')
    path, _ = event.value
    text_chan = wrap_channel(bus.get_object(conn.bus_name, path), 'Text')

    # As far as Gabble's concerned, the two other participants joined
    # immediately after we did.
    handle, handle_, handle__, foobar_handle = conn.RequestHandles(
        cs.HT_CONTACT, jids + ['%s/foobar_gmail.com' % room_jid])
    q.expect('dbus-signal', signal='MembersChangedDetailed',
        predicate=lambda e: e.args[0:4] == [[foobar_handle], [], [], []])
    q.expect('dbus-signal', signal='MembersChangedDetailed',
        predicate=lambda e: e.args[0:4] == [[handle], [], [], []])

    group_props = text_chan.Properties.GetAll(cs.CHANNEL_IFACE_GROUP)
    assertEquals(handle_, group_props['SelfHandle'])
    assertSameSets([handle, handle_, foobar_handle], group_props['Members'])

if __name__ == '__main__':
    exec_test(test)
<commit_msg>muc/name-conflict.py: clarify RequestHandles call<commit_after># vim: fileencoding=utf-8 :
"""
Test gabble trying alternative nicknames when the nick you wanted is already in
use in a MUC you try to join.
"""

import dbus

from gabbletest import (
    exec_test, make_muc_presence, sync_stream, elem,
    )
from servicetest import (
    call_async, unwrap, sync_dbus, assertEquals, assertSameSets, wrap_channel,
    EventPattern,
    )
import constants as cs
import ns

def test(q, bus, conn, stream):
    test_join(q, bus, conn, stream, 'chat@conf.localhost', False)
    test_join(q, bus, conn, stream, 'chien@conf.localhost', True)

    test_gtalk_weirdness(q, bus, conn, stream,
        'private-chat-massive-uuid@groupchat.google.com')

def test_join(q, bus, conn, stream, room_jid, transient_conflict):
    """
    Tells Gabble to join a MUC, but make the first nick it tries conflict with
    an existing member of the MUC.  If transient_conflict is True, then when
    Gabble successfully joins with a different nick the originally conflicting
    user turns out not actually to be in the room (they left while we were
    retrying).
    """
    # Implementation detail: Gabble uses the first part of your jid (if you
    # don't have an alias) as your room nickname, and appends an underscore a
    # few times before giving up.
    member, member_ = [room_jid + '/' + x for x in ['test', 'test_']]

    call_async(q, conn.Requests, 'CreateChannel',
        dbus.Dictionary({ cs.CHANNEL_TYPE: cs.CHANNEL_TYPE_TEXT,
                          cs.TARGET_HANDLE_TYPE: cs.HT_ROOM,
                          cs.TARGET_ID: room_jid,
                        }, signature='sv'))

    # Gabble first tries to join as test
    q.expect('stream-presence', to=member)

    # MUC says no: there's already someone called test in room_jid
    presence = elem('presence', from_=member, type='error')(
        elem(ns.MUC, 'x'),
        elem('error', type='cancel')(
          elem(ns.STANZA, 'conflict'),
        ))
    stream.send(presence)

    # Gabble tries again as test_
    q.expect('stream-presence', to=member_)

    # MUC says yes!

    if not transient_conflict:
        # Send the other member of the room's presence. This is the nick we
        # originally wanted.
        stream.send(make_muc_presence('owner', 'moderator', room_jid, 'test'))

    # If gabble erroneously thinks the other user's presence is our own, it'll
    # think that it's got the whole userlist now. If so, syncing here will make
    # CreateChannel incorrectly return here.
    sync_stream(q, stream)
    sync_dbus(bus, q, conn)

    # Send presence for own membership of room.
    stream.send(make_muc_presence('none', 'participant', room_jid, 'test_'))

    # Only now should we have finished joining the room.
    event = q.expect('dbus-return', method='CreateChannel')
    path, props = event.value
    text_chan = wrap_channel(bus.get_object(conn.bus_name, path), 'Text')
    group_props = unwrap(text_chan.Properties.GetAll(cs.CHANNEL_IFACE_GROUP))

    t, t_ = conn.RequestHandles(cs.HT_CONTACT, [member, member_])

    # Check that Gabble think our nickname in the room is test_, not test
    muc_self_handle = group_props['SelfHandle']
    assert muc_self_handle == t_, (muc_self_handle, t_, t)

    members = group_props['Members']

    if transient_conflict:
        # The user we originally conflicted with isn't actually here; check
        # there's exactly one member (test_).
        assert members == [t_], (members, t_, t)
    else:
        # Check there are exactly two members (test and test_)
        assertSameSets([t, t_], members)

    # In either case, there should be no pending members.
    assert len(group_props['LocalPendingMembers']) == 0, group_props
    assert len(group_props['RemotePendingMembers']) == 0, group_props

    # Check that test_'s handle owner is us, and that test (if it's there) has
    # no owner.
    handle_owners = group_props['HandleOwners']
    assertEquals (conn.GetSelfHandle(), handle_owners[t_])
    if not transient_conflict:
        assertEquals (0, handle_owners[t])

    # test that closing the channel results in an unavailable message to the
    # right jid
    text_chan.Close()

    event = q.expect('stream-presence', to=member_)
    assertEquals('unavailable', event.stanza['type'])

def test_gtalk_weirdness(q, bus, conn, stream, room_jid):
    """
    There's a strange bug in the Google Talk MUC server where it sends the
    <conflict/> stanza twice. This has been reported to their server team; but
    in any case it triggered a crazy bug in Gabble, so here's a regression test.
    """

    # Implementation detail: Gabble uses the first part of your jid (if you
    # don't have an alias) as your room nickname, and appends an underscore a
    # few times before giving up.
    jids = ['%s/test%s' % (room_jid, x) for x in ['', '_', '__']]
    member, member_, member__ = jids

    # Gabble should never get as far as trying to join as 'test__' since
    # joining as 'test_' will succeed.
    q.forbid_events([ EventPattern('stream-presence', to=member__) ])

    call_async(q, conn.Requests, 'CreateChannel',
        dbus.Dictionary({ cs.CHANNEL_TYPE: cs.CHANNEL_TYPE_TEXT,
                          cs.TARGET_HANDLE_TYPE: cs.HT_ROOM,
                          cs.TARGET_ID: room_jid,
                        }, signature='sv'))

    # Gabble first tries to join as test
    q.expect('stream-presence', to=member)

    # Google Talk says no from 'test', twice.
    presence = elem('presence', from_=member, type='error')(
        elem(ns.MUC, 'x'),
        elem('error', type='cancel')(
          elem(ns.STANZA, 'conflict'),
        ))
    stream.send(presence)
    stream.send(presence)

    # Gabble should try to join again as test_
    q.expect('stream-presence', to=member_)

    # Since 'test_' is not in use in the MUC, joining should succeed. According
    # to XEP-0045 §7.1.3 <http://xmpp.org/extensions/xep-0045.html#enter-pres>:
    #  The service MUST first send the complete list of the existing occupants
    #  to the new occupant and only then send the new occupant's own presence
    #  to the new occupant
    # but groupchat.google.com cheerfully violates this.
    stream.send(make_muc_presence('none', 'participant', room_jid, 'test_'))

    # Here's some other random person, who owns the MUC.
    stream.send(make_muc_presence('owner', 'moderator', room_jid, 'foobar_gmail.com'))
    # And here's our hypothetical other self.
    stream.send(make_muc_presence('none', 'participant', room_jid, 'test'))

    # The Gabble bug makes this time out: because Gabble thinks it's joining as
    # test__ it ignores the presence for test_, since it's not flagged with
    # code='210' to say “this is you”. (This is acceptable behaviour by the
    # server: it only needs to include code='210' if it's assigned the client a
    # name other than the one it asked for.
    #
    # The forbidden stream-presence event above doesn't blow up here because
    # servicetest doesn't process events on the 'stream-*' queue at all when
    # we're not waiting for one. But during disconnection in the test clean-up,
    # the forbidden event is encountered and correctly flagged up.
    event = q.expect('dbus-return', method='CreateChannel')
    path, _ = event.value
    text_chan = wrap_channel(bus.get_object(conn.bus_name, path), 'Text')

    # As far as Gabble's concerned, the two other participants joined
    # immediately after we did.  We can't request handles for them before we
    # try to join the MUC, because until we do so, Gabble doesn't know that
    # room_jid is a MUC, and so considers these three JIDs to be different
    # resources of the same contact. There is no race between this method
    # returning and MembersChangedDetailed firing, because libdbus reorders
    # messages when you make blocking calls.
    handle, handle_, handle__, foobar_handle = conn.RequestHandles(
        cs.HT_CONTACT, jids + ['%s/foobar_gmail.com' % room_jid])

    q.expect('dbus-signal', signal='MembersChangedDetailed',
        predicate=lambda e: e.args[0:4] == [[foobar_handle], [], [], []])
    q.expect('dbus-signal', signal='MembersChangedDetailed',
        predicate=lambda e: e.args[0:4] == [[handle], [], [], []])

    group_props = text_chan.Properties.GetAll(cs.CHANNEL_IFACE_GROUP)
    assertEquals(handle_, group_props['SelfHandle'])
    assertSameSets([handle, handle_, foobar_handle], group_props['Members'])

if __name__ == '__main__':
    exec_test(test)
<|endoftext|>


<commit_before>from glob import glob
import json
import pandas
import requests
import numpy
import pickle
import os
import re

home = os.environ["HOME"]
base = "%s/data/pubmed" %os.environ["LAB"]
outfolder = "%s/methods" %(base)
repo_folder = "%s/repos" %(base)
scripts = "%s/SCRIPT/repofish/analysis/methods" %(home)
counts_folder = "%s/counts" %(base)

if not os.path.exists(counts_folder):
    os.mkdir(counts_folder)

urls = pickle.load(open("%s/inputs_categorized.pkl" %outfolder,"rb"))["repos"]
len(urls)
#5570

jobfile = "%s/count_extensions.job" %(scripts)
filey = open(jobfile,"w")

for entry in urls:
    pmid = entry[0]
    url = entry[1]
    filey.writelines("python %s/5.count_extensions.py %s %s %s\n" %(scripts,pmid,url,counts_folder))

filey.close()

<commit_msg>adding script to run counts on sherlock (because it actually works :P)<commit_after>from glob import glob
import json
import pandas
import requests
import numpy
import pickle
import os
import re

# FOR RUNNING ON TACC WITH LAUNCH (fails to clone at once) ########################################
home = os.environ["HOME"]
base = "%s/data/pubmed" %os.environ["LAB"]
outfolder = "%s/methods" %(base)
repo_folder = "%s/repos" %(base)
scripts = "%s/SCRIPT/repofish/analysis/methods" %(home)
counts_folder = "%s/counts" %(base)

if not os.path.exists(counts_folder):
    os.mkdir(counts_folder)

urls = pickle.load(open("%s/inputs_categorized.pkl" %outfolder,"rb"))["repos"]
len(urls)
#5570

jobfile = "%s/count_extensions.job" %(scripts)
filey = open(jobfile,"w")

for entry in urls:
    pmid = entry[0]
    url = entry[1]
    filey.writelines("python %s/5.count_extensions.py %s %s %s\n" %(scripts,pmid,url,counts_folder))

filey.close()

# FOR RUNNING ON SHERLOCK ########################################################################
home = os.environ["HOME"]
base = "%s/DATA/PUBMED" %os.environ["SCRATCH"]
outfolder = "%s/methods" %(base)
scripts = "%s/SCRIPT/python/repofish/analysis/methods" %(home)
counts_folder = "%s/counts" %(base)

if not os.path.exists(counts_folder):
    os.mkdir(counts_folder)

urls = pickle.load(open("%s/inputs_categorized.pkl" %outfolder,"rb"))["repos"]
len(urls)
#5570

count=0
for entry in urls:
    pmid = entry[0]
    url = entry[1]
    job_id = "%s_%s" %(pmid,count)
    jobfile = "%s/.job/%s.job" %(scripts,job_id)
    filey = open(jobfile,"w")
    filey.writelines("#!/bin/bash\n")
    filey.writelines("#SBATCH --job-name=%s\n" %(job_id))
    filey.writelines("#SBATCH --output=.out/%s.out\n" %(job_id))
    filey.writelines("#SBATCH --error=.out/%s.err\n" %(job_id))
    filey.writelines("#SBATCH --time=30:00\n")
    filey.writelines("#SBATCH --mem=4000\n")
    filey.writelines("python %s/5.count_extensions.py %s %s %s\n" %(scripts,pmid,url,counts_folder))
    filey.close()
    os.system("sbatch -p russpold --qos=russpold " + "%s/.job/%s.job" %(scripts,job_id))
    count+=1

<|endoftext|>