<commit_before># coding=utf-8
import hashlib
import hmac
import base64
import json
import requests
import time

from modules.ExchangeApi import ExchangeApi
from modules.ExchangeApi import ApiError
from modules.Bitfinex2Poloniex import Bitfinex2Poloniex

class Bitfinex(ExchangeApi):
    def __init__(self, cfg):
        self.cfg = cfg
        self.url = 'https://api.bitfinex.com/v1/'
        self.key = self.cfg.get("API", "apikey", None)
        self.secret = self.cfg.get("API", "secret", None)
        self.symbols = []
        self.ticker = {}
        self.tickerTime = 0
        self.usedCurrencies = []
        self.timeout = int(self.cfg.get("BOT", "timeout", 30, 1, 180))

    @property
    def _nonce(self):
        '''
        Returns a nonce
        Used in authentication
        '''
        return str(time.time())

    def _sign_payload(self, payload):
        j = json.dumps(payload)
        data = base64.standard_b64encode(j.encode('utf8'))

        h = hmac.new(self.secret.encode('utf8'), data, hashlib.sha384)
        signature = h.hexdigest()
        return {
            "X-BFX-APIKEY": self.key,
            "X-BFX-SIGNATURE": signature,
            "X-BFX-PAYLOAD": data
        }

    def _request(self, request, command, payload=None, verify=True):
        try:
            r = {}
            if (request == 'get'):
                r = requests.get(self.url + command, timeout=self.timeout)
            else:
                r = requests.post(self.url + command, headers=payload, verify=verify, timeout=self.timeout)

            if r.status_code != 200:
                if (r.status_code == 502 or r.status_code in range(520, 527, 1)):
                    raise ApiError('API Error ' + str(r.status_code) +
                                   ': The web server reported a bad gateway or gateway timeout error.')
                else:
                    raise ApiError('API Error ' + str(r.status_code) + ': ' + r.text)

            return r.json()

        except Exception as ex:
            ex.message = ex.message if ex.message else str(ex)
            ex.message = "{0} Requesting {1}".format(ex.message, self.url + command)
            raise ex

    def _post(self, command, payload, verify=True):
        return self._request('post', command, payload, verify)

    def _get(self, command):
        return self._request('get', command)

    def _getSymbols(self):
        '''
        A list of symbol names. Currently "btcusd", "ltcusd", "ltcbtc", ...
        https://bitfinex.readme.io/v1/reference#rest-public-symbols
        '''
        if len(self.symbols) == 0:
            bfxResp = self._get('symbols')
            allCurrencies = self.cfg.get_all_currencies()
            for symbol in bfxResp:
                base = symbol[3:].upper()
                curr = symbol[:3].upper()
                if base in ['USD', 'BTC'] and curr in allCurrencies:
                    self.symbols.append(symbol)

        return self.symbols

    def return_open_loan_offers(self):
        '''
        Returns active loan offers
        https://bitfinex.readme.io/v1/reference#rest-auth-offers
        '''
        signed_payload = self._sign_payload({
            "request": "/v1/offers",
            "nonce": self._nonce
        })

        bfxResp = self._post("offers", signed_payload)
        resp = Bitfinex2Poloniex.convertOpenLoanOffers(bfxResp)

        return resp

    def return_loan_orders(self, currency, limit=0):
        command = ('lendbook/' + currency + '?limit_asks=' + str(limit) + '&limit_bids=' + str(limit))
        bfxResp = self._get(command)
        resp = Bitfinex2Poloniex.convertLoanOrders(bfxResp)

        return resp

    def return_active_loans(self):
        '''
        Returns own active loan offers
        https://bitfinex.readme.io/v1/reference#rest-auth-offers
        '''
        signed_payload = self._sign_payload({
            "request": "/v1/credits",
            "nonce": self._nonce
        })

        bfxResp = self._post("credits", signed_payload)
        resp = Bitfinex2Poloniex.convertActiveLoans(bfxResp)

        return resp

    def return_ticker(self):
        '''
        The ticker is a high level overview of the state of the market
        https://bitfinex.readme.io/v1/reference#rest-public-ticker
        '''
        t = int(time.time())
        if (t - self.tickerTime < 60):
            return self.ticker

        for symbol in self._getSymbols():
            base = symbol[3:].upper()
            curr = symbol[:3].upper()
            if base in ['BTC', 'USD'] and (curr == 'BTC' or curr in self.usedCurrencies):
                couple = (base + '_' + curr)
                coupleReverse = (curr + '_' + base)

                try:
                    ticker = self._get('pubticker/'+symbol)
                except ApiError as e:
                    if not self.ticker[couple]:
                        raise e

                if 'message' in ticker:
                    raise ApiError("Error: %s (%s)".format(ticker['message'], symbol))

                self.ticker[couple] = {
                    "last": ticker['last_price'],
                    "lowestAsk": ticker['ask'],
                    "highestBid": ticker['bid'],
                    "percentChange": "",
                    "baseVolume": str(float(ticker['volume'])*float(ticker['mid'])),
                    "quoteVolume": ticker['volume']
                }
                self.ticker[coupleReverse] = {
                    "last": 1 / float(self.ticker[couple]['last']),
                    "lowestAsk": 1 / float(self.ticker[couple]['lowestAsk']),
                    "highestBid": 1 / float(self.ticker[couple]['highestBid'])
                }

        self.tickerTime = t

        return self.ticker

    def return_available_account_balances(self, account):
        '''
        Returns own balances sorted by account
        https://bitfinex.readme.io/v1/reference#rest-auth-wallet-balances
        '''
        signed_payload = self._sign_payload({
            "request": "/v1/balances",
            "nonce": self._nonce
        })

        bfxResp = self._post("balances", signed_payload)
        filtered_response = [x for x in bfxResp if x['type'] != 'conversion']
        balances = Bitfinex2Poloniex.convertAccountBalances(filtered_response, account)

        if 'lending' in balances:
            self.usedCurrencies = []
            for curr in balances['lending']:
                self.usedCurrencies.append(curr)

        return balances

    def cancel_loan_offer(self, currency, order_number):
        '''
        Cancels an offer
        https://bitfinex.readme.io/v1/reference#rest-auth-cancel-offer
        '''
        signed_payload = self._sign_payload({
            "request": "/v1/offer/cancel",
            "offer_id": order_number,
            "nonce": self._nonce
        })

        bfxResp = self._post("offer/cancel", signed_payload)

        success = 0
        message = ''
        try:
            if bfxResp['id'] == order_number:
                success = 1
                message = "Loan offer canceled ({:.4f} @ {:.4f}%).".format(float(bfxResp['remaining_amount']),
                                                                           float(bfxResp['rate'])/365)
        except Exception as e:
            message = "Error canceling offer: ", str(e)
            success = 0

        return {"success": success, "message": message}

    def create_loan_offer(self, currency, amount, duration, auto_renew, lending_rate):
        '''
        Creates a loan offer for a given currency.
        https://bitfinex.readme.io/v1/reference#rest-auth-new-offer
        '''

        signed_payload = self._sign_payload({
            "request": "/v1/offer/new",
            "currency": currency,
            "amount": str(amount),
            "rate": str(lending_rate * 36500),
            "period": int(duration),
            "direction": "lend",
            "nonce": self._nonce
        })

        try:
            bfxResp = self._post("offer/new", signed_payload)
            plxResp = {"success": 0, "message": "Error", "orderID": 0}
            if bfxResp['id']:
                plxResp['orderId'] = bfxResp['id']
                plxResp['success'] = 1
                plxResp['message'] = "Loan order placed."
            return plxResp

        except Exception as e:
                msg = str(e)
                # "Invalid offer: incorrect amount, minimum is 50 dollar or equivalent in USD"
                if "Invalid offer: incorrect amount, minimum is 50" in msg:
                    usd_min = 50
                    cur_min = usd_min
                    if currency != 'USD':
                        cur_min = usd_min / float(self.return_ticker()['USD_'+currency]['lowestAsk'])

                    raise Exception("Error create_loan_offer: Amount must be at least " + str(cur_min) + " " + currency)
                else:
                    raise e

    def return_balances(self):
        '''
        Returns balances of exchange wallet
        https://bitfinex.readme.io/v1/reference#rest-auth-wallet-balances
        '''
        balances = self.return_available_account_balances('exchange')
        return balances['exchange']

    def transfer_balance(self, currency, amount, from_account, to_account):
        '''
        Transfers values from one account/wallet to another
        https://bitfinex.readme.io/v1/reference#rest-auth-transfer-between-wallets
        '''
        accountMap = {
            'margin': 'trading',
            'lending': 'deposit',
            'exchange': 'exchange'
        }
        signed_payload = self._sign_payload({
            "request": "/v1/transfer",
            "currency": currency,
            "amount": amount,
            "walletfrom": accountMap[from_account],
            "walletto": accountMap[to_account],
            "nonce": self._nonce
        })

        bfxResp = self._post("transfer", signed_payload)
        plxResp = {
            "status":  1 if bfxResp[0]['status'] == "success" else 0,
            "message": bfxResp[0]['message']
        }

        return plxResp

    def return_lending_history(self, start, stop, limit=500):
        '''
        Retrieves balance ledger entries. Search funding payments in it and returns
        it as history.
        https://bitfinex.readme.io/v1/reference#rest-auth-balance-history
        '''
        history = []
        idInc = 0
        allCurrencies = self.cfg.get_all_currencies()
        for curr in allCurrencies:
            signed_payload = self._sign_payload({
                "request": "/v1/history",
                "currency": curr,
                "since": str(start),
                "until": str(stop),
                "limit": limit,
                "wallet": "deposit",
                "nonce": self._nonce
            })
            bfxResp = self._post('history', signed_payload)
            for entry in bfxResp:
                if 'Margin Funding Payment' in entry['description']:
                    amount = float(entry['amount'])
                    history.append({
                        "id": int(float(entry['timestamp'])) + idInc,
                        "currency": curr,
                        "rate": "0.0",
                        "amount": "0.0",
                        "duration": "0.0",
                        "interest": str(amount/0.85),
                        "fee": str(amount-amount/0.85),
                        "earned": str(amount),
                        "open": Bitfinex2Poloniex.convertTimestamp(entry['timestamp']),
                        "close": Bitfinex2Poloniex.convertTimestamp(entry['timestamp'])
                    })
                    idInc += 1

        return history
<commit_msg>Fix Bitfinex nonce calcualtion<commit_after># coding=utf-8
import hashlib
import hmac
import base64
import json
import requests
import time

from modules.ExchangeApi import ExchangeApi
from modules.ExchangeApi import ApiError
from modules.Bitfinex2Poloniex import Bitfinex2Poloniex

class Bitfinex(ExchangeApi):
    def __init__(self, cfg):
        self.cfg = cfg
        self.url = 'https://api.bitfinex.com/v1/'
        self.key = self.cfg.get("API", "apikey", None)
        self.secret = self.cfg.get("API", "secret", None)
        self.symbols = []
        self.ticker = {}
        self.tickerTime = 0
        self.usedCurrencies = []
        self.timeout = int(self.cfg.get("BOT", "timeout", 30, 1, 180))

    @property
    def _nonce(self):
        '''
        Returns a nonce
        Used in authentication
        '''
        return str(int(round(time.time() * 1000)))

    def _sign_payload(self, payload):
        j = json.dumps(payload)
        data = base64.standard_b64encode(j.encode('utf8'))

        h = hmac.new(self.secret.encode('utf8'), data, hashlib.sha384)
        signature = h.hexdigest()
        return {
            "X-BFX-APIKEY": self.key,
            "X-BFX-SIGNATURE": signature,
            "X-BFX-PAYLOAD": data
        }

    def _request(self, request, command, payload=None, verify=True):
        try:
            r = {}
            if (request == 'get'):
                r = requests.get(self.url + command, timeout=self.timeout)
            else:
                r = requests.post(self.url + command, headers=payload, verify=verify, timeout=self.timeout)

            if r.status_code != 200:
                if (r.status_code == 502 or r.status_code in range(520, 527, 1)):
                    raise ApiError('API Error ' + str(r.status_code) +
                                   ': The web server reported a bad gateway or gateway timeout error.')
                else:
                    raise ApiError('API Error ' + str(r.status_code) + ': ' + r.text)

            return r.json()

        except Exception as ex:
            ex.message = ex.message if ex.message else str(ex)
            ex.message = "{0} Requesting {1}".format(ex.message, self.url + command)
            raise ex

    def _post(self, command, payload, verify=True):
        return self._request('post', command, payload, verify)

    def _get(self, command):
        return self._request('get', command)

    def _getSymbols(self):
        '''
        A list of symbol names. Currently "btcusd", "ltcusd", "ltcbtc", ...
        https://bitfinex.readme.io/v1/reference#rest-public-symbols
        '''
        if len(self.symbols) == 0:
            bfxResp = self._get('symbols')
            allCurrencies = self.cfg.get_all_currencies()
            for symbol in bfxResp:
                base = symbol[3:].upper()
                curr = symbol[:3].upper()
                if base in ['USD', 'BTC'] and curr in allCurrencies:
                    self.symbols.append(symbol)

        return self.symbols

    def return_open_loan_offers(self):
        '''
        Returns active loan offers
        https://bitfinex.readme.io/v1/reference#rest-auth-offers
        '''
        signed_payload = self._sign_payload({
            "request": "/v1/offers",
            "nonce": self._nonce
        })

        bfxResp = self._post("offers", signed_payload)
        resp = Bitfinex2Poloniex.convertOpenLoanOffers(bfxResp)

        return resp

    def return_loan_orders(self, currency, limit=0):
        command = ('lendbook/' + currency + '?limit_asks=' + str(limit) + '&limit_bids=' + str(limit))
        bfxResp = self._get(command)
        resp = Bitfinex2Poloniex.convertLoanOrders(bfxResp)

        return resp

    def return_active_loans(self):
        '''
        Returns own active loan offers
        https://bitfinex.readme.io/v1/reference#rest-auth-offers
        '''
        signed_payload = self._sign_payload({
            "request": "/v1/credits",
            "nonce": self._nonce
        })

        bfxResp = self._post("credits", signed_payload)
        resp = Bitfinex2Poloniex.convertActiveLoans(bfxResp)

        return resp

    def return_ticker(self):
        '''
        The ticker is a high level overview of the state of the market
        https://bitfinex.readme.io/v1/reference#rest-public-ticker
        '''
        t = int(time.time())
        if (t - self.tickerTime < 60):
            return self.ticker

        for symbol in self._getSymbols():
            base = symbol[3:].upper()
            curr = symbol[:3].upper()
            if base in ['BTC', 'USD'] and (curr == 'BTC' or curr in self.usedCurrencies):
                couple = (base + '_' + curr)
                coupleReverse = (curr + '_' + base)

                try:
                    ticker = self._get('pubticker/'+symbol)
                except ApiError as e:
                    if not self.ticker[couple]:
                        raise e

                if 'message' in ticker:
                    raise ApiError("Error: %s (%s)".format(ticker['message'], symbol))

                self.ticker[couple] = {
                    "last": ticker['last_price'],
                    "lowestAsk": ticker['ask'],
                    "highestBid": ticker['bid'],
                    "percentChange": "",
                    "baseVolume": str(float(ticker['volume'])*float(ticker['mid'])),
                    "quoteVolume": ticker['volume']
                }
                self.ticker[coupleReverse] = {
                    "last": 1 / float(self.ticker[couple]['last']),
                    "lowestAsk": 1 / float(self.ticker[couple]['lowestAsk']),
                    "highestBid": 1 / float(self.ticker[couple]['highestBid'])
                }

        self.tickerTime = t

        return self.ticker

    def return_available_account_balances(self, account):
        '''
        Returns own balances sorted by account
        https://bitfinex.readme.io/v1/reference#rest-auth-wallet-balances
        '''
        signed_payload = self._sign_payload({
            "request": "/v1/balances",
            "nonce": self._nonce
        })

        bfxResp = self._post("balances", signed_payload)
        filtered_response = [x for x in bfxResp if x['type'] != 'conversion']
        balances = Bitfinex2Poloniex.convertAccountBalances(filtered_response, account)

        if 'lending' in balances:
            self.usedCurrencies = []
            for curr in balances['lending']:
                self.usedCurrencies.append(curr)

        return balances

    def cancel_loan_offer(self, currency, order_number):
        '''
        Cancels an offer
        https://bitfinex.readme.io/v1/reference#rest-auth-cancel-offer
        '''
        signed_payload = self._sign_payload({
            "request": "/v1/offer/cancel",
            "offer_id": order_number,
            "nonce": self._nonce
        })

        bfxResp = self._post("offer/cancel", signed_payload)

        success = 0
        message = ''
        try:
            if bfxResp['id'] == order_number:
                success = 1
                message = "Loan offer canceled ({:.4f} @ {:.4f}%).".format(float(bfxResp['remaining_amount']),
                                                                           float(bfxResp['rate'])/365)
        except Exception as e:
            message = "Error canceling offer: ", str(e)
            success = 0

        return {"success": success, "message": message}

    def create_loan_offer(self, currency, amount, duration, auto_renew, lending_rate):
        '''
        Creates a loan offer for a given currency.
        https://bitfinex.readme.io/v1/reference#rest-auth-new-offer
        '''

        signed_payload = self._sign_payload({
            "request": "/v1/offer/new",
            "currency": currency,
            "amount": str(amount),
            "rate": str(lending_rate * 36500),
            "period": int(duration),
            "direction": "lend",
            "nonce": self._nonce
        })

        try:
            bfxResp = self._post("offer/new", signed_payload)
            plxResp = {"success": 0, "message": "Error", "orderID": 0}
            if bfxResp['id']:
                plxResp['orderId'] = bfxResp['id']
                plxResp['success'] = 1
                plxResp['message'] = "Loan order placed."
            return plxResp

        except Exception as e:
                msg = str(e)
                # "Invalid offer: incorrect amount, minimum is 50 dollar or equivalent in USD"
                if "Invalid offer: incorrect amount, minimum is 50" in msg:
                    usd_min = 50
                    cur_min = usd_min
                    if currency != 'USD':
                        cur_min = usd_min / float(self.return_ticker()['USD_'+currency]['lowestAsk'])

                    raise Exception("Error create_loan_offer: Amount must be at least " + str(cur_min) + " " + currency)
                else:
                    raise e

    def return_balances(self):
        '''
        Returns balances of exchange wallet
        https://bitfinex.readme.io/v1/reference#rest-auth-wallet-balances
        '''
        balances = self.return_available_account_balances('exchange')
        return balances['exchange']

    def transfer_balance(self, currency, amount, from_account, to_account):
        '''
        Transfers values from one account/wallet to another
        https://bitfinex.readme.io/v1/reference#rest-auth-transfer-between-wallets
        '''
        accountMap = {
            'margin': 'trading',
            'lending': 'deposit',
            'exchange': 'exchange'
        }
        signed_payload = self._sign_payload({
            "request": "/v1/transfer",
            "currency": currency,
            "amount": amount,
            "walletfrom": accountMap[from_account],
            "walletto": accountMap[to_account],
            "nonce": self._nonce
        })

        bfxResp = self._post("transfer", signed_payload)
        plxResp = {
            "status":  1 if bfxResp[0]['status'] == "success" else 0,
            "message": bfxResp[0]['message']
        }

        return plxResp

    def return_lending_history(self, start, stop, limit=500):
        '''
        Retrieves balance ledger entries. Search funding payments in it and returns
        it as history.
        https://bitfinex.readme.io/v1/reference#rest-auth-balance-history
        '''
        history = []
        idInc = 0
        allCurrencies = self.cfg.get_all_currencies()
        for curr in allCurrencies:
            signed_payload = self._sign_payload({
                "request": "/v1/history",
                "currency": curr,
                "since": str(start),
                "until": str(stop),
                "limit": limit,
                "wallet": "deposit",
                "nonce": self._nonce
            })
            bfxResp = self._post('history', signed_payload)
            for entry in bfxResp:
                if 'Margin Funding Payment' in entry['description']:
                    amount = float(entry['amount'])
                    history.append({
                        "id": int(float(entry['timestamp'])) + idInc,
                        "currency": curr,
                        "rate": "0.0",
                        "amount": "0.0",
                        "duration": "0.0",
                        "interest": str(amount/0.85),
                        "fee": str(amount-amount/0.85),
                        "earned": str(amount),
                        "open": Bitfinex2Poloniex.convertTimestamp(entry['timestamp']),
                        "close": Bitfinex2Poloniex.convertTimestamp(entry['timestamp'])
                    })
                    idInc += 1

        return history
<|endoftext|>


<commit_before>import os

import discord
import tokage
from discord.ext import commands
from discord.ext.commands import Bot

TOKEN = os.environ.get("BOT_TOKEN")
extensions = ["roll", "roles", "utils", "search", "cancer"]
startup_extensions = ["Cogs." + extension for extension in extensions]

class SynBot(Bot):
    def __init__(self):
        game = discord.Game(name="s!help | syn help", type=2)
        prefix = commands.when_mentioned_or("syn ", "s!")
        super().__init__(command_prefix=prefix, description="Misc Bot", game=game)

    async def close(self):
        await self.t_client.cleanup()
        await super().close()

    async def on_ready(self):
        self.t_client = tokage.Client()

        print('Logged in!')
        print(self.user.name)
        print(self.user.id)
        print('------')
        print("Cogs loaded:")
        for extension in startup_extensions:
            try:
                self.load_extension(str(extension))
                print('"%s" loaded successfully' % extension.split(".")[1])
            except Exception as e:
                exc = '{}: {}'.format(type(e).__name__, e)
                print('Failed to load extension {}\n{}'.format(extension, exc))
        print('------')

SynBot().run(TOKEN)
<commit_msg>load anime cog on startup<commit_after>import os

import discord
import tokage
from discord.ext import commands
from discord.ext.commands import Bot

TOKEN = os.environ.get("BOT_TOKEN")
extensions = ["roll", "roles", "utils", "search", "cancer", "myanimelist"]
startup_extensions = ["Cogs." + extension for extension in extensions]

class SynBot(Bot):
    def __init__(self):
        game = discord.Game(name="s!help | syn help", type=2)
        prefix = commands.when_mentioned_or("syn ", "s!")
        super().__init__(command_prefix=prefix, description="Misc Bot", game=game)

    async def close(self):
        await self.t_client.cleanup()
        await super().close()

    async def on_ready(self):
        self.t_client = tokage.Client()

        print('Logged in!')
        print(self.user.name)
        print(self.user.id)
        print('------')
        print("Cogs loaded:")
        for extension in startup_extensions:
            try:
                self.load_extension(str(extension))
                print('"%s" loaded successfully' % extension.split(".")[1])
            except Exception as e:
                exc = '{}: {}'.format(type(e).__name__, e)
                print('Failed to load extension {}\n{}'.format(extension, exc))
        print('------')

SynBot().run(TOKEN)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.db import models

class CacheKeyManager(models.Manager):
    def get_keys(self, site_id=None, language=None):
        ret = self.none()
        if not site_id and not language:
            # Both site and language are None - return everything
            ret = self.all()
        elif not site_id:
            ret = self.filter(language=language)
        elif not language:
            ret = self.filter(site=site_id)
        else:
            # Filter by site_id *and* by language.
            ret = self.filter(site=site_id).filter(language=language)
        return ret

    def get_or_create(self, kwargs):
        try:
            return super(CacheKeyManager, self).get_or_create(**kwargs)
        except CacheKey.MultipleObjectsReturned:
            # Truncate the table, we don't want a funny cache object to cause
            # mayhem!
            CacheKey.object.all().delete()
            return super(CacheKeyManager, self).get_or_create(**kwargs)

class CacheKey(models.Model):
    '''
    This is to store a "set" of cache keys in a fashion where it's accessible
    by multiple processes / machines.
    Multiple Django instances will then share the keys.
    This allows for selective invalidation of the menu trees (per site, per
    language), in the cache.
    '''
    language = models.CharField(max_length=255)
    site = models.PositiveIntegerField()
    key = models.CharField(max_length=255)
    objects = CacheKeyManager()
<commit_msg>Forgot to add ** to the method signature...<commit_after># -*- coding: utf-8 -*-
from django.db import models

class CacheKeyManager(models.Manager):
    def get_keys(self, site_id=None, language=None):
        ret = self.none()
        if not site_id and not language:
            # Both site and language are None - return everything
            ret = self.all()
        elif not site_id:
            ret = self.filter(language=language)
        elif not language:
            ret = self.filter(site=site_id)
        else:
            # Filter by site_id *and* by language.
            ret = self.filter(site=site_id).filter(language=language)
        return ret

    def get_or_create(self, **kwargs):
        try:
            return super(CacheKeyManager, self).get_or_create(**kwargs)
        except CacheKey.MultipleObjectsReturned:
            # Truncate the table, we don't want a funny cache object to cause
            # mayhem!
            CacheKey.object.all().delete()
            return super(CacheKeyManager, self).get_or_create(**kwargs)

class CacheKey(models.Model):
    '''
    This is to store a "set" of cache keys in a fashion where it's accessible
    by multiple processes / machines.
    Multiple Django instances will then share the keys.
    This allows for selective invalidation of the menu trees (per site, per
    language), in the cache.
    '''
    language = models.CharField(max_length=255)
    site = models.PositiveIntegerField()
    key = models.CharField(max_length=255)
    objects = CacheKeyManager()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import pytest

from web3.utils.transactions import (
    wait_for_transaction_receipt,
)
from web3.utils.string import (
    force_text,
    force_bytes,
)

@pytest.fixture()
def math_contract(web3_tester, MathContract):
    deploy_txn = MathContract.deploy()
    deploy_receipt = web3_tester.eth.getTransactionReceipt(deploy_txn)
    assert deploy_receipt is not None
    _math_contract = MathContract(address=deploy_receipt['contractAddress'])
    return _math_contract

@pytest.fixture()
def string_contract(web3_tester, StringContract):
    deploy_txn = StringContract.deploy(args=["Caqalai"])
    deploy_receipt = web3_tester.eth.getTransactionReceipt(deploy_txn)
    assert deploy_receipt is not None
    _math_contract = StringContract(address=deploy_receipt['contractAddress'])
    return _math_contract

def test_transacting_with_contract_no_arguments(web3_tester, math_contract):
    initial_value = math_contract.call().counter()

    txn_hash = math_contract.transact().increment()
    txn_receipt = web3_tester.eth.getTransactionReceipt(txn_hash)
    assert txn_receipt is not None

    final_value = math_contract.call().counter()

    assert final_value - initial_value == 1

@pytest.mark.parametrize(
    'transact_args,transact_kwargs',
    (
        ((5,), {}),
        (tuple(), {'amt': 5}),
    ),
)
def test_transacting_with_contract_with_arguments(web3_tester,
                                                  math_contract,
                                                  transact_args,
                                                  transact_kwargs):
    initial_value = math_contract.call().counter()

    txn_hash = math_contract.transact().increment(*transact_args, **transact_kwargs)
    txn_receipt = web3_tester.eth.getTransactionReceipt(txn_hash)
    assert txn_receipt is not None

    final_value = math_contract.call().counter()

    assert final_value - initial_value == 5

def test_transacting_with_contract_with_string_argument(web3_tester, string_contract):
    # eth_abi will pass as raw bytes, no encoding
    # unless we encode ourselves
    txn_hash = string_contract.transact().setValue(force_bytes("ÄLÄMÖLÖ"))
    txn_receipt = web3_tester.eth.getTransactionReceipt(txn_hash)
    assert txn_receipt is not None

    final_value = string_contract.call().getValue()

    assert force_bytes(final_value) == force_bytes("ÄLÄMÖLÖ")

def test_transacting_with_contract_with_string_argument(web3_tester, string_contract):

    # eth_abi will pass as raw bytes, no encoding
    # unless we encode ourselves
    txn_hash = string_contract.transact().setValue(force_bytes("ÄLÄMÖLÖ"))
    txn_receipt = web3_tester.eth.getTransactionReceipt(txn_hash)
    assert txn_receipt is not None

    final_value = string_contract.call().getValue()

    assert force_bytes(final_value) == force_bytes("ÄLÄMÖLÖ")

def test_transacting_with_contract_respects_explicit_gas(web3,
                                                         STRING_CONTRACT,
                                                         skip_if_testrpc,
                                                         wait_for_block):
    skip_if_testrpc(web3)

    wait_for_block(web3)

    StringContract = web3.eth.contract(**STRING_CONTRACT)

    deploy_txn = StringContract.deploy(args=["Caqalai"])
    deploy_receipt = wait_for_transaction_receipt(web3, deploy_txn, 30)
    assert deploy_receipt is not None
    string_contract = StringContract(address=deploy_receipt['contractAddress'])

    # eth_abi will pass as raw bytes, no encoding
    # unless we encode ourselves
    txn_hash = string_contract.transact({'gas': 200000}).setValue(force_bytes("ÄLÄMÖLÖ"))
    txn_receipt = wait_for_transaction_receipt(web3, txn_hash, 30)
    assert txn_receipt is not None

    final_value = string_contract.call().getValue()
    assert force_bytes(final_value) == force_bytes("ÄLÄMÖLÖ")

    txn = web3.eth.getTransaction(txn_hash)
    assert txn['gas'] == 200000

def test_auto_gas_computation_when_transacting(web3,
                                               STRING_CONTRACT,
                                               skip_if_testrpc,
                                               wait_for_block):
    skip_if_testrpc(web3)

    wait_for_block(web3)

    StringContract = web3.eth.contract(**STRING_CONTRACT)

    deploy_txn = StringContract.deploy(args=["Caqalai"])
    deploy_receipt = wait_for_transaction_receipt(web3, deploy_txn, 30)
    assert deploy_receipt is not None
    string_contract = StringContract(address=deploy_receipt['contractAddress'])

    gas_estimate = string_contract.estimateGas().setValue(force_bytes("ÄLÄMÖLÖ"))

    # eth_abi will pass as raw bytes, no encoding
    # unless we encode ourselves
    txn_hash = string_contract.transact().setValue(force_bytes("ÄLÄMÖLÖ"))
    txn_receipt = wait_for_transaction_receipt(web3, txn_hash, 30)
    assert txn_receipt is not None

    final_value = string_contract.call().getValue()
    assert force_bytes(final_value) == force_bytes("ÄLÄMÖLÖ")

    txn = web3.eth.getTransaction(txn_hash)
    assert txn['gas'] == gas_estimate + 100000
<commit_msg>tests to make sure defaultAccount is used correctly<commit_after># -*- coding: utf-8 -*-

import pytest

from web3.utils.transactions import (
    wait_for_transaction_receipt,
)
from web3.utils.string import (
    force_text,
    force_bytes,
)

@pytest.fixture()
def math_contract(web3_tester, MathContract):
    deploy_txn = MathContract.deploy()
    deploy_receipt = web3_tester.eth.getTransactionReceipt(deploy_txn)
    assert deploy_receipt is not None
    _math_contract = MathContract(address=deploy_receipt['contractAddress'])
    return _math_contract

@pytest.fixture()
def string_contract(web3_tester, StringContract):
    deploy_txn = StringContract.deploy(args=["Caqalai"])
    deploy_receipt = web3_tester.eth.getTransactionReceipt(deploy_txn)
    assert deploy_receipt is not None
    _math_contract = StringContract(address=deploy_receipt['contractAddress'])
    return _math_contract

def test_transacting_with_contract_no_arguments(web3_tester, math_contract):
    initial_value = math_contract.call().counter()

    txn_hash = math_contract.transact().increment()
    txn_receipt = web3_tester.eth.getTransactionReceipt(txn_hash)
    assert txn_receipt is not None

    final_value = math_contract.call().counter()

    assert final_value - initial_value == 1

@pytest.mark.parametrize(
    'transact_args,transact_kwargs',
    (
        ((5,), {}),
        (tuple(), {'amt': 5}),
    ),
)
def test_transacting_with_contract_with_arguments(web3_tester,
                                                  math_contract,
                                                  transact_args,
                                                  transact_kwargs):
    initial_value = math_contract.call().counter()

    txn_hash = math_contract.transact().increment(*transact_args, **transact_kwargs)
    txn_receipt = web3_tester.eth.getTransactionReceipt(txn_hash)
    assert txn_receipt is not None

    final_value = math_contract.call().counter()

    assert final_value - initial_value == 5

def test_deploy_when_default_account_is_different_than_coinbase(web3_tester,
                                                                STRING_CONTRACT):
    web3_tester.eth.defaultAccount = web3_tester.eth.accounts[1]
    assert web3_tester.eth.defaultAccount != web3_tester.eth.coinbase

    StringContract = web3_tester.eth.contract(**STRING_CONTRACT)

    deploy_txn = StringContract.deploy(args=["Caqalai"])
    deploy_receipt = wait_for_transaction_receipt(web3_tester, deploy_txn, 30)
    assert deploy_receipt['from'] == web3_tester.eth.defaultAccount

def test_transact_when_default_account_is_different_than_coinbase(web3_tester,
                                                                  math_contract,
                                                                  transact_args,
                                                                  transact_kwargs):
    web3_tester.eth.defaultAccount = web3_tester.eth.accounts[1]
    assert web3_tester.eth.defaultAccount != web3_tester.eth.coinbase

    txn_hash = math_contract.transact().increment(*transact_args, **transact_kwargs)
    txn_receipt = web3_tester.eth.getTransactionReceipt(txn_hash)
    assert txn_receipt['from'] == web3_tester.eth.defaultAccount

def test_transacting_with_contract_with_string_argument(web3_tester, string_contract):
    # eth_abi will pass as raw bytes, no encoding
    # unless we encode ourselves
    txn_hash = string_contract.transact().setValue(force_bytes("ÄLÄMÖLÖ"))
    txn_receipt = web3_tester.eth.getTransactionReceipt(txn_hash)
    assert txn_receipt is not None

    final_value = string_contract.call().getValue()

    assert force_bytes(final_value) == force_bytes("ÄLÄMÖLÖ")

def test_transacting_with_contract_with_string_argument(web3_tester, string_contract):

    # eth_abi will pass as raw bytes, no encoding
    # unless we encode ourselves
    txn_hash = string_contract.transact().setValue(force_bytes("ÄLÄMÖLÖ"))
    txn_receipt = web3_tester.eth.getTransactionReceipt(txn_hash)
    assert txn_receipt is not None

    final_value = string_contract.call().getValue()

    assert force_bytes(final_value) == force_bytes("ÄLÄMÖLÖ")

def test_transacting_with_contract_respects_explicit_gas(web3,
                                                         STRING_CONTRACT,
                                                         skip_if_testrpc,
                                                         wait_for_block):
    skip_if_testrpc(web3)

    wait_for_block(web3)

    StringContract = web3.eth.contract(**STRING_CONTRACT)

    deploy_txn = StringContract.deploy(args=["Caqalai"])
    deploy_receipt = wait_for_transaction_receipt(web3, deploy_txn, 30)
    assert deploy_receipt is not None
    string_contract = StringContract(address=deploy_receipt['contractAddress'])

    # eth_abi will pass as raw bytes, no encoding
    # unless we encode ourselves
    txn_hash = string_contract.transact({'gas': 200000}).setValue(force_bytes("ÄLÄMÖLÖ"))
    txn_receipt = wait_for_transaction_receipt(web3, txn_hash, 30)
    assert txn_receipt is not None

    final_value = string_contract.call().getValue()
    assert force_bytes(final_value) == force_bytes("ÄLÄMÖLÖ")

    txn = web3.eth.getTransaction(txn_hash)
    assert txn['gas'] == 200000

def test_auto_gas_computation_when_transacting(web3,
                                               STRING_CONTRACT,
                                               skip_if_testrpc,
                                               wait_for_block):
    skip_if_testrpc(web3)

    wait_for_block(web3)

    StringContract = web3.eth.contract(**STRING_CONTRACT)

    deploy_txn = StringContract.deploy(args=["Caqalai"])
    deploy_receipt = wait_for_transaction_receipt(web3, deploy_txn, 30)
    assert deploy_receipt is not None
    string_contract = StringContract(address=deploy_receipt['contractAddress'])

    gas_estimate = string_contract.estimateGas().setValue(force_bytes("ÄLÄMÖLÖ"))

    # eth_abi will pass as raw bytes, no encoding
    # unless we encode ourselves
    txn_hash = string_contract.transact().setValue(force_bytes("ÄLÄMÖLÖ"))
    txn_receipt = wait_for_transaction_receipt(web3, txn_hash, 30)
    assert txn_receipt is not None

    final_value = string_contract.call().getValue()
    assert force_bytes(final_value) == force_bytes("ÄLÄMÖLÖ")

    txn = web3.eth.getTransaction(txn_hash)
    assert txn['gas'] == gas_estimate + 100000
<|endoftext|>


<commit_before>import praw
import os
import praw.helpers

reddit_username = os.environ['REDDIT_USERNAME']
r = praw.Reddit(user_agent='beetlejuicing bot by /u/{}'.format(reddit_username))
submissions = r.get_subreddit('todayilearned').get_hot(limit=100)

i = 0
for s in submissions:
    print "Thread {}...".format(i)
    i += 1
    for c in praw.helpers.flatten_tree(s.comments):
        if 'author' not in dir(c):
            continue
        # my username godlikeme
        if ((('God' not in c.body) and ('god' not in c.body) or ('like' not in c.body)) and ('avatar' not in c.body)) or (c.score < 5):
            continue
        print "====="
        print u"{} says: {}".format(c.author, c.body)
<commit_msg>More experimenting, now with comment_stream<commit_after>import praw
import os
import praw.helpers

reddit_username = os.environ['REDDIT_USERNAME']
r = praw.Reddit(user_agent='beetlejuicing bot by /u/{}'.format(reddit_username))
saved_comments = []

i = 0
for c in praw.helpers.comment_stream(r, subreddit='askreddit'):
    if i % 500 == 0:
        print "Comment {}...".format(i)
    i += 1
    if 'author' not in dir(c):
        continue
    # my username godlikeme
    if ('god likes' not in c.body) and ('God likes' not in c.body) and ('god hates' not in c.body) and ('God hates' not in c.body):
        # and ('avatar' not in c.body):
        continue

    print "====="
    print u"{} says: {}".format(c.author, c.body)
    saved_comments.append(c)
<|endoftext|>


<commit_before>import discord
import asyncio
import json
from myUser import *
from helpers import *
client = discord.Client()

#Initializing list that stores myUser class members of the server
userList = []
loadStats("stats.txt",userList)

@client.event
async def on_member_ban(member):
    if member.id == client.user.id:
        for channel in member.server.channels:
            await client.send_message(channel, "You dicks")

#gets the missed messages until message.author.id == Tiger bot
@client.event
async def on_ready():
    counterMessage = 0
    print("Logged in as " + client.user.name + ", ID = " + client.user.id)
    #goes through every channel in the server, updating karma
    for server in client.servers:
        for channel in server.channels:
            if channel.type == "text":
                #goes though past 1000 messages, until message.author.id == Tiger
                async for missedMessage in client.logs_from(channel,limit = 1000):
                    counterMessage = missedMessage
                    if missedMessage.author.id != client.user.id:
                        # client.messages.append(missedMessage)
                        print("Added " + missedMessage.author.name +"\'s message")
                        addMessage(userList, missedMessage.author.id, missedMessage.author.name,\
                         missedMessage.content)
                         #if the message has reactions
                        if missedMessage.reactions:
                            for missedReaction in missedMessage.reactions:
                                if not(type(missedReaction.emoji) is str): #checks if emoji is emoji class or str

                                    if missedReaction.emoji.name == "upvote":
                                        upvoteList = await client.get_reaction_users(missedReaction)
                                        #goes through people who reacted with upvote, updates their upvotesGiven
                                        for upvotingUsers in upvoteList:
                                            #if someone upvoted their own post
                                            if upvotingUsers.id == missedMessage.author.id:
                                                for member in missedReaction.emoji.server.members:
                                                    if member.id == missedMessage.author.id:
                                                        await client.remove_reaction(missedReaction.message, missedReaction.emoji, member)
                                                        updateKarma(userList,member.id, member.name, 'downvote')
                                            else:
                                                updateUpvotes(userList, upvotingUsers.id, upvotingUsers.name, 'added')
                                        updateKarma(userList,missedMessage.author.id,\
                                        missedMessage.author.name,"upvote",missedReaction.count)

                                    if missedReaction.emoji.name == "downvote":
                                        upvoteList = await client.get_reaction_users(missedReaction)
                                        #goes through people who reacted with downvote, updates thier downvotes given
                                        for upvotingUsers in upvoteList:
                                            #if someone downvoted their own post
                                            if upvotingUsers.id == missedMessage.author.id:
                                                for member in missedReaction.emoji.server.members:
                                                    if member.id == missedMessage.author.id:
                                                        await client.remove_reaction(missedReaction.message, missedReaction.emoji, member)
                                                        updateKarma(userList,member.id, member.name, 'upvote')
                                            else:
                                                updateDownvotes(userList, upvotingUsers.id, upvotingUsers.name, 'added')
                                        updateKarma(userList, missedMessage.author.id, \
                                        missedMessage.author.name, "downvote", missedReaction.count)
                    else:
                        break

    for server in client.servers:
        for channel in server.channels:
                async for incrementMessage in client.logs_from(channel,limit = 1000, before = counterMessage):
                    # client.messages.append(incrementMessage)
                    if incrementMessage.reactions:
                        for incrementReaction in incrementMessage.reactions:
                            if not(type(incrementReaction.emoji) is str): #checks if emoji is emoji class or str
                                if incrementReaction.emoji.name == "upvote":
                                    upvoteList = await client.get_reaction_users(incrementReaction)
                                    #goes through people who reacted with upvote, updates their upvotesGiven
                                    for upvotingUsers in upvoteList:
                                        #if someone upvoted their own post
                                        if upvotingUsers.id == incrementMessage.author.id:
                                            for member in incrementReaction.emoji.server.members:
                                                if member.id == incrementMessage.author.id:
                                                    await client.remove_reaction(incrementReaction.message, incrementReaction.emoji, member)
                                                    updateKarma(userList,member.id, member.name, 'downvote')
                                    for keithUser in userList:
                                        if keithUser.id == incrementMessage.author.id:
                                            keithUser.karma += incrementReaction.count
                                            print("incremented on enter")

                                if incrementReaction.emoji.name == "downvote":
                                    downvoteList = await client.get_reaction_users(incrementReaction)
                                    #goes through people who reacted with downvote, updates thier downvotes given
                                    for downvotingUsers in downvoteList:
                                        #if someone downvoted their own post
                                        if downvotingUsers.id == incrementMessage.author.id:
                                            for member in incrementReaction.emoji.server.members:
                                                if member.id == incrementMessage.author.id:
                                                    await client.remove_reaction(incrementReaction.message, incrementReaction.emoji, member)
                                                    updateKarma(userList,member.id, member.name, 'upvote')
                                    for keithUser in userList:
                                        if keithUser.id == incrementMessage.author.id:
                                            keithUser.karma -= incrementReaction.count
                                            print("decremented on exit")
    for server in client.servers:
        for channel in server.channels:
            async for missedMessage in client.logs_from(channel,limit = 1000):
                client.messages.append(missedMessage)
    print("Done Adding messages")

@client.event
async def on_reaction_add(reaction,user):
    if(reaction.emoji.name == "upvote"):
        print("upvote given")
        updateUpvotes(userList,user.id, user.name, "added")
        updateKarma(userList,reaction.message.author.id,reaction.message.author.name,"upvote")
        # prevents users from upvoting/downvoting themselves
        if(reaction.message.author.id == user.id):
            for member in reaction.emoji.server.members:
                if member.id == user.id:
                    await client.remove_reaction(reaction.message, reaction.emoji, member)
                    break

    elif reaction.emoji.name == "downvote":
        print("downvote given")
        updateDownvotes(userList,user.id,user.name, "added")
        updateKarma(userList,reaction.message.author.id,reaction.message.author.name,"downvote")
        # prevents users from upvoting/downvoting themselves
        if(reaction.message.author.id == user.id):
            for member in reaction.emoji.server.members:
                if member.id == user.id:
                    await client.remove_reaction(reaction.message, reaction.emoji, member)
                    break

@client.event
async def on_reaction_remove(reaction,user):
    if(reaction.emoji.name == "upvote"):
        print("upvote removed")
        updateUpvotes(userList, user.id, user.name, "removed")
        updateKarma(userList,reaction.message.author.id,reaction.message.author.name,"downvote")

    elif reaction.emoji.name == "downvote":
        print("downvote removed")
        updateDownvotes(userList,user.id, user.name, "removed")
        updateKarma(userList,reaction.message.author.id, reaction.message.author.name, "upvote")

@client.event
async def on_message(message):
    #runs on every message, updates users 'smarts' statistics
    for user in userList:
        if(user.id == message.author.id):
            user.addContent(message.content)
            break
    else:
        userList.append(myUser(message.author.id, message.author.name))
        userList[-1].addContent(message.content)

    if "why tiger" in message.content.lower() or "tiger why did you beat your wife?"\
     in message.content.lower() or "tiger why did you beat your wife" in message.content.lower():
        await client.send_message(message.channel, "Not because I\'m bad at golf!")
    #the trigger to print 'smarts' stats
    elif message.content.lower().startswith("smarts", 0, 6):
        #if message contains @notificaions, will print for all mentioned
        if message.mentions:
            for user in message.mentions:
                for user1 in userList:
                    if(user1.id == user.id):
                        await client.send_message(message.channel, content = \
                            'Username: ' + user1.getName() + '\n' + \
                            '   Average letters per word: ' + str(user1.getLettersPerWord()) + '\n' + \
                            '   Average words per message: ' + str(user1.getWordsPerMessage()) + '\n' + \
                            '   Smarts: ' + str(((user1.getLettersPerWord() + user1.getWordsPerMessage())//2) % 10))
                        break
                else:
                    await client.send_message(message.channel, content = user.name + ' is apparently too dumb to have typed anything.')
        #if no one is mentioned, will print for message author
        else:
            for user in userList:
                if(user.id == message.author.id):
                    await client.send_message(message.channel, content = \
                        'Username: ' + user.getName() + '\n' + \
                        '   Average letters per word: ' + str(user.getLettersPerWord()) + '\n' + \
                        '   Average words per message: ' + str(user.getWordsPerMessage()) + '\n' + \
                        '   Smarts: ' + str(((user.getLettersPerWord() + user.getWordsPerMessage())//2) % 10))
                    break
    #the trigger to print 'karma' stats
    elif message.content.lower().startswith("karma",0,5):
        if message.mentions:
            for user in message.mentions:
                for keithUser in userList:
                    if(keithUser.id == user.id):
                        await client.send_message(message.channel, content = \
                            'Username: ' + keithUser.getName() + '\n' + \
                            # '   Approximate ratio of Upvotes/Downvotes given to others: ' + str(keithUser.getRatio()) + '\n' + \
                            '   Karma: ' + str(keithUser.getKarma()))
                        break
                else:
                    await client.send_message(message.channel, content = \
                    user.name + ' is a tumblr ho and therefore has no karma yet')
        #if no one is mentioned, will print for message author
        else:
            for keithUser in userList:
                if(keithUser.id == message.author.id):
                    await client.send_message(message.channel, content = \
                            'Username: ' + keithUser.getName() + '\n' + \
                            # '   Approximate ratio of Upvotes/Downvotes given to others: ' + str(keithUser.getRatio()) + '\n' + \
                            '   Karma: ' + str(keithUser.getKarma()))
                    break

    # on bot mention
    elif message.content.lower().strip('<>@!').startswith(str(client.user.id)):
        # I ALONE CAN TRIGGER THESE
        if(message.author.id == "165582042426376192"):
            if("beat it" in message.content.lower()):
                for server in client.servers:
                    for channel in server.channels:
                        if str(channel.type) == "text":
                            async for decrementMessage in client.logs_from(channel,limit = 1000):
                                if decrementMessage.reactions:
                                    for decrementReaction in decrementMessage.reactions:
                                        if not(type(decrementReaction.emoji) is str): #checks if emoji is emoji class or str
                                            if decrementReaction.emoji.name == "upvote":
                                                for keithUser in userList:
                                                    if keithUser.id == decrementMessage.author.id:
                                                        keithUser.karma -= decrementReaction.count
                                                        print("decremented on exit")
                                            if decrementReaction.emoji.name == "downvote":
                                                for keithUser in userList:
                                                    if keithUser.id == decrementMessage.author.id:
                                                        keithUser.karma += decrementReaction.count
                                                        print("incremented on exit")
                            await client.send_message(channel, content = "I'm goin', don't mind me")

                print("Exited normaly")
                storeStats("stats.txt",userList)
                await client.logout()
        if("source" in message.content.lower()[message.content.find(' '):]):
            await client.send_message(message.channel, content = "https://github.com/keithZmudzinski/Tiger-Woods-Discord-Bot")

client.run("MjEzMDIxMjIxNTk0MzMzMTg0.Co0YOQ.k0yWmRvt7BlB_HVz9ltB8fuvUos")
<commit_msg>on_ready changes<commit_after>import discord
import asyncio
import json
from myUser import *
from helpers import *
client = discord.Client()

#Initializing list that stores myUser class members of the server
userList = []
loadStats("stats.txt",userList)

@client.event
async def on_member_ban(member):
    if member.id == client.user.id:
        for channel in member.server.channels:
            await client.send_message(channel, "You dicks")

#gets the missed messages until message.author.id == Tiger bot
@client.event
async def on_ready():
    counterMessage = 0
    print("Logged in as " + client.user.name + ", ID = " + client.user.id)
    #goes through every channel in the server, updating karma
    for server in client.servers:
        for channel in server.channels:
            async for missedMessage in client.logs_from(channel,limit = 2000):
                client.messages.append(missedMessage)

    for server in client.servers:
        for channel in server.channels:
            if str(channel.type) == "text":
                #goes though past 1000 messages, until message.author.id == Tiger
                async for missedMessage in client.logs_from(channel,limit = 1000):
                    # counterMessage = missedMessage
                    if missedMessage.author.id != client.user.id:
                        print("Added " + missedMessage.author.name +"\'s message")
                        addMessage(userList, missedMessage.author.id, missedMessage.author.name,\
                         missedMessage.content)
                         #if the message has reactions
                        if missedMessage.reactions:
                            for missedReaction in missedMessage.reactions:
                                if not(type(missedReaction.emoji) is str): #checks if emoji is emoji class or str

                                    if missedReaction.emoji.name == "upvote":
                                        upvoteList = await client.get_reaction_users(missedReaction)
                                        #goes through people who reacted with upvote, updates their upvotesGiven
                                        for upvotingUsers in upvoteList:
                                            #if someone upvoted their own post
                                            if upvotingUsers.id == missedMessage.author.id:
                                                for member in missedReaction.emoji.server.members:
                                                    if member.id == missedMessage.author.id:
                                                        await client.remove_reaction(missedReaction.message, missedReaction.emoji, member)
                                            else:
                                                updateUpvotes(userList, upvotingUsers.id, upvotingUsers.name, 'added')
                                        updateKarma(userList,missedMessage.author.id,\
                                        missedMessage.author.name,"upvote",missedReaction.count)

                                    if missedReaction.emoji.name == "downvote":
                                        upvoteList = await client.get_reaction_users(missedReaction)
                                        #goes through people who reacted with downvote, updates thier downvotes given
                                        for upvotingUsers in upvoteList:
                                            #if someone downvoted their own post
                                            if upvotingUsers.id == missedMessage.author.id:
                                                for member in missedReaction.emoji.server.members:
                                                    if member.id == missedMessage.author.id:
                                                        await client.remove_reaction(missedReaction.message, missedReaction.emoji, member)
                                            else:
                                                updateDownvotes(userList, upvotingUsers.id, upvotingUsers.name, 'added')
                                        updateKarma(userList, missedMessage.author.id, \
                                        missedMessage.author.name, "downvote", missedReaction.count)
                    else:
                        break
    for server in client.servers:
        for channel in server.channels:
            if str(channel.type) == "text":
                #finds the most recent message from Tiger
                counterMessage = discord.utils.get(client.messages, id = "213021221594333184")
                #counts the last 1000 messages made before Tiger last exited, undos the exit decrements with updated information
                async for incrementMessage in client.logs_from(channel,limit = 1000, before = counterMessage):
                    if incrementMessage.reactions:
                        for incrementReaction in incrementMessage.reactions:
                            if not(type(incrementReaction.emoji) is str): #checks if emoji is emoji class or str
                                if incrementReaction.emoji.name == "upvote":
                                    upvoteList = await client.get_reaction_users(incrementReaction)
                                    #goes through people who reacted with upvote, updates their upvotesGiven
                                    for upvotingUsers in upvoteList:
                                        #if someone upvoted their own post
                                        if upvotingUsers.id == incrementMessage.author.id:
                                            for member in incrementReaction.emoji.server.members:
                                                if member.id == incrementMessage.author.id:
                                                    await client.remove_reaction(incrementReaction.message, incrementReaction.emoji, member)
                                    updateKarma(userList, incrementMessage.author.id, incrementMessage.author.name, "upvote", incrementReaction.count)
                                    print(incrementMessage.author.name + " incremented on enter")
                                if incrementReaction.emoji.name == "downvote":
                                    downvoteList = await client.get_reaction_users(incrementReaction)
                                    #goes through people who reacted with downvote, updates thier downvotes given
                                    for downvotingUsers in downvoteList:
                                        #if someone downvoted their own post
                                        if downvotingUsers.id == incrementMessage.author.id:
                                            for member in incrementReaction.emoji.server.members:
                                                if member.id == incrementMessage.author.id:
                                                    await client.remove_reaction(incrementReaction.message, incrementReaction.emoji, member)
                                    updateKarma(userList, incrementMessage.author.id, incrementMessage.author.name, "downvote", incrementReaction.count)
                                    print(incrementMessage.author.name + " decremented on enter")
    print("Done Adding messages")

@client.event
async def on_reaction_add(reaction,user):
    if(reaction.emoji.name == "upvote"):
        print("upvote given")
        updateUpvotes(userList,user.id, user.name, "added")
        updateKarma(userList,reaction.message.author.id,reaction.message.author.name,"upvote")
        # prevents users from upvoting/downvoting themselves
        if(reaction.message.author.id == user.id):
            for member in reaction.emoji.server.members:
                if member.id == user.id:
                    await client.remove_reaction(reaction.message, reaction.emoji, member)
                    break

    elif reaction.emoji.name == "downvote":
        print("downvote given")
        updateDownvotes(userList,user.id,user.name, "added")
        updateKarma(userList,reaction.message.author.id,reaction.message.author.name,"downvote")
        # prevents users from upvoting/downvoting themselves
        if(reaction.message.author.id == user.id):
            for member in reaction.emoji.server.members:
                if member.id == user.id:
                    await client.remove_reaction(reaction.message, reaction.emoji, member)
                    break

@client.event
async def on_reaction_remove(reaction,user):
    if(reaction.emoji.name == "upvote"):
        print("upvote removed")
        updateUpvotes(userList, user.id, user.name, "removed")
        updateKarma(userList,reaction.message.author.id,reaction.message.author.name,"downvote")

    elif reaction.emoji.name == "downvote":
        print("downvote removed")
        updateDownvotes(userList,user.id, user.name, "removed")
        updateKarma(userList,reaction.message.author.id, reaction.message.author.name, "upvote")

@client.event
async def on_message(message):
    #runs on every message, updates users 'smarts' statistics
    for user in userList:
        if(user.id == message.author.id):
            user.addContent(message.content)
            break
    else:
        userList.append(myUser(message.author.id, message.author.name))
        userList[-1].addContent(message.content)

    if "why tiger" in message.content.lower() or "tiger why did you beat your wife?"\
     in message.content.lower() or "tiger why did you beat your wife" in message.content.lower():
        await client.send_message(message.channel, "Not because I\'m bad at golf!")
    #the trigger to print 'smarts' stats
    elif message.content.lower().startswith("smarts", 0, 6):
        #if message contains @notificaions, will print for all mentioned
        if message.mentions:
            for user in message.mentions:
                for user1 in userList:
                    if(user1.id == user.id):
                        await client.send_message(message.channel, content = \
                            'Username: ' + user1.getName() + '\n' + \
                            '   Average letters per word: ' + str(user1.getLettersPerWord()) + '\n' + \
                            '   Average words per message: ' + str(user1.getWordsPerMessage()) + '\n' + \
                            '   Smarts: ' + str(((user1.getLettersPerWord() + user1.getWordsPerMessage())//2) % 10))
                        break
                else:
                    await client.send_message(message.channel, content = user.name + ' is apparently too dumb to have typed anything.')
        #if no one is mentioned, will print for message author
        else:
            for user in userList:
                if(user.id == message.author.id):
                    await client.send_message(message.channel, content = \
                        'Username: ' + user.getName() + '\n' + \
                        '   Average letters per word: ' + str(user.getLettersPerWord()) + '\n' + \
                        '   Average words per message: ' + str(user.getWordsPerMessage()) + '\n' + \
                        '   Smarts: ' + str(((user.getLettersPerWord() + user.getWordsPerMessage())//2) % 10))
                    break
    #the trigger to print 'karma' stats
    elif message.content.lower().startswith("karma",0,5):
        if message.mentions:
            for user in message.mentions:
                for keithUser in userList:
                    if(keithUser.id == user.id):
                        await client.send_message(message.channel, content = \
                            'Username: ' + keithUser.getName() + '\n' + \
                            # '   Approximate ratio of Upvotes/Downvotes given to others: ' + str(keithUser.getRatio()) + '\n' + \
                            '   Karma: ' + str(keithUser.getKarma()))
                        break
                else:
                    await client.send_message(message.channel, content = \
                    user.name + ' is a tumblr ho and therefore has no karma yet')
        #if no one is mentioned, will print for message author
        else:
            for keithUser in userList:
                if(keithUser.id == message.author.id):
                    await client.send_message(message.channel, content = \
                            'Username: ' + keithUser.getName() + '\n' + \
                            # '   Approximate ratio of Upvotes/Downvotes given to others: ' + str(keithUser.getRatio()) + '\n' + \
                            '   Karma: ' + str(keithUser.getKarma()))
                    break

    # on bot mention
    elif message.content.lower().strip('<>@!').startswith(str(client.user.id)):
        # I ALONE CAN TRIGGER THESE
        if(message.author.id == "165582042426376192"):
            if("beat it" in message.content.lower()):
                for server in client.servers:
                    for channel in server.channels:
                        if str(channel.type) == "text":
                            async for decrementMessage in client.logs_from(channel,limit = 1000):
                                if decrementMessage.reactions:
                                    for decrementReaction in decrementMessage.reactions:
                                        if not(type(decrementReaction.emoji) is str): #checks if emoji is emoji class or str
                                            if decrementReaction.emoji.name == "upvote":
                                                for keithUser in userList:
                                                    if keithUser.id == decrementMessage.author.id:
                                                        keithUser.karma -= decrementReaction.count
                                                        print("decremented on exit")
                                            if decrementReaction.emoji.name == "downvote":
                                                for keithUser in userList:
                                                    if keithUser.id == decrementMessage.author.id:
                                                        keithUser.karma += decrementReaction.count
                                                        print("incremented on exit")
                            await client.send_message(channel, content = "I'm goin', don't mind me")

                print("Exited normaly")
                storeStats("stats.txt",userList)
                await client.logout()
        if("source" in message.content.lower()[message.content.find(' '):]):
            await client.send_message(message.channel, content = "https://github.com/keithZmudzinski/Tiger-Woods-Discord-Bot")

client.run("MjEzMDIxMjIxNTk0MzMzMTg0.Co0YOQ.k0yWmRvt7BlB_HVz9ltB8fuvUos")
<|endoftext|>


<commit_before>from tests.test_utils import recordtime

from tests.pages.rollups import sign_in_email_auth

@recordtime
def test_email_auth(driver, profile, base_url):
    # login email auth user
    sign_in_email_auth(driver, profile)
    # assert url is research mode service's dashboard
    assert driver.current_url == base_url + '/services/{}/dashboard'.format(profile.notify_research_service_id)
<commit_msg>Update tests for new dashboard URL<commit_after>from tests.test_utils import recordtime

from tests.pages.rollups import sign_in_email_auth

@recordtime
def test_email_auth(driver, profile, base_url):
    # login email auth user
    sign_in_email_auth(driver, profile)
    # assert url is research mode service's dashboard
    assert (
        driver.current_url == base_url + '/services/{}/dashboard'.format(profile.notify_research_service_id)
    ) or (
        driver.current_url == base_url + '/services/{}'.format(profile.notify_research_service_id)
    )
<|endoftext|>


<commit_before>import pytest
import xarray as xr
from xarray.core.pycompat import suppress

distributed = pytest.importorskip('distributed')
da = pytest.importorskip('dask.array')
from distributed.utils_test import cluster, loop

from xarray.tests.test_backends import create_tmp_file
from xarray.tests.test_dataset import create_test_data

from . import assert_allclose, has_scipy, has_netCDF4, has_h5netcdf

ENGINES = []
if has_scipy:
    ENGINES.append('scipy')
if has_netCDF4:
    ENGINES.append('netcdf4')
if has_h5netcdf:
    ENGINES.append('h5netcdf')

@pytest.mark.parametrize('engine', ENGINES)
def test_dask_distributed_integration_test(loop, engine):
    with cluster() as (s, _):
        with distributed.Client(('127.0.0.1', s['port']), loop=loop):
            original = create_test_data()
            with create_tmp_file() as filename:
                original.to_netcdf(filename, engine=engine)
                restored = xr.open_dataset(filename, chunks=3, engine=engine)
                assert isinstance(restored.var1.data, da.Array)
                computed = restored.compute()
                assert_allclose(original, computed)
<commit_msg>Fix dask-distributed tests (#1293)<commit_after>import pytest
import xarray as xr
from xarray.core.pycompat import suppress

distributed = pytest.importorskip('distributed')
da = pytest.importorskip('dask.array')
from distributed.utils_test import cluster, loop

from xarray.tests.test_backends import create_tmp_file, ON_WINDOWS
from xarray.tests.test_dataset import create_test_data

from . import assert_allclose, has_scipy, has_netCDF4, has_h5netcdf

ENGINES = []
if has_scipy:
    ENGINES.append('scipy')
if has_netCDF4:
    ENGINES.append('netcdf4')
if has_h5netcdf:
    ENGINES.append('h5netcdf')

@pytest.mark.parametrize('engine', ENGINES)
def test_dask_distributed_integration_test(loop, engine):
    with cluster() as (s, _):
        with distributed.Client(s['address'], loop=loop):
            original = create_test_data()
            with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename:
                original.to_netcdf(filename, engine=engine)
                with xr.open_dataset(filename, chunks=3, engine=engine) as restored:
                    assert isinstance(restored.var1.data, da.Array)
                    computed = restored.compute()
                    assert_allclose(original, computed)
<|endoftext|>


<commit_before>from telegram.ext import Updater

import infra
import methods
from config import *

updater = Updater(token=BOT_TOKEN)
j = updater.job_queue
dispatcher = updater.dispatcher

infra.logging.setup_logger(dispatcher)
infra.storage.setup_database()
infra.i18n.setup_locale()

delta = datetime.time(hour=22, tzinfo=TIMEZONE)

# TODO: Write flexible datetime
j.run_daily(methods.get_top.send_msg,
            time=datetime.time(12))

for handler in methods.METHODS:
    dispatcher.add_handler(handler)

updater.start_polling()

if LOG_CHAT_ID:
    methods.greeting.send_startup_greeting(updater.bot, LOG_CHAT_ID)

updater.idle()

infra.storage.db.close()
<commit_msg>bot: remove dispatcher variable if favor of updater.dispatcher<commit_after>from telegram.ext import Updater

import infra
import methods
from config import *

updater = Updater(token=BOT_TOKEN)
j = updater.job_queue

infra.logging.setup_logger(updater.dispatcher)
infra.storage.setup_database()
infra.i18n.setup_locale()

delta = datetime.time(hour=22, tzinfo=TIMEZONE)

# TODO: Write flexible datetime
j.run_daily(methods.get_top.send_msg,
            time=datetime.time(12))

for handler in methods.METHODS:
    updater.dispatcher.add_handler(handler)

updater.start_polling()

if LOG_CHAT_ID:
    methods.greeting.send_startup_greeting(updater.bot, LOG_CHAT_ID)

updater.idle()

infra.storage.db.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""This module runs integration tests against the Cloudfeaster
docker image which is typically produced as part of the CI process.
"""

import json
import os
import subprocess
import tempfile
import unittest

import jsonschema
from nose.util import safe_str

class IntegrationTestCase(unittest.TestCase):

    def docker_run(self, *docker_run_args):
        stdout_filename = tempfile.mktemp()

        docker_image_name = os.environ.get('CLF_DOCKER_IMAGE', None)
        self.assertIsNotNone(docker_image_name)

        process_args = [
            'docker',
            'run',
            docker_image_name,
        ]
        process_args.extend(docker_run_args)

        exit_code = subprocess.call(
            process_args,
            stdout=open(stdout_filename, 'w+'),
            stderr=subprocess.STDOUT)

        stdout = []
        with open(stdout_filename, 'r') as fp:
            stdout += [safe_str(fp.read())]

        os.unlink(stdout_filename)

        return (exit_code, stdout)

    def assert_stdout_is_just_json_doc(self, stdout, schema=None, expected_stdout_as_json=None):
        self.assertEqual(1, len(stdout))

        try:
            stdout_as_json = json.loads(stdout[0])
        except Exception as ex:
            self.assertTrue(False, 'stdout is not a json doc - %s' % ex)

        if schema:
            try:
                jsonschema.validate(stdout_as_json, schema)
            except Exception as ex:
                self.assertTrue(False, "stdout output doesn't match jsonschema - %s" % ex)

        if expected_stdout_as_json:
            self.assertEqual(stdout_as_json, expected_stdout_as_json)

        return stdout_as_json

class SpidersDotPyIntegrationTestCase(IntegrationTestCase):

    def test_spiders_dot_py_without_samples(self):
        expected_stdout_as_json = {}

        (exit_code, stdout) = self.docker_run('spiders.py')
        self.assertEqual(exit_code, 0)
        self.assert_stdout_is_just_json_doc(stdout, expected_stdout_as_json=expected_stdout_as_json)

    def test_spiders_dot_py_with_samples(self):
        expected_stdout_as_json = {
          "cloudfeaster.samples.pypi.PyPISpider": {
            "url": "https://pypi.python.org/pypi",
            "authenticating_factors": {
              "password": {
                "pattern": "^.+$"
              }
            },
            "identifying_factors": {
              "username": {
                "pattern": "^.+$"
              }
            },
            "factor_display_order": [
              "username",
              "password"
            ],
            "factor_display_names": {
              "username": {
                "": "username",
                "fr": "Nom d'utilisateur",
                "en": "username",
                "ja": u"ユーザー名"
              },
              "password": {
                "": "password",
                "fr": "mot de passe",
                "en": "Password",
                "ja": u"パスワード"
              }
            },
            "max_concurrent_crawls": 3,
            "max_crawl_time_in_seconds": 30,
            "paranoia_level": "low",
            "ttl_in_seconds": 60
          },
          "cloudfeaster.samples.pythonwheels_spider.PythonWheelsSpider": {
            "max_concurrent_crawls": 3,
            "authenticating_factors": {},
            "url": "https://pythonwheels.com/",
            "factor_display_order": [],
            "max_crawl_time_in_seconds": 30,
            "identifying_factors": {},
            "factor_display_names": {},
            "paranoia_level": "low",
            "ttl_in_seconds": 60
          },
          "cloudfeaster.samples.bank_of_canada_daily_exchange_rates.BankOfCanadaDailyExchangeRatesSpider": {
            "max_concurrent_crawls": 3,
            "authenticating_factors": {},
            "url": "http://www.bankofcanada.ca/rates/exchange/daily-exchange-rates/",
            "factor_display_order": [],
            "max_crawl_time_in_seconds": 30,
            "identifying_factors": {},
            "factor_display_names": {},
            "paranoia_level": "low",
            "ttl_in_seconds": 60
          }
        }

        (exit_code, stdout) = self.docker_run(
            'spiders.py',
            '--samples')
        self.assertEqual(exit_code, 0)
        self.assert_stdout_is_just_json_doc(stdout, expected_stdout_as_json=expected_stdout_as_json)

class SamplesIntegrationTestCase(IntegrationTestCase):

    def test_spider_host_dot_py_on_boc_fx_rate(self):
        schema = {
            '$schema': 'http://json-schema.org/draft-04/schema#',
            'title': 'validate spider output',
            'description': 'validate spider output',
            'type': 'object',
            'properties': {
                'rates_on': {
                    'type': 'string',
                    # 2018-02-07
                    'pattern': r'^\d{4}\-\d{2}\-\d{2}$',
                },
                'rates': {
                    'patternProperties': {
                        '^.+$': {
                            'type': 'number',
                        },
                    },
                    'additionalProperties': False,
                },
                '_status': {
                    'type': 'string',
                    'pattern': 'Ok',
                },
                '_status_code': {
                    'type': 'integer',
                    'minimum': 0,
                    'maximum': 0,
                },
                '_crawl_time': {
                    'type': 'string',
                    # 2018-02-07T23:52:46.266252+00:00
                    'pattern': r'^\d{4}\-\d{2}\-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}\+\d{2}\:\d{2}$',
                },
                '_crawl_time_in_ms': {
                    'type': 'integer',
                    'minimum': 1,
                },
                '_spider': {
                    'type': 'object',
                    'properties': {
                        'version': {
                            'type': 'string',
                            # sha1:2a9793fbec2b6bab331ab2994607b03deedd2193
                            'pattern': r'^sha1:[a-f0-9]{40}$',
                        },
                        'name': {
                            'type': 'string',
                            'pattern': (
                                'cloudfeaster.samples.bank_of_canada_daily_exchange_rates.'
                                'BankOfCanadaDailyExchangeRatesSpider'
                            ),
                        },
                    },
                    'required': [
                        'version',
                        'name',
                    ],
                    'additionalProperties': False,
                },
            },
            'required': [
                'rates_on',
                '_status',
                '_status_code',
                '_crawl_time',
                '_crawl_time_in_ms',
            ],
            'additionalProperties': False,
        }

        (exit_code, stdout) = self.docker_run(
            'spiderhost.py',
            'cloudfeaster.samples.bank_of_canada_daily_exchange_rates.BankOfCanadaDailyExchangeRatesSpider')
        self.assertEqual(exit_code, 0)
        self.assert_stdout_is_just_json_doc(stdout, schema=schema)
<commit_msg>make int tests work on xenial with seccomp<commit_after># -*- coding: utf-8 -*-
"""This module runs integration tests against the Cloudfeaster
docker image which is typically produced as part of the CI process.
"""

import json
import os
import subprocess
import tempfile
import unittest

import jsonschema
from nose.util import safe_str

class IntegrationTestCase(unittest.TestCase):

    def docker_run(self, *docker_run_args):
        stdout_filename = tempfile.mktemp()

        docker_image_name = os.environ.get('CLF_DOCKER_IMAGE', None)
        self.assertIsNotNone(docker_image_name)

        process_args = [
            'docker',
            'run',
            '--rm',
            # After upgrading to Xenial these integration tests
            # started failing with "Chrome failed to start: crashed"
            # after some Googling the problem was traced to Xenial
            # enabling Secure Computing Mode (secomp) for docker
            # containers where Trusty did not have seccomp capability.
            # The referenced issue below is the one that provided the
            # best guidance on how to solve the problem ie. disabling
            # seccomp which is what the next two process args do.
            #
            # https://github.com/SeleniumHQ/docker-selenium/issues/403
            '--security-opt',
            'seccomp:unconfined',
            docker_image_name,
        ]
        process_args.extend(docker_run_args)

        exit_code = subprocess.call(
            process_args,
            stdout=open(stdout_filename, 'w+'),
            stderr=subprocess.STDOUT)

        stdout = []
        with open(stdout_filename, 'r') as fp:
            stdout += [safe_str(fp.read())]

        os.unlink(stdout_filename)

        return (exit_code, stdout)

    def assert_stdout_is_just_json_doc(self, stdout, schema=None, expected_stdout_as_json=None):
        self.assertEqual(1, len(stdout))

        try:
            stdout_as_json = json.loads(stdout[0])
        except Exception as ex:
            self.assertTrue(False, 'stdout is not a json doc - %s' % ex)

        if schema:
            try:
                jsonschema.validate(stdout_as_json, schema)
            except Exception as ex:
                self.assertTrue(False, "stdout output doesn't match jsonschema - %s" % ex)

        if expected_stdout_as_json:
            self.assertEqual(stdout_as_json, expected_stdout_as_json)

        return stdout_as_json

class SpidersDotPyIntegrationTestCase(IntegrationTestCase):

    def test_spiders_dot_py_without_samples(self):
        expected_stdout_as_json = {}

        (exit_code, stdout) = self.docker_run('spiders.py')
        self.assertEqual(exit_code, 0)
        self.assert_stdout_is_just_json_doc(stdout, expected_stdout_as_json=expected_stdout_as_json)

    def test_spiders_dot_py_with_samples(self):
        expected_stdout_as_json = {
          "cloudfeaster.samples.pypi.PyPISpider": {
            "url": "https://pypi.python.org/pypi",
            "authenticating_factors": {
              "password": {
                "pattern": "^.+$"
              }
            },
            "identifying_factors": {
              "username": {
                "pattern": "^.+$"
              }
            },
            "factor_display_order": [
              "username",
              "password"
            ],
            "factor_display_names": {
              "username": {
                "": "username",
                "fr": "Nom d'utilisateur",
                "en": "username",
                "ja": u"ユーザー名"
              },
              "password": {
                "": "password",
                "fr": "mot de passe",
                "en": "Password",
                "ja": u"パスワード"
              }
            },
            "max_concurrent_crawls": 3,
            "max_crawl_time_in_seconds": 30,
            "paranoia_level": "low",
            "ttl_in_seconds": 60
          },
          "cloudfeaster.samples.pythonwheels_spider.PythonWheelsSpider": {
            "max_concurrent_crawls": 3,
            "authenticating_factors": {},
            "url": "https://pythonwheels.com/",
            "factor_display_order": [],
            "max_crawl_time_in_seconds": 30,
            "identifying_factors": {},
            "factor_display_names": {},
            "paranoia_level": "low",
            "ttl_in_seconds": 60
          },
          "cloudfeaster.samples.bank_of_canada_daily_exchange_rates.BankOfCanadaDailyExchangeRatesSpider": {
            "max_concurrent_crawls": 3,
            "authenticating_factors": {},
            "url": "http://www.bankofcanada.ca/rates/exchange/daily-exchange-rates/",
            "factor_display_order": [],
            "max_crawl_time_in_seconds": 30,
            "identifying_factors": {},
            "factor_display_names": {},
            "paranoia_level": "low",
            "ttl_in_seconds": 60
          }
        }

        (exit_code, stdout) = self.docker_run(
            'spiders.py',
            '--samples')
        self.assertEqual(exit_code, 0)
        self.assert_stdout_is_just_json_doc(stdout, expected_stdout_as_json=expected_stdout_as_json)

class SamplesIntegrationTestCase(IntegrationTestCase):

    def test_spider_host_dot_py_on_boc_fx_rate(self):
        schema = {
            '$schema': 'http://json-schema.org/draft-04/schema#',
            'title': 'validate spider output',
            'description': 'validate spider output',
            'type': 'object',
            'properties': {
                'rates_on': {
                    'type': 'string',
                    # 2018-02-07
                    'pattern': r'^\d{4}\-\d{2}\-\d{2}$',
                },
                'rates': {
                    'patternProperties': {
                        '^.+$': {
                            'type': 'number',
                        },
                    },
                    'additionalProperties': False,
                },
                '_status': {
                    'type': 'string',
                    'pattern': 'Ok',
                },
                '_status_code': {
                    'type': 'integer',
                    'minimum': 0,
                    'maximum': 0,
                },
                '_crawl_time': {
                    'type': 'string',
                    # 2018-02-07T23:52:46.266252+00:00
                    'pattern': r'^\d{4}\-\d{2}\-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}\+\d{2}\:\d{2}$',
                },
                '_crawl_time_in_ms': {
                    'type': 'integer',
                    'minimum': 1,
                },
                '_spider': {
                    'type': 'object',
                    'properties': {
                        'version': {
                            'type': 'string',
                            # sha1:2a9793fbec2b6bab331ab2994607b03deedd2193
                            'pattern': r'^sha1:[a-f0-9]{40}$',
                        },
                        'name': {
                            'type': 'string',
                            'pattern': (
                                'cloudfeaster.samples.bank_of_canada_daily_exchange_rates.'
                                'BankOfCanadaDailyExchangeRatesSpider'
                            ),
                        },
                    },
                    'required': [
                        'version',
                        'name',
                    ],
                    'additionalProperties': False,
                },
            },
            'required': [
                'rates_on',
                '_status',
                '_status_code',
                '_crawl_time',
                '_crawl_time_in_ms',
            ],
            'additionalProperties': False,
        }

        (exit_code, stdout) = self.docker_run(
            'spiderhost.py',
            'cloudfeaster.samples.bank_of_canada_daily_exchange_rates.BankOfCanadaDailyExchangeRatesSpider')
        self.assertEqual(exit_code, 0)
        self.assert_stdout_is_just_json_doc(stdout, schema=schema)
<|endoftext|>


<commit_before>""" A pipeline that generates analysis pipelines.

author: Brian Schrader
since: 2015-12-22
"""

from __future__ import print_function
import argparse, pickle, sys

import pyparsing

from queuemanager import Queue
from parser import Parser
from models import Command, LocalJob
from template import make_script

def main():
    """ Given a config file, spit out the script to run the analysis. """
    parser = argparse.ArgumentParser(
        description='A pipeline that generates analysis pipelines.')
    parser.add_argument('input',
                   help='A valid metapipe configuration file.')
    parser.add_argument('-o', '--output',
                   help='An output destination. If none is provided, the results will be printed to stdout.', default=sys.stdout)
    parser.add_argument('-t', '--temp',
                   help='A desired metapipe binary file. This is used to store temp data between generation and execution. (Default: "%(default)s")', default='.metapipe')
    parser.add_argument('-s', '--shell',
                   help='The path to the shell to be used when executing the pipeline. (Default: "%(default)s)"', default='/bin/bash')
    parser.add_argument('-r', '--run',
                   help='Run the pipeline as soon as it\'s ready.', action='store_true')
    args = parser.parse_args()

    try:
        with open(args.input) as f:
            config = f.read()
    except IOError:
        print('No valid config file found.')
        return -1

    parser = Parser(config)

    try:
        commands = parser.consume()
    except ValueError as e:
        print('Syntax Error: Invalid config file. \n%s' % e)
        return

    pipeline = Queue()
    for cmd in commands:
        job = LocalJob(alias=cmd.alias, command=cmd)
        pipeline.push(job)

    with open(args.temp, 'wb') as f:
        pickle.dump(pipeline, f)

    script = make_script(temp=args.temp, shell=args.shell)

    if args.run:
        print('Initiating pipeline...')
        pass

    # TODO: Add the scripts to the output file in the comments.
    try:
        f = open(args.output, 'w')
        args.output = f
    except TypeError:
        pass

    args.output.write(script)
    f.close()

if __name__ == '__main__':
    main()
<commit_msg>Adds option for pbs.<commit_after>""" A pipeline that generates analysis pipelines.

author: Brian Schrader
since: 2015-12-22
"""

from __future__ import print_function
import argparse, pickle, sys

import pyparsing

from queuemanager import Queue
from parser import Parser
from models import Command, LocalJob, PBSJob
from template import make_script

def main():
    """ Given a config file, spit out the script to run the analysis. """
    parser = argparse.ArgumentParser(
        description='A pipeline that generates analysis pipelines.')
    parser.add_argument('input',
                   help='A valid metapipe configuration file.')
    parser.add_argument('-o', '--output',
                   help='An output destination. If none is provided, the results will be printed to stdout.', default=sys.stdout)
    parser.add_argument('-t', '--temp',
                   help='A desired metapipe binary file. This is used to store temp data between generation and execution. (Default: "%(default)s")', default='.metapipe')
    parser.add_argument('-s', '--shell',
                   help='The path to the shell to be used when executing the pipeline. (Default: "%(default)s)"', default='/bin/bash')
#     parser.add_argument('-r', '--run',
#                    help='Run the pipeline as soon as it\'s ready.', action='store_true')
    parser.add_argument('-j', '--job-type',
                   help='The destination for calculations (i.e. local, a PBS ' 'queue on a cluster, etc).\n'
                   'Options: local, pbs. (Default: "%(default)s)"',
                   default='local')
    args = parser.parse_args()

    try:
        with open(args.input) as f:
            config = f.read()
    except IOError:
        print('No valid config file found.')
        return -1

    parser = Parser(config)

    try:
        commands = parser.consume()
    except ValueError as e:
        print('Syntax Error: Invalid config file. \n%s' % e)
        return

    pipeline = Queue()
    for cmd in commands:
        if args.job_type == 'pbs':
            job = LocalJob(alias=cmd.alias, command=cmd)
        else:
            job = PBSJob(alias=cmd.alias, command=cmd)
        pipeline.push(job)

    with open(args.temp, 'wb') as f:
        pickle.dump(pipeline, f)

    script = make_script(temp=args.temp, shell=args.shell)

#     if args.run:
#         print('Initiating pipeline...')

    # TODO: Add the scripts to the output file in the comments.
    try:
        f = open(args.output, 'w')
        args.output = f
    except TypeError:
        pass

    args.output.write(script)
    f.close()

if __name__ == '__main__':
    main()
<|endoftext|>