<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import pywikibot
import re
import collections
from klasa import *

def czescimowy(data):

    data_slownie = data[6] + data[7] + '.' + data[4] + data[5] + '.' + data[0] + data[1] + data[2] + data[3]
    site = pywikibot.getSite()
    allowedPage = pywikibot.Page(site, 'Wikipedysta:AlkamidBot/części_mowy/dozwolone')
    outputPage = pywikibot.Page(site, 'Wikipedysta:AlkamidBot/części_mowy/wszystkie')
    allowedPageText = allowedPage.get()
    tempListAllowed = allowedPageText.split('\n')
    allowedParts = {}
    cnt = 1
    for elem in tempListAllowed:
        if elem == '|-':
            mykey = None
            cnt = 1
        if cnt == 1:
            cnt += 1
        elif cnt == 2:
            mykey = elem[1:]
            allowedParts[mykey] = []
            cnt += 1
        elif cnt == 3:
            if elem[1:] != '':
                tmp = elem[1:].split(', ')
                for a in tmp:
                    allowedParts[mykey].append(a)
            cnt = 1

    lista = getListFromXML(data)

    re_przyslowie = re.compile(r'\'\'{{przysłowie .*?}}\'\'$')
    re_forma = re.compile(r'(\'\'|){{forma (czasownika|rzeczownika|przymiotnika|zaimka|liczebnika|rodzajnika|przysłówka)\|[a-z]*?}}(\'\'|)$')
    re_morfem = re.compile(r'\'\'{{morfem\|[a-z]*?(}}|\|(przyrostek|przedrostek|przyrostkowy|przedrostkowy)}})\'\'$')
    re_ref = re.compile(r'<ref.*?(</ref>|/>)')

    # depending on the position of <ref> in POS, it might leave behind
    # four apostrophes (e.g. in "vergehen"). The regex below is to remove
    # them, but only if they are not preceded/followed by other apostrophes
    # (that would break reflective words, e.g. "apunhalar")
    re_apostrophes = re.compile(r'([^\'])\'\'\s*\'\'([^\'])')

    re_spacje = re.compile(r'\'\'(.*?)\'\'$')
    re_zwrotnyFr = re.compile(r'\'\'\'\'\'(s\'|se ).*?\'\'\', czasownik zwrotny\'\'$')
    re_zwrotny = re.compile(r'\'\'czasownik zwrotny \'\'\'(.*?)(\'\'\'|\')$')
    re_nieprzechodni_dk_ndk = re.compile(r'\'\'czasownik( (nie|)przechodni|)( (nie|)dokonany|)( lub (nie|)dokonany|)\'\' \({{(n|)dk}} (\[\[(.*?)\]\]|\'\'brak\'\')\)$') #potem można usunąć | z "przechodni" i "dokonany" - wymóg, by wszystkie czasowniki posiadały informację o przechodniości i aspekcie
    re_zwrotny_dk_ndk = re.compile(r'\'\'czasownik zwrotny( (nie|)dokonany|)( lub (nie|)dokonany|) \'\'\'(.*?)(się|sobie)\'\'\'\'\' \({{(n|)dk}} (\[\[(.*?)(się|sobie)\]\]|\'\'brak\'\')\)$')
    re_zwrotny_sie_in_title = re.compile(r'\'\'czasownik zwrotny( (nie|)dokonany|)( lub (nie|)dokonany|)\'\' \({{(n|)dk}} (\[\[(.*?)(się|sobie)\]\]|\'\'brak\'\')\)$')

    final = ''
    final_input = ''
    tabelka = '{| class="wikitable"\n!nazwa części mowy\n!przykładowe hasła\n!języki, w których występuje'
    lista_na_wiki = collections.defaultdict()

    pretext = 'W poniższych hasłach trzeba poprawić nazwę części mowy. Jeśli znajduje się tutaj poprawny opis części mowy, można go dopisać na [[Wikipedysta:AlkamidBot/części_mowy/dozwolone|listę dozwolonych]]. Jeśli jakiś błąd powtarza się zbyt często, można dopisać go na [[Wikipedysta:AlkamidBot/części_mowy/zamiana|listę automatycznej zamiany]] lub zgłosić to [[Dyskusja wikipedysty:Alkamid|Alkamidowi]]. Ostatnia aktualizacja wg zrzutu bazy danych z %s.\n' % (data_slownie)

    for a in lista:
        ifex = 0
        try: h = Haslo(a)
        except sectionsNotFound:
            pass
        except WrongHeader:
            pass
        else:
            if h.type == 3:
                for c in h.listLangs:
                    c.pola()
                    if c.type == 1:
                        for d in c.znaczeniaDetail:
                            found = 0
                            d[0] = re.sub(re_ref, '', d[0])
                            d[0] = re.sub(re_apostrophes, r'\1\2', d[0])
                            #generowanie tabelki na wiki
                            s_spacje = re.search(re_spacje, d[0])
                            temp = d[0]

                            if s_spacje:
                                temp = '\'\'%s\'\'' % s_spacje.group(1).strip()

                            if temp in allowedParts:
                                if len(allowedParts[temp]) == 0 or c.lang in allowedParts[temp]:
                                    found = 1

                            s_przyslowie = re.search(re_przyslowie, temp)
                            s_forma = re.search(re_forma, temp)
                            s_morfem = re.search(re_morfem, temp)
                            s_zwrotnyFr = re.search(re_zwrotnyFr, temp)
                            s_zwrotny = re.search(re_zwrotny, temp)
                            s_nieprzechodni_dk_ndk = re.search(re_nieprzechodni_dk_ndk, temp)
                            s_zwrotny_dk_ndk = re.search(re_zwrotny_dk_ndk, temp)
                            if 'się' in a.title:
                                s_zwrotny_sie_in_title = re.search(re_zwrotny_sie_in_title, temp)
                            else:
                                s_zwrotny_sie_in_title = 0

                            if not found and not s_przyslowie and not s_forma and not s_morfem and not s_zwrotny and not s_zwrotnyFr and not s_zwrotny_dk_ndk and not s_nieprzechodni_dk_ndk and not s_zwrotny_sie_in_title:
                                try: lista_na_wiki[temp]
                                except KeyError:
                                    lista_na_wiki[temp] = collections.defaultdict()
                                    lista_na_wiki[temp]['samples'] = []
                                    lista_na_wiki[temp]['langs'] = []
                                    if h.title not in lista_na_wiki[temp]['samples']:
                                        lista_na_wiki[temp]['samples'].append(h.title)
                                    if c.lang not in lista_na_wiki[temp]['langs']:
                                        lista_na_wiki[temp]['langs'].append(c.lang)
                                else:
                                    if h.title not in lista_na_wiki[temp]['samples']:
                                        lista_na_wiki[temp]['samples'].append(h.title)
                                    if c.lang not in lista_na_wiki[temp]['langs']:
                                        lista_na_wiki[temp]['langs'].append(c.lang)
                                ifex = 1
                if ifex:
                    final_input += '\n%s' % (h.title)

    for key in sorted(lista_na_wiki.keys()):
        tabelka += '\n|-\n|%s\n|' % key
        i = 0
        for a in lista_na_wiki[key]['samples']:
            if i<10:
                tabelka += '[[%s]]' % a
                if i<9:
                    tabelka += ', '
                i += 1
        tabelka += '\n|'
        if len(lista_na_wiki[key]['langs'])<10:
            for b in lista_na_wiki[key]['langs']:
                tabelka += '%s, ' % b

    tabelka += '\n|}'
    '''
    outputPage.text = pretext + '\n' + tabelka
    outputPage.save(comment="Aktualizacja listy", botflag=False)
    '''
    with open("output/czescimowy_tabelka.txt", encoding='utf-8', mode='w') as f:
        f.write(tabelka)
    with open("output/czescimowy_input.txt", encoding='utf-8', mode='w') as f:
        f.write(final_input)

<commit_msg>apostrophes sorted out<commit_after># -*- coding: utf-8 -*-

import pywikibot
import re
import collections
from klasa import *

def czescimowy(data):

    data_slownie = data[6:8] + '.' + data[4:6] + '.' + data[0:4]
    site = pywikibot.getSite()
    allowedPage = pywikibot.Page(site, 'Wikipedysta:AlkamidBot/części_mowy/dozwolone')
    outputPage = pywikibot.Page(site, 'Wikipedysta:AlkamidBot/części_mowy/wszystkie')
    allowedPageText = allowedPage.get()
    tempListAllowed = allowedPageText.split('\n')
    allowedParts = {}
    cnt = 1
    for elem in tempListAllowed:
        if elem == '|-':
            mykey = None
            cnt = 1
        if cnt == 1:
            cnt += 1
        elif cnt == 2:
            mykey = elem[1:]
            allowedParts[mykey] = []
            cnt += 1
        elif cnt == 3:
            if elem[1:] != '':
                tmp = elem[1:].split(', ')
                for a in tmp:
                    allowedParts[mykey].append(a)
            cnt = 1

    lista = getListFromXML(data)

    re_przyslowie = re.compile(r'\'\'{{przysłowie .*?}}\'\'$')
    re_forma = re.compile(r'(\'\'|){{forma (czasownika|rzeczownika|przymiotnika|zaimka|liczebnika|rodzajnika|przysłówka)\|[a-z]*?}}(\'\'|)$')
    re_morfem = re.compile(r'\'\'{{morfem\|[a-z]*?(}}|\|(przyrostek|przedrostek|przyrostkowy|przedrostkowy)}})\'\'$')
    re_ref = re.compile(r'<ref.*?(</ref>|/>)')

    # depending on the position of <ref> in POS, it might leave behind
    # four apostrophes (e.g. in "vergehen"). The regex below is to remove
    # them, but only if they are not preceded/followed by other apostrophes
    # (that would break reflective words, e.g. "apunhalar")
    re_apostrophes = re.compile(r'([^\'])\'\'(\s*)\'\'([^\'])')

    re_spacje = re.compile(r'\'\'(.*?)\'\'$')
    re_zwrotnyFr = re.compile(r'\'\'\'\'\'(s\'|se ).*?\'\'\', czasownik zwrotny\'\'$')
    re_zwrotny = re.compile(r'\'\'czasownik zwrotny \'\'\'(.*?)(\'\'\'|\')$')
    re_nieprzechodni_dk_ndk = re.compile(r'\'\'czasownik( (nie|)przechodni|)( (nie|)dokonany|)( lub (nie|)dokonany|)\'\' \({{(n|)dk}} (\[\[(.*?)\]\]|\'\'brak\'\')\)$') #potem można usunąć | z "przechodni" i "dokonany" - wymóg, by wszystkie czasowniki posiadały informację o przechodniości i aspekcie
    re_zwrotny_dk_ndk = re.compile(r'\'\'czasownik zwrotny( (nie|)dokonany|)( lub (nie|)dokonany|) \'\'\'(.*?)(się|sobie)\'\'\'\'\' \({{(n|)dk}} (\[\[(.*?)(się|sobie)\]\]|\'\'brak\'\')\)$')
    re_zwrotny_sie_in_title = re.compile(r'\'\'czasownik zwrotny( (nie|)dokonany|)( lub (nie|)dokonany|)\'\' \({{(n|)dk}} (\[\[(.*?)(się|sobie)\]\]|\'\'brak\'\')\)$')

    final = ''
    final_input = ''
    tabelka = '{| class="wikitable"\n!nazwa części mowy\n!przykładowe hasła\n!języki, w których występuje'
    lista_na_wiki = collections.defaultdict()

    pretext = 'W poniższych hasłach trzeba poprawić nazwę części mowy. Jeśli znajduje się tutaj poprawny opis części mowy, można go dopisać na [[Wikipedysta:AlkamidBot/części_mowy/dozwolone|listę dozwolonych]]. Jeśli jakiś błąd powtarza się zbyt często, można dopisać go na [[Wikipedysta:AlkamidBot/części_mowy/zamiana|listę automatycznej zamiany]] lub zgłosić to [[Dyskusja wikipedysty:Alkamid|Alkamidowi]]. Ostatnia aktualizacja wg zrzutu bazy danych z %s.\n' % (data_slownie)

    for a in lista:
        ifex = 0
        try: h = Haslo(a)
        except sectionsNotFound:
            pass
        except WrongHeader:
            pass
        else:
            if h.type == 3:
                for c in h.listLangs:
                    c.pola()
                    if c.type == 1:
                        for d in c.znaczeniaDetail:
                            found = 0
                            d[0] = re.sub(re_ref, '', d[0])
                            d[0] = re.sub(re_apostrophes, r'\1\2\3', d[0])
                            #generowanie tabelki na wiki
                            s_spacje = re.search(re_spacje, d[0])
                            temp = d[0]

                            if s_spacje:
                                temp = '\'\'%s\'\'' % s_spacje.group(1).strip()

                            if temp in allowedParts:
                                if len(allowedParts[temp]) == 0 or c.lang in allowedParts[temp]:
                                    found = 1

                            s_przyslowie = re.search(re_przyslowie, temp)
                            s_forma = re.search(re_forma, temp)
                            s_morfem = re.search(re_morfem, temp)
                            s_zwrotnyFr = re.search(re_zwrotnyFr, temp)
                            s_zwrotny = re.search(re_zwrotny, temp)
                            s_nieprzechodni_dk_ndk = re.search(re_nieprzechodni_dk_ndk, temp)
                            s_zwrotny_dk_ndk = re.search(re_zwrotny_dk_ndk, temp)
                            if 'się' in a.title:
                                s_zwrotny_sie_in_title = re.search(re_zwrotny_sie_in_title, temp)
                            else:
                                s_zwrotny_sie_in_title = 0

                            if not found and not s_przyslowie and not s_forma and not s_morfem and not s_zwrotny and not s_zwrotnyFr and not s_zwrotny_dk_ndk and not s_nieprzechodni_dk_ndk and not s_zwrotny_sie_in_title:
                                try: lista_na_wiki[temp]
                                except KeyError:
                                    lista_na_wiki[temp] = collections.defaultdict()
                                    lista_na_wiki[temp]['samples'] = []
                                    lista_na_wiki[temp]['langs'] = []
                                    if h.title not in lista_na_wiki[temp]['samples']:
                                        lista_na_wiki[temp]['samples'].append(h.title)
                                    if c.lang not in lista_na_wiki[temp]['langs']:
                                        lista_na_wiki[temp]['langs'].append(c.lang)
                                else:
                                    if h.title not in lista_na_wiki[temp]['samples']:
                                        lista_na_wiki[temp]['samples'].append(h.title)
                                    if c.lang not in lista_na_wiki[temp]['langs']:
                                        lista_na_wiki[temp]['langs'].append(c.lang)
                                ifex = 1
                if ifex:
                    final_input += '\n%s' % (h.title)

    for key in sorted(lista_na_wiki.keys()):
        tabelka += '\n|-\n|%s\n|' % key
        i = 0
        for a in lista_na_wiki[key]['samples']:
            if i<10:
                tabelka += '[[%s]]' % a
                if i<9:
                    tabelka += ', '
                i += 1
        tabelka += '\n|'
        if len(lista_na_wiki[key]['langs'])<10:
            for b in lista_na_wiki[key]['langs']:
                tabelka += '%s, ' % b

    tabelka += '\n|}'

    outputPage.text = pretext + '\n' + tabelka
    outputPage.save(comment="Aktualizacja listy", botflag=False)

    with open("output/czescimowy_tabelka.txt", encoding='utf-8', mode='w') as f:
        f.write(tabelka)
    with open("output/czescimowy_input.txt", encoding='utf-8', mode='w') as f:
        f.write(final_input)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import datetime
from model.contact import Contact
import random,pytest,string

now_time = datetime.datetime.now()

def random_string(prefix,maxlen):
    symbols = string.ascii_letters + string.digits# + " "*10 #добавим 10 пробелов + string.punctuation
    return prefix + "".join([random.choice(symbols) for i in range (random.randrange(maxlen))])

def random_phone(prefix,maxlen):
    symbols = string.digits + "(" + ")" + "-"
    return prefix + "".join([random.choice(symbols) for i in range (random.randrange(maxlen))])

testdata = [Contact(firstname=random_string("firstname" , 10),
                     middlename=random_string("middlename" , 10),
                     lastname=random_string("lastname" , 10),
                     nickname=random_string("nickname" , 10),
                     title=random_string("firstname" , 10),
                     company=random_string("firstname" , 10),
                     address=random_string("firstname" , 10),
                     home=random_phone("" , 10),
                     mobile=random_phone("" , 10),
                     work=random_phone("" , 10),
                     fax=random_phone("" , 10),
                     email2=random_string("email2" , 10),
                     email3=random_string("email3" , 10),
                     homepage=random_string("homepage" , 10),
                     address2=random_string("address2" , 10),
                     phone2=random_phone("" , 10),
                     notes=random_string("notes" , 10))
             for i in range (5)
            ]

@pytest.mark.parametrize("group_param",testdata,ids=[repr(x) for x in testdata])
def test_add_contact(app,group_param):
    old_contacts = app.contact.get_contacts_list()
    #[TEST]
    app.contact.create(group_param)
    assert len(old_contacts) + 1  == app.contact.count() # хеш функция - считает количество элементов
    new_contacts = app.contact.get_contacts_list()
    old_contacts.append(group_param)
    o=sorted(old_contacts,key=Contact.id_or_max)
    n=sorted(new_contacts,key=Contact.id_or_max)
    assert sorted(old_contacts,key=Contact.id_or_max)  ==  sorted(new_contacts,key=Contact.id_or_max)

<commit_msg>Добавлена функция генерации случайного email с правилами имя@сайт.домен<commit_after># -*- coding: utf-8 -*-
import datetime
from model.contact import Contact
import random,pytest,string

now_time = datetime.datetime.now()

def random_string(prefix,maxlen):
    symbols = string.ascii_letters + string.digits# + " "*10 #добавим 10 пробелов + string.punctuation
    return prefix + "".join([random.choice(symbols) for i in range (random.randrange(maxlen))])

def random_phone(prefix,maxlen):
    symbols = string.digits + "(" + ")" + "-"
    return prefix + "".join([random.choice(symbols) for i in range (random.randrange(maxlen))])

def random_email(prefix,maxlen):
    symbols =  string.ascii_letters + string.digits
    domen=string.ascii_lowercase #Константное перечисление букв из набора ASCII в нижнем регистре.
    return prefix + ("".join([random.choice(symbols) for i in range (random.randrange(maxlen//2))]))+"@"+\
           ("".join([random.choice(symbols) for i in range (random.randrange(maxlen//2))]))+"."+\
            ("".join([random.choice(domen) for i in range (random.randint(2, 3))])) # Return a random integer N such that a <= N <= b.

testdata = [Contact(firstname=random_string("firstname" , 10),
                     middlename=random_string("middlename" , 10),
                     lastname=random_string("lastname" , 10),
                     nickname=random_string("nickname" , 10),
                     title=random_string("firstname" , 10),
                     company=random_string("firstname" , 10),
                     address=random_string("firstname" , 10),
                     home=random_phone("" , 10),
                     mobile=random_phone("" , 10),
                     work=random_phone("" , 10),
                     fax=random_phone("" , 10),
                     email2=random_email("email2" , 15),
                     email3=random_email("email3" , 15),
                     homepage=random_string("homepage" , 10),
                     address2=random_string("address2" , 10),
                     phone2=random_phone("" , 10),
                     notes=random_string("notes" , 10))
             for i in range (5)
            ]

@pytest.mark.parametrize("group_param",testdata,ids=[repr(x) for x in testdata])
def test_add_contact(app,group_param):
    old_contacts = app.contact.get_contacts_list()
    #[TEST]
    app.contact.create(group_param)
    assert len(old_contacts) + 1  == app.contact.count() # хеш функция - считает количество элементов
    new_contacts = app.contact.get_contacts_list()
    old_contacts.append(group_param)
    o=sorted(old_contacts,key=Contact.id_or_max)
    n=sorted(new_contacts,key=Contact.id_or_max)
    assert sorted(old_contacts,key=Contact.id_or_max)  ==  sorted(new_contacts,key=Contact.id_or_max)

<|endoftext|>


<commit_before>#!/usr/bin/env python

import optparse
import requests
import subprocess

from maas_common import metric, status_ok, status_err

OVERVIEW_URL = "http://%s:%s/api/overview"
NODES_URL = "http://%s:%s/api/nodes"
CLUSTERED = True
CLUSTER_SIZE = 3

OVERVIEW_METRICS = {"queue_totals": ("messages",
                                     "messages_ready",
                                     "messages_unacknowledged"),
                    "message_stats": ("get",
                                      "ack",
                                      "deliver_get",
                                      "deliver",
                                      "publish")}
NODES_METRICS = ("proc_used",
                 "proc_total",
                 "fd_used",
                 "fd_total",
                 "sockets_used",
                 "sockets_total",
                 "mem_used",
                 "mem_limit",
                 "mem_alarm",
                 "disk_free_alarm",
                 "uptime")

def hostname():
    """Return the name of the current host/node."""
    return subprocess.check_output(['hostname']).strip()

def parse_args():
    parser = optparse.OptionParser(
        usage='%prog [-h] [-H hostname] [-P port] [-u username] [-p password]'
    )
    parser.add_option('-H', '--host', action='store', dest='host',
                      default='localhost',
                      help='Host address to use when connecting')
    parser.add_option('-P', '--port', action='store', dest='port',
                      default='15672',
                      help='Port to use when connecting')
    parser.add_option('-U', '--username', action='store', dest='username',
                      default='guest',
                      help='Username to use for authentication')
    parser.add_option('-p', '--password', action='store', dest='password',
                      default='guest',
                      help='Password to use for authentication')
    parser.add_option('-n', '--name', action='store', dest='name',
                      default=None,
                      help='Hostname to check for cluster membership')
    return parser.parse_args()

def main():
    (options, _) = parse_args()
    metrics = {}
    s = requests.Session()  # Make a Session to store the authenticate creds
    s.auth = (options.username, options.password)

    try:
        r = s.get(OVERVIEW_URL % (options.host, options.port))
    except requests.exceptions.ConnectionError as e:
        status_err(str(e))

    if r.ok:
        resp_json = r.json()  # Parse the JSON once
        for k in OVERVIEW_METRICS:
            if k in resp_json:
                for i in OVERVIEW_METRICS[k]:
                    if i in resp_json[k]:
                        metrics[i] = resp_json[k][i]
    else:
        status_err('Received status {0} from RabbitMQ API'.format(
            r.status_code))

    try:
        r = s.get(NODES_URL % (options.host, options.port))
    except requests.exceptions.ConnectionError as e:
        status_err(str(e))

    # Either use the option provided by the commandline flag or the current
    # hostname
    name = options.name or hostname()
    is_cluster_member = False
    if r.ok:
        resp_json = r.json()
        for i in NODES_METRICS:
            if i in resp_json[0]:
                metrics[i] = resp_json[0][i]

        # Ensure this node is a member of the cluster
        is_cluster_member = any(name == n['name'] for n in resp_json)
        # Gather the queue lengths for all nodes in the cluster
        queues = [n['run_queue'] for n in resp_json]
        # Grab the first queue length
        first = queues.pop()
        # Check that all other queues are equal to it
        if not all(first == q for q in queues):
            # If they're not, the queues are not synchronized
            print "status err cluster not replicated across all nodes"
    else:
        status_err('Received status {0} from RabbitMQ API'.format(
            r.status_code))

    if CLUSTERED:
        if len(r.json()) < CLUSTER_SIZE:
            status_err('cluster too small')
        if not is_cluster_member:
            status_err('{0} not a member of the cluster'.format(name))
    else:
        status_ok()

    for k, v in metrics.items():
        metric(k, 'int64', v)

if __name__ == "__main__":
    main()
<commit_msg>Check the provided hostname is at the end of the node name<commit_after>#!/usr/bin/env python

import optparse
import requests
import subprocess

from maas_common import metric, status_ok, status_err

OVERVIEW_URL = "http://%s:%s/api/overview"
NODES_URL = "http://%s:%s/api/nodes"
CLUSTERED = True
CLUSTER_SIZE = 3

OVERVIEW_METRICS = {"queue_totals": ("messages",
                                     "messages_ready",
                                     "messages_unacknowledged"),
                    "message_stats": ("get",
                                      "ack",
                                      "deliver_get",
                                      "deliver",
                                      "publish")}
NODES_METRICS = ("proc_used",
                 "proc_total",
                 "fd_used",
                 "fd_total",
                 "sockets_used",
                 "sockets_total",
                 "mem_used",
                 "mem_limit",
                 "mem_alarm",
                 "disk_free_alarm",
                 "uptime")

def hostname():
    """Return the name of the current host/node."""
    return subprocess.check_output(['hostname']).strip()

def parse_args():
    parser = optparse.OptionParser(
        usage='%prog [-h] [-H hostname] [-P port] [-u username] [-p password]'
    )
    parser.add_option('-H', '--host', action='store', dest='host',
                      default='localhost',
                      help='Host address to use when connecting')
    parser.add_option('-P', '--port', action='store', dest='port',
                      default='15672',
                      help='Port to use when connecting')
    parser.add_option('-U', '--username', action='store', dest='username',
                      default='guest',
                      help='Username to use for authentication')
    parser.add_option('-p', '--password', action='store', dest='password',
                      default='guest',
                      help='Password to use for authentication')
    parser.add_option('-n', '--name', action='store', dest='name',
                      default=None,
                      help='Hostname to check for cluster membership')
    return parser.parse_args()

def main():
    (options, _) = parse_args()
    metrics = {}
    s = requests.Session()  # Make a Session to store the authenticate creds
    s.auth = (options.username, options.password)

    try:
        r = s.get(OVERVIEW_URL % (options.host, options.port))
    except requests.exceptions.ConnectionError as e:
        status_err(str(e))

    if r.ok:
        resp_json = r.json()  # Parse the JSON once
        for k in OVERVIEW_METRICS:
            if k in resp_json:
                for i in OVERVIEW_METRICS[k]:
                    if i in resp_json[k]:
                        metrics[i] = resp_json[k][i]
    else:
        status_err('Received status {0} from RabbitMQ API'.format(
            r.status_code))

    try:
        r = s.get(NODES_URL % (options.host, options.port))
    except requests.exceptions.ConnectionError as e:
        status_err(str(e))

    # Either use the option provided by the commandline flag or the current
    # hostname
    name = '@' + (options.name or hostname())
    is_cluster_member = False
    if r.ok:
        resp_json = r.json()
        for i in NODES_METRICS:
            if i in resp_json[0]:
                metrics[i] = resp_json[0][i]

        # Ensure this node is a member of the cluster
        is_cluster_member = any(n['name'].endswith(name) for n in resp_json)
        # Gather the queue lengths for all nodes in the cluster
        queues = [n['run_queue'] for n in resp_json]
        # Grab the first queue length
        first = queues.pop()
        # Check that all other queues are equal to it
        if not all(first == q for q in queues):
            # If they're not, the queues are not synchronized
            print "status err cluster not replicated across all nodes"
    else:
        status_err('Received status {0} from RabbitMQ API'.format(
            r.status_code))

    if CLUSTERED:
        if len(r.json()) < CLUSTER_SIZE:
            status_err('cluster too small')
        if not is_cluster_member:
            status_err('{0} not a member of the cluster'.format(name))

    status_ok()

    for k, v in metrics.items():
        metric(k, 'int64', v)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import locale
import re
import os
import sys
import stat
from glob import glob
from os.path import basename, join, splitext, isdir, isfile, exists
from io import open
from subprocess import call, check_call

from conda_build.config import build_prefix, build_python, PY3K
from conda_build import external
from conda_build import environ
from conda_build import utils
from conda_build import source
from conda.compat import lchmod

if sys.platform.startswith('linux'):
    from conda_build import elf
elif sys.platform == 'darwin':
    from conda_build import macho

SHEBANG_PAT = re.compile(r'^#!.+$', re.M)

def is_obj(path):
    assert sys.platform != 'win32'
    return bool((sys.platform.startswith('linux') and elf.is_elf(path)) or
                (sys.platform == 'darwin' and macho.is_macho(path)))

def fix_shebang(f, osx_is_app=False):
    path = join(build_prefix, f)
    if is_obj(path):
        return
    elif os.path.islink(path):
        return
    with open(path, encoding=locale.getpreferredencoding()) as fi:
        try:
            data = fi.read()
        except UnicodeDecodeError: # file is binary
            return
    m = SHEBANG_PAT.match(data)
    if not (m and 'python' in m.group()):
        return

    py_exec = (build_prefix + '/python.app/Contents/MacOS/python'
               if sys.platform == 'darwin' and osx_is_app else
               build_prefix + '/bin/' + basename(build_python))
    new_data = SHEBANG_PAT.sub('#!' + py_exec, data, count=1)
    if new_data == data:
        return
    print("updating shebang:", f)
    with open(path, 'w', encoding=locale.getpreferredencoding()) as fo:
        fo.write(new_data)
    os.chmod(path, int('755', 8))

def write_pth(egg_path):
    fn = basename(egg_path)
    with open(join(environ.SP_DIR,
                   '%s.pth' % (fn.split('-')[0])), 'w', encoding='utf-8') as fo:
        fo.write('./%s\n' % fn)

def remove_easy_install_pth(preserve_egg_dir=False):
    """
    remove the need for easy-install.pth and finally remove easy-install.pth
    itself
    """
    sp_dir = environ.SP_DIR
    for egg_path in glob(join(sp_dir, '*-py*.egg')):
        if isdir(egg_path):
            if preserve_egg_dir:
                write_pth(egg_path)
                continue

            print('found egg dir:', egg_path)
            try:
                os.rename(join(egg_path, 'EGG-INFO/PKG-INFO'),
                          egg_path + '-info')
            except OSError:
                pass
            utils.rm_rf(join(egg_path, 'EGG-INFO'))
            for fn in os.listdir(egg_path):
                if fn == '__pycache__':
                    utils.rm_rf(join(egg_path, fn))
                else:
                    os.rename(join(egg_path, fn), join(sp_dir, fn))

        elif isfile(egg_path):
            print('found egg:', egg_path)
            write_pth(egg_path)

    utils.rm_rf(join(sp_dir, 'easy-install.pth'))

def rm_py_along_so():
    "remove .py (.pyc) files alongside .so or .pyd files"
    for root, dirs, files in os.walk(build_prefix):
        for fn in files:
            if fn.endswith(('.so', '.pyd')):
                name, unused_ext = splitext(fn)
                for ext in '.py', '.pyc':
                    if name + ext in files:
                        os.unlink(join(root, name + ext))

def compile_missing_pyc():
    sp_dir = environ.SP_DIR

    need_compile = False
    for root, dirs, files in os.walk(sp_dir):
        for fn in files:
            if fn.endswith('.py') and fn + 'c' not in files:
                need_compile = True
    if need_compile:
        print('compiling .pyc files...')
        utils._check_call([build_python, '-Wi', join(environ.STDLIB_DIR,
                                                     'compileall.py'),
                           '-q', '-x', 'port_v3', sp_dir])

def post_process(preserve_egg_dir=False):
    remove_easy_install_pth(preserve_egg_dir=preserve_egg_dir)
    rm_py_along_so()
    if not PY3K:
        compile_missing_pyc()

def osx_ch_link(path, link):
    assert path.startswith(build_prefix + '/')
    reldir = utils.rel_lib(path[len(build_prefix) + 1:])

    if link.startswith((build_prefix + '/lib', 'lib', '@executable_path/')):
        return '@loader_path/%s/%s' % (reldir, basename(link))

    if link == '/usr/local/lib/libgcc_s.1.dylib':
        return '/usr/lib/libgcc_s.1.dylib'

def mk_relative_osx(path):
    assert sys.platform == 'darwin' and is_obj(path)
    macho.install_name_change(path, osx_ch_link)

    if path.endswith('.dylib'):
        # note that not every MachO binaries is a "dynamically linked shared
        # library" which have an identification name, a .so C extensions
        # extensions is a "bundle".  One can verify this using the "file"
        # command.
        names = macho.otool(path)
        if names:
            args = ['install_name_tool', '-id', basename(names[0]), path]
            print(' '.join(args))
            check_call(args)

    for name in macho.otool(path):
        assert not name.startswith(build_prefix), path

def mk_relative(f, binary_relocation=True):
    assert sys.platform != 'win32'
    if f.startswith('bin/'):
        fix_shebang(f)

    if not binary_relocation:
        return

    path = join(build_prefix, f)
    if sys.platform.startswith('linux') and is_obj(path):
        rpath = '$ORIGIN/' + utils.rel_lib(f)
        patchelf = external.find_executable('patchelf')
        print('patchelf: file: %s\n    setting rpath to: %s' % (path, rpath))
        call([patchelf, '--set-rpath', rpath, path])

    if sys.platform == 'darwin' and is_obj(path):
        mk_relative_osx(path)

def fix_permissions(files):
    for root, dirs, unused_files in os.walk(build_prefix):
        for dn in dirs:
            os.chmod(join(root, dn), int('755', 8))

    for f in files:
        path = join(build_prefix, f)
        st = os.lstat(path)
        lchmod(path, stat.S_IMODE(st.st_mode) | stat.S_IWUSR) # chmod u+w

def post_build(files, binary_relocation=True):
    print('number of files:', len(files))
    fix_permissions(files)
    for f in files:
        if sys.platform != 'win32':
            mk_relative(f, binary_relocation=binary_relocation)

def get_build_metadata(m):
    if exists(join(source.WORK_DIR, '__conda_version__.txt')):
        with open(join(source.WORK_DIR, '__conda_version__.txt')) as f:
            version = f.read().strip()
            print("Setting version from __conda_version__.txt: %s" % version)
            m.meta['package']['version'] = version
    if exists(join(source.WORK_DIR, '__conda_buildnum__.txt')):
        with open(join(source.WORK_DIR, '__conda_buildnum__.txt')) as f:
            build_number = f.read().strip()
            print("Setting build number from __conda_buildnum__.txt: %s" %
                  build_number)
            m.meta['build']['number'] = build_number
    if exists(join(source.WORK_DIR, '__conda_buildstr__.txt')):
        with open(join(source.WORK_DIR, '__conda_buildstr__.txt')) as f:
            buildstr = f.read().strip()
            print("Setting version from __conda_buildstr__.txt: %s" % buildstr)
            m.meta['build']['string'] = buildstr
<commit_msg>Print a message when the binary relocation logic is being skipped<commit_after>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import locale
import re
import os
import sys
import stat
from glob import glob
from os.path import basename, join, splitext, isdir, isfile, exists
from io import open
from subprocess import call, check_call

from conda_build.config import build_prefix, build_python, PY3K
from conda_build import external
from conda_build import environ
from conda_build import utils
from conda_build import source
from conda.compat import lchmod

if sys.platform.startswith('linux'):
    from conda_build import elf
elif sys.platform == 'darwin':
    from conda_build import macho

SHEBANG_PAT = re.compile(r'^#!.+$', re.M)

def is_obj(path):
    assert sys.platform != 'win32'
    return bool((sys.platform.startswith('linux') and elf.is_elf(path)) or
                (sys.platform == 'darwin' and macho.is_macho(path)))

def fix_shebang(f, osx_is_app=False):
    path = join(build_prefix, f)
    if is_obj(path):
        return
    elif os.path.islink(path):
        return
    with open(path, encoding=locale.getpreferredencoding()) as fi:
        try:
            data = fi.read()
        except UnicodeDecodeError: # file is binary
            return
    m = SHEBANG_PAT.match(data)
    if not (m and 'python' in m.group()):
        return

    py_exec = (build_prefix + '/python.app/Contents/MacOS/python'
               if sys.platform == 'darwin' and osx_is_app else
               build_prefix + '/bin/' + basename(build_python))
    new_data = SHEBANG_PAT.sub('#!' + py_exec, data, count=1)
    if new_data == data:
        return
    print("updating shebang:", f)
    with open(path, 'w', encoding=locale.getpreferredencoding()) as fo:
        fo.write(new_data)
    os.chmod(path, int('755', 8))

def write_pth(egg_path):
    fn = basename(egg_path)
    with open(join(environ.SP_DIR,
                   '%s.pth' % (fn.split('-')[0])), 'w', encoding='utf-8') as fo:
        fo.write('./%s\n' % fn)

def remove_easy_install_pth(preserve_egg_dir=False):
    """
    remove the need for easy-install.pth and finally remove easy-install.pth
    itself
    """
    sp_dir = environ.SP_DIR
    for egg_path in glob(join(sp_dir, '*-py*.egg')):
        if isdir(egg_path):
            if preserve_egg_dir:
                write_pth(egg_path)
                continue

            print('found egg dir:', egg_path)
            try:
                os.rename(join(egg_path, 'EGG-INFO/PKG-INFO'),
                          egg_path + '-info')
            except OSError:
                pass
            utils.rm_rf(join(egg_path, 'EGG-INFO'))
            for fn in os.listdir(egg_path):
                if fn == '__pycache__':
                    utils.rm_rf(join(egg_path, fn))
                else:
                    os.rename(join(egg_path, fn), join(sp_dir, fn))

        elif isfile(egg_path):
            print('found egg:', egg_path)
            write_pth(egg_path)

    utils.rm_rf(join(sp_dir, 'easy-install.pth'))

def rm_py_along_so():
    "remove .py (.pyc) files alongside .so or .pyd files"
    for root, dirs, files in os.walk(build_prefix):
        for fn in files:
            if fn.endswith(('.so', '.pyd')):
                name, unused_ext = splitext(fn)
                for ext in '.py', '.pyc':
                    if name + ext in files:
                        os.unlink(join(root, name + ext))

def compile_missing_pyc():
    sp_dir = environ.SP_DIR

    need_compile = False
    for root, dirs, files in os.walk(sp_dir):
        for fn in files:
            if fn.endswith('.py') and fn + 'c' not in files:
                need_compile = True
    if need_compile:
        print('compiling .pyc files...')
        utils._check_call([build_python, '-Wi', join(environ.STDLIB_DIR,
                                                     'compileall.py'),
                           '-q', '-x', 'port_v3', sp_dir])

def post_process(preserve_egg_dir=False):
    remove_easy_install_pth(preserve_egg_dir=preserve_egg_dir)
    rm_py_along_so()
    if not PY3K:
        compile_missing_pyc()

def osx_ch_link(path, link):
    assert path.startswith(build_prefix + '/')
    reldir = utils.rel_lib(path[len(build_prefix) + 1:])

    if link.startswith((build_prefix + '/lib', 'lib', '@executable_path/')):
        return '@loader_path/%s/%s' % (reldir, basename(link))

    if link == '/usr/local/lib/libgcc_s.1.dylib':
        return '/usr/lib/libgcc_s.1.dylib'

def mk_relative_osx(path):
    assert sys.platform == 'darwin' and is_obj(path)
    macho.install_name_change(path, osx_ch_link)

    if path.endswith('.dylib'):
        # note that not every MachO binaries is a "dynamically linked shared
        # library" which have an identification name, a .so C extensions
        # extensions is a "bundle".  One can verify this using the "file"
        # command.
        names = macho.otool(path)
        if names:
            args = ['install_name_tool', '-id', basename(names[0]), path]
            print(' '.join(args))
            check_call(args)

    for name in macho.otool(path):
        assert not name.startswith(build_prefix), path

def mk_relative(f, binary_relocation=True):
    assert sys.platform != 'win32'
    if f.startswith('bin/'):
        fix_shebang(f)

    if not binary_relocation:
        return

    path = join(build_prefix, f)
    if sys.platform.startswith('linux') and is_obj(path):
        rpath = '$ORIGIN/' + utils.rel_lib(f)
        patchelf = external.find_executable('patchelf')
        print('patchelf: file: %s\n    setting rpath to: %s' % (path, rpath))
        call([patchelf, '--set-rpath', rpath, path])

    if sys.platform == 'darwin' and is_obj(path):
        mk_relative_osx(path)

def fix_permissions(files):
    for root, dirs, unused_files in os.walk(build_prefix):
        for dn in dirs:
            os.chmod(join(root, dn), int('755', 8))

    for f in files:
        path = join(build_prefix, f)
        st = os.lstat(path)
        lchmod(path, stat.S_IMODE(st.st_mode) | stat.S_IWUSR) # chmod u+w

def post_build(files, binary_relocation=True):
    print('number of files:', len(files))
    fix_permissions(files)
    if not binary_relocation:
        print("Skipping binary relocation logic")
    for f in files:
        if sys.platform != 'win32':
            mk_relative(f, binary_relocation=binary_relocation)

def get_build_metadata(m):
    if exists(join(source.WORK_DIR, '__conda_version__.txt')):
        with open(join(source.WORK_DIR, '__conda_version__.txt')) as f:
            version = f.read().strip()
            print("Setting version from __conda_version__.txt: %s" % version)
            m.meta['package']['version'] = version
    if exists(join(source.WORK_DIR, '__conda_buildnum__.txt')):
        with open(join(source.WORK_DIR, '__conda_buildnum__.txt')) as f:
            build_number = f.read().strip()
            print("Setting build number from __conda_buildnum__.txt: %s" %
                  build_number)
            m.meta['build']['number'] = build_number
    if exists(join(source.WORK_DIR, '__conda_buildstr__.txt')):
        with open(join(source.WORK_DIR, '__conda_buildstr__.txt')) as f:
            buildstr = f.read().strip()
            print("Setting version from __conda_buildstr__.txt: %s" % buildstr)
            m.meta['build']['string'] = buildstr
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from zope.interface import implementer
import deform.exception
import deform.widget

from substanced.form import FormView as SubstanceDFormView
from substanced.form import FormError

from pontus.interfaces import IFormView
from pontus.view import ElementaryView, merge_dicts
from pontus.default_behavior import Cancel

try:
      basestring
except NameError:
      basestring = str

@implementer(IFormView)
class FormView(ElementaryView, SubstanceDFormView):

    title = 'Form View'
    chmod = []

    def __init__(self, context, request, parent=None, wizard=None, stepid=None, **kwargs):
        self.schema = self.schema.clone()
        SubstanceDFormView.__init__(self, context, request)
        ElementaryView.__init__(self, context, request, parent, wizard, stepid, **kwargs)
        self.buttons = [behavior.title for behavior in self.behaviorinstances.values()]

    def setviewid(self, viewid):
        ElementaryView.setviewid(self, viewid)
        self.formid = viewid

    def get_view_requirements(self):
        form, reqts = self._build_form()
        result = {}
        result['js_links'] = list(reqts['js'])
        result['css_links'] = list(reqts['css'])
        _requirements = self.requirements_copy
        result = merge_dicts(_requirements, result)
        return result

    def update(self,):
        self.init_stepid(self.schema)  # TODO: in before_update?
        form, reqts = self._build_form()
        form.formid = self.viewid + '_' + form.formid
        for c in form.children:
            c.oid = c.oid + form.formid

        item = None
        result = {}
        posted_formid = None
        error = False
        if '__formid__' in self.request.POST:
            posted_formid = self.request.POST['__formid__']

        if posted_formid is not None and posted_formid == form.formid:
            for button in form.buttons:
                if button.name in self.request.POST:
                    try:
                        if (button.name in self.behaviorinstances) and isinstance(self.behaviorinstances[button.name], Cancel):
                            # bypass form validation for Cancel behavior
                            validated = {}
                        else :
                            controls = self.request.POST.items()
                            validated = form.validate(controls)

                    except deform.exception.ValidationFailure as e:
                        fail = getattr(self, '%s_failure' % button.name, None)
                        if fail is None:
                            fail = self._failure
                        item = fail(e, form)
                        error = True
                    else:
                        try:
                            behavior = self.behaviorinstances[button.name]
                            item = behavior.execute(self.context,
                                    self.request, validated)
                            self.finished_successfully = True
                        except FormError as e:
                            snippet = '
Failed: %s
' % e
                            self.request.sdiapi.flash(snippet, 'danger',
                                                      allow_duplicate=True)
                            item = self.adapt_item(form.render(validated),
                                                   form.formid)
                            error = True

                    break

        if item is None:
            if not self.finished_successfully:
                item = self.show(form)

        if isinstance(item, dict):
            if error:
                item['isactive'] = True

            result['coordinates'] = {self.coordinates: [item]}
            result['js_links'] = list(reqts['js'])
            result['css_links'] = list(reqts['css'])
        else:
            result = item

        return result

    def before(self, form):
        if self.chmod:
            self._chmod(form, self.chmod)

    def show(self, form):
        result = self.default_data()
        body = None
        if result is None:
            body = form.render(readonly=False)
        else:
            body = form.render(appstruct=result, readonly=False)

        return self.adapt_item(body, form.formid)

    def default_data(self):
        return None

    def _failure(self, e, form):
        return self.adapt_item(e.render(), form.formid)

    def _get(self, form, node):
        for child in form.children:
            if child.name == node:
                return child

        return None

    def _chmod(self, form, mask):
        for m in mask:
            node = self._get(form, m[0])
            if node is not None:
                if isinstance(m[1], basestring):
                   if m[1] == 'r':
                       node.widget.readonly = True
                else:
                    self._chmod(node.children[0], m[1])
<commit_msg>We need to create a deform.form.Button ourselves for button title translation purpose<commit_after># -*- coding: utf-8 -*-
from zope.interface import implementer
import deform.exception
import deform.widget
from deform.form import Button

from substanced.form import FormView as SubstanceDFormView
from substanced.form import FormError

from pontus.interfaces import IFormView
from pontus.view import ElementaryView, merge_dicts
from pontus.default_behavior import Cancel

try:
      basestring
except NameError:
      basestring = str

@implementer(IFormView)
class FormView(ElementaryView, SubstanceDFormView):

    title = 'Form View'
    chmod = []

    def __init__(self, context, request, parent=None, wizard=None, stepid=None, **kwargs):
        self.schema = self.schema.clone()
        SubstanceDFormView.__init__(self, context, request)
        ElementaryView.__init__(self, context, request, parent, wizard, stepid, **kwargs)
        self.buttons = [Button(name=behavior.title,
                               title=behavior.title) for behavior in self.behaviorinstances.values()]

    def setviewid(self, viewid):
        ElementaryView.setviewid(self, viewid)
        self.formid = viewid

    def get_view_requirements(self):
        form, reqts = self._build_form()
        result = {}
        result['js_links'] = list(reqts['js'])
        result['css_links'] = list(reqts['css'])
        _requirements = self.requirements_copy
        result = merge_dicts(_requirements, result)
        return result

    def update(self,):
        self.init_stepid(self.schema)  # TODO: in before_update?
        form, reqts = self._build_form()
        form.formid = self.viewid + '_' + form.formid
        for c in form.children:
            c.oid = c.oid + form.formid

        item = None
        result = {}
        posted_formid = None
        error = False
        if '__formid__' in self.request.POST:
            posted_formid = self.request.POST['__formid__']

        if posted_formid is not None and posted_formid == form.formid:
            for button in form.buttons:
                if button.name in self.request.POST:
                    try:
                        if (button.name in self.behaviorinstances) and isinstance(self.behaviorinstances[button.name], Cancel):
                            # bypass form validation for Cancel behavior
                            validated = {}
                        else :
                            controls = self.request.POST.items()
                            validated = form.validate(controls)

                    except deform.exception.ValidationFailure as e:
                        fail = getattr(self, '%s_failure' % button.name, None)
                        if fail is None:
                            fail = self._failure
                        item = fail(e, form)
                        error = True
                    else:
                        try:
                            behavior = self.behaviorinstances[button.name]
                            item = behavior.execute(self.context,
                                    self.request, validated)
                            self.finished_successfully = True
                        except FormError as e:
                            snippet = '
Failed: %s
' % e
                            self.request.sdiapi.flash(snippet, 'danger',
                                                      allow_duplicate=True)
                            item = self.adapt_item(form.render(validated),
                                                   form.formid)
                            error = True

                    break

        if item is None:
            if not self.finished_successfully:
                item = self.show(form)

        if isinstance(item, dict):
            if error:
                item['isactive'] = True

            result['coordinates'] = {self.coordinates: [item]}
            result['js_links'] = list(reqts['js'])
            result['css_links'] = list(reqts['css'])
        else:
            result = item

        return result

    def before(self, form):
        if self.chmod:
            self._chmod(form, self.chmod)

    def show(self, form):
        result = self.default_data()
        body = None
        if result is None:
            body = form.render(readonly=False)
        else:
            body = form.render(appstruct=result, readonly=False)

        return self.adapt_item(body, form.formid)

    def default_data(self):
        return None

    def _failure(self, e, form):
        return self.adapt_item(e.render(), form.formid)

    def _get(self, form, node):
        for child in form.children:
            if child.name == node:
                return child

        return None

    def _chmod(self, form, mask):
        for m in mask:
            node = self._get(form, m[0])
            if node is not None:
                if isinstance(m[1], basestring):
                   if m[1] == 'r':
                       node.widget.readonly = True
                else:
                    self._chmod(node.children[0], m[1])
<|endoftext|>


<commit_before>from ..lib.asserts import get_assert_function
from ..exceptions import PackageException
from .install_command import install

p_assert = get_assert_function(PackageException)

DEFAULT_INDEX_URLS = ['https://pypi.python.org/simple/']

class Packages(object):
    """
    this is the public API for downloading packages into an environment.

    unless otherwise specified, all properties in this class are
    mutable: updating them will take immediate effect.
    """

    def __init__(self):
        self._versions = {}
        self._index_urls = list(DEFAULT_INDEX_URLS)

    @property
    def versions(self):
        """versions is a dictionary object of <package_name, version_spec> pairs.

        when a request is made to install a package, it will use the
        version specified in this dictionary.

        * if the package installation specifies a version, it will override
        the version specified here.

        .. code:: python

            # this sets the version to be used in this dictionary to 0.2.3.
            packages.install("uranium", version="==0.2.3")

        TODO: this will also contain entries to packages installed without a specified version.
        the version installed will be updated here.
        """
        # TODO: create a version dictionary,
        # to assert version specs.
        return self._versions

    @property
    def index_urls(self):
        """
        index urls is a list of the urls that Packages queries when
        looking for packages.
        """
        return self._index_urls

    @index_urls.setter
    def index_urls(self, value):
        p_assert(isinstance(value, list),
                 "only lists can be set as a value for indexes")
        self._index_urls = value

    def install(self, name, version=None, develop=False, upgrade=False, install_options=[]):
        """
        install is used when installing a python package into the environment.

        if version is set, the specified version of the package will be installed.

        if develop is set to True, the package will be installed as editable: the source
        in the directory passed will be used when using that package.

        if install_options is provided, it should be a list of options, like
        ["--prefix=/opt/srv", "--install-lib=/opt/srv/lib"]
        """
        if self._is_package_already_installed(name, version):
            return
        p_assert(
            not (develop and version),
            "unable to set both version and develop flags when installing packages"
        )
        if name in self.versions:
            if version is None:
                version = self.versions[name]
            del self.versions[name]
        req_set = install(
            name, upgrade=upgrade, develop=develop, version=version,
            index_urls=self.index_urls, constraint_dict=self.versions,
            install_options=install_options
        )
        if req_set:
            for req in req_set.requirements.values():
                if req.installed_version:
                    self.versions[req.name] = ("==" + req.installed_version)

    @staticmethod
    def _is_package_already_installed(name, version):
        import pkg_resources
        try:
            package_name = name
            if version:
                package_name += version
            pkg_resources.get_distribution(package_name)
            return True
        except Exception:
            pass
        return False
<commit_msg>replace [] with None for install_options default value in __init__.py<commit_after>from ..lib.asserts import get_assert_function
from ..exceptions import PackageException
from .install_command import install

p_assert = get_assert_function(PackageException)

DEFAULT_INDEX_URLS = ['https://pypi.python.org/simple/']

class Packages(object):
    """
    this is the public API for downloading packages into an environment.

    unless otherwise specified, all properties in this class are
    mutable: updating them will take immediate effect.
    """

    def __init__(self):
        self._versions = {}
        self._index_urls = list(DEFAULT_INDEX_URLS)

    @property
    def versions(self):
        """versions is a dictionary object of <package_name, version_spec> pairs.

        when a request is made to install a package, it will use the
        version specified in this dictionary.

        * if the package installation specifies a version, it will override
        the version specified here.

        .. code:: python

            # this sets the version to be used in this dictionary to 0.2.3.
            packages.install("uranium", version="==0.2.3")

        TODO: this will also contain entries to packages installed without a specified version.
        the version installed will be updated here.
        """
        # TODO: create a version dictionary,
        # to assert version specs.
        return self._versions

    @property
    def index_urls(self):
        """
        index urls is a list of the urls that Packages queries when
        looking for packages.
        """
        return self._index_urls

    @index_urls.setter
    def index_urls(self, value):
        p_assert(isinstance(value, list),
                 "only lists can be set as a value for indexes")
        self._index_urls = value

    def install(self, name, version=None, develop=False, upgrade=False, install_options=None):
        """
        install is used when installing a python package into the environment.

        if version is set, the specified version of the package will be installed.

        if develop is set to True, the package will be installed as editable: the source
        in the directory passed will be used when using that package.

        if install_options is provided, it should be a list of options, like
        ["--prefix=/opt/srv", "--install-lib=/opt/srv/lib"]
        """
        if self._is_package_already_installed(name, version):
            return
        p_assert(
            not (develop and version),
            "unable to set both version and develop flags when installing packages"
        )
        if name in self.versions:
            if version is None:
                version = self.versions[name]
            del self.versions[name]
        req_set = install(
            name, upgrade=upgrade, develop=develop, version=version,
            index_urls=self.index_urls, constraint_dict=self.versions,
            install_options=install_options
        )
        if req_set:
            for req in req_set.requirements.values():
                if req.installed_version:
                    self.versions[req.name] = ("==" + req.installed_version)

    @staticmethod
    def _is_package_already_installed(name, version):
        import pkg_resources
        try:
            package_name = name
            if version:
                package_name += version
            pkg_resources.get_distribution(package_name)
            return True
        except Exception:
            pass
        return False
<|endoftext|>


<commit_before>import re
import datetime
import requests
import saveimg

# this code is used to get pixiv id from daily ranking
class Daily(object):
    def __init__(data, date = ""):
        data.date = date

    def get_pids(data):
        date_url = ''
        if len(data.date):
            year, month, day = data.date.split('-')
            if datetime.date(year = int(year), month = int(month), day = int(day)) < datetime.date.today():
                date_url = "&date=" + year + month + day
            else:
                data.date = str(datetime.date.today())

        daily_list_page = requests.get('http://www.pixiv.net/ranking.php?mode=daily' + date_url)
        pattern = re.compile('data-id="(\d*)">')

        return pattern.findall(daily_list_page.text), saveimg.mkdir(date = data.date)

<commit_msg>remove daily_pid.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3

import time

from mediawords.db import connect_to_db
from mediawords.util.log import create_logger
from mediawords.util.process import run_alone

log = create_logger(__name__)

def copy_nonpartitioned_sentences_to_partitions():
    """Gradually copy sentences from "story_sentences_nonpartitioned" to "story_sentences_partitioned"."""

    # How many stories the sentences of which to copy at the same time
    stories_chunk_size = 100 * 1000

    db = connect_to_db()

    # With 512 MB, database can deduplicate (sort) sentences in memory instead of disk
    db.query("SET work_mem TO '512MB'")

    max_stories_id = db.query("SELECT MAX(stories_id) FROM stories").flat()[0]
    if max_stories_id is None:
        raise Exception("Max. stories ID is None.")

    log.info("Max. stories ID: {}".format(max_stories_id))

    for start_stories_id in range(0, max_stories_id + 1, stories_chunk_size):
        # There's an overlap of a single stories_id here but it doesn't matter that much
        end_stories_id = start_stories_id + stories_chunk_size

        log.info("Copying sentences of stories_id BETWEEN {} AND {} to the partitioned table...".format(
            start_stories_id,
            end_stories_id,
        ))

        copied_sentences = db.query(
            'SELECT copy_chunk_of_nonpartitioned_sentences_to_partitions(%(start_stories_id)s, %(end_stories_id)s)',
            {'start_stories_id': start_stories_id, 'end_stories_id': end_stories_id}
        ).flat()[0]

        log.info("Copied {} sentences of stories_id BETWEEN {} AND {} to the partitioned table.".format(
            copied_sentences,
            start_stories_id,
            end_stories_id,
        ))

        log.info("All done!")

        # Weird, but otherwise Ansible deployments to mctest don't work due to this script exit(0)ing right away
        while True:
            time.sleep(1)

if __name__ == '__main__':
    run_alone(copy_nonpartitioned_sentences_to_partitions)
<commit_msg>Fix indentation<commit_after>#!/usr/bin/env python3

import time

from mediawords.db import connect_to_db
from mediawords.util.log import create_logger
from mediawords.util.process import run_alone

log = create_logger(__name__)

def copy_nonpartitioned_sentences_to_partitions():
    """Gradually copy sentences from "story_sentences_nonpartitioned" to "story_sentences_partitioned"."""

    # How many stories the sentences of which to copy at the same time
    stories_chunk_size = 100 * 1000

    db = connect_to_db()

    # With 512 MB, database can deduplicate (sort) sentences in memory instead of disk
    db.query("SET work_mem TO '512MB'")

    max_stories_id = db.query("SELECT MAX(stories_id) FROM stories").flat()[0]
    if max_stories_id is None:
        raise Exception("Max. stories ID is None.")

    log.info("Max. stories ID: {}".format(max_stories_id))

    for start_stories_id in range(0, max_stories_id + 1, stories_chunk_size):
        # There's an overlap of a single stories_id here but it doesn't matter that much
        end_stories_id = start_stories_id + stories_chunk_size

        log.info("Copying sentences of stories_id BETWEEN {} AND {} to the partitioned table...".format(
            start_stories_id,
            end_stories_id,
        ))

        copied_sentences = db.query(
            'SELECT copy_chunk_of_nonpartitioned_sentences_to_partitions(%(start_stories_id)s, %(end_stories_id)s)',
            {'start_stories_id': start_stories_id, 'end_stories_id': end_stories_id}
        ).flat()[0]

        log.info("Copied {} sentences of stories_id BETWEEN {} AND {} to the partitioned table.".format(
            copied_sentences,
            start_stories_id,
            end_stories_id,
        ))

    log.info("All done!")

    # Weird, but otherwise Ansible deployments to mctest don't work due to this script exit(0)ing right away
    while True:
        time.sleep(1)

if __name__ == '__main__':
    run_alone(copy_nonpartitioned_sentences_to_partitions)
<|endoftext|>


<commit_before># Erik Halperin, 6/26/14
# Polls Condor collectors for some data and outputs it to Mongo
# There must be a config.ini with a [poll] section and the options collectors and intervals, the values of which should be seperated by commas
# Having a hard time precisely counting jobs processsed per second, but generally a couple thousand jobs won't take more than 3 seconds

import sys, time
import classad, htcondor
from pymongo import Connection #to connect to mongo
from concurrent import futures #for processing collectors concurrently
from ConfigParser import RawConfigParser #for reading from config.ini

#set up mongo collection
connection = Connection('mc.mwt2.org', 27017)
db = connection.visualization_db
dbc = db.condor_records

#default wait time interval
def_int = 30

def config_parse():
    conf = RawConfigParser()
    conf.read('config.ini')

    collectors = []
    intervals = []
    if conf.has_section('poll'):
        #get collectors
        if conf.has_option('poll', 'collectors'):
            value = conf.get('poll', 'collectors')
            for v in value.split(','):
                collectors.append(v.strip())

        #get wait intervals for collectors
        if conf.has_option('poll', 'intervals'):
            value = conf.get('poll', 'intervals')
            for v in value.split(','):
                intervals.append(int(v.strip()))

    #errors and checking
    if(len(collectors) != len(intervals)):
        sys.stderr.write('config.ini: each collector must have a wait interval\n')
        sys.exit(0)

    new_intvs = [i if i > 0 else def_int for i in intervals]

    return [collectors, new_intvs]

def mongo_store(coll, intv):
    while(True):
        slot_state = coll.query(htcondor.AdTypes.Startd,'true',['Name','RemoteGroup','NodeOnline','JobId','State','RemoteOwner','COLLECTOR_HOST_STRING'])
        timestamp = str(int(time.time()))

        #storing data into mongo
        for slot in slot_state:
            sl = dict(slot)
            if 'JobId' in sl:
                sl['_id'] = sl.pop('JobId') #JobId will become _id for mongo
            else:
                continue
            if 'TargetType' in sl: #TargetType, CurrentTime, and MyType tag along for some reason
                del sl['TargetType']
            if 'CurrentTime' in sl:
                del sl['CurrentTime']
            if 'MyType' in sl:
                del sl['MyType']
            dbc.insert(sl)

        time.sleep(interval)

def main():
    ret_list = config_parse() #collectors = ret_list[0], intervals = ret_list[1]

    with futures.ThreadPoolExecutor(max_workers=3) as executor:
        for collector, interval in zip(ret_list[0],ret_list[1]):
            executor.submit(mongo_store, htcondor.Collector(collector), interval)

main()

<commit_msg>Fixed wait interval problem<commit_after>File Edit Options Buffers Tools Python Help
# Erik Halperin, 6/26/14
# Polls Condor collectors for some data and outputs it to Mongo
# There must be a config.ini with a [poll] section and the options collectors and intervals, the values of which should be seperated by commas
# Having a hard time precisely counting jobs processsed per second, but generally a couple thousand jobs won't take more than 3 seconds

import sys, time
import classad, htcondor
from pymongo import Connection #to connect to mongo
from concurrent import futures #for processing collectors concurrently
from ConfigParser import RawConfigParser #for reading from config.ini

#set up mongo collection
connection = Connection('mc.mwt2.org', 27017)
db = connection.visualization_db
dbc = db.condor_records

#default wait time interval
def_int = 30

def config_parse():
    conf = RawConfigParser()
    conf.read('config.ini')

    collectors = []
    intervals = []
    if conf.has_section('poll'):
        #get collectors
        if conf.has_option('poll', 'collectors'):
            value = conf.get('poll', 'collectors')
            for v in value.split(','):
                collectors.append(v.strip())

        #get wait intervals for collectors
        if conf.has_option('poll', 'intervals'):
            value = conf.get('poll', 'intervals')
            for v in value.split(','):
                intervals.append(int(v.strip()))

    #errors and checking
    if(len(collectors) != len(intervals)):
        sys.stderr.write('config.ini: each collector must have a wait interval\n')
        sys.exit(0)

    new_intvs = [i if i > 0 else def_int for i in intervals]

    return [collectors, new_intvs]

def mongo_store(coll, intv):
    while(True):
        slot_state = coll.query(htcondor.AdTypes.Startd,'true',['Name','RemoteGroup','NodeOnline','JobId','State','RemoteOwner','COLLECTOR_HOST_STRING'])
        timestamp = str(int(time.time()))

        #storing data into mongo
        for slot in slot_state:
            sl = dict(slot)
            if 'JobId' in sl:
                sl['_id'] = sl.pop('JobId') #JobId will become _id for mongo
            else:
                continue
            if 'TargetType' in sl: #TargetType, CurrentTime, and MyType tag along for some reason
                del sl['TargetType']
            if 'CurrentTime' in sl:
                del sl['CurrentTime']
            if 'MyType' in sl:
                del sl['MyType']
            dbc.insert(sl)

        time.sleep(intv)

def main():
    ret_list = config_parse() #collectors = ret_list[0], intervals = ret_list[1]

    with futures.ThreadPoolExecutor(max_workers=len(ret_list[1]) as executor:
        for collector, interval in zip(ret_list[0],ret_list[1]):
            executor.submit(mongo_store, htcondor.Collector(collector), interval)

main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php
#
# Copyright 2006,2007 Frank Scholz <coherence@beebits.net>
# Copyright 2014 Hartmut Goebel <h.goebel@crazy-compilers.com>
#
"""
little helper to get the proper ElementTree package
"""

import re
import exceptions

try:
    import cElementTree as ET
    import elementtree
except ImportError:
    try:
        from elementtree import ElementTree as ET
        import elementtree
    except ImportError:
        # this seems to be necessary with the python2.5 on the Maemo platform
        try:
            from xml.etree import cElementTree as ET
            from xml import etree as elementtree
        except ImportError:
            try:
                from xml.etree import ElementTree as ET
                from xml import etree as elementtree
            except ImportError:
                raise ImportError("ElementTree: no ElementTree module found, "
                                  "critical error")

utf8_escape = re.compile(eval(r'u"[&<>\"]+"'))
escape = re.compile(eval(r'u"[&<>\"\u0080-\uffff]+"'))

def new_encode_entity(text, pattern=utf8_escape):

    def escape_entities(m, map=elementtree.ElementTree._escape_map):
        """
        map reserved and non-ascii characters to numerical entities
        """
        out = []
        append = out.append
        for char in m.group():
            t = map.get(char)
            if t is None:
                t = "&#%d;" % ord(char)
            append(t)
        if type(text) == unicode:
            return ''.join(out)
        else:
            return u''.encode('utf-8').join(out)

    try:
        if type(text) == unicode:
            return elementtree.ElementTree._encode(
                escape.sub(escape_entities, text), 'ascii')
        else:
            return elementtree.ElementTree._encode(
                utf8_escape.sub(escape_entities, text.decode('utf-8')), 'utf-8')
    except TypeError:
        elementtree.ElementTree._raise_serialization_error(text)

elementtree.ElementTree._encode_entity = new_encode_entity

# it seems there are some ElementTree libs out there
# which have the alias XMLParser and some that haven't.
#
# So we just use the XMLTreeBuilder method for now
# if XMLParser isn't available.
if not hasattr(ET, 'XMLParser'):
    def XMLParser(encoding='utf-8'):
        return ET.XMLTreeBuilder()

    ET.XMLParser = XMLParser

namespace_map_update = elementtree.ElementTree._namespace_map.update

class ElementInterface(elementtree.ElementTree._ElementInterface): pass

def indent(elem, level=0):
    """
    generate pretty looking XML, based upon:
    http://effbot.org/zone/element-lib.htm#prettyprint
    """
    i = "\n" + level * "  "
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + "  "
        for elem in elem:
            indent(elem, level + 1)
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

def parse_xml(data, encoding="utf-8", dump_invalid_data=False):
    try:
        parser = ET.XMLParser(encoding=encoding)
    except exceptions.TypeError:
        parser = ET.XMLParser()

    # my version of twisted.web returns page_infos as a dictionary in
    # the second item of the data list
    if isinstance(data, (list, tuple)):
        data = data[0]

    try:
        data = data.encode(encoding)
    except UnicodeDecodeError:
        pass

    # Guess from who we're getting this?
    data = data.replace('\x00', '')
    try:
        parser.feed(data)
    except Exception, error:
        if dump_invalid_data:
            print error, repr(data)
        parser.close()
        raise
    else:
        return ET.ElementTree(parser.close())
<commit_msg>Add a fixme-comment.<commit_after># -*- coding: utf-8 -*-
#
# Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php
#
# Copyright 2006,2007 Frank Scholz <coherence@beebits.net>
# Copyright 2014 Hartmut Goebel <h.goebel@crazy-compilers.com>
#
"""
little helper to get the proper ElementTree package
"""

import re
import exceptions

try:
    import cElementTree as ET
    import elementtree
except ImportError:
    try:
        from elementtree import ElementTree as ET
        import elementtree
    except ImportError:
        # this seems to be necessary with the python2.5 on the Maemo platform
        try:
            from xml.etree import cElementTree as ET
            from xml import etree as elementtree
        except ImportError:
            try:
                from xml.etree import ElementTree as ET
                from xml import etree as elementtree
            except ImportError:
                raise ImportError("ElementTree: no ElementTree module found, "
                                  "critical error")

utf8_escape = re.compile(eval(r'u"[&<>\"]+"'))
escape = re.compile(eval(r'u"[&<>\"\u0080-\uffff]+"'))

def new_encode_entity(text, pattern=utf8_escape):

    def escape_entities(m, map=elementtree.ElementTree._escape_map):
        """
        map reserved and non-ascii characters to numerical entities
        """
        out = []
        append = out.append
        for char in m.group():
            t = map.get(char)
            if t is None:
                t = "&#%d;" % ord(char)
            append(t)
        if type(text) == unicode:
            return ''.join(out)
        else:
            return u''.encode('utf-8').join(out)

    try:
        if type(text) == unicode:
            return elementtree.ElementTree._encode(
                escape.sub(escape_entities, text), 'ascii')
        else:
            return elementtree.ElementTree._encode(
                utf8_escape.sub(escape_entities, text.decode('utf-8')), 'utf-8')
    except TypeError:
        elementtree.ElementTree._raise_serialization_error(text)

elementtree.ElementTree._encode_entity = new_encode_entity

# it seems there are some ElementTree libs out there
# which have the alias XMLParser and some that haven't.
#
# So we just use the XMLTreeBuilder method for now
# if XMLParser isn't available.
if not hasattr(ET, 'XMLParser'):
    def XMLParser(encoding='utf-8'):
        return ET.XMLTreeBuilder()

    ET.XMLParser = XMLParser

namespace_map_update = elementtree.ElementTree._namespace_map.update

class ElementInterface(elementtree.ElementTree._ElementInterface): pass

def indent(elem, level=0):
    """
    generate pretty looking XML, based upon:
    http://effbot.org/zone/element-lib.htm#prettyprint
    """
    i = "\n" + level * "  "
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + "  "
        for elem in elem:
            indent(elem, level + 1)
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

def parse_xml(data, encoding="utf-8", dump_invalid_data=False):
    try:
        parser = ET.XMLParser(encoding=encoding)
    except exceptions.TypeError:
        parser = ET.XMLParser()

    # my version of twisted.web returns page_infos as a dictionary in
    # the second item of the data list
    # :fixme: This must be handled where twisted.web is fetching the data
    if isinstance(data, (list, tuple)):
        data = data[0]

    try:
        data = data.encode(encoding)
    except UnicodeDecodeError:
        pass

    # Guess from who we're getting this?
    data = data.replace('\x00', '')
    try:
        parser.feed(data)
    except Exception, error:
        if dump_invalid_data:
            print error, repr(data)
        parser.close()
        raise
    else:
        return ET.ElementTree(parser.close())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import curses, time, locale, sys, os
import pygame
from pygame import mixer
import pyglet
from doggunk import *

import signal
import sys

def signal_handler(signal, frame):
    curses.noqiflush()

def signal_handler(signal, frame):
    pass

def init_curses(scr):
    #Drawing the dog requires 256 colors, so this is done first
    curses.start_color()
    curses.use_default_colors()
    curses.curs_set(0)
    curses.noecho()
    #Setting up the color pairs for drawing the dog
    curses.init_pair(1, 16, -1)
    curses.init_pair(2, 16, 255)
    curses.init_pair(3, 255, 16)
    curses.init_pair(4, -1, 16)

#Get the path of the script and provide the name of the
# sound file.
scriptpath = os.path.dirname(os.path.abspath(__file__))
musicfile = "mus_dance_of_dog.ogg"

#Setting up signal handlers
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTSTP, signal_handler)

#Setting up pygame's audio parts
#pygame.init()
pygame.mixer.init()
pygame.mixer.music.load(scriptpath + '/' + musicfile)
pygame.mixer.music.play()

#Perform the dog dance
try:
    locale.setlocale(locale.LC_ALL, '')
    scr = curses.initscr()

    init_curses(scr)

    height,width = scr.getmaxyx()

    posx = width / 2 - 9
    posy = height / 2 - 3

    #Keep the dogdance as long as the music plays
    while pygame.mixer.music.get_busy():
        draw_dog1(scr,posx,posy)
        scr.refresh()
        time.sleep(0.2)
        #Draw the next frame
        draw_dog2(scr,posx,posy)
        scr.refresh()
        time.sleep(0.2)
        #Repeat 'till the song stops

    scr.keypad(0)
finally:
#Clean up
    curses.echo()
    curses.nocbreak()
    curses.endwin()
<commit_msg>Code cleanups<commit_after># -*- coding: utf-8 -*-

import curses, time, locale, sys, os, signal
import pygame
from doggunk import *

def signal_handler(signal, frame):
    curses.noqiflush()

def init_curses(scr):
    #Drawing the dog requires 256 colors, so this is done first
    curses.start_color()
    curses.use_default_colors()
    curses.curs_set(0)
    curses.noecho()
    #Setting up the color pairs for drawing the dog
    curses.init_pair(1, 16, -1)
    curses.init_pair(2, 16, 255)
    curses.init_pair(3, 255, 16)
    curses.init_pair(4, -1, 16)

#Get the path of the script and provide the name of the
# sound file.
scriptpath = os.path.dirname(os.path.abspath(__file__))
musicfile = "mus_dance_of_dog.ogg"

#Setting up signal handlers
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTSTP, signal_handler)

#Setting up pygame's audio parts
pygame.mixer.init()
pygame.mixer.music.load(scriptpath + '/' + musicfile)
pygame.mixer.music.play()

#Perform the dog dance
try:
    locale.setlocale(locale.LC_ALL, '')
    scr = curses.initscr()

    init_curses(scr)

    height,width = scr.getmaxyx()

    posx = width / 2 - 9
    posy = height / 2 - 3

    #Keep the dogdance as long as the music plays
    while pygame.mixer.music.get_busy():
        draw_dog1(scr,posx,posy)
        scr.refresh()
        time.sleep(0.2)
        #Draw the next frame
        draw_dog2(scr,posx,posy)
        scr.refresh()
        time.sleep(0.2)
        #Repeat 'till the song stops

    scr.keypad(0)
finally:
#Clean up
    curses.echo()
    curses.nocbreak()
    curses.endwin()
<|endoftext|>


<commit_before>import os
import random
import gzip
import lzma
import json

from Levenshtein import distance as levenshtein_distance

from typing import List, Union

"""
Lookup table handling utilities.
"""

def get_token_replacement(input_token, lookup, max_dist):
    """
    Gets a replacement for an input token sampled from a given lookup table.
    Samples uniformly among all candidates available for an input token.
    Accepts additional filtering criteria based on the maximum edit distance
    between the original and the candidate token.
    """
    candidates = lookup[input_token]

    if max_dist >= 0:
        candidates = [c for c in candidates \
            if levenshtein_distance(input_token, c) <= max_dist]

    if len(candidates) == 0:
        return input_token

    idx = random.randint(0, len(candidates) - 1)
    replacement_token = candidates[idx]

    return replacement_token

def load_lookup(files: Union[str, List[str]]) -> bool:
    """
    Loads a lookup table with replacement candidates from a file
        (or every file in a list of files).
    """
    if isinstance(files, str):
        lut = _load_lookup_from_file(files)
        return lut

    elif isinstance(files, list) or isinstance(files, tuple):
        lut = dict()

        for fn in files:
            lut_part = _load_lookup_from_file(fn)

            # merge dictionaries
            for key, value in lut_part.items():
                if key not in lut:
                    lut[key] = value
                else:
                    lut[key] += value

        return lut
    else:
        return {}

def _load_lookup_from_file(file_name: str):
    """
    Loads a lookup table with replacement candidates from a given file.
    """

    dir_path = os.path.dirname(os.path.realpath(__file__))
    file_path = os.path.join(dir_path, f"{file_name}")

    basename, ext = os.path.splitext(file_name)

    assert ext in [".gz", ".xz", ".json"], \
            f"File extension not supported '{ext}'"

    if ext == ".xz":
        with lzma.open(file_path, 'r') as f:
            typos = json.loads(f.read().decode('utf-8'))
    elif ext == ".gz":
        with gzip.open(file_path, 'r') as f:
            typos = json.loads(f.read().decode('utf-8'))
    elif ext == ".json":
        with open(file_path, 'r') as f:
            typos = json.load(f)
    else:
        typos = {}

    return typos

<commit_msg>added missing docstrings<commit_after>import os
import random
import gzip
import lzma
import json

from Levenshtein import distance as levenshtein_distance

from typing import List, Union

"""
Lookup table handling utilities.
"""

def get_token_replacement(input_token, lookup, max_dist):
    """
    Gets a replacement for an input token sampled from a given lookup table.
    Samples uniformly among all candidates available for an input token.
    Accepts additional filtering criteria based on the maximum edit distance
    between the original and the candidate token.

    :param input_token: a token for which a replacement needs to be obtained
    :param max_dist: max. Levenshtein distance between the original and the
                     perturbed token
    :param lookup: a dictionary (lookup table) of replacement candidates
    :returns: a replacement for an input token
    :rtype: str
    """
    candidates = lookup[input_token]

    if max_dist >= 0:
        candidates = [c for c in candidates \
            if levenshtein_distance(input_token, c) <= max_dist]

    if len(candidates) == 0:
        return input_token

    idx = random.randint(0, len(candidates) - 1)
    replacement_token = candidates[idx]

    return replacement_token

def load_lookup(files: Union[str, List[str]]) -> bool:
    """
    Loads a lookup table with replacement candidates from a file
        (or every file in a list of files).

    :param lookup: a file or a list of files with lookup tables in JSON format
    """
    if isinstance(files, str):
        lut = _load_lookup_from_file(files)
        return lut

    elif isinstance(files, list) or isinstance(files, tuple):
        lut = dict()

        for fn in files:
            lut_part = _load_lookup_from_file(fn)

            # merge dictionaries
            for key, value in lut_part.items():
                if key not in lut:
                    lut[key] = value
                else:
                    lut[key] += value

        return lut
    else:
        return {}

def _load_lookup_from_file(file_name: str):
    """
    Loads a lookup table with replacement candidates from a given file.

    :param file_name: a JSON file (raw or compressed using lzma or gzip
                      algorithm)
    """

    dir_path = os.path.dirname(os.path.realpath(__file__))
    file_path = os.path.join(dir_path, f"{file_name}")

    basename, ext = os.path.splitext(file_name)

    assert ext in [".gz", ".xz", ".json"], \
            f"File extension not supported '{ext}'"

    if ext == ".xz":
        with lzma.open(file_path, 'r') as f:
            typos = json.loads(f.read().decode('utf-8'))
    elif ext == ".gz":
        with gzip.open(file_path, 'r') as f:
            typos = json.loads(f.read().decode('utf-8'))
    elif ext == ".json":
        with open(file_path, 'r') as f:
            typos = json.load(f)
    else:
        typos = {}

    return typos

<|endoftext|>