<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

"""
defs.py

Definitions for mapping between kana/romaji and an internal representation.
"""

#----------------------------------------------------------------------------
# Lemmas
#
# These are unambiguous representations of one or more kana,
#   essentially wapuro-style Kunrei-shiki.
#
# Final data structure is a list of strings.
#----------------------------------------------------------------------------

# lemmas used in native Japanese words
# First section includes base moras, dakuon, handakuon,and youon
# Section section adds dakuon
# dya/dyu/dyo are omitted due to lack of standard use
LEMMA_TAB_BASIC = """\
A I U E O
KA KI KU KE KO KYA KYU KYO
GA GI GU GE GO GYA GYU GYO
SA SI SU SE SO SYA SYU SYO
ZA ZI ZU ZE ZO ZYA ZYU ZYO
TA TI TU TE TO TYA TYU TYO
DA DI DU DE DO
NA NI NU NE NO NYA NYU NYO
HA HI HU HE HO HYA HYU HYO
BA BI BU BE BO BYA BYU BYO
PA PI PU PE PO PYA PYU PYO
MA MI MU ME MO MYA MYU MYO
YA    YU    YO
RA RI RU RE RO RYA RYU RYO
WA          WO
N'

KKA KKI KKU KKE KKO KKYA KKYU KKYO
SSA SSI SSU SSE SSO SSYA SSYU SSYO
TTA TTI TTU TTE TTO TTYA TTYU TTYO
PPA PPI PPU PPE PPO PPYA PPYU PPYO
"""

# lemmas for borrowed words, archaic spellings, etc.
# sections are:
#   -added moras
#   -dakuon for voiced moras
#   -dakuon for added moras
#   -chouon sign
LEMMA_TAB_EXTENDED = """\
    UXI       UXE UXO
VA  VI   VU   VE  VO   VYA VYU VYO
              SYE
              ZYE
    TEXI TOXU
              TYE
TSA TSI       TSE TSO
    DEXI DOXU          DYA DYU DYO
FA  FI        FE  FO   FYA FYU FYO
              YE
    WI        WE

GGA GGI GGU GGE GGO GGYA GGYU GGYO
ZZA ZZI ZZU ZZE ZZO ZZYA ZZYU ZZYO
DDA DDI DDU DDE DDO DDYA DDYU DDYO
HHA HHI HHU HHE HHO HHYA HHYU HHYO
BBA BBI BBU BBE BBO BBYA BBYU BBYO

                 SSYE
                 ZZYE
     TTEXI TTOXU
                 TTYE
TTSA TTSI        TTSE TTSO
     DDEXI DDOXU
FFA  FFI         FFE  FFO FFYA FFYU FFYO

-
"""

SOKUON_CONSONANTS = "KGSZTDHFBP"

# use to handle non-standard use of small kana
WAPURO_SMALL_KANA_PRE =  "XTU"
WAPURO_SMALL_KANA_POST = "XA XI XU XE XO   XYA XYU XYO   XWA"

#----------------------------------------------------------------------------
# Kana tables
#
# Used to map kana <-> lemmas
# Derived from python-romkan, itself based on KAKASI
#   <http://kakasi.namazu.org/>
#----------------------------------------------------------------------------

HIRAGANA_TAB = """\
あ A,  い I,  う U,  え E,  お O,
か KA, き KI, く KU, け KE, こ KO, きゃ KYA, きゅ KYU, きょ KYO,
が GA, ぎ GI, ぐ GU, げ GE, ご GO, ぎゃ GYA, ぎゅ GYU, ぎょ GYO,
さ SA, し SI, す SU, せ SE, そ SO, しゃ SYA, しゅ SYU, しょ SYO,
ざ ZA, じ ZI, ず ZU, ぜ ZE, ぞ ZO, じゃ ZYA, じゅ ZYU, じょ ZYO,
た TA, ち TI, つ TU, て TE, と TO, ちゃ TYA, ちゅ TYU, ちょ TYO,
だ DA, ぢ DI, づ DU, で DE, ど DO,
な NA, に NI, ぬ NU, ね NE, の NO, にゃ NYA, にゅ NYU, にょ NYO,
は HA, ひ HI, ふ HU, へ HE, ほ HO, ひゃ HYA, ひゅ HYU, ひょ HYO,
ば BA, び BI, ぶ BU, べ BE, ぼ BO, びゃ BYA, びゅ BYU, びょ BYO,
ぱ PA, ぴ PI, ぷ PU, ぺ PE, ぽ PO, ぴゃ PYA, ぴゅ PYU, ぴょ PYO,
ま MA, み MI, む MU, め ME, も MO, みゃ MYA, みゅ MYU, みょ MYO,
や YA,        ゆ YU,       よ YO,
ら RA, り RI, る RU, れ RE, ろ RO, りゃ RYA, りゆ RYU, りょ RYO,
わ WA,                     を WO,
ん N',

         うぃ UXI,            うぇ UXE, うぉ UXO,
ゔぁ VA,  ゔぃ VI,   ゔ VU,    ゔぇ VE,  ゔぉ VO,
                             しぇ SYE,
                             じぇ ZYE,
         てぃ TEXI, とぅ TOXU,
                             ちぇ TYE,
つぁ TSA, つぃ TSI,  つぇ TSE,           つぉ TSO,
         でぃ DEXI, どぅ DOXU,                   ぢゃ DYA, ぢゅ DYU, ぢょ DYO,
ふぁ FA,  ふぃ FI,             ふぇ FE,  ふぉ FO,  ふゃ FYA, ふゅ FYU, ふょ FYO,
                              いぇ YE,
         ゐ WI,         ゑ WE,

ー -
"""

HIRAGANA_SOKUON = "っ"
HIRAGANA_SMALL_POST = "ぁ ぃ ぅ ぇ ぉ ゃ ゅ ょ ゎ"

# KATAKANA_TAB = """ァ  xa ア  a  ィ  xi イ  i  ゥ  xu
# ウ  u  ヴ  vu ヴァ va ヴィ vi ヴェ ve
# ヴォ vo ェ  xe エ  e  ォ  xo オ  o

# カ  ka ガ  ga キ  ki キャ kya   キュ kyu
# キョ kyo   ギ  gi ギャ gya   ギュ gyu   ギョ gyo
# ク  ku グ  gu ケ  ke ゲ  ge コ  ko
# ゴ  go

# サ  sa ザ  za シ  si シャ sya   シュ syu
# ショ syo   シェ  sye
# ジ  zi ジャ zya   ジュ zyu   ジョ zyo
# ス  su ズ  zu セ  se ゼ  ze ソ  so
# ゾ  zo

# タ  ta ダ  da チ  ti チャ tya   チュ tyu
# チョ tyo   ヂ  di ヂャ dya   ヂュ dyu   ヂョ dyo
# ティ  ti

# ッ  xtu
# ッヴ vvu   ッヴァ   vva   ッヴィ   vvi
# ッヴェ   vve   ッヴォ   vvo
# ッカ kka   ッガ gga   ッキ kki   ッキャ   kkya
# ッキュ   kkyu  ッキョ   kkyo  ッギ ggi   ッギャ   ggya
# ッギュ   ggyu  ッギョ   ggyo  ック kku   ッグ ggu
# ッケ kke   ッゲ gge   ッコ kko   ッゴ ggo   ッサ ssa
# ッザ zza   ッシ ssi   ッシャ   ssya
# ッシュ   ssyu  ッショ   ssyo  ッシェ   ssye
# ッジ zzi   ッジャ   zzya  ッジュ   zzyu  ッジョ   zzyo
# ッス ssu   ッズ zzu   ッセ sse   ッゼ zze   ッソ sso
# ッゾ zzo   ッタ tta   ッダ dda   ッチ tti   ッティ tti
# ッチャ   ttya  ッチュ   ttyu  ッチョ   ttyo  ッヂ ddi
# ッヂャ   ddya  ッヂュ   ddyu  ッヂョ   ddyo  ッツ ttu
# ッヅ ddu   ッテ tte   ッデ dde   ット tto   ッド ddo
# ッドゥ ddu
# ッハ hha   ッバ bba   ッパ ppa   ッヒ hhi
# ッヒャ   hhya  ッヒュ   hhyu  ッヒョ   hhyo  ッビ bbi
# ッビャ   bbya  ッビュ   bbyu  ッビョ   bbyo  ッピ ppi
# ッピャ   ppya  ッピュ   ppyu  ッピョ   ppyo  ッフ hhu   ッフュ ffu
# ッファ   ffa   ッフィ   ffi   ッフェ   ffe   ッフォ   ffo
# ッブ bbu   ップ ppu   ッヘ hhe   ッベ bbe   ッペ  ppe
# ッホ hho   ッボ bbo   ッポ ppo   ッヤ yya   ッユ yyu
# ッヨ yyo   ッラ rra   ッリ rri   ッリャ   rrya
# ッリュ   rryu  ッリョ   rryo  ッル rru   ッレ rre
# ッロ rro

# ツ  tu ヅ  du テ  te デ  de ト  to
# ド  do ドゥ  du

# ナ  na ニ  ni ニャ nya   ニュ nyu   ニョ nyo
# ヌ  nu ネ  ne ノ  no

# ハ  ha バ  ba パ  pa ヒ  hi ヒャ hya
# ヒュ hyu   ヒョ hyo   ビ  bi ビャ bya   ビュ byu
# ビョ byo   ピ  pi ピャ pya   ピュ pyu   ピョ pyo
# フ  hu ファ fa フィ fi フェ fe フォ fo
# フュ  fu
# ブ  bu プ  pu ヘ  he ベ  be ペ  pe
# ホ  ho ボ  bo ポ  po

# マ  ma ミ  mi ミャ mya   ミュ myu   ミョ myo
# ム  mu メ  me モ  mo

# ャ  xya   ヤ  ya ュ  xyu   ユ  yu ョ  xyo
# ヨ  yo

# ラ  ra リ  ri リャ rya   リュ ryu   リョ ryo
# ル  ru レ  re ロ  ro

# ヮ  xwa   ワ  wa ウィ  wi ヰ wi ヱ  we ウェ we
# ヲ  wo ウォ  wo ン n

# ン   n'
# ディ  dyi
# ー   -
# チェ  tye
# ッチェ   ttye
# ジェ zye
# """

#
# Tests
#

# if __name__ == "__main__":
#     print list(groups(re.split('\s', "a 1 b 2 cc 33,\nddd 444\teee 555")))<commit_msg>Clean up table spacing.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

"""
defs.py

Definitions for mapping between kana/romaji and an internal representation.
"""

#----------------------------------------------------------------------------
# Lemmas
#
# These are unambiguous representations of one or more kana,
#   essentially wapuro-style Kunrei-shiki.
#
# Final data structure is a list of strings.
#----------------------------------------------------------------------------

# lemmas used in native Japanese words
# First section includes base moras, dakuon, handakuon,and youon
# Section section adds dakuon
# dya/dyu/dyo are omitted due to lack of standard use
LEMMA_TAB_BASIC = """\
A  I  U  E  O
KA KI KU KE KO KYA KYU KYO
GA GI GU GE GO GYA GYU GYO
SA SI SU SE SO SYA SYU SYO
ZA ZI ZU ZE ZO ZYA ZYU ZYO
TA TI TU TE TO TYA TYU TYO
DA DI DU DE DO
NA NI NU NE NO NYA NYU NYO
HA HI HU HE HO HYA HYU HYO
BA BI BU BE BO BYA BYU BYO
PA PI PU PE PO PYA PYU PYO
MA MI MU ME MO MYA MYU MYO
YA    YU    YO
RA RI RU RE RO RYA RYU RYO
WA          WO
N'

KKA KKI KKU KKE KKO KKYA KKYU KKYO
SSA SSI SSU SSE SSO SSYA SSYU SSYO
TTA TTI TTU TTE TTO TTYA TTYU TTYO
PPA PPI PPU PPE PPO PPYA PPYU PPYO
"""

# lemmas for borrowed words, archaic spellings, etc.
# sections are:
#   -added moras
#   -dakuon for voiced moras
#   -dakuon for added moras
#   -chouon sign
LEMMA_TAB_EXTENDED = """\
    UXI       UXE UXO
VA  VI   VU   VE  VO   VYA VYU VYO
              SYE
              ZYE
    TEXI TOXU
              TYE
TSA TSI       TSE TSO
    DEXI DOXU          DYA DYU DYO
FA  FI        FE  FO   FYA FYU FYO
              YE
    WI        WE

GGA GGI GGU GGE GGO GGYA GGYU GGYO
ZZA ZZI ZZU ZZE ZZO ZZYA ZZYU ZZYO
DDA DDI DDU DDE DDO DDYA DDYU DDYO
HHA HHI HHU HHE HHO HHYA HHYU HHYO
BBA BBI BBU BBE BBO BBYA BBYU BBYO

                 SSYE
                 ZZYE
     TTEXI TTOXU
                 TTYE
TTSA TTSI        TTSE TTSO
     DDEXI DDOXU
FFA  FFI         FFE  FFO FFYA FFYU FFYO

-
"""

SOKUON_CONSONANTS = "KGSZTDHFBP"

# use to handle non-standard use of small kana
WAPURO_SMALL_KANA_PRE =  "XTU"
WAPURO_SMALL_KANA_POST = "XA XI XU XE XO   XYA XYU XYO   XWA"

#----------------------------------------------------------------------------
# Kana tables
#
# Used to map kana <-> lemmas
# Derived from python-romkan, itself based on KAKASI
#   <http://kakasi.namazu.org/>
#----------------------------------------------------------------------------

HIRAGANA_TAB = """\
あ A,  い I,  う U,  え E,  お O,
か KA, き KI, く KU, け KE, こ KO, きゃ KYA, きゅ KYU, きょ KYO,
が GA, ぎ GI, ぐ GU, げ GE, ご GO, ぎゃ GYA, ぎゅ GYU, ぎょ GYO,
さ SA, し SI, す SU, せ SE, そ SO, しゃ SYA, しゅ SYU, しょ SYO,
ざ ZA, じ ZI, ず ZU, ぜ ZE, ぞ ZO, じゃ ZYA, じゅ ZYU, じょ ZYO,
た TA, ち TI, つ TU, て TE, と TO, ちゃ TYA, ちゅ TYU, ちょ TYO,
だ DA, ぢ DI, づ DU, で DE, ど DO,
な NA, に NI, ぬ NU, ね NE, の NO, にゃ NYA, にゅ NYU, にょ NYO,
は HA, ひ HI, ふ HU, へ HE, ほ HO, ひゃ HYA, ひゅ HYU, ひょ HYO,
ば BA, び BI, ぶ BU, べ BE, ぼ BO, びゃ BYA, びゅ BYU, びょ BYO,
ぱ PA, ぴ PI, ぷ PU, ぺ PE, ぽ PO, ぴゃ PYA, ぴゅ PYU, ぴょ PYO,
ま MA, み MI, む MU, め ME, も MO, みゃ MYA, みゅ MYU, みょ MYO,
や YA,        ゆ YU,       よ YO,
ら RA, り RI, る RU, れ RE, ろ RO, りゃ RYA, りゆ RYU, りょ RYO,
わ WA,                     を WO,
ん N',

         うぃ UXI,            うぇ UXE, うぉ UXO,
ゔぁ VA,  ゔぃ VI,   ゔ VU,    ゔぇ VE,  ゔぉ VO,
                             しぇ SYE,
                             じぇ ZYE,
         てぃ TEXI, とぅ TOXU,
                             ちぇ TYE,
つぁ TSA, つぃ TSI,  つぇ TSE,           つぉ TSO,
         でぃ DEXI, どぅ DOXU,                   ぢゃ DYA, ぢゅ DYU, ぢょ DYO,
ふぁ FA,  ふぃ FI,             ふぇ FE,  ふぉ FO,  ふゃ FYA, ふゅ FYU, ふょ FYO,
                              いぇ YE,
         ゐ WI,         ゑ WE,

ー -
"""

HIRAGANA_SOKUON = "っ"
HIRAGANA_SMALL_POST = "ぁ ぃ ぅ ぇ ぉ ゃ ゅ ょ ゎ"

# KATAKANA_TAB = """ァ  xa ア  a  ィ  xi イ  i  ゥ  xu
# ウ  u  ヴ  vu ヴァ va ヴィ vi ヴェ ve
# ヴォ vo ェ  xe エ  e  ォ  xo オ  o

# カ  ka ガ  ga キ  ki キャ kya   キュ kyu
# キョ kyo   ギ  gi ギャ gya   ギュ gyu   ギョ gyo
# ク  ku グ  gu ケ  ke ゲ  ge コ  ko
# ゴ  go

# サ  sa ザ  za シ  si シャ sya   シュ syu
# ショ syo   シェ  sye
# ジ  zi ジャ zya   ジュ zyu   ジョ zyo
# ス  su ズ  zu セ  se ゼ  ze ソ  so
# ゾ  zo

# タ  ta ダ  da チ  ti チャ tya   チュ tyu
# チョ tyo   ヂ  di ヂャ dya   ヂュ dyu   ヂョ dyo
# ティ  ti

# ッ  xtu
# ッヴ vvu   ッヴァ   vva   ッヴィ   vvi
# ッヴェ   vve   ッヴォ   vvo
# ッカ kka   ッガ gga   ッキ kki   ッキャ   kkya
# ッキュ   kkyu  ッキョ   kkyo  ッギ ggi   ッギャ   ggya
# ッギュ   ggyu  ッギョ   ggyo  ック kku   ッグ ggu
# ッケ kke   ッゲ gge   ッコ kko   ッゴ ggo   ッサ ssa
# ッザ zza   ッシ ssi   ッシャ   ssya
# ッシュ   ssyu  ッショ   ssyo  ッシェ   ssye
# ッジ zzi   ッジャ   zzya  ッジュ   zzyu  ッジョ   zzyo
# ッス ssu   ッズ zzu   ッセ sse   ッゼ zze   ッソ sso
# ッゾ zzo   ッタ tta   ッダ dda   ッチ tti   ッティ tti
# ッチャ   ttya  ッチュ   ttyu  ッチョ   ttyo  ッヂ ddi
# ッヂャ   ddya  ッヂュ   ddyu  ッヂョ   ddyo  ッツ ttu
# ッヅ ddu   ッテ tte   ッデ dde   ット tto   ッド ddo
# ッドゥ ddu
# ッハ hha   ッバ bba   ッパ ppa   ッヒ hhi
# ッヒャ   hhya  ッヒュ   hhyu  ッヒョ   hhyo  ッビ bbi
# ッビャ   bbya  ッビュ   bbyu  ッビョ   bbyo  ッピ ppi
# ッピャ   ppya  ッピュ   ppyu  ッピョ   ppyo  ッフ hhu   ッフュ ffu
# ッファ   ffa   ッフィ   ffi   ッフェ   ffe   ッフォ   ffo
# ッブ bbu   ップ ppu   ッヘ hhe   ッベ bbe   ッペ  ppe
# ッホ hho   ッボ bbo   ッポ ppo   ッヤ yya   ッユ yyu
# ッヨ yyo   ッラ rra   ッリ rri   ッリャ   rrya
# ッリュ   rryu  ッリョ   rryo  ッル rru   ッレ rre
# ッロ rro

# ツ  tu ヅ  du テ  te デ  de ト  to
# ド  do ドゥ  du

# ナ  na ニ  ni ニャ nya   ニュ nyu   ニョ nyo
# ヌ  nu ネ  ne ノ  no

# ハ  ha バ  ba パ  pa ヒ  hi ヒャ hya
# ヒュ hyu   ヒョ hyo   ビ  bi ビャ bya   ビュ byu
# ビョ byo   ピ  pi ピャ pya   ピュ pyu   ピョ pyo
# フ  hu ファ fa フィ fi フェ fe フォ fo
# フュ  fu
# ブ  bu プ  pu ヘ  he ベ  be ペ  pe
# ホ  ho ボ  bo ポ  po

# マ  ma ミ  mi ミャ mya   ミュ myu   ミョ myo
# ム  mu メ  me モ  mo

# ャ  xya   ヤ  ya ュ  xyu   ユ  yu ョ  xyo
# ヨ  yo

# ラ  ra リ  ri リャ rya   リュ ryu   リョ ryo
# ル  ru レ  re ロ  ro

# ヮ  xwa   ワ  wa ウィ  wi ヰ wi ヱ  we ウェ we
# ヲ  wo ウォ  wo ン n

# ン   n'
# ディ  dyi
# ー   -
# チェ  tye
# ッチェ   ttye
# ジェ zye
# """

#
# Tests
#

# if __name__ == "__main__":
#     print list(groups(re.split('\s', "a 1 b 2 cc 33,\nddd 444\teee 555")))<|endoftext|>


<commit_before>from array import array
from contextlib import closing
import sys
import time

from javax.net.ssl import SSLContext, TrustManager, X509TrustManager
from java.net import URL

from org.apache.abdera import Abdera

# Modified from http://tech.pedersen-live.com/2010/10/trusting-all-certificates-in-jython/
# would be best to manage the trust store for something real

class TrustAllX509TrustManager(X509TrustManager):
    """Define a custom TrustManager which will blindly accept all certificates"""
    def checkClientTrusted(self, chain, auth):
        pass

    def checkServerTrusted(self, chain, auth):
        pass

    def getAcceptedIssuers(self):
        return None

def trust_all():
    trust_managers = array(TrustManager, [TrustAllX509TrustManager()])
    trust_all_context = SSLContext.getInstance("SSL")
    trust_all_context.init(None, trust_managers, None)
    SSLContext.setDefault(trust_all_context)

def read_events(feed_url, mark_id=None, read_back=200, poll_time=5.):
    # a given read of the feed is going to be reversed; use the urn as the mark
    count = 0
    abdera = Abdera()
    parser = abdera.getParser()

    def read_chunks(last_id):
        chunk_count = 0
        url = feed_url
        while chunk_count < read_back:
            with closing(URL(url).openStream()) as f:
                print >> sys.stderr, "Reading feed", url
                doc = parser.parse(f)
                feed = doc.getRoot()
                url = str(feed.getLinks("next")[0].href)
                for entry in feed.entries:
                    if last_id == entry.id:
                        return
                    yield entry
                    chunk_count += 1

    # combine loops together FIXME!
    last_id = None
    for entry in reversed(list(read_chunks(mark_id))):
        last_id = entry.id
        count += 1
        yield entry
    print >> sys.stderr, "Read", count, "events so far"
    while True:
        print >> sys.stderr, "Sleeping", poll_time, "then read to", last_id
        time.sleep(poll_time)
        for entry in reversed(list(read_chunks(last_id))):
            last_id = entry.id
            count += 1
            yield entry
        print >> sys.stderr, "Read", count, "events so far"

def main():
    trust_all()
    url = "https://atom.prod.dfw1.us.ci.rackspace.net/nova/events"
    last_updated = None
    seen = set()
    count = 0
    for event in read_events(url):
        if last_updated and event.updated < last_updated:
            print >> sys.stderr, "Out of order!!!!"
        if event.id in seen:
            print >> sys.stderr, "Duplicated", event.id
            continue
        print count, event.id, event.updated, [cat.term for cat in event.categories]

        last_updated = event.updated
        seen.add(event.id)

        if count >= 1000:
            break

        count += 1

main()
<commit_msg>Initial version of nontransactional spout<commit_after>from collections import deque
from contextlib import closing
import time

from backtype.storm.topology.base import BaseRichSpout
from backtype.storm.tuple import Fields, Values
from java.net import URL
from org.apache.abdera import Abdera
from org.slf4j import LoggerFactory

from clamp import PackageProxy
from romper.trust import trust_all_certificates

# still need to implement consistent hashing if this will be used at large scale
# on the other hand, currently we see 1 event/s/data center for status updates, which can be readily handled
# with just one instance of the spout

# Make transactional; see https://github.com/nathanmarz/storm/wiki/Trident-state
# and https://github.com/nathanmarz/storm/wiki/Trident-spouts
# specifically opaque transactional support, such as seen in
# https://github.com/nathanmarz/storm-contrib/blob/master/storm-kafka/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java

class AtomSpout(BaseRichSpout):
    # FIXME doc the conf requirements

    __proxymaker__ = PackageProxy("otter")

    def open(self, conf, context, collector):
        if conf.get("trust_all_certificates"):
            trust_all_certificates()
        self.collector = collector
        self.last_time = time.time()

        # FIXME get last_id from ZK; also consider how to partition
        # with consistent hashing
        self.readers = [AtomReader(feed_url) for feed_url in conf["atom_feeds"]]

    def nextTuple(self):
        for reader in self.readers:
            for event in reader.read_events():
                self.collector.emit(Values([event.updated, event]))
        # FIXME update last_id per feed in ZK

        # FIXME low pri, but should make configurable
        # Also, in the event of lengthy disconnect we may also see rate limiting
        time.sleep(1.)

    def declareOutputFields(self, declarer):
        declarer.declare(Fields(["ts", "event"]))

# NOTE keep this as a distinct, testable piece of code

class SeenWindow(object):
    """Keeps track of what has been seen for `window` number of items"""

    def __init__(self, window=500):
        self.seen = set()
        self.purgeq = deque()
        self.window = window

    def add(self, item):
        self.seen.add(item)
        self.purgeq.append(item)
        if len(self.purgeq) > self.window:
            old = self.purgeq.popLeft()
            self.seen.remove(old)

    def __contains__(self, item):
        return item in self.seen

class AtomReader(object):

    def __init__(self, feed_url, read_back_pages=200, last_id=None):
        self.feed_url = feed_url
        self.read_back_pages = read_back_pages
        self.last_id = last_id
        self.parser = Abdera().getParser()
        self.seen = SeenWindow()
        self.last_updated = None
        self.total_events = 0
        self.log = LoggerFactory.getLogger(AtomReader)

    def _read_pages(self):
        """Reads pages from the feed, from newest to oldest (reverse chronology)"""
        page_count = 0
        url = self.feed_url
        while page_count < self.read_back_pages:  # FIXME what if we exhaust our pages?
            with closing(URL(url).openStream()) as f:
                self.log.debug("Reading feed: {}", url)
                doc = self.parser.parse(f)
                feed = doc.getRoot()
                url = str(feed.getLinks("next")[0].href)
                for entry in feed.entries:
                    if self.last_id == entry.id:
                        return  # done given read back to last_id
                    yield entry
                    page_count += 1

    def read_events(self):
        """Deliver events from oldest to newest, with some minimal sanity checking"""
        count = 0
        for event in reversed(list(self._read_pages())):
            if self.last_updated and event.updated < self.last_updated:
                self.log.warn("Ignoring out of order event in feed {}: {} ({}) is older than previous event {}",
                         self.feed_url, event.id, event.updated, self.last_updated)
                continue
            if event.id in self.seen:
                self.log.warn("Ignoring duplicated event in feed {}: {}", self.feed_url, event.id)
                continue
            self.last_updated = event.updated
            self.seen.add(event.id)
            self.last_id = event.id
            count += 1
            yield event

        # FIXME storm supports metrics. use that functionality.
        self.total_events += count
        self.log.debug("Read {} of {} events from feed {}", count, self.total_events, self.feed_url)
<|endoftext|>


<commit_before>from PyQt4 import QtCore, QtGui

from utils import *
import shared

class AccountMixin (object):
    def accountColor (self):
        if not self.isEnabled:
            return QtGui.QBrush(QtGui.QColor(128, 128, 128))
        elif self.type == "chan":
            return QtGui.QBrush(QtGui.QColor(216, 119, 0))
        elif self.type == "mailinglist" or self.type == "subscription":
            return QtGui.QBrush(QtGui.QColor(137, 04, 177))
        else:
            return QtGui.QBrush(QtGui.QApplication.palette().text().color())

    def setAddress(self, address):
        self.address = str(address)
        self.updateText()

    def setUnreadCount(self, cnt):
        self.unreadCount = int(cnt)
        self.updateText()

    def setEnabled(self, enabled):
        self.isEnabled = enabled
        self.updateText()

    def setType(self):
        if shared.safeConfigGetBoolean(self.address, 'chan'):
            self.type = "chan"
        elif shared.safeConfigGetBoolean(self.address, 'mailinglist'):
            self.type = "mailinglist"
        else:
            self.type = "normal"

    def updateText(self):
        pass

class Ui_FolderWidget(QtGui.QTreeWidgetItem, AccountMixin):
    folderWeight = {"inbox": 1, "sent": 2, "trash": 3}
    def __init__(self, parent, pos = 0, address = "", folderName = "", unreadCount = 0):
        super(QtGui.QTreeWidgetItem, self).__init__()
        self.initialised = False
        self.setAddress(address)
        self.setFolderName(folderName)
        self.setUnreadCount(unreadCount)
        self.initialised = True
        self.updateText()
        parent.insertChild(pos, self)

    def setFolderName(self, fname):
        self.folderName = str(fname)
        self.updateText()

    def updateText(self):
        if not self.initialised:
            return
        text = QtGui.QApplication.translate("MainWindow", self.folderName)
        font = QtGui.QFont()
        if self.unreadCount > 0:
            text += " (" + str(self.unreadCount) + ")"
            font.setBold(True)
        else:
            font.setBold(False)
        self.setFont(0, font)
        self.setText(0, text)
        self.setToolTip(0, text)
 #       self.setData(0, QtCore.Qt.UserRole, [self.address, self.folderName])

    # inbox, sent, thrash first, rest alphabetically
    def __lt__(self, other):
        if (isinstance(other, Ui_FolderWidget)):
            if self.folderName in self.folderWeight:
                x = self.folderWeight[self.folderName]
            else:
                x = 4
            if other.folderName in self.folderWeight:
                y = self.folderWeight[other.folderName]
            else:
                y = 4
            reverse = False
            if self.treeWidget().header().sortIndicatorOrder() == QtCore.Qt.DescendingOrder:
                reverse = True
            if x == y:
                return self.folderName < other.folderName
            else:
                return (x >= y if reverse else x < y)

        return super(QtGui.QTreeWidgetItem, self).__lt__(other)

class Ui_AddressWidget(QtGui.QTreeWidgetItem, AccountMixin):
    def __init__(self, parent, pos = 0, address = "", unreadCount = 0, enabled = True):
        super(QtGui.QTreeWidgetItem, self).__init__()
        parent.insertTopLevelItem(pos, self)
        # only set default when creating
        #super(QtGui.QTreeWidgetItem, self).setExpanded(shared.config.getboolean(self.address, 'enabled'))
        self.initialised = False
        self.setAddress(address)
        self.setEnabled(enabled)
        self.setUnreadCount(unreadCount)
        self.setType()
        self.initialised = True
        self.setExpanded(enabled) # does updateText

    def updateText(self):
        if not self.initialised:
            return
        text = unicode(shared.config.get(self.address, 'label'), 'utf-8)') + ' (' + self.address + ')'

        font = QtGui.QFont()
        if self.unreadCount > 0:
            # only show message count if the child doesn't show
            if not self.isExpanded():
                text += " (" + str(self.unreadCount) + ")"
            font.setBold(True)
        else:
            font.setBold(False)
        self.setFont(0, font)

        #set text color
        brush = self.accountColor()
        brush.setStyle(QtCore.Qt.NoBrush)
        self.setForeground(0, brush)

        self.setIcon(0, avatarize(self.address))
        self.setText(0, text)
        self.setToolTip(0, text)
#        self.setData(0, QtCore.Qt.UserRole, [self.address, "inbox"])

    def setExpanded(self, expand):
        super(Ui_AddressWidget, self).setExpanded(expand)
        self.updateText()

    def edit(self):
        self.setText(0, shared.config.get(self.address, 'label'))
        super(QtGui.QAbstractItemView, self).edit()

    # label (or address) alphabetically, disabled at the end
    def __lt__(self, other):
        if (isinstance(other, Ui_AddressWidget)):
            reverse = False
            if self.treeWidget().header().sortIndicatorOrder() == QtCore.Qt.DescendingOrder:
                reverse = True
            if self.isEnabled == other.isEnabled:
                if self.type == other.type:
                    if shared.config.get(self.address, 'label'):
                        x = shared.config.get(self.address, 'label').decode('utf-8').lower()
                    else:
                        x = self.address.decode('utf-8').lower()
                    if shared.config.get(other.address, 'label'):
                        y = shared.config.get(other.address, 'label').decode('utf-8').lower()
                    else:
                        y = other.address.decode('utf-8').lower()
                    return x < y
                else:
                    return (reverse if self.type == "mailinglist" else not reverse)
#            else:
            return (not reverse if self.isEnabled else reverse)

        return super(QtGui.QTreeWidgetItem, self).__lt__(other)

class Ui_SubscriptionWidget(Ui_AddressWidget, AccountMixin):
    def __init__(self, parent, pos = 0, address = "", unreadCount = 0, label = "", enabled = True):
        super(QtGui.QTreeWidgetItem, self).__init__()
        parent.insertTopLevelItem(pos, self)
        # only set default when creating
        #super(QtGui.QTreeWidgetItem, self).setExpanded(shared.config.getboolean(self.address, 'enabled'))
        self.initialised = False
        self.setAddress(address)
        self.setEnabled(enabled)
        self.setType()
        self.setLabel(label)
        self.setUnreadCount (unreadCount)
        self.initialised = True
        self.setExpanded(enabled) # does updateText

    def setLabel(self, label):
        self.label = label

    def setType(self):
        self.type = "subscription"

    def edit(self):
        self.setText(0, self.label)
        super(QtGui.QAbstractItemView, self).edit()

    def updateText(self):
        if not self.initialised:
            return
        text = unicode(self.label, 'utf-8)') + ' (' + self.address + ')'

        font = QtGui.QFont()
        if self.unreadCount > 0:
            # only show message count if the child doesn't show
            if not self.isExpanded():
                text += " (" + str(self.unreadCount) + ")"
            font.setBold(True)
        else:
            font.setBold(False)
        self.setFont(0, font)

        #set text color
        brush = self.accountColor()
        brush.setStyle(QtCore.Qt.NoBrush)
        self.setForeground(0, brush)

        self.setIcon(0, avatarize(self.address))
        self.setText(0, text)
        self.setToolTip(0, text)
#        self.setData(0, QtCore.Qt.UserRole, [self.address, "inbox"])

    # label (or address) alphabetically, disabled at the end
    def __lt__(self, other):
        if (isinstance(other, Ui_SubscriptionWidget)):
            reverse = False
            if self.treeWidget().header().sortIndicatorOrder() == QtCore.Qt.DescendingOrder:
                reverse = True
            if self.isEnabled == other.isEnabled:
                if self.label:
                    x = self.label.decode('utf-8').lower()
                else:
                    x = self.address.decode('utf-8').lower()
                if other.label:
                    y = other.label.decode('utf-8').lower()
                else:
                    y = other.address.decode('utf-8').lower()
                return x < y
#            else:
            return (not reverse if self.isEnabled else reverse)

        return super(QtGui.QTreeWidgetItem, self).__lt__(other)

class Ui_AddressBookWidgetItem(QtGui.QTableWidgetItem, AccountMixin):
    _types = {'normal': 0, 'chan': 1, 'subscription': 2}

    def __init__ (self, text, type = 'normal'):
        super(QtGui.QTableWidgetItem, self).__init__(text)
        self.label = text
        self.type = type
        try:
            self.typeNum = self._types[self.type]
        except:
            self.type = 0
        self.setEnabled(True)
        brush = self.accountColor()
        brush.setStyle(QtCore.Qt.NoBrush)
        self.setForeground(brush)

    def __lt__ (self, other):
        if (isinstance(other, Ui_AddressBookWidgetItem)):
            reverse = False
            if self.tableWidget().horizontalHeader().sortIndicatorOrder() == QtCore.Qt.DescendingOrder:
                reverse = True
            if self.typeNum == other.typeNum:
                return self.label.decode('utf-8').lower() < other.label.decode('utf-8').lower()
            else:
                return (not reverse if self.typeNum < other.typeNum else reverse)
        return super(QtGui.QTableWidgetItem, self).__lt__(other)

class Ui_AddressBookWidgetItemLabel(Ui_AddressBookWidgetItem):
    def __init__ (self, address, label, type):
        Ui_AddressBookWidgetItem.__init__(self, label, type)
        self.label = label
        self.setIcon(avatarize(address))

class Ui_AddressBookWidgetItemAddress(Ui_AddressBookWidgetItem):
    def __init__ (self, address, label, type):
        Ui_AddressBookWidgetItem.__init__(self, address, type)
        self.label = label
        self.setFlags(QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)<commit_msg>Show tooltips on addressbook<commit_after>from PyQt4 import QtCore, QtGui

from utils import *
import shared

class AccountMixin (object):
    def accountColor (self):
        if not self.isEnabled:
            return QtGui.QBrush(QtGui.QColor(128, 128, 128))
        elif self.type == "chan":
            return QtGui.QBrush(QtGui.QColor(216, 119, 0))
        elif self.type == "mailinglist" or self.type == "subscription":
            return QtGui.QBrush(QtGui.QColor(137, 04, 177))
        else:
            return QtGui.QBrush(QtGui.QApplication.palette().text().color())

    def setAddress(self, address):
        self.address = str(address)
        self.updateText()

    def setUnreadCount(self, cnt):
        self.unreadCount = int(cnt)
        self.updateText()

    def setEnabled(self, enabled):
        self.isEnabled = enabled
        self.updateText()

    def setType(self):
        if shared.safeConfigGetBoolean(self.address, 'chan'):
            self.type = "chan"
        elif shared.safeConfigGetBoolean(self.address, 'mailinglist'):
            self.type = "mailinglist"
        else:
            self.type = "normal"

    def updateText(self):
        pass

class Ui_FolderWidget(QtGui.QTreeWidgetItem, AccountMixin):
    folderWeight = {"inbox": 1, "sent": 2, "trash": 3}
    def __init__(self, parent, pos = 0, address = "", folderName = "", unreadCount = 0):
        super(QtGui.QTreeWidgetItem, self).__init__()
        self.initialised = False
        self.setAddress(address)
        self.setFolderName(folderName)
        self.setUnreadCount(unreadCount)
        self.initialised = True
        self.updateText()
        parent.insertChild(pos, self)

    def setFolderName(self, fname):
        self.folderName = str(fname)
        self.updateText()

    def updateText(self):
        if not self.initialised:
            return
        text = QtGui.QApplication.translate("MainWindow", self.folderName)
        font = QtGui.QFont()
        if self.unreadCount > 0:
            text += " (" + str(self.unreadCount) + ")"
            font.setBold(True)
        else:
            font.setBold(False)
        self.setFont(0, font)
        self.setText(0, text)
        self.setToolTip(0, text)
 #       self.setData(0, QtCore.Qt.UserRole, [self.address, self.folderName])

    # inbox, sent, thrash first, rest alphabetically
    def __lt__(self, other):
        if (isinstance(other, Ui_FolderWidget)):
            if self.folderName in self.folderWeight:
                x = self.folderWeight[self.folderName]
            else:
                x = 4
            if other.folderName in self.folderWeight:
                y = self.folderWeight[other.folderName]
            else:
                y = 4
            reverse = False
            if self.treeWidget().header().sortIndicatorOrder() == QtCore.Qt.DescendingOrder:
                reverse = True
            if x == y:
                return self.folderName < other.folderName
            else:
                return (x >= y if reverse else x < y)

        return super(QtGui.QTreeWidgetItem, self).__lt__(other)

class Ui_AddressWidget(QtGui.QTreeWidgetItem, AccountMixin):
    def __init__(self, parent, pos = 0, address = "", unreadCount = 0, enabled = True):
        super(QtGui.QTreeWidgetItem, self).__init__()
        parent.insertTopLevelItem(pos, self)
        # only set default when creating
        #super(QtGui.QTreeWidgetItem, self).setExpanded(shared.config.getboolean(self.address, 'enabled'))
        self.initialised = False
        self.setAddress(address)
        self.setEnabled(enabled)
        self.setUnreadCount(unreadCount)
        self.setType()
        self.initialised = True
        self.setExpanded(enabled) # does updateText

    def updateText(self):
        if not self.initialised:
            return
        text = unicode(shared.config.get(self.address, 'label'), 'utf-8)') + ' (' + self.address + ')'

        font = QtGui.QFont()
        if self.unreadCount > 0:
            # only show message count if the child doesn't show
            if not self.isExpanded():
                text += " (" + str(self.unreadCount) + ")"
            font.setBold(True)
        else:
            font.setBold(False)
        self.setFont(0, font)

        #set text color
        brush = self.accountColor()
        brush.setStyle(QtCore.Qt.NoBrush)
        self.setForeground(0, brush)

        self.setIcon(0, avatarize(self.address))
        self.setText(0, text)
        self.setToolTip(0, text)
#        self.setData(0, QtCore.Qt.UserRole, [self.address, "inbox"])

    def setExpanded(self, expand):
        super(Ui_AddressWidget, self).setExpanded(expand)
        self.updateText()

    def edit(self):
        self.setText(0, shared.config.get(self.address, 'label'))
        super(QtGui.QAbstractItemView, self).edit()

    # label (or address) alphabetically, disabled at the end
    def __lt__(self, other):
        if (isinstance(other, Ui_AddressWidget)):
            reverse = False
            if self.treeWidget().header().sortIndicatorOrder() == QtCore.Qt.DescendingOrder:
                reverse = True
            if self.isEnabled == other.isEnabled:
                if self.type == other.type:
                    if shared.config.get(self.address, 'label'):
                        x = shared.config.get(self.address, 'label').decode('utf-8').lower()
                    else:
                        x = self.address.decode('utf-8').lower()
                    if shared.config.get(other.address, 'label'):
                        y = shared.config.get(other.address, 'label').decode('utf-8').lower()
                    else:
                        y = other.address.decode('utf-8').lower()
                    return x < y
                else:
                    return (reverse if self.type == "mailinglist" else not reverse)
#            else:
            return (not reverse if self.isEnabled else reverse)

        return super(QtGui.QTreeWidgetItem, self).__lt__(other)

class Ui_SubscriptionWidget(Ui_AddressWidget, AccountMixin):
    def __init__(self, parent, pos = 0, address = "", unreadCount = 0, label = "", enabled = True):
        super(QtGui.QTreeWidgetItem, self).__init__()
        parent.insertTopLevelItem(pos, self)
        # only set default when creating
        #super(QtGui.QTreeWidgetItem, self).setExpanded(shared.config.getboolean(self.address, 'enabled'))
        self.initialised = False
        self.setAddress(address)
        self.setEnabled(enabled)
        self.setType()
        self.setLabel(label)
        self.setUnreadCount (unreadCount)
        self.initialised = True
        self.setExpanded(enabled) # does updateText

    def setLabel(self, label):
        self.label = label

    def setType(self):
        self.type = "subscription"

    def edit(self):
        self.setText(0, self.label)
        super(QtGui.QAbstractItemView, self).edit()

    def updateText(self):
        if not self.initialised:
            return
        text = unicode(self.label, 'utf-8)') + ' (' + self.address + ')'

        font = QtGui.QFont()
        if self.unreadCount > 0:
            # only show message count if the child doesn't show
            if not self.isExpanded():
                text += " (" + str(self.unreadCount) + ")"
            font.setBold(True)
        else:
            font.setBold(False)
        self.setFont(0, font)

        #set text color
        brush = self.accountColor()
        brush.setStyle(QtCore.Qt.NoBrush)
        self.setForeground(0, brush)

        self.setIcon(0, avatarize(self.address))
        self.setText(0, text)
        self.setToolTip(0, text)
#        self.setData(0, QtCore.Qt.UserRole, [self.address, "inbox"])

    # label (or address) alphabetically, disabled at the end
    def __lt__(self, other):
        if (isinstance(other, Ui_SubscriptionWidget)):
            reverse = False
            if self.treeWidget().header().sortIndicatorOrder() == QtCore.Qt.DescendingOrder:
                reverse = True
            if self.isEnabled == other.isEnabled:
                if self.label:
                    x = self.label.decode('utf-8').lower()
                else:
                    x = self.address.decode('utf-8').lower()
                if other.label:
                    y = other.label.decode('utf-8').lower()
                else:
                    y = other.address.decode('utf-8').lower()
                return x < y
#            else:
            return (not reverse if self.isEnabled else reverse)

        return super(QtGui.QTreeWidgetItem, self).__lt__(other)

class Ui_AddressBookWidgetItem(QtGui.QTableWidgetItem, AccountMixin):
    _types = {'normal': 0, 'chan': 1, 'subscription': 2}

    def __init__ (self, text, type = 'normal'):
        super(QtGui.QTableWidgetItem, self).__init__(text)
        self.label = text
        self.type = type
        try:
            self.typeNum = self._types[self.type]
        except:
            self.type = 0
        self.setEnabled(True)
        brush = self.accountColor()
        brush.setStyle(QtCore.Qt.NoBrush)
        self.setForeground(brush)

    def __lt__ (self, other):
        if (isinstance(other, Ui_AddressBookWidgetItem)):
            reverse = False
            if self.tableWidget().horizontalHeader().sortIndicatorOrder() == QtCore.Qt.DescendingOrder:
                reverse = True
            if self.typeNum == other.typeNum:
                return self.label.decode('utf-8').lower() < other.label.decode('utf-8').lower()
            else:
                return (not reverse if self.typeNum < other.typeNum else reverse)
        return super(QtGui.QTableWidgetItem, self).__lt__(other)

class Ui_AddressBookWidgetItemLabel(Ui_AddressBookWidgetItem):
    def __init__ (self, address, label, type):
        Ui_AddressBookWidgetItem.__init__(self, label, type)
        self.setIcon(avatarize(address))
        self.setToolTip(label + " (" + address + ")")

class Ui_AddressBookWidgetItemAddress(Ui_AddressBookWidgetItem):
    def __init__ (self, address, label, type):
        Ui_AddressBookWidgetItem.__init__(self, address, type)
        self.setFlags(QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
        self.setToolTip(label + " (" + address + ")")<|endoftext|>


<commit_before># This file is part of cclib (http://cclib.sf.net), a library for parsing
# and interpreting the results of computational chemistry packages.
#
# Copyright (C) 2006-2014, the cclib development team
#
# The library is free software, distributed under the terms of
# the GNU Lesser General Public version 2.1 or later. You should have
# received a copy of the license along with cclib. You can also access
# the full license online at http://www.gnu.org/copyleft/lgpl.html.

import numpy

import bettertest

class GenericSPTest(bettertest.TestCase):
    """Restricted single point unittest."""

    # In STO-3G, H has 1, C has 5 (1 S and 4 SP).
    nbasisdict = {1:1, 6:5}

    # Approximate B3LYP energy of dvb after SCF in STO-3G.
    b3lyp_energy = -10365

    def testnatom(self):
        """Is the number of atoms equal to 20?"""
        self.assertEquals(self.data.natom, 20)

    def testatomnos(self):
        """Are the atomnos correct?"""
        self.failUnless(numpy.alltrue([numpy.issubdtype(atomno,int) for atomno in self.data.atomnos]))
        # This will work only for numpy
        #self.assertEquals(self.data.atomnos.dtype.char, 'i')
        self.assertEquals(self.data.atomnos.shape, (20,) )
        self.assertEquals(sum(self.data.atomnos==6) + sum(self.data.atomnos==1), 20)

    def testatomcharges(self):
        """Are atomcharges (at least Mulliken) consistent with natom and sum to zero?"""
        for type in set(['mulliken'] + list(self.data.atomcharges.keys())):
            charges = self.data.atomcharges[type]
            self.assertEquals(len(charges), self.data.natom)
            self.assertInside(sum(charges), 0.0, 0.001)

    def testatomcoords(self):
        """Are the dimensions of atomcoords 1 x natom x 3?"""
        self.assertEquals(self.data.atomcoords.shape,(1,self.data.natom,3))

    def testcharge_and_mult(self):
        """Are the charge and multiplicity correct?"""
        self.assertEquals(self.data.charge, 0)
        self.assertEquals(self.data.mult, 1)

    def testnbasis(self):
        """Is the number of basis set functions correct?"""
        count = sum([self.nbasisdict[n] for n in self.data.atomnos])
        self.assertEquals(self.data.nbasis, count)

    def testatombasis(self):
        """Are the indices in atombasis the right amount and unique?"""
        all = []
        for i,atom in enumerate(self.data.atombasis):
            self.assertEquals(len(atom), self.nbasisdict[self.data.atomnos[i]])
            all += atom
        # Test if there are as many indices as atomic orbitals.
        self.assertEquals(len(all), self.data.nbasis)
        # Check if all are different (every orbital indexed once).
        self.assertEquals(len(set(all)), len(all))

    def testcoreelectrons(self):
        """Are the coreelectrons all 0?"""
        ans = numpy.zeros(self.data.natom, 'i')
        self.assertArrayEquals(self.data.coreelectrons, ans)

    def testnormalisesym(self):
        """Did this subclass overwrite normalisesym?"""
        self.assertNotEquals(self.logfile.normalisesym("A"),"ERROR: This should be overwritten by this subclass")

    def testsymlabels(self):
        """Are all the symmetry labels either Ag/u or Bg/u?"""
        sumwronglabels = sum([x not in ['Ag','Bu','Au','Bg'] for x in self.data.mosyms[0]])
        self.assertEquals(sumwronglabels,0)

    def testhomos(self):
        """Is the index of the HOMO equal to 34?"""
        self.assertArrayEquals(self.data.homos, numpy.array([34],"i"),"%s != array([34],'i')" % numpy.array_repr(self.data.homos))

    def testscfvaluetype(self):
        """Are scfvalues and its elements the right type??"""
        self.assertEquals(type(self.data.scfvalues),type([]))
        self.assertEquals(type(self.data.scfvalues[0]),type(numpy.array([])))

    def testscfenergy(self):
        """Is the SCF energy within 40eV of target?"""
        self.assertInside(self.data.scfenergies[-1], self.b3lyp_energy, 40, "Final scf energy: %f not %i +- 40eV" %(self.data.scfenergies[-1], self.b3lyp_energy))

    def testscftargetdim(self):
        """Do the scf targets have the right dimensions?"""
        self.assertEquals(self.data.scftargets.shape,(len(self.data.scfvalues),len(self.data.scfvalues[0][0])))

    def testlengthmoenergies(self):
        """Is the number of evalues equal to nmo?"""
        self.assertEquals(len(self.data.moenergies[0]), self.data.nmo)

    def testtypemoenergies(self):
        """Is moenergies a list containing one numpy array?"""
        self.assertEquals(type(self.data.moenergies), type([]))
        self.assertEquals(type(self.data.moenergies[0]), type(numpy.array([])))

    def testdimmocoeffs(self):
        """Are the dimensions of mocoeffs equal to 1 x nmo x nbasis?"""
        self.assertEquals(type(self.data.mocoeffs), type([]))
        self.assertEquals(len(self.data.mocoeffs), 1)
        self.assertEquals(self.data.mocoeffs[0].shape,
                          (self.data.nmo, self.data.nbasis))

    def testdimaooverlaps(self):
        """Are the dims of the overlap matrix consistent with nbasis?"""
        # ADF has the attribute fooverlaps instead of aooverlaps.
        if not hasattr(self.data, "aooverlaps") and hasattr(self.data, "fooverlaps"):
            self.data.aooverlaps = self.data.fooverlaps
        self.assertEquals(self.data.aooverlaps.shape,(self.data.nbasis,self.data.nbasis))

    def testaooverlaps(self):
        """Are the first row and colm of the overlap matrix identical?"""
        # ADF has the attribute fooverlaps instead of aooverlaps.
        if not hasattr(self.data, "aooverlaps") and hasattr(self.data, "fooverlaps"):
            self.data.aooverlaps = self.data.fooverlaps
        self.assertEquals(sum(self.data.aooverlaps[0,:] -
                              self.data.aooverlaps[:,0]),
                          0)

class ADFSPTest(GenericSPTest):
    """ADF restricted single point unittest."""

    # ADF parser does not extract atombasis.
    def testatombasis(self):
        """Are the indices in atombasis the right amount and unique? PASS"""
        self.assertEquals(1, 1)

    def testscfenergy(self):
        """Is the SCF energy within 1eV of -140eV?"""
        self.assertInside(self.data.scfenergies[-1],-140,1,"Final scf energy: %f not -140+-1eV" % self.data.scfenergies[-1])

class GamessUKSPTest(GenericSPTest):
    """GAMESS-UK restricted single point unittest."""

class GamessUSSPTest(GenericSPTest):
    """GAMESS-US restricted single point unittest."""

    old_tests = ["GAMESS/GAMESS-US/dvb_sp_2006.02.22.r2.out.gz"]

class GaussianSPTest(GenericSPTest):
    """Gaussian restricted single point unittest."""

    # Molecular mass of DVB in mD.
    molecularmass = 130078.25

    def testatommasses(self):
        """Do the atom masses sum up to the molecular mass (130078.25+-0.1mD)?"""
        mm = 1000*sum(self.data.atommasses)
        self.assertInside(mm, 130078.25, 0.1, "Molecule mass: %f not 130078 +- 0.1mD" %mm)

class JaguarSPTest(GenericSPTest):
    """Jaguar restricted single point unittest."""

    # Data file does not contain enough information. Can we make a new one?
    def testatombasis(self):
        """Are the indices in atombasis the right amount and unique? PASS"""
        self.assertEquals(1, 1)

    # Jaguar prints only 10 virtual MOs by default. Can we re-run with full output?
    def testlengthmoenergies(self):
        """Is the number of evalues equal to the number of occ. MOs + 10?"""
        self.assertEquals(len(self.data.moenergies[0]), self.data.homos[0]+11)

    # Data file does not contain enough information. Can we make a new one?
    def testdimmocoeffs(self):
        """Are the dimensions of mocoeffs equal to 1 x nmo x nbasis? PASS"""
        self.assertEquals(1, 1)

class MolproSPTest(GenericSPTest):
    """Molpro restricted single point unittest."""

    def testsymlabels(self):
        """Are all the symmetry labels either Ag/u or Bg/u? PASS"""
        self.assertEquals(1,1)

class OrcaSPTest(GenericSPTest):
    """ORCA restricted single point unittest."""

    # Approximate B3LYP energy of dvb after SCF in 3-21G.
    b3lyp_energy = -10470

    # ORCA has no support for symmetry yet.
    def testsymlabels(self):
        """Are all the symmetry labels either Ag/u or Bg/u? PASS"""
        self.assertEquals(1,1)

class PCGamessSPTest(GenericSPTest):
    """PC-GAMESS restricted single point unittest."""

if __name__=="__main__":

    from testall import testall
    testall(modules=["SP"])
<commit_msg>SP test: remove ORCA-specific SCF energy now that STO-3G is used<commit_after># This file is part of cclib (http://cclib.sf.net), a library for parsing
# and interpreting the results of computational chemistry packages.
#
# Copyright (C) 2006-2014, the cclib development team
#
# The library is free software, distributed under the terms of
# the GNU Lesser General Public version 2.1 or later. You should have
# received a copy of the license along with cclib. You can also access
# the full license online at http://www.gnu.org/copyleft/lgpl.html.

import numpy

import bettertest

class GenericSPTest(bettertest.TestCase):
    """Restricted single point unittest."""

    # In STO-3G, H has 1, C has 5 (1 S and 4 SP).
    nbasisdict = {1:1, 6:5}

    # Approximate B3LYP energy of dvb after SCF in STO-3G.
    b3lyp_energy = -10365

    def testnatom(self):
        """Is the number of atoms equal to 20?"""
        self.assertEquals(self.data.natom, 20)

    def testatomnos(self):
        """Are the atomnos correct?"""
        self.failUnless(numpy.alltrue([numpy.issubdtype(atomno,int) for atomno in self.data.atomnos]))
        # This will work only for numpy
        #self.assertEquals(self.data.atomnos.dtype.char, 'i')
        self.assertEquals(self.data.atomnos.shape, (20,) )
        self.assertEquals(sum(self.data.atomnos==6) + sum(self.data.atomnos==1), 20)

    def testatomcharges(self):
        """Are atomcharges (at least Mulliken) consistent with natom and sum to zero?"""
        for type in set(['mulliken'] + list(self.data.atomcharges.keys())):
            charges = self.data.atomcharges[type]
            self.assertEquals(len(charges), self.data.natom)
            self.assertInside(sum(charges), 0.0, 0.001)

    def testatomcoords(self):
        """Are the dimensions of atomcoords 1 x natom x 3?"""
        self.assertEquals(self.data.atomcoords.shape,(1,self.data.natom,3))

    def testcharge_and_mult(self):
        """Are the charge and multiplicity correct?"""
        self.assertEquals(self.data.charge, 0)
        self.assertEquals(self.data.mult, 1)

    def testnbasis(self):
        """Is the number of basis set functions correct?"""
        count = sum([self.nbasisdict[n] for n in self.data.atomnos])
        self.assertEquals(self.data.nbasis, count)

    def testatombasis(self):
        """Are the indices in atombasis the right amount and unique?"""
        all = []
        for i,atom in enumerate(self.data.atombasis):
            self.assertEquals(len(atom), self.nbasisdict[self.data.atomnos[i]])
            all += atom
        # Test if there are as many indices as atomic orbitals.
        self.assertEquals(len(all), self.data.nbasis)
        # Check if all are different (every orbital indexed once).
        self.assertEquals(len(set(all)), len(all))

    def testcoreelectrons(self):
        """Are the coreelectrons all 0?"""
        ans = numpy.zeros(self.data.natom, 'i')
        self.assertArrayEquals(self.data.coreelectrons, ans)

    def testnormalisesym(self):
        """Did this subclass overwrite normalisesym?"""
        self.assertNotEquals(self.logfile.normalisesym("A"),"ERROR: This should be overwritten by this subclass")

    def testsymlabels(self):
        """Are all the symmetry labels either Ag/u or Bg/u?"""
        sumwronglabels = sum([x not in ['Ag','Bu','Au','Bg'] for x in self.data.mosyms[0]])
        self.assertEquals(sumwronglabels,0)

    def testhomos(self):
        """Is the index of the HOMO equal to 34?"""
        self.assertArrayEquals(self.data.homos, numpy.array([34],"i"),"%s != array([34],'i')" % numpy.array_repr(self.data.homos))

    def testscfvaluetype(self):
        """Are scfvalues and its elements the right type??"""
        self.assertEquals(type(self.data.scfvalues),type([]))
        self.assertEquals(type(self.data.scfvalues[0]),type(numpy.array([])))

    def testscfenergy(self):
        """Is the SCF energy within 40eV of target?"""
        self.assertInside(self.data.scfenergies[-1], self.b3lyp_energy, 40, "Final scf energy: %f not %i +- 40eV" %(self.data.scfenergies[-1], self.b3lyp_energy))

    def testscftargetdim(self):
        """Do the scf targets have the right dimensions?"""
        self.assertEquals(self.data.scftargets.shape,(len(self.data.scfvalues),len(self.data.scfvalues[0][0])))

    def testlengthmoenergies(self):
        """Is the number of evalues equal to nmo?"""
        self.assertEquals(len(self.data.moenergies[0]), self.data.nmo)

    def testtypemoenergies(self):
        """Is moenergies a list containing one numpy array?"""
        self.assertEquals(type(self.data.moenergies), type([]))
        self.assertEquals(type(self.data.moenergies[0]), type(numpy.array([])))

    def testdimmocoeffs(self):
        """Are the dimensions of mocoeffs equal to 1 x nmo x nbasis?"""
        self.assertEquals(type(self.data.mocoeffs), type([]))
        self.assertEquals(len(self.data.mocoeffs), 1)
        self.assertEquals(self.data.mocoeffs[0].shape,
                          (self.data.nmo, self.data.nbasis))

    def testdimaooverlaps(self):
        """Are the dims of the overlap matrix consistent with nbasis?"""
        # ADF has the attribute fooverlaps instead of aooverlaps.
        if not hasattr(self.data, "aooverlaps") and hasattr(self.data, "fooverlaps"):
            self.data.aooverlaps = self.data.fooverlaps
        self.assertEquals(self.data.aooverlaps.shape,(self.data.nbasis,self.data.nbasis))

    def testaooverlaps(self):
        """Are the first row and colm of the overlap matrix identical?"""
        # ADF has the attribute fooverlaps instead of aooverlaps.
        if not hasattr(self.data, "aooverlaps") and hasattr(self.data, "fooverlaps"):
            self.data.aooverlaps = self.data.fooverlaps
        self.assertEquals(sum(self.data.aooverlaps[0,:] -
                              self.data.aooverlaps[:,0]),
                          0)

class ADFSPTest(GenericSPTest):
    """ADF restricted single point unittest."""

    # ADF parser does not extract atombasis.
    def testatombasis(self):
        """Are the indices in atombasis the right amount and unique? PASS"""
        self.assertEquals(1, 1)

    def testscfenergy(self):
        """Is the SCF energy within 1eV of -140eV?"""
        self.assertInside(self.data.scfenergies[-1],-140,1,"Final scf energy: %f not -140+-1eV" % self.data.scfenergies[-1])

class GamessUKSPTest(GenericSPTest):
    """GAMESS-UK restricted single point unittest."""

class GamessUSSPTest(GenericSPTest):
    """GAMESS-US restricted single point unittest."""

    old_tests = ["GAMESS/GAMESS-US/dvb_sp_2006.02.22.r2.out.gz"]

class GaussianSPTest(GenericSPTest):
    """Gaussian restricted single point unittest."""

    # Molecular mass of DVB in mD.
    molecularmass = 130078.25

    def testatommasses(self):
        """Do the atom masses sum up to the molecular mass (130078.25+-0.1mD)?"""
        mm = 1000*sum(self.data.atommasses)
        self.assertInside(mm, 130078.25, 0.1, "Molecule mass: %f not 130078 +- 0.1mD" %mm)

class JaguarSPTest(GenericSPTest):
    """Jaguar restricted single point unittest."""

    # Data file does not contain enough information. Can we make a new one?
    def testatombasis(self):
        """Are the indices in atombasis the right amount and unique? PASS"""
        self.assertEquals(1, 1)

    # Jaguar prints only 10 virtual MOs by default. Can we re-run with full output?
    def testlengthmoenergies(self):
        """Is the number of evalues equal to the number of occ. MOs + 10?"""
        self.assertEquals(len(self.data.moenergies[0]), self.data.homos[0]+11)

    # Data file does not contain enough information. Can we make a new one?
    def testdimmocoeffs(self):
        """Are the dimensions of mocoeffs equal to 1 x nmo x nbasis? PASS"""
        self.assertEquals(1, 1)

class MolproSPTest(GenericSPTest):
    """Molpro restricted single point unittest."""

    def testsymlabels(self):
        """Are all the symmetry labels either Ag/u or Bg/u? PASS"""
        self.assertEquals(1,1)

class OrcaSPTest(GenericSPTest):
    """ORCA restricted single point unittest."""

    # ORCA has no support for symmetry yet.
    def testsymlabels(self):
        """Are all the symmetry labels either Ag/u or Bg/u? PASS"""
        self.assertEquals(1,1)

class PCGamessSPTest(GenericSPTest):
    """PC-GAMESS restricted single point unittest."""

if __name__=="__main__":

    from testall import testall
    testall(modules=["SP"])
<|endoftext|>


<commit_before>from PyQt4 import QtGui, QtCore
import os
import json

class FileList(QtGui.QListWidget):
    def __init__(self, dir_path):
        super(FileList, self).__init__()
        self.dir_path = dir_path
        self.name_timestamp_dict = {}
        self.load_sweeps_in_dir()
        for name, dir_path in self.name_timestamp_dict.items():
            item = QtGui.QListWidgetItem(name, parent=self)
            # QtCore.Qt.UserRole is simply the integer 32 and denotes the role
            # of the data. That seems super weird.
            item.setData(QtCore.Qt.UserRole, dir_path)
            self.addItem(item)

    def load_sweeps_in_dir(self):
        for dirpath, dirnames, fnames in os.walk(self.dir_path, followlinks=False):
            if 'meta.json' in fnames:
                json_path = os.path.join(dirpath, 'meta.json')
                with open(json_path) as json_file:
                    meta = json.load(json_file)
                self.name_timestamp_dict[meta['name']] = dirpath

<commit_msg>Newest entry is selected by default in file list<commit_after>from PyQt4 import QtGui, QtCore
import os
import json

class FileList(QtGui.QListWidget):
    def __init__(self, dir_path):
        super(FileList, self).__init__()
        self.dir_path = dir_path
        self.name_timestamp_dict = {}
        self.load_sweeps_in_dir()
        for name, dir_path in self.name_timestamp_dict.items():
            item = QtGui.QListWidgetItem(name, parent=self)
            # QtCore.Qt.UserRole is simply the integer 32 and denotes the role
            # of the data. That seems super weird.
            item.setData(QtCore.Qt.UserRole, dir_path)
            self.addItem(item)
        self.setCurrentRow(0)

    def load_sweeps_in_dir(self):
        for dirpath, dirnames, fnames in os.walk(self.dir_path, followlinks=False):
            if 'meta.json' in fnames:
                json_path = os.path.join(dirpath, 'meta.json')
                with open(json_path) as json_file:
                    meta = json.load(json_file)
                self.name_timestamp_dict[meta['name']] = dirpath

<|endoftext|>


<commit_before>import importlib
import inspect
import tkinter as tk

import os
from tkinter import ttk

from agents.base_agent import BaseAgent, BOT_CONFIG_MODULE_HEADER, AGENT_MODULE_KEY
from utils.agent_creator import import_agent, get_base_import_package
from utils.rlbot_config_parser import PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_CONFIG_KEY, PARTICIPANT_BOT_KEY, \
    PARTICIPANT_RLBOT_KEY, PARTICIPANT_BOT_SKILL_KEY

class BaseAgentFrame(tk.Frame):

    config_object = None  # The config that is
    overall_config = None  # The config that is shared by all agent frames.
    overall_index = -1  # The index that grabs data from the overall_config
    team_index = -1  # The index representing what team the agent belongs to.
    parent = None  # The parent frame
    agent_config_path = None  # The config path for the agent config file
    agent_class = None  # The class for the agent
    agent_config = None

    def __init__(self, parent, team_index, *args, **kwargs):
        tk.Frame.__init__(self, parent, *args, *kwargs)
        self.team_index = team_index
        self.parent = parent

    def initialize_widgets(self):
        pass

    def refresh_widgets(self):
        pass

    @staticmethod
    def grid_items(start_row=0, start_index=0, *widgets):
        """Grid all items found in WIDGETS starting in row START_ROW from index START_INDEX"""
        for row, widget_list in enumerate(widgets):
            row += start_row
            for column, widget in enumerate(widget_list):
                column += start_index
                widget.grid(row=row, column=column, sticky="nsew")

    def get_agent_config_path(self):
        return os.path.realpath(self.overall_config.get(PARTICIPANT_CONFIGURATION_HEADER,
                                                        PARTICIPANT_CONFIG_KEY, self.overall_index))

    def set_agent_config_path(self, config_path):
        self.overall_config.set_value(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_CONFIG_KEY,
                                      config_path, self.overall_index)

    def is_participant_bot(self):
        return self.overall_config.getboolean(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_BOT_KEY, self.overall_index)

    def set_is_participant_bot(self, is_bot):
        return self.overall_config.set_value(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_BOT_KEY,
                                             is_bot, self.overall_index)

    def is_participant_custom_bot(self):
        return self.overall_config.getboolean(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_RLBOT_KEY, self.overall_index)

    def set_is_participant_custom_bot(self, is_rlbot):
        return self.overall_config.set_value(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_RLBOT_KEY,
                                             is_rlbot, self.overall_index)

    def get_bot_skill(self):
        return self.overall_config.getfloat(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_BOT_SKILL_KEY, self.overall_index)

    def set_bot_skill(self, bot_skill):
        return self.overall_config.set_value(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_BOT_SKILL_KEY,
                                             bot_skill, self.overall_index)

    def load_agent_config(self, agent_class=None):
        if self.is_participant_bot() and self.is_participant_custom_bot():
            agent_config_path = self.get_agent_config_path()
            base_agent_config = BaseAgent.create_agent_configurations()
            if agent_config_path is None:
                if agent_class is not None:
                    self.agent_class = agent_class
                    self.agent_config = self.agent_class.create_agent_configurations()
                else:
                    self.agent_config = base_agent_config
            else:
                base_agent_config.parse_file(agent_config_path)
                bot_module = base_agent_config.get(BOT_CONFIG_MODULE_HEADER, AGENT_MODULE_KEY)
                base_import = get_base_import_package(os.path.dirname(agent_config_path))
                self.agent_class = import_agent(base_import + '.' + bot_module)
                self.agent_config = self.agent_class.create_agent_configurations()
                self.agent_config.parse_file(agent_config_path)

    def load_config(self, overall_config_file, overall_index):
        self.overall_config = overall_config_file
        self.overall_index = overall_index
        self.load_agent_config()

    def load_agent_from_path(self, agent_file_path):
        module = get_base_import_package(agent_file_path)
        self.agent_class = import_agent(module)
<commit_msg>code cleanup<commit_after>import tkinter as tk

import os

from agents.base_agent import BaseAgent, BOT_CONFIG_MODULE_HEADER, AGENT_MODULE_KEY
from utils.agent_creator import import_agent, get_base_import_package
from utils.rlbot_config_parser import PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_CONFIG_KEY, PARTICIPANT_BOT_KEY, \
    PARTICIPANT_RLBOT_KEY, PARTICIPANT_BOT_SKILL_KEY

class BaseAgentFrame(tk.Frame):

    config_object = None  # The config that is
    overall_config = None  # The config that is shared by all agent frames.
    overall_index = -1  # The index that grabs data from the overall_config
    team_index = -1  # The index representing what team the agent belongs to.
    parent = None  # The parent frame
    agent_config_path = None  # The config path for the agent config file
    agent_class = None  # The class for the agent
    agent_config = None

    def __init__(self, parent, team_index, *args, **kwargs):
        tk.Frame.__init__(self, parent, *args, *kwargs)
        self.team_index = team_index
        self.parent = parent

    def initialize_widgets(self):
        pass

    def refresh_widgets(self):
        pass

    @staticmethod
    def grid_items(start_row=0, start_index=0, *widgets):
        """Grid all items found in WIDGETS starting in row START_ROW from index START_INDEX"""
        for row, widget_list in enumerate(widgets):
            row += start_row
            for column, widget in enumerate(widget_list):
                column += start_index
                widget.grid(row=row, column=column, sticky="nsew")

    def get_agent_config_path(self):
        return os.path.realpath(self.overall_config.get(PARTICIPANT_CONFIGURATION_HEADER,
                                                        PARTICIPANT_CONFIG_KEY, self.overall_index))

    def set_agent_config_path(self, config_path):
        self.overall_config.set_value(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_CONFIG_KEY,
                                      config_path, self.overall_index)

    def is_participant_bot(self):
        return self.overall_config.getboolean(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_BOT_KEY, self.overall_index)

    def set_is_participant_bot(self, is_bot):
        return self.overall_config.set_value(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_BOT_KEY,
                                             is_bot, self.overall_index)

    def is_participant_custom_bot(self):
        return self.overall_config.getboolean(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_RLBOT_KEY, self.overall_index)

    def set_is_participant_custom_bot(self, is_rlbot):
        return self.overall_config.set_value(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_RLBOT_KEY,
                                             is_rlbot, self.overall_index)

    def get_bot_skill(self):
        return self.overall_config.getfloat(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_BOT_SKILL_KEY, self.overall_index)

    def set_bot_skill(self, bot_skill):
        return self.overall_config.set_value(PARTICIPANT_CONFIGURATION_HEADER, PARTICIPANT_BOT_SKILL_KEY,
                                             bot_skill, self.overall_index)

    def load_agent_config(self, agent_class=None):
        """
        Loads the config specific to the agent.
        This only happens if the frame is for a custom agent otherwise this method is skipped
        :param agent_class: If passed in and there is not a config file this agent is used.
        :return:
        """
        if self.is_participant_bot() and self.is_participant_custom_bot():
            agent_config_path = self.get_agent_config_path()
            base_agent_config = BaseAgent.create_agent_configurations()
            if agent_config_path is None:
                if agent_class is not None:
                    self.agent_class = agent_class
                    self.agent_config = self.agent_class.create_agent_configurations()
                else:
                    self.agent_config = base_agent_config
            else:
                base_agent_config.parse_file(agent_config_path)
                bot_module = base_agent_config.get(BOT_CONFIG_MODULE_HEADER, AGENT_MODULE_KEY)
                base_import = get_base_import_package(os.path.dirname(agent_config_path))
                self.agent_class = import_agent(base_import + '.' + bot_module)
                self.agent_config = self.agent_class.create_agent_configurations()
                self.agent_config.parse_file(agent_config_path)

    def load_config(self, overall_config_file, overall_index):
        """Loads the config data into the agent"""
        self.overall_config = overall_config_file
        self.overall_index = overall_index
        self.load_agent_config()

    def load_agent_from_path(self, agent_file_path):
        module = get_base_import_package(agent_file_path)
        self.agent_class = import_agent(module)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys
import os
import shutil
import datetime
import tempfile
import tarfile

import click
import requests
from six import exec_, print_

from . import StrategyExecutor
from . import api
from .data import LocalDataProxy
from .logger import user_log
from .trading_params import TradingParams
from .utils.click_helper import Date
from .utils import dummy_func

@click.group()
@click.option('-v', '--verbose', count=True)
@click.pass_context
def cli(ctx, verbose):
    ctx.obj["VERBOSE"] = verbose

def entry_point():
    cli(obj={})

@cli.command()
@click.option('-d', '--data-bundle-path', default=os.path.expanduser("~/.rqalpha"), type=click.Path())
def update_bundle(data_bundle_path):
    """update data bundle, download if not found"""
    day = datetime.date.today() - datetime.timedelta(days=1)
    tmp = os.path.join(tempfile.gettempdir(), 'rq.bundle')

    while True:
        url = 'http://7xjci3.com1.z0.glb.clouddn.com/bundles/rqbundle_%04d%02d%02d.tar.bz2' % (day.year, day.month, day.day)
        print_('try {} ...'.format(url))
        r = requests.get(url, stream=True)
        if r.status_code != 200:
            day = day - datetime.timedelta(days=1)
            continue

        out = open(tmp, 'wb')
        total_length = int(r.headers.get('content-length'))

        with click.progressbar(length=total_length, label='downloading ...') as bar:
            for data in r.iter_content(chunk_size=8192):
                bar.update(len(data))
                out.write(data)

        out.close()
        break

    shutil.rmtree(data_bundle_path, ignore_errors=True)
    os.mkdir(data_bundle_path)
    tar = tarfile.open(tmp, 'r:bz2')
    tar.extractall(data_bundle_path)
    os.remove(tmp)

@cli.command()
@click.option('-f', '--strategy-file', type=click.Path(exists=True), required=True)
@click.option('-s', '--start-date', type=Date(), required=True)
@click.option('-e', '--end-date', type=Date(), required=True)
@click.option('-o', '--output-file', type=click.Path(writable=True))
@click.option('--draw-result/--no-draw-result', default=True)
@click.option('-d', '--data-bundle-path', default=os.path.expanduser("~/.rqalpha"), type=click.Path())
def run(strategy_file, start_date, end_date, output_file, draw_result, data_bundle_path):
    '''run strategy from file
    '''
    if not os.path.exists(data_bundle_path):
        print_("data bundle not found. Run `%s update_bundle` to download data bundle." % sys.argv[0])
        return

    with open(strategy_file) as f:
        source_code = f.read()

    results_df = run_strategy(source_code, strategy_file, start_date, end_date, data_bundle_path)

    if output_file is not None:
        results_df.to_pickle(output_file)

    if draw_result:
        import matplotlib
        import matplotlib.pyplot as plt
        plt.style.use('ggplot')

        f, ax = plt.subplots(num=strategy_file, figsize=(16, 8))

        ax.get_xaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
        ax.get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
        ax.grid(b=True, which='minor', linewidth=.2)
        ax.grid(b=True, which='major', linewidth=1)

        ax.plot(results_df["total_returns"], label="strategy", alpha=1, linewidth=2, color="#aa4643")
        ax.plot(results_df["benchmark_total_returns"], label="benchmark", alpha=1, linewidth=2, color="#4572a7")

        plt.legend()
        plt.show()

@cli.command()
@click.option('-d', '--directory', default="./", type=click.Path(), required=True)
def generate_examples(directory):
    '''generate example strategies to target folder
    '''
    source_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "examples")

    shutil.copytree(source_dir, os.path.join(directory, "examples"))

def run_strategy(source_code, strategy_filename, start_date, end_date, data_bundle_path):
    scope = {
        "logger": user_log,
        "print": print_,
    }
    scope.update(api.__dict__)
    code = compile(source_code, strategy_filename, 'exec')
    exec_(code, scope)

    try:
        data_proxy = LocalDataProxy(data_bundle_path)
    except FileNotFoundError:
        print_("data bundle might crash. Run `%s update_bundle` to redownload data bundle." % sys.argv[0])
        sys.exit()

    trading_cal = data_proxy.get_trading_dates(start_date, end_date)
    trading_params = TradingParams(trading_cal, start_date=start_date.date(), end_date=end_date.date())

    executor = StrategyExecutor(
        init=scope.get("init", dummy_func),
        before_trading=scope.get("before_trading", dummy_func),
        handle_bar=scope.get("handle_bar", dummy_func),

        trading_params=trading_params,
        data_proxy=data_proxy,
    )

    results_df = executor.execute()

    return results_df

if __name__ == '__main__':
    entry_point()
<commit_msg>export only specific api to user module<commit_after># -*- coding: utf-8 -*-

import sys
import os
import shutil
import datetime
import tempfile
import tarfile

import click
import requests
from six import exec_, print_

from . import StrategyExecutor
from . import api
from .data import LocalDataProxy
from .logger import user_log
from .trading_params import TradingParams
from .utils.click_helper import Date
from .utils import dummy_func

@click.group()
@click.option('-v', '--verbose', count=True)
@click.pass_context
def cli(ctx, verbose):
    ctx.obj["VERBOSE"] = verbose

def entry_point():
    cli(obj={})

@cli.command()
@click.option('-d', '--data-bundle-path', default=os.path.expanduser("~/.rqalpha"), type=click.Path())
def update_bundle(data_bundle_path):
    """update data bundle, download if not found"""
    day = datetime.date.today() - datetime.timedelta(days=1)
    tmp = os.path.join(tempfile.gettempdir(), 'rq.bundle')

    while True:
        url = 'http://7xjci3.com1.z0.glb.clouddn.com/bundles/rqbundle_%04d%02d%02d.tar.bz2' % (day.year, day.month, day.day)
        print_('try {} ...'.format(url))
        r = requests.get(url, stream=True)
        if r.status_code != 200:
            day = day - datetime.timedelta(days=1)
            continue

        out = open(tmp, 'wb')
        total_length = int(r.headers.get('content-length'))

        with click.progressbar(length=total_length, label='downloading ...') as bar:
            for data in r.iter_content(chunk_size=8192):
                bar.update(len(data))
                out.write(data)

        out.close()
        break

    shutil.rmtree(data_bundle_path, ignore_errors=True)
    os.mkdir(data_bundle_path)
    tar = tarfile.open(tmp, 'r:bz2')
    tar.extractall(data_bundle_path)
    os.remove(tmp)

@cli.command()
@click.option('-f', '--strategy-file', type=click.Path(exists=True), required=True)
@click.option('-s', '--start-date', type=Date(), required=True)
@click.option('-e', '--end-date', type=Date(), required=True)
@click.option('-o', '--output-file', type=click.Path(writable=True))
@click.option('--draw-result/--no-draw-result', default=True)
@click.option('-d', '--data-bundle-path', default=os.path.expanduser("~/.rqalpha"), type=click.Path())
def run(strategy_file, start_date, end_date, output_file, draw_result, data_bundle_path):
    '''run strategy from file
    '''
    if not os.path.exists(data_bundle_path):
        print_("data bundle not found. Run `%s update_bundle` to download data bundle." % sys.argv[0])
        return

    with open(strategy_file) as f:
        source_code = f.read()

    results_df = run_strategy(source_code, strategy_file, start_date, end_date, data_bundle_path)

    if output_file is not None:
        results_df.to_pickle(output_file)

    if draw_result:
        import matplotlib
        import matplotlib.pyplot as plt
        plt.style.use('ggplot')

        f, ax = plt.subplots(num=strategy_file, figsize=(16, 8))

        ax.get_xaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
        ax.get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
        ax.grid(b=True, which='minor', linewidth=.2)
        ax.grid(b=True, which='major', linewidth=1)

        ax.plot(results_df["total_returns"], label="strategy", alpha=1, linewidth=2, color="#aa4643")
        ax.plot(results_df["benchmark_total_returns"], label="benchmark", alpha=1, linewidth=2, color="#4572a7")

        plt.legend()
        plt.show()

@cli.command()
@click.option('-d', '--directory', default="./", type=click.Path(), required=True)
def generate_examples(directory):
    '''generate example strategies to target folder
    '''
    source_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "examples")

    shutil.copytree(source_dir, os.path.join(directory, "examples"))

def run_strategy(source_code, strategy_filename, start_date, end_date, data_bundle_path):
    scope = {
        "logger": user_log,
        "print": print_,
    }
    scope.update({export_name: getattr(api, export_name) for export_name in api.__all__})
    code = compile(source_code, strategy_filename, 'exec')
    exec_(code, scope)

    try:
        data_proxy = LocalDataProxy(data_bundle_path)
    except FileNotFoundError:
        print_("data bundle might crash. Run `%s update_bundle` to redownload data bundle." % sys.argv[0])
        sys.exit()

    trading_cal = data_proxy.get_trading_dates(start_date, end_date)
    trading_params = TradingParams(trading_cal, start_date=start_date.date(), end_date=end_date.date())

    executor = StrategyExecutor(
        init=scope.get("init", dummy_func),
        before_trading=scope.get("before_trading", dummy_func),
        handle_bar=scope.get("handle_bar", dummy_func),

        trading_params=trading_params,
        data_proxy=data_proxy,
    )

    results_df = executor.execute()

    return results_df

if __name__ == '__main__':
    entry_point()
<|endoftext|>


<commit_before><commit_msg>namespace declaration<commit_after>__import__('pkg_resources').declare_namespace(__name__)<|endoftext|>