<commit_before>from steam.enums.base import SteamIntEnum

class EResult(SteamIntEnum):
    Invalid = 0
    OK = 1
    Fail = 2
    NoConnection = 3
    InvalidPassword = 5
    LoggedInElsewhere = 6
    InvalidProtocolVer = 7
    InvalidParam = 8
    FileNotFound = 9
    Busy = 10
    InvalidState = 11
    InvalidName = 12
    InvalidEmail = 13
    DuplicateName = 14
    AccessDenied = 15
    Timeout = 16
    Banned = 17
    AccountNotFound = 18
    InvalidSteamID = 19
    ServiceUnavailable = 20
    NotLoggedOn = 21
    Pending = 22
    EncryptionFailure = 23
    InsufficientPrivilege = 24
    LimitExceeded = 25
    Revoked = 26
    Expired = 27
    AlreadyRedeemed = 28
    DuplicateRequest = 29
    AlreadyOwned = 30
    IPNotFound = 31
    PersistFailed = 32
    LockingFailed = 33
    LogonSessionReplaced = 34
    ConnectFailed = 35
    HandshakeFailed = 36
    IOFailure = 37
    RemoteDisconnect = 38
    ShoppingCartNotFound = 39
    Blocked = 40
    Ignored = 41
    NoMatch = 42
    AccountDisabled = 43
    ServiceReadOnly = 44
    AccountNotFeatured = 45
    AdministratorOK = 46
    ContentVersion = 47
    TryAnotherCM = 48
    PasswordRequiredToKickSession = 49
    AlreadyLoggedInElsewhere = 50
    Suspended = 51
    Cancelled = 52
    DataCorruption = 53
    DiskFull = 54
    RemoteCallFailed = 55
    PasswordUnset = 56
    ExternalAccountUnlinked = 57
    PSNTicketInvalid = 58
    ExternalAccountAlreadyLinked = 59
    RemoteFileConflict = 60
    IllegalPassword = 61
    SameAsPreviousValue = 62
    AccountLogonDenied = 63
    CannotUseOldPassword = 64
    InvalidLoginAuthCode = 65
    AccountLogonDeniedNoMail = 66
    HardwareNotCapableOfIPT = 67
    IPTInitError = 68
    ParentalControlRestricted = 69
    FacebookQueryError = 70
    ExpiredLoginAuthCode = 71
    IPLoginRestrictionFailed = 72
    AccountLockedDown = 73
    AccountLogonDeniedVerifiedEmailRequired = 74
    NoMatchingURL = 75
    BadResponse = 76
    RequirePasswordReEntry = 77
    ValueOutOfRange = 78
    UnexpectedError = 79
    Disabled = 80
    InvalidCEGSubmission = 81
    RestrictedDevice = 82
    RegionLocked = 83
    RateLimitExceeded = 84
    AccountLoginDeniedNeedTwoFactor = 85
    ItemDeleted = 86
    AccountLoginDeniedThrottle = 87
    TwoFactorCodeMismatch = 88
    TwoFactorActivationCodeMismatch = 89
    AccountAssociatedToMultiplePartners = 90
    NotModified = 91
    NoMobileDevice = 92
    TimeNotSynced = 93
    SMSCodeFailed = 94
    AccountLimitExceeded = 95
    AccountActivityLimitExceeded = 96
    PhoneActivityLimitExceeded = 97
    RefundToWallet = 98
    EmailSendFailure = 99
    NotSettled = 100
    NeedCaptcha = 101
    GSLTDenied = 102
    GSOwnerDenied = 103
    InvalidItemType = 104

class EUniverse(SteamIntEnum):
    Invalid = 0
    Public = 1
    Beta = 2
    Internal = 3
    Dev = 4
    Max = 5

class EType(SteamIntEnum):
    Invalid = 0
    Individual = 1
    Multiseat = 2
    GameServer = 3
    AnonGameServer = 4
    Pending = 5
    ContentServer = 6
    Clan = 7
    Chat = 8
    ConsoleUser = 9
    AnonUser = 10
    Max = 11

class EServerType(SteamIntEnum):
    Invalid = -1
    First = 0
    GM = 1
    BUM = 2
    AM = 3
    BS = 4
    VS = 5
    ATS = 6
    CM = 7
    FBS = 8
    BoxMonitor = 9
    SS = 10
    DRMS = 11
    HubOBSOLETE = 12
    Console = 13
    PICS = 14
    Client = 15
    BootstrapOBSOLETE = 16
    DP = 17
    WG = 18
    SM = 19
    UFS = 21
    Util = 23
    DSS = 24
    Community = 24
    P2PRelayOBSOLETE = 25
    AppInformation = 26
    Spare = 27
    FTS = 28
    EPM = 29
    PS = 30
    IS = 31
    CCS = 32
    DFS = 33
    LBS = 34
    MDS = 35
    CS = 36
    GC = 37
    NS = 38
    OGS = 39
    WebAPI = 40
    UDS = 41
    MMS = 42
    GMS = 43
    KGS = 44
    UCM = 45
    RM = 46
    FS = 47
    Econ = 48
    Backpack = 49
    UGS = 50
    Store = 51
    MoneyStats = 52
    CRE = 53
    UMQ = 54
    Workshop = 55
    BRP = 56
    GCH = 57
    MPAS = 58
    Trade = 59
    Secrets = 60
    Logsink = 61
    Market = 62
    Quest = 63
    WDS = 64
    ACS = 65
    PNP = 66
    Max = 67

class EOSType(SteamIntEnum):
    Unknown = -1
    UMQ = -400

    PS3 = -300

    MacOSUnknown = -102
    MacOS104 = -101
    MacOS105 = -100
    MacOS1058 = -99
    MacOS106 = -95
    MacOS1063 = -94
    MacOS1064_slgu = -93
    MacOS1067 = -92
    MacOS107 = -90
    MacOS108 = -89
    MacOS109 = -88
    MacOS1010 = -87

    LinuxUnknown = -203
    Linux22 = -202
    Linux24 = -201
    Linux26 = -200
    Linux32 = -199
    Linux35 = -198
    Linux36 = -197
    Linux310 = -196

    WinUnknown = 0
    Win311 = 1
    Win95 = 2
    Win98 = 3
    WinME = 4
    WinNT = 5
    Win200 = 6
    WinXP = 7
    Win2003 = 8
    WinVista = 9
    Win7 = 10
    Win2008 = 11
    Win2012 = 12
    Win8 = 13
    Win81 = 14
    Win2012R2 = 15
    Win10 = 16

    WinMAX = 15

    Max = 26

class EPersonaState(SteamIntEnum):
    Offline = 0
    Online = 1
    Busy = 2
    Away = 3
    Snooze = 4
    LookingToTrade = 5
    LookingToPlay = 6
    Max = 7

class EFriendRelationship(SteamIntEnum):
    No = 0
    Blocked = 1
    RequestRecipient = 2
    Friend = 3
    RequestInitiator = 4
    Ignored = 5
    IgnoredFriend = 6
    SuggestedFriend = 7
    Max = 8

class EAccountFlags(SteamIntEnum):
    NormalUser = 0
    PersonaNameSet = 1
    Unbannable = 2
    PasswordSet = 4
    Support = 8
    Admin = 16
    Supervisor = 32
    AppEditor = 64
    HWIDSet = 128
    PersonalQASet = 256
    VacBeta = 512
    Debug = 1024
    Disabled = 2048
    LimitedUser = 4096
    LimitedUserForce = 8192
    EmailValidated = 16384
    MarketingTreatment = 32768
    OGGInviteOptOut = 65536
    ForcePasswordChange = 131072
    ForceEmailVerification = 262144
    LogonExtraSecurity = 524288
    LogonExtraSecurityDisabled = 1048576
    Steam2MigrationComplete = 2097152
    NeedLogs = 4194304
    Lockdown = 8388608
    MasterAppEditor = 16777216
    BannedFromWebAPI = 33554432
    ClansOnlyFromFriends = 67108864
    GlobalModerator = 134217728

class EFriendFlags(SteamIntEnum):
    No = 0
    Blocked = 1
    FriendshipRequested = 2
    Immediate = 4
    ClanMember = 8
    OnGameServer = 16
    RequestingFriendship = 128
    RequestingInfo = 256
    Ignored = 512
    IgnoredFriend = 1024
    Suggested = 2048
    FlagAll = 65535

class EPersonaStateFlag(SteamIntEnum):
    HasRichPresence = 1
    InJoinableGame = 2
    OnlineUsingWeb = 256
    OnlineUsingMobile = 512
    OnlineUsingBigPicture = 1024

class EClientPersonaStateFlag(SteamIntEnum):
    Status = 1
    PlayerName = 2
    QueryPort = 4
    SourceID = 8
    Presence = 16
    Metadata = 32
    LastSeen = 64
    ClanInfo = 128
    GameExtraInfo = 256
    GameDataBlob = 512
    ClanTag = 1024
    Facebook = 2048

# Do not remove
from sys import modules
from enum import EnumMeta

__all__ = list(map(lambda y: y.__name__,
              filter(lambda x: x.__class__ is EnumMeta, modules[__name__].__dict__.values()),
              ))

del modules, EnumMeta
<commit_msg>added EResult.IPBanned<commit_after>from steam.enums.base import SteamIntEnum

class EResult(SteamIntEnum):
    Invalid = 0
    OK = 1
    Fail = 2
    NoConnection = 3
    InvalidPassword = 5
    LoggedInElsewhere = 6
    InvalidProtocolVer = 7
    InvalidParam = 8
    FileNotFound = 9
    Busy = 10
    InvalidState = 11
    InvalidName = 12
    InvalidEmail = 13
    DuplicateName = 14
    AccessDenied = 15
    Timeout = 16
    Banned = 17
    AccountNotFound = 18
    InvalidSteamID = 19
    ServiceUnavailable = 20
    NotLoggedOn = 21
    Pending = 22
    EncryptionFailure = 23
    InsufficientPrivilege = 24
    LimitExceeded = 25
    Revoked = 26
    Expired = 27
    AlreadyRedeemed = 28
    DuplicateRequest = 29
    AlreadyOwned = 30
    IPNotFound = 31
    PersistFailed = 32
    LockingFailed = 33
    LogonSessionReplaced = 34
    ConnectFailed = 35
    HandshakeFailed = 36
    IOFailure = 37
    RemoteDisconnect = 38
    ShoppingCartNotFound = 39
    Blocked = 40
    Ignored = 41
    NoMatch = 42
    AccountDisabled = 43
    ServiceReadOnly = 44
    AccountNotFeatured = 45
    AdministratorOK = 46
    ContentVersion = 47
    TryAnotherCM = 48
    PasswordRequiredToKickSession = 49
    AlreadyLoggedInElsewhere = 50
    Suspended = 51
    Cancelled = 52
    DataCorruption = 53
    DiskFull = 54
    RemoteCallFailed = 55
    PasswordUnset = 56
    ExternalAccountUnlinked = 57
    PSNTicketInvalid = 58
    ExternalAccountAlreadyLinked = 59
    RemoteFileConflict = 60
    IllegalPassword = 61
    SameAsPreviousValue = 62
    AccountLogonDenied = 63
    CannotUseOldPassword = 64
    InvalidLoginAuthCode = 65
    AccountLogonDeniedNoMail = 66
    HardwareNotCapableOfIPT = 67
    IPTInitError = 68
    ParentalControlRestricted = 69
    FacebookQueryError = 70
    ExpiredLoginAuthCode = 71
    IPLoginRestrictionFailed = 72
    AccountLockedDown = 73
    AccountLogonDeniedVerifiedEmailRequired = 74
    NoMatchingURL = 75
    BadResponse = 76
    RequirePasswordReEntry = 77
    ValueOutOfRange = 78
    UnexpectedError = 79
    Disabled = 80
    InvalidCEGSubmission = 81
    RestrictedDevice = 82
    RegionLocked = 83
    RateLimitExceeded = 84
    AccountLoginDeniedNeedTwoFactor = 85
    ItemDeleted = 86
    AccountLoginDeniedThrottle = 87
    TwoFactorCodeMismatch = 88
    TwoFactorActivationCodeMismatch = 89
    AccountAssociatedToMultiplePartners = 90
    NotModified = 91
    NoMobileDevice = 92
    TimeNotSynced = 93
    SMSCodeFailed = 94
    AccountLimitExceeded = 95
    AccountActivityLimitExceeded = 96
    PhoneActivityLimitExceeded = 97
    RefundToWallet = 98
    EmailSendFailure = 99
    NotSettled = 100
    NeedCaptcha = 101
    GSLTDenied = 102
    GSOwnerDenied = 103
    InvalidItemType = 104
    IPBanned = 105

class EUniverse(SteamIntEnum):
    Invalid = 0
    Public = 1
    Beta = 2
    Internal = 3
    Dev = 4
    Max = 5

class EType(SteamIntEnum):
    Invalid = 0
    Individual = 1
    Multiseat = 2
    GameServer = 3
    AnonGameServer = 4
    Pending = 5
    ContentServer = 6
    Clan = 7
    Chat = 8
    ConsoleUser = 9
    AnonUser = 10
    Max = 11

class EServerType(SteamIntEnum):
    Invalid = -1
    First = 0
    GM = 1
    BUM = 2
    AM = 3
    BS = 4
    VS = 5
    ATS = 6
    CM = 7
    FBS = 8
    BoxMonitor = 9
    SS = 10
    DRMS = 11
    HubOBSOLETE = 12
    Console = 13
    PICS = 14
    Client = 15
    BootstrapOBSOLETE = 16
    DP = 17
    WG = 18
    SM = 19
    UFS = 21
    Util = 23
    DSS = 24
    Community = 24
    P2PRelayOBSOLETE = 25
    AppInformation = 26
    Spare = 27
    FTS = 28
    EPM = 29
    PS = 30
    IS = 31
    CCS = 32
    DFS = 33
    LBS = 34
    MDS = 35
    CS = 36
    GC = 37
    NS = 38
    OGS = 39
    WebAPI = 40
    UDS = 41
    MMS = 42
    GMS = 43
    KGS = 44
    UCM = 45
    RM = 46
    FS = 47
    Econ = 48
    Backpack = 49
    UGS = 50
    Store = 51
    MoneyStats = 52
    CRE = 53
    UMQ = 54
    Workshop = 55
    BRP = 56
    GCH = 57
    MPAS = 58
    Trade = 59
    Secrets = 60
    Logsink = 61
    Market = 62
    Quest = 63
    WDS = 64
    ACS = 65
    PNP = 66
    Max = 67

class EOSType(SteamIntEnum):
    Unknown = -1
    UMQ = -400

    PS3 = -300

    MacOSUnknown = -102
    MacOS104 = -101
    MacOS105 = -100
    MacOS1058 = -99
    MacOS106 = -95
    MacOS1063 = -94
    MacOS1064_slgu = -93
    MacOS1067 = -92
    MacOS107 = -90
    MacOS108 = -89
    MacOS109 = -88
    MacOS1010 = -87

    LinuxUnknown = -203
    Linux22 = -202
    Linux24 = -201
    Linux26 = -200
    Linux32 = -199
    Linux35 = -198
    Linux36 = -197
    Linux310 = -196

    WinUnknown = 0
    Win311 = 1
    Win95 = 2
    Win98 = 3
    WinME = 4
    WinNT = 5
    Win200 = 6
    WinXP = 7
    Win2003 = 8
    WinVista = 9
    Win7 = 10
    Win2008 = 11
    Win2012 = 12
    Win8 = 13
    Win81 = 14
    Win2012R2 = 15
    Win10 = 16

    WinMAX = 15

    Max = 26

class EPersonaState(SteamIntEnum):
    Offline = 0
    Online = 1
    Busy = 2
    Away = 3
    Snooze = 4
    LookingToTrade = 5
    LookingToPlay = 6
    Max = 7

class EFriendRelationship(SteamIntEnum):
    No = 0
    Blocked = 1
    RequestRecipient = 2
    Friend = 3
    RequestInitiator = 4
    Ignored = 5
    IgnoredFriend = 6
    SuggestedFriend = 7
    Max = 8

class EAccountFlags(SteamIntEnum):
    NormalUser = 0
    PersonaNameSet = 1
    Unbannable = 2
    PasswordSet = 4
    Support = 8
    Admin = 16
    Supervisor = 32
    AppEditor = 64
    HWIDSet = 128
    PersonalQASet = 256
    VacBeta = 512
    Debug = 1024
    Disabled = 2048
    LimitedUser = 4096
    LimitedUserForce = 8192
    EmailValidated = 16384
    MarketingTreatment = 32768
    OGGInviteOptOut = 65536
    ForcePasswordChange = 131072
    ForceEmailVerification = 262144
    LogonExtraSecurity = 524288
    LogonExtraSecurityDisabled = 1048576
    Steam2MigrationComplete = 2097152
    NeedLogs = 4194304
    Lockdown = 8388608
    MasterAppEditor = 16777216
    BannedFromWebAPI = 33554432
    ClansOnlyFromFriends = 67108864
    GlobalModerator = 134217728

class EFriendFlags(SteamIntEnum):
    No = 0
    Blocked = 1
    FriendshipRequested = 2
    Immediate = 4
    ClanMember = 8
    OnGameServer = 16
    RequestingFriendship = 128
    RequestingInfo = 256
    Ignored = 512
    IgnoredFriend = 1024
    Suggested = 2048
    FlagAll = 65535

class EPersonaStateFlag(SteamIntEnum):
    HasRichPresence = 1
    InJoinableGame = 2
    OnlineUsingWeb = 256
    OnlineUsingMobile = 512
    OnlineUsingBigPicture = 1024

class EClientPersonaStateFlag(SteamIntEnum):
    Status = 1
    PlayerName = 2
    QueryPort = 4
    SourceID = 8
    Presence = 16
    Metadata = 32
    LastSeen = 64
    ClanInfo = 128
    GameExtraInfo = 256
    GameDataBlob = 512
    ClanTag = 1024
    Facebook = 2048

# Do not remove
from sys import modules
from enum import EnumMeta

__all__ = list(map(lambda y: y.__name__,
              filter(lambda x: x.__class__ is EnumMeta, modules[__name__].__dict__.values()),
              ))

del modules, EnumMeta
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

"""
WPTools Fetch module.
"""

from __future__ import print_function

import pycurl
import requests
import sys
import time

from . import __title__, __contact__, __version__
from io import BytesIO
from string import Template

class WPToolsFetch:

    ENDPOINT = "http://en.wikipedia.org"
    ACTION_QUERY = Template(("${API}/w/api.php?action=query"
                             "&titles=${titles}"
                             "&format=json"
                             "&formatversion=2"
                             "&prop=revisions"  # latest revision
                             "&rvprop=content"  # content of latest revision
                             "&redirects"
                             "&continue="))
    QUERY = {"html": Template(("${WIKI}/w/api.php?action=parse"
                               "&format=json"
                               "&page=${page}"
                               "&prop=text"
                               "&contentmodel=text"
                               "&disablelimitreport="
                               "&disableeditsection="
                               "&disabletoc=")),
             "images": Template(("${WIKI}/w/api.php?action=query"
                                 "&titles=${page}"
                                 "&format=json"
                                 "&format=json"
                                 "&formatversion=2"
                                 "&prop=images")),
             "parsetree": Template(("${WIKI}/w/api.php?action=parse"
                                    "&format=json"
                                    "&page=${page}"
                                    "&prop=parsetree"
                                    "&disablelimitreport="
                                    "&disableeditsection="
                                    "&disabletoc="
                                    "&contentmodel=text")),
             "wikitext": Template(("${WIKI}/w/api.php?action=parse"
                                   "&format=json"
                                   "&page=${page}"
                                   "&prop=wikitext"
                                   "&contentmodel=wikitext"
                                   "&disablelimitreport="
                                   "&disableeditsection="
                                   "&disabletoc="))}
    RETRY_SLEEP = 2
    RETRY_MAX = 3
    TIMEOUT = 30

    def __init__(self, wiki=ENDPOINT, lead=False, verbose=False):
        self.wiki = wiki or self.ENDPOINT
        self.lead = lead
        self.verbose = verbose
        self.curl_setup()
        self.title = None
        self.retries = 0

    def __del__(self):
        self.cobj.close()
        if self.verbose:
            print("connection closed.", file=sys.stderr)

    def curl(self, url):
        """speed"""
        crl = self.cobj
        try:
            crl.setopt(crl.URL, url)
        except UnicodeEncodeError:
            crl.setopt(crl.URL, url.encode('utf-8'))
        return self.curl_perform(crl)

    def curl_perform(self, crl):
        """try curl, retry after sleep up to max retries"""
        if self.retries >= self.RETRY_MAX:
            return "RETRY_MAX (%d) exceeded!" % self.RETRY_MAX
        try:
            bfr = BytesIO()
            crl.setopt(crl.WRITEFUNCTION, bfr.write)
            crl.perform()
            if self.verbose:
                self.curl_report(crl)
            body = bfr.getvalue()
            bfr.close()
            self.retries = 0
            return body
        except Exception as detail:
            print("RETRY Caught exception: %s" % detail)
            self.retries += 1
            time.sleep(self.RETRY_SLEEP)
            self.curl_perform(crl)

    def curl_report(self, crl):
        kbps = crl.getinfo(crl.SPEED_DOWNLOAD) / 1000.0
        out = {"url": crl.getinfo(crl.EFFECTIVE_URL),
               "user-agent": self.user_agent(),
               "content": crl.getinfo(crl.CONTENT_TYPE),
               "status": crl.getinfo(crl.RESPONSE_CODE),
               "bytes": crl.getinfo(crl.SIZE_DOWNLOAD),
               "seconds": "%5.3f" % crl.getinfo(crl.TOTAL_TIME),
               "kB/s": "%3.1f" % kbps}
        print("WPToolsFetch HTTP", file=sys.stderr)
        for key in out:
            print("    %s: %s" % (key, out[key]), file=sys.stderr)
        print(file=sys.stderr)

    def curl_setup(self):
        crl = pycurl.Curl()
        # crl.setopt(crl.URL, wiki)
        crl.setopt(crl.USERAGENT, self.user_agent())
        crl.setopt(crl.FOLLOWLOCATION, True)
        crl.setopt(crl.CONNECTTIMEOUT, self.TIMEOUT)
        self.cobj = crl

    def html(self, title):
        """get HTML keeping connection open"""
        return self.curl(self.query('html', title))

    def query(self, content, page):
        self.title = page
        page = page.replace(" ", "+")
        page = page[0].upper() + page[1:]
        qry = self.QUERY[content].substitute(WIKI=self.wiki, page=page)
        if self.lead:
            return qry + "&section=0"
        return qry

    def request(self, url, hdr, output):
        print("GET %s\nHDR %s\nOUT %s" % (url, hdr, output), file=sys.stderr)
        r = requests.get(url, headers=hdr, timeout=self.TIMEOUT)
        if r.status_code != 200:
            raise ValueError("HTTP status code = %d" % r.status_code)
        return r.content

    def user_agent(self):
        return "%s/%s (+%s)" % (__title__, __version__, __contact__)

def get_html(title, lead=False, test=False, wiki=WPToolsFetch.ENDPOINT,
             verbose=False):
    obj = WPToolsFetch(wiki, lead, verbose)
    qry = obj.query('html', title)
    if test:
        return qry
    return obj.curl(qry)

def get_infobox():
    pass

def get_images(title, lead=False, test=False, wiki=WPToolsFetch.ENDPOINT,
               verbose=False):
    obj = WPToolsFetch(wiki, lead, verbose)
    qry = obj.query('images', title)
    if test:
        return qry
    return obj.curl(qry)

def get_parsetree(title, lead, test, wiki, verbose=False):
    obj = WPToolsFetch(wiki, lead, verbose)
    qry = obj.query('parsetree', title)
    if test:
        return qry
    return obj.curl(qry)

def get_wikitext(title, lead, test, wiki, verbose=False):
    obj = WPToolsFetch(wiki, lead, verbose)
    url = obj.query('wikitext', title)
    if test:
        return url
    return obj.curl(url)
<commit_msg>added default args for consistency in wptools.fetch<commit_after># -*- coding:utf-8 -*-

"""
WPTools Fetch module.
"""

from __future__ import print_function

import pycurl
import requests
import sys
import time

from . import __title__, __contact__, __version__
from io import BytesIO
from string import Template

class WPToolsFetch:

    ENDPOINT = "http://en.wikipedia.org"
    ACTION_QUERY = Template(("${API}/w/api.php?action=query"
                             "&titles=${titles}"
                             "&format=json"
                             "&formatversion=2"
                             "&prop=revisions"  # latest revision
                             "&rvprop=content"  # content of latest revision
                             "&redirects"
                             "&continue="))
    QUERY = {"html": Template(("${WIKI}/w/api.php?action=parse"
                               "&format=json"
                               "&page=${page}"
                               "&prop=text"
                               "&contentmodel=text"
                               "&disablelimitreport="
                               "&disableeditsection="
                               "&disabletoc=")),
             "images": Template(("${WIKI}/w/api.php?action=query"
                                 "&titles=${page}"
                                 "&format=json"
                                 "&format=json"
                                 "&formatversion=2"
                                 "&prop=images")),
             "parsetree": Template(("${WIKI}/w/api.php?action=parse"
                                    "&format=json"
                                    "&page=${page}"
                                    "&prop=parsetree"
                                    "&disablelimitreport="
                                    "&disableeditsection="
                                    "&disabletoc="
                                    "&contentmodel=text")),
             "wikitext": Template(("${WIKI}/w/api.php?action=parse"
                                   "&format=json"
                                   "&page=${page}"
                                   "&prop=wikitext"
                                   "&contentmodel=wikitext"
                                   "&disablelimitreport="
                                   "&disableeditsection="
                                   "&disabletoc="))}
    RETRY_SLEEP = 2
    RETRY_MAX = 3
    TIMEOUT = 30

    def __init__(self, wiki=ENDPOINT, lead=False, verbose=False):
        self.wiki = wiki or self.ENDPOINT
        self.lead = lead
        self.verbose = verbose
        self.curl_setup()
        self.title = None
        self.retries = 0

    def __del__(self):
        self.cobj.close()
        if self.verbose:
            print("connection closed.", file=sys.stderr)

    def curl(self, url):
        """speed"""
        crl = self.cobj
        try:
            crl.setopt(crl.URL, url)
        except UnicodeEncodeError:
            crl.setopt(crl.URL, url.encode('utf-8'))
        return self.curl_perform(crl)

    def curl_perform(self, crl):
        """try curl, retry after sleep up to max retries"""
        if self.retries >= self.RETRY_MAX:
            return "RETRY_MAX (%d) exceeded!" % self.RETRY_MAX
        try:
            bfr = BytesIO()
            crl.setopt(crl.WRITEFUNCTION, bfr.write)
            crl.perform()
            if self.verbose:
                self.curl_report(crl)
            body = bfr.getvalue()
            bfr.close()
            self.retries = 0
            return body
        except Exception as detail:
            print("RETRY Caught exception: %s" % detail)
            self.retries += 1
            time.sleep(self.RETRY_SLEEP)
            self.curl_perform(crl)

    def curl_report(self, crl):
        kbps = crl.getinfo(crl.SPEED_DOWNLOAD) / 1000.0
        out = {"url": crl.getinfo(crl.EFFECTIVE_URL),
               "user-agent": self.user_agent(),
               "content": crl.getinfo(crl.CONTENT_TYPE),
               "status": crl.getinfo(crl.RESPONSE_CODE),
               "bytes": crl.getinfo(crl.SIZE_DOWNLOAD),
               "seconds": "%5.3f" % crl.getinfo(crl.TOTAL_TIME),
               "kB/s": "%3.1f" % kbps}
        print("WPToolsFetch HTTP", file=sys.stderr)
        for key in out:
            print("    %s: %s" % (key, out[key]), file=sys.stderr)
        print(file=sys.stderr)

    def curl_setup(self):
        crl = pycurl.Curl()
        # crl.setopt(crl.URL, wiki)
        crl.setopt(crl.USERAGENT, self.user_agent())
        crl.setopt(crl.FOLLOWLOCATION, True)
        crl.setopt(crl.CONNECTTIMEOUT, self.TIMEOUT)
        self.cobj = crl

    def html(self, title):
        """get HTML keeping connection open"""
        return self.curl(self.query('html', title))

    def query(self, content, page):
        self.title = page
        page = page.replace(" ", "+")
        page = page[0].upper() + page[1:]
        qry = self.QUERY[content].substitute(WIKI=self.wiki, page=page)
        if self.lead:
            return qry + "&section=0"
        return qry

    def request(self, url, hdr, output):
        print("GET %s\nHDR %s\nOUT %s" % (url, hdr, output), file=sys.stderr)
        r = requests.get(url, headers=hdr, timeout=self.TIMEOUT)
        if r.status_code != 200:
            raise ValueError("HTTP status code = %d" % r.status_code)
        return r.content

    def user_agent(self):
        return "%s/%s (+%s)" % (__title__, __version__, __contact__)

def get_html(title, lead=False, test=False, wiki=WPToolsFetch.ENDPOINT,
             verbose=False):
    obj = WPToolsFetch(wiki, lead, verbose)
    qry = obj.query('html', title)
    if test:
        return qry
    return obj.curl(qry)

def get_infobox():
    pass

def get_images(title, lead=False, test=False, wiki=WPToolsFetch.ENDPOINT,
               verbose=False):
    obj = WPToolsFetch(wiki, lead, verbose)
    qry = obj.query('images', title)
    if test:
        return qry
    return obj.curl(qry)

def get_parsetree(title, lead=False, test=False, wiki=WPToolsFetch.ENDPOINT,
                  verbose=False):
    obj = WPToolsFetch(wiki, lead, verbose)
    qry = obj.query('parsetree', title)
    if test:
        return qry
    return obj.curl(qry)

def get_wikitext(title, lead=False, test=False, wiki=WPToolsFetch.ENDPOINT,
                 verbose=False):
    obj = WPToolsFetch(wiki, lead, verbose)
    url = obj.query('wikitext', title)
    if test:
        return url
    return obj.curl(url)
<|endoftext|>


<commit_before>import glob
import sys
import argparse
import datetime
import os.path
import tempfile
import werkzeug.serving
import werkzeug.wsgi

tmpdir = tempfile.gettempdir()
parser = argparse.ArgumentParser("Ekklesia Portal runserver.py")

parser.add_argument("-b", "--bind", default="ekklesia-portal-localhost", help="hostname / IP to bind to, default ekklesia-portal-localhost")
parser.add_argument("-p", "--http_port", default=8080, help="HTTP port to use, default 8080")
parser.add_argument("-d", "--debug", action="store_true", help="enable werkzeug debugger / reloader")
parser.add_argument("-s", "--stackdump", action="store_true", help=f"write stackdumps to temp dir {tmpdir} on SIGQUIT")
parser.add_argument("-c", "--config-file", help=f"path to config file in YAML / JSON format")

def stackdump_setup():
    import codecs
    import logging
    import sys
    logg = logging.getLogger(__name__)
    import threading
    import traceback
    try:
        import IPython.core.ultratb as ultratb
    except:
        ultratb = None

    if ultratb is None:
        logg.warn("IPython not installed, stack dumps not available!")
    else:
        logg.info("IPython installed, write stack dumps to tmpdir with: `kill -QUIT <ekklesia_portal_pid>`")

        def dumpstacks(signal, frame):
            print("dumping stack")
            filepath = os.path.join(tmpdir, "ekklesia_portal_threadstatus")
            id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
            full = ["-" * 80]
            tb_formatter = ultratb.ListTB(color_scheme="Linux")
            for thread_id, stack in sys._current_frames().items():
                thread_name = id2name.get(thread_id, "")
                stacktrace = traceback.extract_stack(stack)
                stb = tb_formatter.structured_traceback(Exception, Exception(), stacktrace)[8:-1]
                if stb:
                    formatted_trace = tb_formatter.stb2text(stb).strip()
                    with codecs.open("{}.{}".format(filepath, thread_id), "w", encoding='utf8') as wf:
                        wf.write("\n{}".format(formatted_trace))
                    if len(stb) > 4:
                        short_stb = stb[:2] + ["..."] + stb[-2:]
                    else:
                        short_stb = stb
                    formatted_trace_short = tb_formatter.stb2text(short_stb).strip()
                    full.append("# Thread: %s(%d)" % (thread_name, thread_id))
                    full.append(formatted_trace_short)
                    full.append("-" * 80)

            with codecs.open(filepath, "w", encoding='utf8') as wf:
                wf.write("\n".join(full))

        import signal
        signal.signal(signal.SIGQUIT, dumpstacks)

def run():
    from ekklesia_portal.app import make_wsgi_app
    args = parser.parse_args()
    print("cmdline args:", args)
    wsgi_app = make_wsgi_app(args.config_file)

    wrapped_app = werkzeug.wsgi.SharedDataMiddleware(wsgi_app, {
        '/static': ("ekklesia_portal", 'static'),
        '/static/deform': ("deform", 'static')
    })

    if args.stackdump:
        stackdump_setup()

    # use ipdb as default breakpoint() hook (Python 3.7 feature)
    try:
        import ipdb
    except ImportError:
        pass
    else:
        sys.breakpointhook = ipdb.set_trace

    with open(os.path.join(tmpdir, "ekklesia_portal.started"), "w") as wf:
        wf.write(datetime.datetime.now().isoformat())
        wf.write("\n")

    # reload when translation MO files change
    extra_reload_files = glob.glob('src/ekklesia_portal/translations/**/*.mo', recursive=True)
    werkzeug.serving.run_simple(args.bind, args.http_port, wrapped_app, use_reloader=args.debug, extra_files=extra_reload_files, use_debugger=args.debug)

if __name__ == "__main__":
    run()
<commit_msg>use localhost as default param for -b (bind)<commit_after>import glob
import sys
import argparse
import datetime
import os.path
import tempfile
import werkzeug.serving
import werkzeug.wsgi

tmpdir = tempfile.gettempdir()
parser = argparse.ArgumentParser("Ekklesia Portal runserver.py")

parser.add_argument("-b", "--bind", default="localhost", help="hostname / IP to bind to, default ekklesia-portal-localhost")
parser.add_argument("-p", "--http_port", default=8080, help="HTTP port to use, default 8080")
parser.add_argument("-d", "--debug", action="store_true", help="enable werkzeug debugger / reloader")
parser.add_argument("-s", "--stackdump", action="store_true", help=f"write stackdumps to temp dir {tmpdir} on SIGQUIT")
parser.add_argument("-c", "--config-file", help=f"path to config file in YAML / JSON format")

def stackdump_setup():
    import codecs
    import logging
    import sys
    logg = logging.getLogger(__name__)
    import threading
    import traceback
    try:
        import IPython.core.ultratb as ultratb
    except:
        ultratb = None

    if ultratb is None:
        logg.warn("IPython not installed, stack dumps not available!")
    else:
        logg.info("IPython installed, write stack dumps to tmpdir with: `kill -QUIT <ekklesia_portal_pid>`")

        def dumpstacks(signal, frame):
            print("dumping stack")
            filepath = os.path.join(tmpdir, "ekklesia_portal_threadstatus")
            id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
            full = ["-" * 80]
            tb_formatter = ultratb.ListTB(color_scheme="Linux")
            for thread_id, stack in sys._current_frames().items():
                thread_name = id2name.get(thread_id, "")
                stacktrace = traceback.extract_stack(stack)
                stb = tb_formatter.structured_traceback(Exception, Exception(), stacktrace)[8:-1]
                if stb:
                    formatted_trace = tb_formatter.stb2text(stb).strip()
                    with codecs.open("{}.{}".format(filepath, thread_id), "w", encoding='utf8') as wf:
                        wf.write("\n{}".format(formatted_trace))
                    if len(stb) > 4:
                        short_stb = stb[:2] + ["..."] + stb[-2:]
                    else:
                        short_stb = stb
                    formatted_trace_short = tb_formatter.stb2text(short_stb).strip()
                    full.append("# Thread: %s(%d)" % (thread_name, thread_id))
                    full.append(formatted_trace_short)
                    full.append("-" * 80)

            with codecs.open(filepath, "w", encoding='utf8') as wf:
                wf.write("\n".join(full))

        import signal
        signal.signal(signal.SIGQUIT, dumpstacks)

def run():
    from ekklesia_portal.app import make_wsgi_app
    args = parser.parse_args()
    print("cmdline args:", args)
    wsgi_app = make_wsgi_app(args.config_file)

    wrapped_app = werkzeug.wsgi.SharedDataMiddleware(wsgi_app, {
        '/static': ("ekklesia_portal", 'static'),
        '/static/deform': ("deform", 'static')
    })

    if args.stackdump:
        stackdump_setup()

    # use ipdb as default breakpoint() hook (Python 3.7 feature)
    try:
        import ipdb
    except ImportError:
        pass
    else:
        sys.breakpointhook = ipdb.set_trace

    with open(os.path.join(tmpdir, "ekklesia_portal.started"), "w") as wf:
        wf.write(datetime.datetime.now().isoformat())
        wf.write("\n")

    # reload when translation MO files change
    extra_reload_files = glob.glob('src/ekklesia_portal/translations/**/*.mo', recursive=True)
    werkzeug.serving.run_simple(args.bind, args.http_port, wrapped_app, use_reloader=args.debug, extra_files=extra_reload_files, use_debugger=args.debug)

if __name__ == "__main__":
    run()
<|endoftext|>


<commit_before>from ..migrator import upgrade_step

def number(value):
    value = value.lower().replace(' ', '')
    value = value.replace('x10^', 'e')
    if value in ('', 'unknown'):
        return None
    try:
        return int(value)
    except ValueError:
        return float(value)

@upgrade_step('biosample', '', '2')
def biosample_0_2(value, system):
    # http://redmine.encodedcc.org/issues/794
    if 'starting_amount' in value:
        new_value = number(value['starting_amount'])
        if new_value is None:
            del value['starting_amount']
        else:
            value['starting_amount'] = new_value

@upgrade_step('biosample', '2', '3')
def biosample_2_3(value, system):
    # http://redmine.encodedcc.org/issues/940

    go_mapping = {
        "nucleus": "GO:0005634",
        "cytosol": "GO:0005829",
        "chromatin": "GO:0000785",
        "membrane fraction": "GO:0016020",
        "mitochondria": "GO:0005739",
        "nuclear matrix": "GO:0016363",
        "nucleolus": "GO:0005730",
        "nucleoplasm": "GO:0005654",
        "polysome": "GO:0005844"
    }

    if 'subcellular_fraction' in value:
        value['subcellular_fraction_term_id']  = go_mapping[value['subcellular_fraction']]

        if value['subcellular_fraction']== "membrane":
            value['subcellular_fraction'] = "membrane fraction"

        value['subcellular_fraction_term_name'] = value['subcellular_fraction']
        del value['subcellular_fraction']

<commit_msg>Fixed dictionary.<commit_after>from ..migrator import upgrade_step

def number(value):
    value = value.lower().replace(' ', '')
    value = value.replace('x10^', 'e')
    if value in ('', 'unknown'):
        return None
    try:
        return int(value)
    except ValueError:
        return float(value)

@upgrade_step('biosample', '', '2')
def biosample_0_2(value, system):
    # http://redmine.encodedcc.org/issues/794
    if 'starting_amount' in value:
        new_value = number(value['starting_amount'])
        if new_value is None:
            del value['starting_amount']
        else:
            value['starting_amount'] = new_value

@upgrade_step('biosample', '2', '3')
def biosample_2_3(value, system):
    # http://redmine.encodedcc.org/issues/940

    go_mapping = {
        "nucleus": "GO:0005634",
        "cytosol": "GO:0005829",
        "chromatin": "GO:0000785",
        "membrane": "GO:0016020",
        "mitochondria": "GO:0005739",
        "nuclear matrix": "GO:0016363",
        "nucleolus": "GO:0005730",
        "nucleoplasm": "GO:0005654",
        "polysome": "GO:0005844"
    }

    if 'subcellular_fraction' in value:
        value['subcellular_fraction_term_id']  = go_mapping[value['subcellular_fraction']]

        if value['subcellular_fraction']== "membrane":
            value['subcellular_fraction'] = "membrane fraction"

        value['subcellular_fraction_term_name'] = value['subcellular_fraction']
        del value['subcellular_fraction']

<|endoftext|>


<commit_before>from pprint import pprint

import systextil.models as sys_mod

from utils.functions import get_client_ip

import produto.functions as pro_fun
import produto.classes as pro_cla

from estoque import classes
from estoque import models
from estoque import queries

class Transfere():

    def __init__(
            self, cursor, tipo, nivel, ref, tam, cor, qtd,
            deposito_origem, deposito_destino,
            nova_ref, novo_tam, nova_cor,
            num_doc, descricao, request, cria_num_doc=True):
        self.can_exec = False

        self.cursor = cursor
        self.tipo = tipo
        self.nivel = nivel
        self.ref = ref
        self.tam = tam
        self.cor = cor
        self.qtd = qtd
        self.deposito_origem = deposito_origem
        self.deposito_destino = deposito_destino
        self.nova_ref = nova_ref
        self.novo_tam = novo_tam
        self.nova_cor = nova_cor
        self.num_doc = num_doc
        self.descricao = descricao
        self.request = request
        self.cria_num_doc = cria_num_doc

        self.valid_entries()

        self.calc_vars()
        self.can_exec = True

    def calc_vars(self):
        if self.tem_trans_saida:
            self.estoque_origem = self.get_estoque(self.deposito_origem)
            self.novo_estoque_origem = self.estoque_origem - self.qtd
        else:
            self.estoque_origem = 0
            self.novo_estoque_origem = 0

        if self.tem_trans_entrada:
            self.estoque_destino = self.get_estoque(self.deposito_destino)
            self.novo_estoque_destino = self.estoque_destino + self.qtd
        else:
            self.estoque_destino = 0
            self.novo_estoque_destino = 0

        produto = queries.get_preco_medio_niv_ref_cor_tam(
            self.cursor, self.nivel, self.ref, self.cor, self.tam)
        try:
            self.preco_medio = produto[0]['preco_medio']
        except Exception:
            raise ValueError(
                f'Não encontrado o preço médio do item {self.str_item}.')

    def valid_entries(self):
        self.valid_tipo()
        self.valid_configuracao()
        self.valid_item()
        self.valid_quant()
        self.valid_deps()
        self.valid_num_doc()

    def valid_tipo(self):
        if isinstance(self.tipo, models.TipoMovStq):
            self.tip_mov = self.tipo
        else:
            try:
                self.tip_mov = models.TipoMovStq.objects.get(codigo=self.tipo)
            except models.TipoMovStq.DoesNotExist as e:
                raise ValueError(
                    f'Tipo de movimento de estoque "{self.tipo}" '
                    'não cadastrado.')

    def valid_item(self):
        objs_prod = pro_cla.ObjsProduto(
            self.nivel, self.ref, self.tam, self.cor)

        self.str_item = objs_prod.str_item
        self.produto_item = objs_prod.produto_item

    def valid_quant(self):
        if self.qtd <= 0:
            raise ValueError(
                'Quantidade deve ser maior que zero.')

    def valid_deps(self):
        if not self.tem_trans_saida:
            self.deposito_origem = 0

        if not self.tem_trans_entrada:
            self.deposito_destino = 0

        if self.deposito_origem == self.deposito_destino:
            raise ValueError('Depósitos devem ser diferentes.')

    def valid_num_doc(self):
        obj_doc_mov_stq = classes.ObjDocMovStq(
            self.num_doc, self.descricao, self.request.user,
            cria=self.cria_num_doc)
        if obj_doc_mov_stq.num_doc != '0':
            self.num_doc = obj_doc_mov_stq.num_doc
            self.doc_mov_stq = obj_doc_mov_stq.doc_mov_stq

    def get_estoque(self, deposito_field):
        l_estoque = queries.get_estoque_dep_niv_ref_cor_tam(
            self.cursor, deposito_field,
            self.nivel, self.ref, self.cor, self.tam)
        if len(l_estoque) == 0:
            return 0
        else:
            return l_estoque[0]['estoque']

    def valid_transacao(self, codigo, ent_sai, descricao):
        try:
            tipo_trans = sys_mod.TipoTransacao.validos.get(
                codigo_transacao=codigo)
        except sys_mod.TipoTransacao.DoesNotExist as e:
            raise ValueError(
                f'Não encontrada transação de {descricao} '
                f'"{codigo}" do tipo de movimento de estoque "{self.tipo}".')

        if tipo_trans.entrada_saida not in ent_sai:
            raise ValueError(
                f'Transação de {descricao} "{codigo}" '
                f'do tipo de movimento de estoque "{self.tipo}" é do '
                'tipo da operação errado.')

        if tipo_trans.entrada_saida == 'T':
            return 'S'
        else:
            return tipo_trans.entrada_saida

    def valid_configuracao(self):
        self.trans_saida = self.tip_mov.trans_saida
        self.tem_trans_saida = self.trans_saida != 0

        self.trans_entrada = self.tip_mov.trans_entrada
        self.tem_trans_entrada = self.trans_entrada != 0

        if self.tem_trans_saida:
            self.trans_saida_e_s = self.valid_transacao(
                self.trans_saida, 'ST', 'saída')

        if self.tem_trans_entrada:
            self.trans_entrada_e_s = self.valid_transacao(
                self.trans_entrada, 'E', 'entrada')

    def exec(self):
        if not self.can_exec:
            raise ValueError(
                'Execução impedida por algum erro de inicialização.')

        if self.tem_trans_saida:
            self.insert(
                'saída',
                self.deposito_origem,
                self.trans_saida,
                self.trans_saida_e_s,
                )

        if self.tem_trans_entrada:
            self.insert(
                'entrada',
                self.deposito_destino,
                self.trans_entrada,
                self.trans_entrada_e_s,
                )

        mov_stq = models.MovStq(
            tipo_mov=self.tip_mov,
            item=self.produto_item,
            quantidade=self.qtd,
            deposito_origem=self.deposito_origem,
            deposito_destino=self.deposito_destino,
            documento=self.doc_mov_stq,
            usuario=self.request.user,
        )
        mov_stq.save()

    def insert(self, descr, dep, trans, e_s):
        if not queries.insert_transacao(
                self.cursor,
                dep,
                self.nivel,
                self.ref,
                self.tam,
                self.cor,
                self.num_doc,
                trans,
                e_s,
                self.qtd,
                self.preco_medio,
                self.request.user,
                get_client_ip(self.request)
                ):
            raise ValueError(
                f'Erro ao inserir transação de {descr}.')
<commit_msg>Iniciando validação para renomear<commit_after>from pprint import pprint

import systextil.models as sys_mod

from utils.functions import get_client_ip

import produto.functions as pro_fun
import produto.classes as pro_cla

from estoque import classes
from estoque import models
from estoque import queries

class Transfere():

    def __init__(
            self, cursor, tipo, nivel, ref, tam, cor, qtd,
            deposito_origem, deposito_destino,
            nova_ref, novo_tam, nova_cor,
            num_doc, descricao, request, cria_num_doc=True):
        self.can_exec = False

        self.cursor = cursor
        self.tipo = tipo
        self.nivel = nivel
        self.ref = ref
        self.tam = tam
        self.cor = cor
        self.qtd = qtd
        self.deposito_origem = deposito_origem
        self.deposito_destino = deposito_destino
        self.nova_ref = nova_ref
        self.novo_tam = novo_tam
        self.nova_cor = nova_cor
        self.num_doc = num_doc
        self.descricao = descricao
        self.request = request
        self.cria_num_doc = cria_num_doc

        self.valid_entries()

        self.calc_vars()
        self.can_exec = True

    def calc_vars(self):
        if self.tem_trans_saida:
            self.estoque_origem = self.get_estoque(self.deposito_origem)
            self.novo_estoque_origem = self.estoque_origem - self.qtd
        else:
            self.estoque_origem = 0
            self.novo_estoque_origem = 0

        if self.tem_trans_entrada:
            self.estoque_destino = self.get_estoque(self.deposito_destino)
            self.novo_estoque_destino = self.estoque_destino + self.qtd
        else:
            self.estoque_destino = 0
            self.novo_estoque_destino = 0

        produto = queries.get_preco_medio_niv_ref_cor_tam(
            self.cursor, self.nivel, self.ref, self.cor, self.tam)
        try:
            self.preco_medio = produto[0]['preco_medio']
        except Exception:
            raise ValueError(
                f'Não encontrado o preço médio do item {self.str_item}.')

    def valid_entries(self):
        self.valid_tipo()
        self.valid_configuracao()
        self.valid_item()
        self.valid_quant()
        self.valid_deps()
        if self.tip_mov.renomeia:
            self.valid_novo_item()
        self.valid_num_doc()

    def valid_tipo(self):
        if isinstance(self.tipo, models.TipoMovStq):
            self.tip_mov = self.tipo
        else:
            try:
                self.tip_mov = models.TipoMovStq.objects.get(codigo=self.tipo)
            except models.TipoMovStq.DoesNotExist as e:
                raise ValueError(
                    f'Tipo de movimento de estoque "{self.tipo}" '
                    'não cadastrado.')

    def valid_item(self):
        objs_prod = pro_cla.ObjsProduto(
            self.nivel, self.ref, self.tam, self.cor)

        self.str_item = objs_prod.str_item
        self.produto_item = objs_prod.produto_item

    def valid_quant(self):
        if self.qtd <= 0:
            raise ValueError(
                'Quantidade deve ser maior que zero.')

    def valid_deps(self):
        if not self.tem_trans_saida:
            self.deposito_origem = 0

        if not self.tem_trans_entrada:
            self.deposito_destino = 0

        if self.deposito_origem == self.deposito_destino:
            raise ValueError('Depósitos devem ser diferentes.')

    def valid_novo_item(self):
        if (len(self.nova_ref) + len(self.novo_tam) +
                len(self.nova_cor)) == 0:
            raise ValueError('Algum novo código deve ser indicado.')

    def valid_num_doc(self):
        obj_doc_mov_stq = classes.ObjDocMovStq(
            self.num_doc, self.descricao, self.request.user,
            cria=self.cria_num_doc)
        if obj_doc_mov_stq.num_doc != '0':
            self.num_doc = obj_doc_mov_stq.num_doc
            self.doc_mov_stq = obj_doc_mov_stq.doc_mov_stq

    def get_estoque(self, deposito_field):
        l_estoque = queries.get_estoque_dep_niv_ref_cor_tam(
            self.cursor, deposito_field,
            self.nivel, self.ref, self.cor, self.tam)
        if len(l_estoque) == 0:
            return 0
        else:
            return l_estoque[0]['estoque']

    def valid_transacao(self, codigo, ent_sai, descricao):
        try:
            tipo_trans = sys_mod.TipoTransacao.validos.get(
                codigo_transacao=codigo)
        except sys_mod.TipoTransacao.DoesNotExist as e:
            raise ValueError(
                f'Não encontrada transação de {descricao} '
                f'"{codigo}" do tipo de movimento de estoque "{self.tipo}".')

        if tipo_trans.entrada_saida not in ent_sai:
            raise ValueError(
                f'Transação de {descricao} "{codigo}" '
                f'do tipo de movimento de estoque "{self.tipo}" é do '
                'tipo da operação errado.')

        if tipo_trans.entrada_saida == 'T':
            return 'S'
        else:
            return tipo_trans.entrada_saida

    def valid_configuracao(self):
        self.trans_saida = self.tip_mov.trans_saida
        self.tem_trans_saida = self.trans_saida != 0

        self.trans_entrada = self.tip_mov.trans_entrada
        self.tem_trans_entrada = self.trans_entrada != 0

        if self.tem_trans_saida:
            self.trans_saida_e_s = self.valid_transacao(
                self.trans_saida, 'ST', 'saída')

        if self.tem_trans_entrada:
            self.trans_entrada_e_s = self.valid_transacao(
                self.trans_entrada, 'E', 'entrada')

    def exec(self):
        if not self.can_exec:
            raise ValueError(
                'Execução impedida por algum erro de inicialização.')

        if self.tem_trans_saida:
            self.insert(
                'saída',
                self.deposito_origem,
                self.trans_saida,
                self.trans_saida_e_s,
                )

        if self.tem_trans_entrada:
            self.insert(
                'entrada',
                self.deposito_destino,
                self.trans_entrada,
                self.trans_entrada_e_s,
                )

        mov_stq = models.MovStq(
            tipo_mov=self.tip_mov,
            item=self.produto_item,
            quantidade=self.qtd,
            deposito_origem=self.deposito_origem,
            deposito_destino=self.deposito_destino,
            documento=self.doc_mov_stq,
            usuario=self.request.user,
        )
        mov_stq.save()

    def insert(self, descr, dep, trans, e_s):
        if not queries.insert_transacao(
                self.cursor,
                dep,
                self.nivel,
                self.ref,
                self.tam,
                self.cor,
                self.num_doc,
                trans,
                e_s,
                self.qtd,
                self.preco_medio,
                self.request.user,
                get_client_ip(self.request)
                ):
            raise ValueError(
                f'Erro ao inserir transação de {descr}.')
<|endoftext|>


<commit_before>from pprint import pprint

import systextil.models as sys_mod

from utils.functions import get_client_ip

import produto.functions as pro_fun
import produto.classes as pro_cla

from estoque import classes
from estoque import models
from estoque import queries

class Transfere():

    def __init__(
            self, cursor, nivel, ref, tam, cor, qtd,
            deposito_origem, deposito_destino,
            num_doc, descricao, request):
        self.can_exec = False

        self.cursor = cursor
        self.nivel = nivel
        self.ref = ref
        self.tam = tam
        self.cor = cor
        self.qtd = qtd
        self.deposito_origem = deposito_origem
        self.deposito_destino = deposito_destino
        self.num_doc = num_doc
        self.descricao = descricao
        self.request = request

        self.valid_entries()
        self.valid_configuracao()

        self.calc_vars()
        self.can_exec = True

    def calc_vars(self):
        self.estoque_origem = self.get_estoque(self.deposito_origem)
        self.estoque_destino = self.get_estoque(self.deposito_destino)
        self.novo_estoque_origem = self.estoque_origem - self.qtd
        self.novo_estoque_destino = self.estoque_destino + self.qtd

        produto = queries.get_preco_medio_niv_ref_cor_tam(
            self.cursor, self.nivel, self.ref, self.cor, self.tam)
        try:
            self.preco_medio = produto[0]['preco_medio']
        except Exception:
            raise ValueError(
                f'Não encontrado o preço médio do item {self.str_item}.')

    def valid_entries(self):
        self.valid_item()
        self.valid_quant()
        self.valid_deps()
        self.valid_num_doc()

    def valid_item(self):
        objs_prod = pro_cla.ObjsProduto(
            self.nivel, self.ref, self.tam, self.cor)

        self.str_item = objs_prod.str_item
        self.produto_item = objs_prod.produto_item

    def valid_quant(self):
        if self.qtd <= 0:
            raise ValueError(
                'Quantidade deve ser maior que zero.')

    def valid_deps(self):
        if self.deposito_origem == self.deposito_destino:
            raise ValueError('Depósitos devem ser diferentes.')

    def valid_num_doc(self):
        obj_doc_mov_stq = classes.ObjDocMovStq(
            self.num_doc, self.descricao, self.request.user)
        self.num_doc = obj_doc_mov_stq.num_doc
        self.doc_mov_stq = obj_doc_mov_stq.doc_mov_stq

    def get_estoque(self, deposito_field):
        l_estoque = queries.get_estoque_dep_niv_ref_cor_tam(
            self.cursor, deposito_field,
            self.nivel, self.ref, self.cor, self.tam)
        if len(l_estoque) == 0:
            return 0
        else:
            return l_estoque[0]['estoque']

    def valid_transacao(self, codigo, ent_sai, descricao):
        try:
            tipo_trans = sys_mod.TipoTransacao.objects.get(
                codigo_transacao=codigo)
        except sys_mod.TipoTransacao.DoesNotExist as e:
            raise ValueError(
                f'Não encontrada transação de {descricao} '
                f'"{codigo}" do tipo de movimento de estoque "TRANSF".')

        if tipo_trans.entrada_saida not in ent_sai:
            raise ValueError(
                f'Transação de {descricao} "{codigo}" '
                'do tipo de movimento de estoque "TRANSF" é do '
                'tipo da operação errado.')

        if tipo_trans.entrada_saida == 'T':
            return 'S'
        else:
            return tipo_trans.entrada_saida

    def valid_configuracao(self):
        try:
            tip_mov = models.TipoMovStq.objects.get(codigo='TRANSF')
        except models.TipoMovStq.DoesNotExist as e:
            raise ValueError(
                'Tipo de movimento de estoque "TRANSF" não cadastrado.')

        self.trans_saida = tip_mov.trans_saida
        self.trans_entrada = tip_mov.trans_entrada

        self.trans_saida_e_s = self.valid_transacao(
            self.trans_saida, 'ST', 'saída')
        self.trans_entrada_e_s = self.valid_transacao(
            self.trans_entrada, 'E', 'entrada')

    def exec(self):
        if not self.can_exec:
            raise ValueError(
                'Execução impedida por algum erro de inicialização.')

        if not queries.insert_transacao(
                self.cursor,
                self.deposito_origem,
                self.nivel,
                self.ref,
                self.tam,
                self.cor,
                self.num_doc,
                self.trans_saida,
                self.trans_saida_e_s,
                self.qtd,
                self.preco_medio,
                self.request.user,
                get_client_ip(self.request)
                ):
            raise ValueError(
                'Erro ao inserir transação de saída.')
<commit_msg>criando def insert<commit_after>from pprint import pprint

import systextil.models as sys_mod

from utils.functions import get_client_ip

import produto.functions as pro_fun
import produto.classes as pro_cla

from estoque import classes
from estoque import models
from estoque import queries

class Transfere():

    def __init__(
            self, cursor, nivel, ref, tam, cor, qtd,
            deposito_origem, deposito_destino,
            num_doc, descricao, request):
        self.can_exec = False

        self.cursor = cursor
        self.nivel = nivel
        self.ref = ref
        self.tam = tam
        self.cor = cor
        self.qtd = qtd
        self.deposito_origem = deposito_origem
        self.deposito_destino = deposito_destino
        self.num_doc = num_doc
        self.descricao = descricao
        self.request = request

        self.valid_entries()
        self.valid_configuracao()

        self.calc_vars()
        self.can_exec = True

    def calc_vars(self):
        self.estoque_origem = self.get_estoque(self.deposito_origem)
        self.estoque_destino = self.get_estoque(self.deposito_destino)
        self.novo_estoque_origem = self.estoque_origem - self.qtd
        self.novo_estoque_destino = self.estoque_destino + self.qtd

        produto = queries.get_preco_medio_niv_ref_cor_tam(
            self.cursor, self.nivel, self.ref, self.cor, self.tam)
        try:
            self.preco_medio = produto[0]['preco_medio']
        except Exception:
            raise ValueError(
                f'Não encontrado o preço médio do item {self.str_item}.')

    def valid_entries(self):
        self.valid_item()
        self.valid_quant()
        self.valid_deps()
        self.valid_num_doc()

    def valid_item(self):
        objs_prod = pro_cla.ObjsProduto(
            self.nivel, self.ref, self.tam, self.cor)

        self.str_item = objs_prod.str_item
        self.produto_item = objs_prod.produto_item

    def valid_quant(self):
        if self.qtd <= 0:
            raise ValueError(
                'Quantidade deve ser maior que zero.')

    def valid_deps(self):
        if self.deposito_origem == self.deposito_destino:
            raise ValueError('Depósitos devem ser diferentes.')

    def valid_num_doc(self):
        obj_doc_mov_stq = classes.ObjDocMovStq(
            self.num_doc, self.descricao, self.request.user)
        self.num_doc = obj_doc_mov_stq.num_doc
        self.doc_mov_stq = obj_doc_mov_stq.doc_mov_stq

    def get_estoque(self, deposito_field):
        l_estoque = queries.get_estoque_dep_niv_ref_cor_tam(
            self.cursor, deposito_field,
            self.nivel, self.ref, self.cor, self.tam)
        if len(l_estoque) == 0:
            return 0
        else:
            return l_estoque[0]['estoque']

    def valid_transacao(self, codigo, ent_sai, descricao):
        try:
            tipo_trans = sys_mod.TipoTransacao.objects.get(
                codigo_transacao=codigo)
        except sys_mod.TipoTransacao.DoesNotExist as e:
            raise ValueError(
                f'Não encontrada transação de {descricao} '
                f'"{codigo}" do tipo de movimento de estoque "TRANSF".')

        if tipo_trans.entrada_saida not in ent_sai:
            raise ValueError(
                f'Transação de {descricao} "{codigo}" '
                'do tipo de movimento de estoque "TRANSF" é do '
                'tipo da operação errado.')

        if tipo_trans.entrada_saida == 'T':
            return 'S'
        else:
            return tipo_trans.entrada_saida

    def valid_configuracao(self):
        try:
            tip_mov = models.TipoMovStq.objects.get(codigo='TRANSF')
        except models.TipoMovStq.DoesNotExist as e:
            raise ValueError(
                'Tipo de movimento de estoque "TRANSF" não cadastrado.')

        self.trans_saida = tip_mov.trans_saida
        self.trans_entrada = tip_mov.trans_entrada

        self.trans_saida_e_s = self.valid_transacao(
            self.trans_saida, 'ST', 'saída')
        self.trans_entrada_e_s = self.valid_transacao(
            self.trans_entrada, 'E', 'entrada')

    def exec(self):
        if not self.can_exec:
            raise ValueError(
                'Execução impedida por algum erro de inicialização.')

        self.insert(
            'saída',
            self.deposito_origem,
            self.trans_saida,
            self.trans_saida_e_s,
            )

    def insert(self, descr, dep, trans, e_s):
        if not queries.insert_transacao(
                self.cursor,
                dep,
                self.nivel,
                self.ref,
                self.tam,
                self.cor,
                self.num_doc,
                trans,
                e_s,
                self.qtd,
                self.preco_medio,
                self.request.user,
                get_client_ip(self.request)
                ):
            raise ValueError(
                f'Erro ao inserir transação de {descr}.')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2010 Radim Rehurek <radimrehurek@seznam.cz>
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html

"""USAGE: %(program)s WIKI_XML_DUMP OUTPUT_PREFIX

Convert articles from a Wikipedia dump to (sparse) vectors. The input is a bz2-compressed \
dump of Wikipedia articles, in XML format.

This actually creates three files:
 * OUTPUT_PREFIX_wordids.txt: mapping between words and their integer ids
 * OUTPUT_PREFIX_bow.mm: bag-of-words (word counts) representation, in Matrix Matrix format
 * OUTPUT_PREFIX_tfidf.mm: TF-IDF representation

The output Matrix Market files can then be compressed (e.g., by bzip2) to save \
disk space; gensim's corpus iterators can work with compressed input, too.

Example: ./wikicorpus.py ~/gensim/results/enwiki-20100622-pages-articles.xml.bz2 ~/gensim/results/wiki_en
"""

import logging
import itertools
import sys
import os.path
import re
import bz2

from gensim import interfaces, matutils, utils
from gensim.corpora.dictionary import Dictionary # for constructing word->id mappings

logger = logging.getLogger('wikicorpus')
logger.setLevel(logging.INFO)

ARTICLE_MIN_CHARS = 500

RE_P0 = re.compile('<!--.*?-->', re.DOTALL | re.UNICODE)
RE_P1 = re.compile('<ref([> ].*?)(</ref>|/>)', re.DOTALL | re.UNICODE)
RE_P2 = re.compile("(\n\[\[[a-z][a-z][\w-]*:[^:\]]+\]\])+$", re.UNICODE)
RE_P3 = re.compile("{{([^}{]*)}}", re.DOTALL | re.UNICODE)
RE_P4 = re.compile("{{([^}]*)}}", re.DOTALL | re.UNICODE)
RE_P5 = re.compile('\[(\w+):\/\/(.*?)(( (.*?))|())\]', re.UNICODE)
RE_P6 = re.compile("\[([^][]*)\|([^][]*)\]", re.DOTALL | re.UNICODE)
RE_P7 = re.compile('\n\[\[[iI]mage(.*?)(\|.*?)*\|(.*?)\]\]', re.UNICODE)
RE_P8 = re.compile('\n\[\[[fF]ile(.*?)(\|.*?)*\|(.*?)\]\]', re.UNICODE)
RE_P9 = re.compile('<nowiki([> ].*?)(</nowiki>|/>)', re.DOTALL | re.UNICODE)
RE_P10 = re.compile('<math([> ].*?)(</math>|/>)', re.DOTALL | re.UNICODE)
RE_P11 = re.compile('<(.*?)>', re.DOTALL | re.UNICODE)
RE_P12 = re.compile('\n(({\|)|(\|-)|(\|}))(.*?)(?=\n)', re.UNICODE)
RE_P13 = re.compile('\n(\||\!)(.*?\|)*([^|]*?)', re.UNICODE)

def filterWiki(raw):
    """
    Filter out wiki mark-up from utf8 string `raw`, leaving only text.
    """
    # the parsing the wiki markup is not perfect, but sufficient for our purposes
    # contributions to improving this code are welcome :)
    text = utils.decode_htmlentities(unicode(raw, 'utf8', 'ignore'))
    text = utils.decode_htmlentities(text) # '&nbsp;' --> '\xa0'
    text = re.sub(RE_P2, "", text) # remove the last list (=languages)
    # the wiki markup is recursive (markup inside markup etc)
    # instead of writing a recursive grammar, here we deal with that by removing
    # markup in a loop, starting with inner-most expressions and working outwards,
    # as long as something changes.
    iters = 0
    while True:
        old, iters = text, iters + 1
        text = re.sub(RE_P0, "", text) # remove comments
        text = re.sub(RE_P1, '', text) # remove footnotes
        text = re.sub(RE_P9, "", text) # remove outside links
        text = re.sub(RE_P10, "", text) # remove math content
        text = re.sub(RE_P11, "", text) # remove all remaining tags
        # remove templates (no recursion)
        text = re.sub(RE_P3, '', text)
        text = re.sub(RE_P4, '', text)
        text = re.sub(RE_P5, '\\3', text) # remove urls, keep description
        text = re.sub(RE_P7, '\n\\3', text) # simplify images, keep description only
        text = re.sub(RE_P8, '\n\\3', text) # simplify files, keep description only
        text = re.sub(RE_P6, '\\2', text) # simplify links, keep description only
        # remove table markup
        text = text.replace('||', '\n|') # each table cell on a separate line
        text = re.sub(RE_P12, '\n', text) # remove formatting lines
        text = re.sub(RE_P13, '\n\\3', text) # leave only cell content
        # remove empty mark-up
        text = text.replace('[]', '')
        if old == text or iters > 2: # stop if nothing changed between two iterations or after a fixed number of iterations
            break

    # the following is needed to make the tokenizer see '[[socialist]]s' as a single word 'socialists'
    # TODO is this really desirable?
    text = text.replace('[', '').replace(']', '') # promote all remaining markup to plain text
    return text

def tokenize(content):
    """
    Tokenize a piece of text from wikipedia. The input string `content` is assumed
    to be mark-up free (see `filterWiki()`).

    Return tokens as utf8 bytestrings.
    """
    # TODO maybe ignore tokens with non-latin characters? (no chinese, arabic, russian etc.)
    return [token.encode('utf8') for token in utils.tokenize(content, lower = True, errors = 'ignore')
            if len(token) <= 15 and not token.startswith('_')]

class WikiCorpus(interfaces.CorpusABC):
    """
    Treat a wikipedia articles dump (*articles.xml.bz2) as a (read-only) corpus.

    The documents are extracted on-the-fly, so that the whole (massive) dump
    can stay compressed on disk.

    >>> wiki = WikiCorpus('enwiki-20100622-pages-articles.xml.bz2') # create word->word_id mapping, takes almost 8h
    >>> wiki.saveAsText('wiki_en_vocab200k') # another 8h, creates a file in MatrixMarket format plus file with id->word

    """
    def __init__(self, fname, noBelow = 20, keep_words = 200000, dictionary = None):
        """
        Initialize the corpus. This scans the corpus once, to determine its
        vocabulary (only the first `keep_words` most frequent words that
        appear in at least `noBelow` documents are kept).
        """
        self.fname = fname
        if dictionary is None:
            self.dictionary = Dictionary(self.getArticles())
            self.dictionary.filterExtremes(noBelow = noBelow, noAbove = 0.1, keepN = keep_words)
        else:
            self.dictionary = dictionary

    def __len__(self):
        return self.numDocs

    def __iter__(self):
        """
        The function that defines a corpus -- iterating over the corpus yields
        vectors, one for each document.
        """
        for docNo, text in enumerate(self.getArticles()):
            yield self.dictionary.doc2bow(text, allowUpdate = False)

    def saveDictionary(self, fname):
        """
        Store id->word mapping to a file, in format `id[TAB]word_utf8[TAB]document frequency[NEWLINE]`.
        """
        logger.info("saving dictionary mapping to %s" % fname)
        fout = open(fname, 'w')
        for token, tokenId in sorted(self.dictionary.token2id.iteritems()):
            fout.write("%i\t%s\t%i\n" % (tokenId, token, self.dictionary.docFreq[tokenId]))
        fout.close()

    @staticmethod
    def loadDictionary(fname):
        """
        Load previously stored mapping between words and their ids.

        The result can be used as the `id2word` parameter for input to transformations.
        """
        result = {}
        for lineNo, line in enumerate(open(fname)):
            cols = line[:-1].split('\t')
            if len(cols) == 2:
                wordId, word = cols
            elif len(cols) == 3:
                wordId, word, docFreq = cols
            else:
                continue
            result[int(wordId)] = word # docFreq not used
        return result

    def saveAsText(self, fname):
        """
        Store the corpus to disk, in a human-readable text format.

        This actually saves two files:

        1. Document-term co-occurence frequency counts (bag-of-words), as
           a Matrix Market file `fname_bow.mm`.
        2. Token to integer mapping, as a text file `fname_wordids.txt`.

        """
        self.saveDictionary(fname + '_wordids.txt')
        matutils.MmWriter.writeCorpus(fname + '_bow.mm', self, progressCnt = 10000)

    def getArticles(self):
        """
        Iterate over the dump, returning text version of each article.

        Only articles of sufficient length are returned (short articles & redirects
        etc are ignored).
        """
        articles, intext = 0, False
        for lineno, line in enumerate(bz2.BZ2File(self.fname)):
            if line.startswith('      <text'):
                intext = True
                line = line[line.find('>') + 1 : ]
                lines = [line]
            elif intext:
                lines.append(line)
            pos = line.find('</text>') # can be on the same line as <text>
            if pos >= 0:
                intext = False
                if not lines:
                    continue
                lines[-1] = line[:pos]
                text = filterWiki(''.join(lines))
                if len(text) > ARTICLE_MIN_CHARS: # article redirects are pruned here
                    articles += 1
                    yield tokenize(text) # split text into tokens

        self.numDocs = articles # cache corpus length
#endclass WikiCorpus

class VocabTransform(interfaces.TransformationABC):
    """
    Convert corpus to another, with different feature ids.

    Given a mapping between old ids and new ids (some old ids may be missing,
    i.e. the mapping need not be a bijection), this will wrap a
    corpus so that iterating over VocabTransform[corpus] returns the same vectors but with
    the new ids.

    Old features that have no counterpart in the new ids are discarded. This
    can be used to filter vocabulary of a corpus::

    >>> old2new = dict((oldid, newid) for newid, oldid in enumerate(remaining_ids))
    >>> id2word = dict((newid, oldid2token[oldid]) for oldid, newid in old2new.iteritems())
    >>> vt = VocabTransform(old2new, id2token)

    """
    def __init__(self, old2new, id2token = None):
        self.old2new = old2new
        self.id2token = id2token

    def __getitem__(self, bow):
        """
        Return representation with the ids transformed.
        """
        # if the input vector is in fact a corpus, return a transformed corpus as a result
        if utils.isCorpus(bow):
            return self._apply(bow)

        return [(self.old2new[oldid], weight) for oldid, weight in bow if oldid in self.old2new]
#endclass VocabTransform

if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s')
    logging.root.setLevel(level = logging.INFO)
    logging.info("running %s" % ' '.join(sys.argv))

    program = os.path.basename(sys.argv[0])

    # check and process input arguments
    if len(sys.argv) < 3:
        print globals()['__doc__'] % locals()
        sys.exit(1)
    input, output = sys.argv[1:3]

    # build dictionary. only keep 200k most frequent words (out of total ~7m unique tokens)
    # takes about 8h on a macbook pro
    wiki = WikiCorpus(input, keep_words = 200000)

    # save dictionary and bag-of-words
    # another ~8h
    wiki.saveAsText(output)
    del wiki

    # initialize corpus reader and word->id mapping
    from gensim.corpora import MmCorpus
    id2token = WikiCorpus.loadDictionary(output + '_wordids.txt')
    mm = MmCorpus(output + '_bow.mm')

    # build tfidf
    # ~20min
    from gensim.models import TfidfModel
    tfidf = TfidfModel(mm, id2word = id2token, normalize = True)

    # save tfidf vectors in matrix market format
    # ~1.5h; result file is 14GB! bzip2'ed down to 4.5GB
    MmCorpus.saveCorpus(output + '_tfidf.mm', tfidf[mm], progressCnt = 10000)

    logging.info("finished running %s" % program)

    # running lsi (chunks=20000, numTopics=400) on wiki_tfidf then takes about 14h.
<commit_msg>adding extra log to wikicorpus<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2010 Radim Rehurek <radimrehurek@seznam.cz>
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html

"""USAGE: %(program)s WIKI_XML_DUMP OUTPUT_PREFIX

Convert articles from a Wikipedia dump to (sparse) vectors. The input is a bz2-compressed \
dump of Wikipedia articles, in XML format.

This actually creates three files:
 * OUTPUT_PREFIX_wordids.txt: mapping between words and their integer ids
 * OUTPUT_PREFIX_bow.mm: bag-of-words (word counts) representation, in Matrix Matrix format
 * OUTPUT_PREFIX_tfidf.mm: TF-IDF representation

The output Matrix Market files can then be compressed (e.g., by bzip2) to save \
disk space; gensim's corpus iterators can work with compressed input, too.

Example: ./wikicorpus.py ~/gensim/results/enwiki-20100622-pages-articles.xml.bz2 ~/gensim/results/wiki_en
"""

import logging
import itertools
import sys
import os.path
import re
import bz2

from gensim import interfaces, matutils, utils
from gensim.corpora.dictionary import Dictionary # for constructing word->id mappings

logger = logging.getLogger('wikicorpus')
logger.setLevel(logging.INFO)

ARTICLE_MIN_CHARS = 500

RE_P0 = re.compile('<!--.*?-->', re.DOTALL | re.UNICODE)
RE_P1 = re.compile('<ref([> ].*?)(</ref>|/>)', re.DOTALL | re.UNICODE)
RE_P2 = re.compile("(\n\[\[[a-z][a-z][\w-]*:[^:\]]+\]\])+$", re.UNICODE)
RE_P3 = re.compile("{{([^}{]*)}}", re.DOTALL | re.UNICODE)
RE_P4 = re.compile("{{([^}]*)}}", re.DOTALL | re.UNICODE)
RE_P5 = re.compile('\[(\w+):\/\/(.*?)(( (.*?))|())\]', re.UNICODE)
RE_P6 = re.compile("\[([^][]*)\|([^][]*)\]", re.DOTALL | re.UNICODE)
RE_P7 = re.compile('\n\[\[[iI]mage(.*?)(\|.*?)*\|(.*?)\]\]', re.UNICODE)
RE_P8 = re.compile('\n\[\[[fF]ile(.*?)(\|.*?)*\|(.*?)\]\]', re.UNICODE)
RE_P9 = re.compile('<nowiki([> ].*?)(</nowiki>|/>)', re.DOTALL | re.UNICODE)
RE_P10 = re.compile('<math([> ].*?)(</math>|/>)', re.DOTALL | re.UNICODE)
RE_P11 = re.compile('<(.*?)>', re.DOTALL | re.UNICODE)
RE_P12 = re.compile('\n(({\|)|(\|-)|(\|}))(.*?)(?=\n)', re.UNICODE)
RE_P13 = re.compile('\n(\||\!)(.*?\|)*([^|]*?)', re.UNICODE)

def filterWiki(raw):
    """
    Filter out wiki mark-up from utf8 string `raw`, leaving only text.
    """
    # the parsing the wiki markup is not perfect, but sufficient for our purposes
    # contributions to improving this code are welcome :)
    text = utils.decode_htmlentities(unicode(raw, 'utf8', 'ignore'))
    text = utils.decode_htmlentities(text) # '&nbsp;' --> '\xa0'
    text = re.sub(RE_P2, "", text) # remove the last list (=languages)
    # the wiki markup is recursive (markup inside markup etc)
    # instead of writing a recursive grammar, here we deal with that by removing
    # markup in a loop, starting with inner-most expressions and working outwards,
    # as long as something changes.
    iters = 0
    while True:
        old, iters = text, iters + 1
        text = re.sub(RE_P0, "", text) # remove comments
        text = re.sub(RE_P1, '', text) # remove footnotes
        text = re.sub(RE_P9, "", text) # remove outside links
        text = re.sub(RE_P10, "", text) # remove math content
        text = re.sub(RE_P11, "", text) # remove all remaining tags
        # remove templates (no recursion)
        text = re.sub(RE_P3, '', text)
        text = re.sub(RE_P4, '', text)
        text = re.sub(RE_P5, '\\3', text) # remove urls, keep description
        text = re.sub(RE_P7, '\n\\3', text) # simplify images, keep description only
        text = re.sub(RE_P8, '\n\\3', text) # simplify files, keep description only
        text = re.sub(RE_P6, '\\2', text) # simplify links, keep description only
        # remove table markup
        text = text.replace('||', '\n|') # each table cell on a separate line
        text = re.sub(RE_P12, '\n', text) # remove formatting lines
        text = re.sub(RE_P13, '\n\\3', text) # leave only cell content
        # remove empty mark-up
        text = text.replace('[]', '')
        if old == text or iters > 2: # stop if nothing changed between two iterations or after a fixed number of iterations
            break

    # the following is needed to make the tokenizer see '[[socialist]]s' as a single word 'socialists'
    # TODO is this really desirable?
    text = text.replace('[', '').replace(']', '') # promote all remaining markup to plain text
    return text

def tokenize(content):
    """
    Tokenize a piece of text from wikipedia. The input string `content` is assumed
    to be mark-up free (see `filterWiki()`).

    Return tokens as utf8 bytestrings.
    """
    # TODO maybe ignore tokens with non-latin characters? (no chinese, arabic, russian etc.)
    return [token.encode('utf8') for token in utils.tokenize(content, lower = True, errors = 'ignore')
            if len(token) <= 15 and not token.startswith('_')]

class WikiCorpus(interfaces.CorpusABC):
    """
    Treat a wikipedia articles dump (*articles.xml.bz2) as a (read-only) corpus.

    The documents are extracted on-the-fly, so that the whole (massive) dump
    can stay compressed on disk.

    >>> wiki = WikiCorpus('enwiki-20100622-pages-articles.xml.bz2') # create word->word_id mapping, takes almost 8h
    >>> wiki.saveAsText('wiki_en_vocab200k') # another 8h, creates a file in MatrixMarket format plus file with id->word

    """
    def __init__(self, fname, noBelow = 20, keep_words = 200000, dictionary = None):
        """
        Initialize the corpus. This scans the corpus once, to determine its
        vocabulary (only the first `keep_words` most frequent words that
        appear in at least `noBelow` documents are kept).
        """
        self.fname = fname
        if dictionary is None:
            self.dictionary = Dictionary(self.getArticles())
            self.dictionary.filterExtremes(noBelow = noBelow, noAbove = 0.1, keepN = keep_words)
        else:
            self.dictionary = dictionary

    def __len__(self):
        return self.numDocs

    def __iter__(self):
        """
        The function that defines a corpus -- iterating over the corpus yields
        vectors, one for each document.
        """
        for docNo, text in enumerate(self.getArticles()):
            yield self.dictionary.doc2bow(text, allowUpdate = False)

    def saveDictionary(self, fname):
        """
        Store id->word mapping to a file, in format `id[TAB]word_utf8[TAB]document frequency[NEWLINE]`.
        """
        logger.info("saving dictionary mapping to %s" % fname)
        fout = open(fname, 'w')
        for token, tokenId in sorted(self.dictionary.token2id.iteritems()):
            fout.write("%i\t%s\t%i\n" % (tokenId, token, self.dictionary.docFreq[tokenId]))
        fout.close()

    @staticmethod
    def loadDictionary(fname):
        """
        Load previously stored mapping between words and their ids.

        The result can be used as the `id2word` parameter for input to transformations.
        """
        result = {}
        for lineNo, line in enumerate(open(fname)):
            cols = line[:-1].split('\t')
            if len(cols) == 2:
                wordId, word = cols
            elif len(cols) == 3:
                wordId, word, docFreq = cols
            else:
                continue
            result[int(wordId)] = word # docFreq not used
        return result

    def saveAsText(self, fname):
        """
        Store the corpus to disk, in a human-readable text format.

        This actually saves two files:

        1. Document-term co-occurence frequency counts (bag-of-words), as
           a Matrix Market file `fname_bow.mm`.
        2. Token to integer mapping, as a text file `fname_wordids.txt`.

        """
        self.saveDictionary(fname + '_wordids.txt')
        matutils.MmWriter.writeCorpus(fname + '_bow.mm', self, progressCnt = 10000)

    def getArticles(self):
        """
        Iterate over the dump, returning text version of each article.

        Only articles of sufficient length are returned (short articles & redirects
        etc are ignored).
        """
        articles, articles_all = 0, 0
        intext, positions = False, 0
        for lineno, line in enumerate(bz2.BZ2File(self.fname)):
            if line.startswith('      <text'):
                intext = True
                line = line[line.find('>') + 1 : ]
                lines = [line]
            elif intext:
                lines.append(line)
            pos = line.find('</text>') # can be on the same line as <text>
            if pos >= 0:
                articles_all += 1
                intext = False
                if not lines:
                    continue
                lines[-1] = line[:pos]
                text = filterWiki(''.join(lines))
                if len(text) > ARTICLE_MIN_CHARS: # article redirects are pruned here
                    articles += 1
                    result = tokenize(text) # split text into tokens
                    positions += len(result)
                    yield result

        logger.info("finished iterating over Wikipedia corpus of %i documents with %i positions"
                     "(total %i articles before pruning)" %
                     (articles, positions, articles_all))
        self.numDocs = articles # cache corpus length
#endclass WikiCorpus

class VocabTransform(interfaces.TransformationABC):
    """
    Convert corpus to another, with different feature ids.

    Given a mapping between old ids and new ids (some old ids may be missing,
    i.e. the mapping need not be a bijection), this will wrap a
    corpus so that iterating over VocabTransform[corpus] returns the same vectors but with
    the new ids.

    Old features that have no counterpart in the new ids are discarded. This
    can be used to filter vocabulary of a corpus::

    >>> old2new = dict((oldid, newid) for newid, oldid in enumerate(remaining_ids))
    >>> id2word = dict((newid, oldid2token[oldid]) for oldid, newid in old2new.iteritems())
    >>> vt = VocabTransform(old2new, id2token)

    """
    def __init__(self, old2new, id2token = None):
        self.old2new = old2new
        self.id2token = id2token

    def __getitem__(self, bow):
        """
        Return representation with the ids transformed.
        """
        # if the input vector is in fact a corpus, return a transformed corpus as a result
        if utils.isCorpus(bow):
            return self._apply(bow)

        return [(self.old2new[oldid], weight) for oldid, weight in bow if oldid in self.old2new]
#endclass VocabTransform

if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s')
    logging.root.setLevel(level = logging.INFO)
    logging.info("running %s" % ' '.join(sys.argv))

    program = os.path.basename(sys.argv[0])

    # check and process input arguments
    if len(sys.argv) < 3:
        print globals()['__doc__'] % locals()
        sys.exit(1)
    input, output = sys.argv[1:3]

    # build dictionary. only keep 200k most frequent words (out of total ~7m unique tokens)
    # takes about 8h on a macbook pro
    wiki = WikiCorpus(input, keep_words = 200000)

    # save dictionary and bag-of-words
    # another ~8h
    wiki.saveAsText(output)
    del wiki

    # initialize corpus reader and word->id mapping
    from gensim.corpora import MmCorpus
    id2token = WikiCorpus.loadDictionary(output + '_wordids.txt')
    mm = MmCorpus(output + '_bow.mm')

    # build tfidf
    # ~20min
    from gensim.models import TfidfModel
    tfidf = TfidfModel(mm, id2word = id2token, normalize = True)

    # save tfidf vectors in matrix market format
    # ~1.5h; result file is 14GB! bzip2'ed down to 4.5GB
    MmCorpus.saveCorpus(output + '_tfidf.mm', tfidf[mm], progressCnt = 10000)

    logging.info("finished running %s" % program)

    # running lsi (chunks=20000, numTopics=400) on wiki_tfidf then takes about 14h.
<|endoftext|>


<commit_before>import pika
import json
import sys
import logging
from elasticsearch import Elasticsearch
from elasticsearch_dsl import Search
from elasticsearch_dsl.query import Q
import traceback
import replayer
import dateutil
import copy
import datetime
from graccreq.oim import projects, OIMTopology
import StringIO
from graccreq.correct import Corrections

def SummaryReplayerFactory(msg, parameters, config):

    # Start profiling
    if 'General' in config and 'Profile' in config['General'] and config['General']['Profile']:
        logging.debug("Staring profiler")
        import cProfile
        pr = cProfile.Profile()
        pr.enable()

    try:
        replayer = SummaryReplayer(msg, parameters, config)
        replayer.run()
    except Exception, e:
        logging.error(traceback.format_exc())

    if 'General' in config and 'Profile' in config['General'] and config['General']['Profile']:
        logging.debug("Stopping profiler")
        import pstats
        pr.disable()
        s = StringIO.StringIO()
        sortby = 'cumulative'
        ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
        ps.print_stats()
        f = open('/tmp/profile.txt', 'a')
        f.write(s.getvalue())
        f.close()
        print s.getvalue()

class SummaryReplayer(replayer.Replayer):
    def __init__(self, message, parameters, config):
        super(SummaryReplayer, self).__init__(message, parameters)
        self._config = config

        # Initialize the project information
        self.project = projects.OIMProjects()

        # Initialize the OIM Topology information
        self.topology = OIMTopology.OIMTopology()

        # Initiatlize name corrections
        self.corrections = []
        for c in self._config.get('Corrections',[]):
            self.corrections.append(Corrections(uri=self._config['ElasticSearch'].get('uri','http://localhost:9200'),
                                                index=c['index'],
                                                doc_type=c['doc_type'],
                                                match_fields=c['match_fields'],
                                                source_field=c['source_field'],
                                                dest_field=c['dest_field']))

    def run(self):

        logging.debug("Beggining run of SummaryReplayer")
        self.createConnection()

        # Send start message to control channel, if requested
        self.sendControlMessage({'status': 'ok', 'stage': 'starting'})

        # Query elsaticsearch and send the data to the destination
        logging.info("Sending response to %s with routing key %s" % (self.msg['destination'], self.msg['routing_key']))
        try:
            for day in self._queryElasticsearch(self.msg['from'], self.msg['to'], None):
                for record in day:
                    for c in self.corrections:
                        c.correct(record)
                    self.addProperties(record)
                    self.sendMessage(json.dumps(record))
        except Exception, e:
            logging.error("Exception caught in query ES: %s" % str(e))
            logging.error(traceback.format_exc())
            raise e

        self.sendControlMessage({'status': 'ok', 'stage': 'finished'})

        self.conn.close()

    def on_return(self, channel, method, properties, body):
        sys.stderr.write("Got returned message\n")

    def addProperties(self, record):

        returned_doc = self.project.parseDoc(record)
        topology_doc = self.topology.generate_dict_for_gracc(record)
        record.update(returned_doc)
        record.update(topology_doc)
        return record

    def _queryElasticsearch(self, from_date, to_date, query):
        logging.debug("Connecting to ES")
        client = Elasticsearch(timeout=300)

        # For summaries, we only summarize full days, so strip the time from the from & to
        # Round the date up, so we get the entire last day they requested.
        from_date = dateutil.parser.parse(from_date).date()
        to_date = dateutil.parser.parse(to_date).date() + datetime.timedelta(days=1)

        logging.debug("Beginning search")
        s = Search(using=client, index=self._config['ElasticSearch']['raw_index'])
        s = s.filter('range', **{'EndTime': {'from': from_date, 'to': to_date }})
        s = s.query('bool', must=[Q('match', _type=self._config['ElasticSearch']['raw_type'])])

        # Fill in the unique terms and metrics
        unique_terms = [["EndTime", 0], ["VOName", "N/A"], ["ProjectName", "N/A"], ["DN", "N/A"], ["Processors", 1], ["ResourceType", "N/A"], ["CommonName", "N/A"], ["Host_description", "N/A"], ["Resource_ExitCode", 0], ["Grid", "N/A"], ["ReportableVOName", "N/A"], ["ProbeName", "N/A"], ["SiteName", "N/A"]]
        metrics = ["WallDuration", "CpuDuration_user", "CpuDuration_system", "CoreHours"]

        # If the terms are missing, set as "N/A"
        curBucket = s.aggs.bucket(unique_terms[0][0], 'date_histogram', field=unique_terms[0][0], interval="day")
        new_unique_terms = unique_terms[1:]

        for term in new_unique_terms:
        	curBucket = curBucket.bucket(term[0], 'terms', field=term[0], missing=term[1], size=(2**31)-1)

        for metric in metrics:
        	curBucket.metric(metric, 'sum', field=metric)

        response = s.execute()

        def recurseBucket(curData, curBucket, index, data):
            """
            Recursively process the buckets down the nested aggregations

            :param curData: Current
            :param bucket curBucket: A elasticsearch bucket object
            :param int index: Index of the unique_terms that we are processing
            """
            curTerm = unique_terms[index][0]

            # Check if we are at the end of the list
            if not curBucket[curTerm]['buckets']:
                # Make a copy of the data
                nowData = copy.deepcopy(curData)
                data.append(nowData)
            else:
                # Get the current key, and add it to the data
                for bucket in curBucket[curTerm]['buckets']:
                    nowData = copy.deepcopy(curData)
                    nowData[curTerm] = bucket['key']
                    if index == (len(unique_terms) - 1):
                        # reached the end of the unique terms
                        for metric in metrics:
                            nowData[metric] = bucket[metric].value
                            # Add the doc count
                        nowData["Count"] = bucket['doc_count']
                        data.append(nowData)
                    else:
                        recurseBucket(nowData, bucket, index+1, data)

        # We only want to hold onto 1 day's worth of summaries
        print len(response.aggregations['EndTime']['buckets'])
        for day in response.aggregations['EndTime']['buckets']:
            data = []
            recurseBucket({"EndTime": day['key_as_string']}, day, 1, data)
            yield data

        # Aggregate on njobs, WallDuration, CpuUserDuration, CpuSystemDuration
        # Or, unique keys on EndTime (day), VO, ProjectName, CommonName, DN, ResourceType, HostDescription, ApplicationExitCode, Grid, Cores

<commit_msg>Add Njobs<commit_after>import pika
import json
import sys
import logging
from elasticsearch import Elasticsearch
from elasticsearch_dsl import Search
from elasticsearch_dsl.query import Q
import traceback
import replayer
import dateutil
import copy
import datetime
from graccreq.oim import projects, OIMTopology
import StringIO
from graccreq.correct import Corrections

def SummaryReplayerFactory(msg, parameters, config):

    # Start profiling
    if 'General' in config and 'Profile' in config['General'] and config['General']['Profile']:
        logging.debug("Staring profiler")
        import cProfile
        pr = cProfile.Profile()
        pr.enable()

    try:
        replayer = SummaryReplayer(msg, parameters, config)
        replayer.run()
    except Exception, e:
        logging.error(traceback.format_exc())

    if 'General' in config and 'Profile' in config['General'] and config['General']['Profile']:
        logging.debug("Stopping profiler")
        import pstats
        pr.disable()
        s = StringIO.StringIO()
        sortby = 'cumulative'
        ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
        ps.print_stats()
        f = open('/tmp/profile.txt', 'a')
        f.write(s.getvalue())
        f.close()
        print s.getvalue()

class SummaryReplayer(replayer.Replayer):
    def __init__(self, message, parameters, config):
        super(SummaryReplayer, self).__init__(message, parameters)
        self._config = config

        # Initialize the project information
        self.project = projects.OIMProjects()

        # Initialize the OIM Topology information
        self.topology = OIMTopology.OIMTopology()

        # Initiatlize name corrections
        self.corrections = []
        for c in self._config.get('Corrections',[]):
            self.corrections.append(Corrections(uri=self._config['ElasticSearch'].get('uri','http://localhost:9200'),
                                                index=c['index'],
                                                doc_type=c['doc_type'],
                                                match_fields=c['match_fields'],
                                                source_field=c['source_field'],
                                                dest_field=c['dest_field']))

    def run(self):

        logging.debug("Beggining run of SummaryReplayer")
        self.createConnection()

        # Send start message to control channel, if requested
        self.sendControlMessage({'status': 'ok', 'stage': 'starting'})

        # Query elsaticsearch and send the data to the destination
        logging.info("Sending response to %s with routing key %s" % (self.msg['destination'], self.msg['routing_key']))
        try:
            for day in self._queryElasticsearch(self.msg['from'], self.msg['to'], None):
                for record in day:
                    for c in self.corrections:
                        c.correct(record)
                    self.addProperties(record)
                    self.sendMessage(json.dumps(record))
        except Exception, e:
            logging.error("Exception caught in query ES: %s" % str(e))
            logging.error(traceback.format_exc())
            raise e

        self.sendControlMessage({'status': 'ok', 'stage': 'finished'})

        self.conn.close()

    def on_return(self, channel, method, properties, body):
        sys.stderr.write("Got returned message\n")

    def addProperties(self, record):

        returned_doc = self.project.parseDoc(record)
        topology_doc = self.topology.generate_dict_for_gracc(record)
        record.update(returned_doc)
        record.update(topology_doc)
        return record

    def _queryElasticsearch(self, from_date, to_date, query):
        logging.debug("Connecting to ES")
        client = Elasticsearch(timeout=300)

        # For summaries, we only summarize full days, so strip the time from the from & to
        # Round the date up, so we get the entire last day they requested.
        from_date = dateutil.parser.parse(from_date).date()
        to_date = dateutil.parser.parse(to_date).date() + datetime.timedelta(days=1)

        logging.debug("Beginning search")
        s = Search(using=client, index=self._config['ElasticSearch']['raw_index'])
        s = s.filter('range', **{'EndTime': {'from': from_date, 'to': to_date }})
        s = s.query('bool', must=[Q('match', _type=self._config['ElasticSearch']['raw_type'])])

        # Fill in the unique terms and metrics
        unique_terms = [["EndTime", 0], ["VOName", "N/A"], ["ProjectName", "N/A"], ["DN", "N/A"], ["Processors", 1], ["ResourceType", "N/A"], ["CommonName", "N/A"], ["Host_description", "N/A"], ["Resource_ExitCode", 0], ["Grid", "N/A"], ["ReportableVOName", "N/A"], ["ProbeName", "N/A"], ["SiteName", "N/A"]]
        metrics = ["WallDuration", "CpuDuration_user", "CpuDuration_system", "CoreHours", "Njobs"]

        # If the terms are missing, set as "N/A"
        curBucket = s.aggs.bucket(unique_terms[0][0], 'date_histogram', field=unique_terms[0][0], interval="day")
        new_unique_terms = unique_terms[1:]

        for term in new_unique_terms:
        	curBucket = curBucket.bucket(term[0], 'terms', field=term[0], missing=term[1], size=(2**31)-1)

        for metric in metrics:
        	curBucket.metric(metric, 'sum', field=metric)

        response = s.execute()

        def recurseBucket(curData, curBucket, index, data):
            """
            Recursively process the buckets down the nested aggregations

            :param curData: Current
            :param bucket curBucket: A elasticsearch bucket object
            :param int index: Index of the unique_terms that we are processing
            """
            curTerm = unique_terms[index][0]

            # Check if we are at the end of the list
            if not curBucket[curTerm]['buckets']:
                # Make a copy of the data
                nowData = copy.deepcopy(curData)
                data.append(nowData)
            else:
                # Get the current key, and add it to the data
                for bucket in curBucket[curTerm]['buckets']:
                    nowData = copy.deepcopy(curData)
                    nowData[curTerm] = bucket['key']
                    if index == (len(unique_terms) - 1):
                        # reached the end of the unique terms
                        for metric in metrics:
                            nowData[metric] = bucket[metric].value
                            # Add the doc count
                        nowData["Count"] = bucket['doc_count']
                        data.append(nowData)
                    else:
                        recurseBucket(nowData, bucket, index+1, data)

        # We only want to hold onto 1 day's worth of summaries
        print len(response.aggregations['EndTime']['buckets'])
        for day in response.aggregations['EndTime']['buckets']:
            data = []
            recurseBucket({"EndTime": day['key_as_string']}, day, 1, data)
            yield data

        # Aggregate on njobs, WallDuration, CpuUserDuration, CpuSystemDuration
        # Or, unique keys on EndTime (day), VO, ProjectName, CommonName, DN, ResourceType, HostDescription, ApplicationExitCode, Grid, Cores

<|endoftext|>