<commit_before># Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
# Copyright (c) 2006-2007 Sippy Software, Inc. All rights reserved.
#
# This file is part of SIPPY, a free RFC3261 SIP stack and B2BUA.
#
# SIPPY is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# For a license to use the SIPPY software under conditions
# other than those described here, or to purchase support for this
# software, please contact Sippy Software, Inc. by e-mail at the
# following addresses: sales@sippysoft.com.
#
# SIPPY is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.

from SipURL import SipURL
from string import maketrans

class SipAddress(object):
    name = None
    url = None
    params = None
    hadbrace = None
    transtable = maketrans('-.!%*_+`\'~', 'a' * 10)

    def __init__(self, address = None, name = None, url = None, params = None, hadbrace = None):
        self.params = {}
        self.hadbrace = True
        if address == None:
            self.name = name
            self.url = url
            if params != None:
                self.params = params
            if hadbrace != None:
                self.hadbrace = hadbrace
            return
        # simple 'sip:foo' case
        if address.lower().startswith('sip:') and address.find('<') == -1:
            parts = address.split(';', 1)
            self.url = SipURL(parts[0])
            if len(parts) == 2:
                for l in parts[1].split(';'):
                    if not l:
                        continue
                    k, v = l.split('=')
                    self.params[k] = v
            self.hadbrace = False
            return
        self.name, url = address.split('<', 1)
        self.name = self.name.strip()
        if self.name.startswith('"'):
            self.name = self.name[1:]
        if self.name.endswith('"'):
            self.name = self.name[:-1]
        url, paramstring = url.split('>', 1)
        self.url = SipURL(url)
        paramstring = paramstring.strip()
        if paramstring:
            for l in paramstring.split(';'):
                if not l:
                    continue
                k, v = l.split('=')
                self.params[k] = v

    def __str__(self):
        return self.localStr()

    def localStr(self, local_addr = None, local_port = None):
        if self.hadbrace:
            od = '<'
            cd = '>'
        else:
            od = ''
            cd = ''
        s = ''
        if self.name != None and len(self.name) > 0:
            if not self.name.translate(self.transtable).isalnum():
                s += '"%s" ' % self.name
            else:
                s += self.name + ' '
            od = '<'
            cd = '>'
        s += od + self.url.localStr(local_addr, local_port) + cd
        for k, v in self.params.items():
            s += ';' + k + '=' + v
        return s

    def getCopy(self):
        return SipAddress(name = self.name, url = self.url.getCopy(), params = self.params.copy(), hadbrace = self.hadbrace)

    def getParam(self, name):
        try:
            return self.params[name]
        except KeyError:
            return None

    def setParam(self, name, value):
        self.params[name] = value

    def delParam(self, name):
        del self.params[name]
<commit_msg>o Add support for parameters without a value, like in RFC3842:<commit_after># Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
# Copyright (c) 2006-2007 Sippy Software, Inc. All rights reserved.
#
# This file is part of SIPPY, a free RFC3261 SIP stack and B2BUA.
#
# SIPPY is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# For a license to use the SIPPY software under conditions
# other than those described here, or to purchase support for this
# software, please contact Sippy Software, Inc. by e-mail at the
# following addresses: sales@sippysoft.com.
#
# SIPPY is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.

from SipURL import SipURL
from string import maketrans

class SipAddress(object):
    name = None
    url = None
    params = None
    params_order = None
    hadbrace = None
    transtable = maketrans('-.!%*_+`\'~', 'a' * 10)

    def __init__(self, address = None, name = None, url = None, params = None, hadbrace = None, \
      params_order = None):
        self.params = {}
        self.params_order = []
        self.hadbrace = True
        if address == None:
            self.name = name
            self.url = url
            if params != None:
                self.params = params
            if params_order != None:
                self.params_order = params_order
            if hadbrace != None:
                self.hadbrace = hadbrace
            return
        # simple 'sip:foo' case
        if address.lower().startswith('sip:') and address.find('<') == -1:
            parts = address.split(';', 1)
            self.url = SipURL(parts[0])
            if len(parts) == 2:
                for l in parts[1].split(';'):
                    if not l:
                        continue
                    k_v = l.split('=', 1)
                    if len(k_v) == 2:
                        k, v = k_v
                    else:
                        k = k_v[0]
                        v = None
                    self.params[k] = v
                    if self.params_order.count(k) > 0:
                        self.params_order.remove(k)
                    self.params_order.append(k)
            self.hadbrace = False
            return
        self.name, url = address.split('<', 1)
        self.name = self.name.strip()
        if self.name.startswith('"'):
            self.name = self.name[1:]
        if self.name.endswith('"'):
            self.name = self.name[:-1]
        url, paramstring = url.split('>', 1)
        self.url = SipURL(url)
        paramstring = paramstring.strip()
        if paramstring:
            for l in paramstring.split(';'):
                if not l:
                    continue
                k_v = l.split('=', 1)
                if len(k_v) == 2:
                    k, v = k_v
                else:
                    k = k_v[0]
                    v = None
                self.params[k] = v
                if self.params_order.count(k) > 0:
                    self.params_order.remove(k)
                self.params_order.append(k)

    def __str__(self):
        return self.localStr()

    def localStr(self, local_addr = None, local_port = None):
        if self.hadbrace:
            od = '<'
            cd = '>'
        else:
            od = ''
            cd = ''
        s = ''
        if self.name != None and len(self.name) > 0:
            if not self.name.translate(self.transtable).isalnum():
                s += '"%s" ' % self.name
            else:
                s += self.name + ' '
            od = '<'
            cd = '>'
        s += od + self.url.localStr(local_addr, local_port) + cd
        for k in self.params_order:
            v = self.params[k]
            if v != None:
                s += ';' + k + '=' + v
            else:
                s += ';' + k
        return s

    def getCopy(self):
        return SipAddress(name = self.name, url = self.url.getCopy(), params = self.params.copy(), \
          hadbrace = self.hadbrace, params_order = self.params_order[:])

    def getParam(self, name):
        return self.params.get(name, None)

    def setParam(self, name, value = None):
        self.params[name] = value
        if self.params_order.count(name) == 0:
            self.params_order.append(name)

    def delParam(self, name):
        del self.params[name]
        self.params_order.remove(name)
<|endoftext|>


<commit_before># Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
# Copyright (c) 2006-2007 Sippy Software, Inc. All rights reserved.
#
# This file is part of SIPPY, a free RFC3261 SIP stack and B2BUA.
#
# SIPPY is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# For a license to use the SIPPY software under conditions
# other than those described here, or to purchase support for this
# software, please contact Sippy Software, Inc. by e-mail at the
# following addresses: sales@sippysoft.com.
#
# SIPPY is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.

from SipMsg import SipMsg
from SipHeader import SipHeader
from SipCSeq import SipCSeq
from SipTo import SipTo
from SipResponse import SipResponse
from SipURL import SipURL
from SipAddress import SipAddress
from SipExpires import SipExpires

class SipRequest(SipMsg):
    method = None
    ruri = None
    sipver = None
    user_agent = None

    def __init__(self, buf = None, method = None, ruri = None, sipver = 'SIP/2.0', to = None, fr0m = None, via = None, cseq = None, \
                 callid = None, maxforwards = None, body = None, contact = None, routes = (), target = None, cguid = None,
                 user_agent = None):
        SipMsg.__init__(self, buf)
        if buf != None:
            return

        self.method = method
        self.ruri = ruri
        if target == None:
            if len(routes) == 0:
                self.setTarget(self.ruri.getAddr())
            else:
                self.setTarget(routes[0].getAddr())
        else:
            self.setTarget(target)
        self.sipver = sipver
        self.appendHeader(SipHeader(name = 'via', body = via))
        if via == None:
            self.getHFBody('via').genBranch()
        self.appendHeaders([SipHeader(name = 'route', body = x) for x in routes])
        self.appendHeader(SipHeader(name = 'max-forwards', body = maxforwards))
        self.appendHeader(SipHeader(name = 'from', body = fr0m))
        if to == None:
            to = SipTo(address = SipAddress(url = ruri))
        self.appendHeader(SipHeader(name = 'to', body = to))
        self.appendHeader(SipHeader(name = 'call-id', body = callid))
        self.appendHeader(SipHeader(name = 'cseq', body = SipCSeq(cseq = cseq, method = method)))
        if contact != None:
            self.appendHeader(SipHeader(name = 'contact', body = contact))
        if method in ('INVITE', 'REGISTER'):
            self.appendHeader(SipHeader(name = 'expires'))
        if user_agent != None:
            self.user_agent = user_agent
            self.appendHeader(SipHeader(name = 'user-agent', bodys = user_agent))
        else:
            self.appendHeader(SipHeader(name = 'user-agent'))
        if cguid != None:
            self.appendHeader(SipHeader(name = 'cisco-guid', body = cguid))
            self.appendHeader(SipHeader(name = 'h323-conf-id', body = cguid))
        if body != None:
            self.setBody(body)

    def setSL(self, startline):
        self.method, ruri, self.sipver = startline.split()
        self.ruri = SipURL(ruri)

    def getSL(self):
        return self.method + ' ' + str(self.ruri) + ' ' + self.sipver

    def getMethod(self):
        return self.method

    def getRURI(self):
        return self.ruri

    def setRURI(self, ruri):
        self.ruri = ruri

    def genResponse(self, scode, reason, body = None, server = None):
        # Should be done at the transaction level
        # to = self.getHF('to').getBody().getCopy()
        # if code > 100 and to.getTag() == None:
        #    to.genTag()
        return SipResponse(scode = scode, reason = reason, sipver = self.sipver, fr0m = self.getHFBody('from').getCopy(), \
                           callid = self.getHFBody('call-id').getCopy(), vias = [x.getCopy() for x in self.getHFBodys('via')], \
                           to = self.getHFBody('to').getCopy(), cseq = self.getHFBody('cseq').getCopy(), \
                           rrs = [x.getCopy() for x in self.getHFBodys('record-route')], body = body, \
                           server = server)

    def genACK(self, to = None):
        if to == None:
            to = self.getHFBody('to').getCopy()
        maxforwards = self.getHFBodys('max-forwards')
        if len(maxforwards) > 0:
            maxforward = maxforwards[0].getCopy()
        else:
            maxforward = None
        return SipRequest(method = 'ACK', ruri = self.ruri.getCopy(), sipver = self.sipver, \
                          fr0m = self.getHFBody('from').getCopy(), to = to, \
                          via = self.getHFBody('via').getCopy(), callid = self.getHFBody('call-id').getCopy(), \
                          cseq = self.getHFBody('cseq').getCSeqNum(), maxforwards = maxforward, \
                          user_agent = self.user_agent)

    def genCANCEL(self):
        maxforwards = self.getHFBodys('max-forwards')
        if len(maxforwards) > 0:
            maxforward = maxforwards[0].getCopy()
        else:
            maxforward = None
        return SipRequest(method = 'CANCEL', ruri = self.ruri.getCopy(), sipver = self.sipver, \
                          fr0m = self.getHFBody('from').getCopy(), to = self.getHFBody('to').getCopy(), \
                          via = self.getHFBody('via').getCopy(), callid = self.getHFBody('call-id').getCopy(), \
                          cseq = self.getHFBody('cseq').getCSeqNum(), maxforwards = maxforward, \
                          routes = [x.getCopy() for x in self.getHFBodys('route')], target = self.getTarget(), \
                          user_agent = self.user_agent)

    def genRequest(self, method, cseq = None):
        if cseq == None:
            cseq = self.getHFBody('cseq').getCSeqNum()
        maxforwards = self.getHFBodys('max-forwards')
        if len(maxforwards) > 0:
            maxforward = maxforwards[0].getCopy()
        else:
            maxforward = None
        return SipRequest(method = method, ruri = self.ruri.getCopy(), sipver = self.sipver, \
                          fr0m = self.getHFBody('from').getCopy(), to = self.getHFBody('to').getCopy(), \
                          via = self.getHFBody('via').getCopy(), callid = self.getHFBody('call-id').getCopy(), \
                          cseq = cseq, maxforwards = maxforward, \
                          user_agent = self.user_agent)
<commit_msg>Add support for 'Expires' HF into the constructor and genRequest method.<commit_after># Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
# Copyright (c) 2006-2007 Sippy Software, Inc. All rights reserved.
#
# This file is part of SIPPY, a free RFC3261 SIP stack and B2BUA.
#
# SIPPY is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# For a license to use the SIPPY software under conditions
# other than those described here, or to purchase support for this
# software, please contact Sippy Software, Inc. by e-mail at the
# following addresses: sales@sippysoft.com.
#
# SIPPY is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.

from SipMsg import SipMsg
from SipHeader import SipHeader
from SipCSeq import SipCSeq
from SipTo import SipTo
from SipResponse import SipResponse
from SipURL import SipURL
from SipAddress import SipAddress
from SipExpires import SipExpires

class SipRequest(SipMsg):
    method = None
    ruri = None
    sipver = None
    user_agent = None

    def __init__(self, buf = None, method = None, ruri = None, sipver = 'SIP/2.0', to = None, fr0m = None, via = None, cseq = None, \
                 callid = None, maxforwards = None, body = None, contact = None, routes = (), target = None, cguid = None,
                 user_agent = None, expires = None):
        SipMsg.__init__(self, buf)
        if buf != None:
            return
        self.method = method
        self.ruri = ruri
        if target == None:
            if len(routes) == 0:
                self.setTarget(self.ruri.getAddr())
            else:
                self.setTarget(routes[0].getAddr())
        else:
            self.setTarget(target)
        self.sipver = sipver
        self.appendHeader(SipHeader(name = 'via', body = via))
        if via == None:
            self.getHFBody('via').genBranch()
        self.appendHeaders([SipHeader(name = 'route', body = x) for x in routes])
        self.appendHeader(SipHeader(name = 'max-forwards', body = maxforwards))
        self.appendHeader(SipHeader(name = 'from', body = fr0m))
        if to == None:
            to = SipTo(address = SipAddress(url = ruri))
        self.appendHeader(SipHeader(name = 'to', body = to))
        self.appendHeader(SipHeader(name = 'call-id', body = callid))
        self.appendHeader(SipHeader(name = 'cseq', body = SipCSeq(cseq = cseq, method = method)))
        if contact != None:
            self.appendHeader(SipHeader(name = 'contact', body = contact))
        if method in ('INVITE', 'REGISTER'):
            if expires == None:
                expires = SipHeader(name = 'expires')
            else:
                expires = SipHeader(name = 'expires', body = expires)
            self.appendHeader(expires)
        if user_agent != None:
            self.user_agent = user_agent
            self.appendHeader(SipHeader(name = 'user-agent', bodys = user_agent))
        else:
            self.appendHeader(SipHeader(name = 'user-agent'))
        if cguid != None:
            self.appendHeader(SipHeader(name = 'cisco-guid', body = cguid))
            self.appendHeader(SipHeader(name = 'h323-conf-id', body = cguid))
        if body != None:
            self.setBody(body)

    def setSL(self, startline):
        self.method, ruri, self.sipver = startline.split()
        self.ruri = SipURL(ruri)

    def getSL(self):
        return self.method + ' ' + str(self.ruri) + ' ' + self.sipver

    def getMethod(self):
        return self.method

    def getRURI(self):
        return self.ruri

    def setRURI(self, ruri):
        self.ruri = ruri

    def genResponse(self, scode, reason, body = None, server = None):
        # Should be done at the transaction level
        # to = self.getHF('to').getBody().getCopy()
        # if code > 100 and to.getTag() == None:
        #    to.genTag()
        return SipResponse(scode = scode, reason = reason, sipver = self.sipver, fr0m = self.getHFBody('from').getCopy(), \
                           callid = self.getHFBody('call-id').getCopy(), vias = [x.getCopy() for x in self.getHFBodys('via')], \
                           to = self.getHFBody('to').getCopy(), cseq = self.getHFBody('cseq').getCopy(), \
                           rrs = [x.getCopy() for x in self.getHFBodys('record-route')], body = body, \
                           server = server)

    def genACK(self, to = None):
        if to == None:
            to = self.getHFBody('to').getCopy()
        maxforwards = self.getHFBodys('max-forwards')
        if len(maxforwards) > 0:
            maxforward = maxforwards[0].getCopy()
        else:
            maxforward = None
        return SipRequest(method = 'ACK', ruri = self.ruri.getCopy(), sipver = self.sipver, \
                          fr0m = self.getHFBody('from').getCopy(), to = to, \
                          via = self.getHFBody('via').getCopy(), callid = self.getHFBody('call-id').getCopy(), \
                          cseq = self.getHFBody('cseq').getCSeqNum(), maxforwards = maxforward, \
                          user_agent = self.user_agent)

    def genCANCEL(self):
        maxforwards = self.getHFBodys('max-forwards')
        if len(maxforwards) > 0:
            maxforward = maxforwards[0].getCopy()
        else:
            maxforward = None
        return SipRequest(method = 'CANCEL', ruri = self.ruri.getCopy(), sipver = self.sipver, \
                          fr0m = self.getHFBody('from').getCopy(), to = self.getHFBody('to').getCopy(), \
                          via = self.getHFBody('via').getCopy(), callid = self.getHFBody('call-id').getCopy(), \
                          cseq = self.getHFBody('cseq').getCSeqNum(), maxforwards = maxforward, \
                          routes = [x.getCopy() for x in self.getHFBodys('route')], target = self.getTarget(), \
                          user_agent = self.user_agent)

    def genRequest(self, method, cseq = None):
        if cseq == None:
            cseq = self.getHFBody('cseq').getCSeqNum()
        maxforwards = self.getHFBodys('max-forwards')
        if len(maxforwards) > 0:
            maxforward = maxforwards[0].getCopy()
        else:
            maxforward = None
        expires = self.getHFBodys('expires')
        if len(expires) > 0:
            expires = expires[0].getCopy()
        else:
            expires = None
        return SipRequest(method = method, ruri = self.ruri.getCopy(), sipver = self.sipver, \
                          fr0m = self.getHFBody('from').getCopy(), to = self.getHFBody('to').getCopy(), \
                          via = self.getHFBody('via').getCopy(), callid = self.getHFBody('call-id').getCopy(), \
                          cseq = cseq, maxforwards = maxforward, \
                          user_agent = self.user_agent, expires = expires)
<|endoftext|>


<commit_before># Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
# Copyright (c) 2006-2007 Sippy Software, Inc. All rights reserved.
#
# This file is part of SIPPY, a free RFC3261 SIP stack and B2BUA.
#
# SIPPY is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# For a license to use the SIPPY software under conditions
# other than those described here, or to purchase support for this
# software, please contact Sippy Software, Inc. by e-mail at the
# following addresses: sales@sippysoft.com.
#
# SIPPY is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.

from SipMsg import SipMsg
from SipHeader import SipHeader
from SipCSeq import SipCSeq
from SipTo import SipTo
from SipResponse import SipResponse
from SipURL import SipURL
from SipAddress import SipAddress
from SipExpires import SipExpires

class SipRequest(SipMsg):
    method = None
    ruri = None
    sipver = None
    user_agent = None

    def __init__(self, buf = None, method = None, ruri = None, sipver = 'SIP/2.0', to = None, fr0m = None, via = None, cseq = None, \
                 callid = None, maxforwards = None, body = None, contact = None, routes = (), target = None, cguid = None,
                 user_agent = None, expires = None):
        SipMsg.__init__(self, buf)
        if buf != None:
            return
        self.method = method
        self.ruri = ruri
        if target == None:
            if len(routes) == 0:
                self.setTarget(self.ruri.getAddr())
            else:
                self.setTarget(routes[0].getAddr())
        else:
            self.setTarget(target)
        self.sipver = sipver
        self.appendHeader(SipHeader(name = 'via', body = via))
        if via == None:
            self.getHFBody('via').genBranch()
        self.appendHeaders([SipHeader(name = 'route', body = x) for x in routes])
        self.appendHeader(SipHeader(name = 'max-forwards', body = maxforwards))
        self.appendHeader(SipHeader(name = 'from', body = fr0m))
        if to == None:
            to = SipTo(address = SipAddress(url = ruri))
        self.appendHeader(SipHeader(name = 'to', body = to))
        self.appendHeader(SipHeader(name = 'call-id', body = callid))
        self.appendHeader(SipHeader(name = 'cseq', body = SipCSeq(cseq = cseq, method = method)))
        if contact != None:
            self.appendHeader(SipHeader(name = 'contact', body = contact))
        if expires == None and method == 'INVITE':
            expires = SipHeader(name = 'expires')
            self.appendHeader(expires)
        elif expires != None:
            expires = SipHeader(name = 'expires', body = expires)
            self.appendHeader(expires)
        if user_agent != None:
            self.user_agent = user_agent
            self.appendHeader(SipHeader(name = 'user-agent', bodys = user_agent))
        else:
            self.appendHeader(SipHeader(name = 'user-agent'))
        if cguid != None:
            self.appendHeader(SipHeader(name = 'cisco-guid', body = cguid))
            self.appendHeader(SipHeader(name = 'h323-conf-id', body = cguid))
        if body != None:
            self.setBody(body)

    def setSL(self, startline):
        self.method, ruri, self.sipver = startline.split()
        self.ruri = SipURL(ruri)

    def getSL(self):
        return self.method + ' ' + str(self.ruri) + ' ' + self.sipver

    def getMethod(self):
        return self.method

    def getRURI(self):
        return self.ruri

    def setRURI(self, ruri):
        self.ruri = ruri

    def genResponse(self, scode, reason, body = None, server = None):
        # Should be done at the transaction level
        # to = self.getHF('to').getBody().getCopy()
        # if code > 100 and to.getTag() == None:
        #    to.genTag()
        return SipResponse(scode = scode, reason = reason, sipver = self.sipver, fr0m = self.getHFBody('from').getCopy(), \
                           callid = self.getHFBody('call-id').getCopy(), vias = [x.getCopy() for x in self.getHFBodys('via')], \
                           to = self.getHFBody('to').getCopy(), cseq = self.getHFBody('cseq').getCopy(), \
                           rrs = [x.getCopy() for x in self.getHFBodys('record-route')], body = body, \
                           server = server)

    def genACK(self, to = None):
        if to == None:
            to = self.getHFBody('to').getCopy()
        maxforwards = self.getHFBodys('max-forwards')
        if len(maxforwards) > 0:
            maxforward = maxforwards[0].getCopy()
        else:
            maxforward = None
        return SipRequest(method = 'ACK', ruri = self.ruri.getCopy(), sipver = self.sipver, \
                          fr0m = self.getHFBody('from').getCopy(), to = to, \
                          via = self.getHFBody('via').getCopy(), callid = self.getHFBody('call-id').getCopy(), \
                          cseq = self.getHFBody('cseq').getCSeqNum(), maxforwards = maxforward, \
                          user_agent = self.user_agent)

    def genCANCEL(self):
        maxforwards = self.getHFBodys('max-forwards')
        if len(maxforwards) > 0:
            maxforward = maxforwards[0].getCopy()
        else:
            maxforward = None
        return SipRequest(method = 'CANCEL', ruri = self.ruri.getCopy(), sipver = self.sipver, \
                          fr0m = self.getHFBody('from').getCopy(), to = self.getHFBody('to').getCopy(), \
                          via = self.getHFBody('via').getCopy(), callid = self.getHFBody('call-id').getCopy(), \
                          cseq = self.getHFBody('cseq').getCSeqNum(), maxforwards = maxforward, \
                          routes = [x.getCopy() for x in self.getHFBodys('route')], target = self.getTarget(), \
                          user_agent = self.user_agent)

    def genRequest(self, method, cseq = None):
        if cseq == None:
            cseq = self.getHFBody('cseq').getCSeqNum()
        maxforwards = self.getHFBodys('max-forwards')
        if len(maxforwards) > 0:
            maxforward = maxforwards[0].getCopy()
        else:
            maxforward = None
        expires = self.getHFBodys('expires')
        if len(expires) > 0:
            expires = expires[0].getCopy()
        else:
            expires = None
        return SipRequest(method = method, ruri = self.ruri.getCopy(), sipver = self.sipver, \
                          fr0m = self.getHFBody('from').getCopy(), to = self.getHFBody('to').getCopy(), \
                          via = self.getHFBody('via').getCopy(), callid = self.getHFBody('call-id').getCopy(), \
                          cseq = cseq, maxforwards = maxforward, \
                          user_agent = self.user_agent, expires = expires)
<commit_msg>Use getHFBCopy(s) methods where appropriate for better speed and resilence.<commit_after># Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
# Copyright (c) 2006-2007 Sippy Software, Inc. All rights reserved.
#
# This file is part of SIPPY, a free RFC3261 SIP stack and B2BUA.
#
# SIPPY is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# For a license to use the SIPPY software under conditions
# other than those described here, or to purchase support for this
# software, please contact Sippy Software, Inc. by e-mail at the
# following addresses: sales@sippysoft.com.
#
# SIPPY is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.

from SipMsg import SipMsg
from SipHeader import SipHeader
from SipCSeq import SipCSeq
from SipTo import SipTo
from SipResponse import SipResponse
from SipURL import SipURL
from SipAddress import SipAddress
from SipExpires import SipExpires

class SipRequest(SipMsg):
    method = None
    ruri = None
    sipver = None
    user_agent = None

    def __init__(self, buf = None, method = None, ruri = None, sipver = 'SIP/2.0', to = None, fr0m = None, via = None, cseq = None, \
                 callid = None, maxforwards = None, body = None, contact = None, routes = (), target = None, cguid = None,
                 user_agent = None, expires = None):
        SipMsg.__init__(self, buf)
        if buf != None:
            return
        self.method = method
        self.ruri = ruri
        if target == None:
            if len(routes) == 0:
                self.setTarget(self.ruri.getAddr())
            else:
                self.setTarget(routes[0].getAddr())
        else:
            self.setTarget(target)
        self.sipver = sipver
        self.appendHeader(SipHeader(name = 'via', body = via))
        if via == None:
            self.getHFBody('via').genBranch()
        self.appendHeaders([SipHeader(name = 'route', body = x) for x in routes])
        self.appendHeader(SipHeader(name = 'max-forwards', body = maxforwards))
        self.appendHeader(SipHeader(name = 'from', body = fr0m))
        if to == None:
            to = SipTo(address = SipAddress(url = ruri))
        self.appendHeader(SipHeader(name = 'to', body = to))
        self.appendHeader(SipHeader(name = 'call-id', body = callid))
        self.appendHeader(SipHeader(name = 'cseq', body = SipCSeq(cseq = cseq, method = method)))
        if contact != None:
            self.appendHeader(SipHeader(name = 'contact', body = contact))
        if expires == None and method == 'INVITE':
            expires = SipHeader(name = 'expires')
            self.appendHeader(expires)
        elif expires != None:
            expires = SipHeader(name = 'expires', body = expires)
            self.appendHeader(expires)
        if user_agent != None:
            self.user_agent = user_agent
            self.appendHeader(SipHeader(name = 'user-agent', bodys = user_agent))
        else:
            self.appendHeader(SipHeader(name = 'user-agent'))
        if cguid != None:
            self.appendHeader(SipHeader(name = 'cisco-guid', body = cguid))
            self.appendHeader(SipHeader(name = 'h323-conf-id', body = cguid))
        if body != None:
            self.setBody(body)

    def setSL(self, startline):
        self.method, ruri, self.sipver = startline.split()
        self.ruri = SipURL(ruri)

    def getSL(self):
        return self.method + ' ' + str(self.ruri) + ' ' + self.sipver

    def getMethod(self):
        return self.method

    def getRURI(self):
        return self.ruri

    def setRURI(self, ruri):
        self.ruri = ruri

    def genResponse(self, scode, reason, body = None, server = None):
        # Should be done at the transaction level
        # to = self.getHF('to').getBody().getCopy()
        # if code > 100 and to.getTag() == None:
        #    to.genTag()
        return SipResponse(scode = scode, reason = reason, sipver = self.sipver, fr0m = self.getHFBCopy('from'), \
                           callid = self.getHFBCopy('call-id'), vias = self.getHFBCopys('via'), \
                           to = self.getHFBCopy('to'), cseq = self.getHFBCopy('cseq'), \
                           rrs = self.getHFBCopys('record-route'), body = body, \
                           server = server)

    def genACK(self, to = None):
        if to == None:
            to = self.getHFBody('to').getCopy()
        maxforwards = self.getHFBodys('max-forwards')
        if len(maxforwards) > 0:
            maxforward = maxforwards[0].getCopy()
        else:
            maxforward = None
        return SipRequest(method = 'ACK', ruri = self.ruri.getCopy(), sipver = self.sipver, \
                          fr0m = self.getHFBCopy('from'), to = to, \
                          via = self.getHFBCopy('via'), callid = self.getHFBCopy('call-id'), \
                          cseq = self.getHFBody('cseq').getCSeqNum(), maxforwards = maxforward, \
                          user_agent = self.user_agent)

    def genCANCEL(self):
        maxforwards = self.getHFBodys('max-forwards')
        if len(maxforwards) > 0:
            maxforward = maxforwards[0].getCopy()
        else:
            maxforward = None
        return SipRequest(method = 'CANCEL', ruri = self.ruri.getCopy(), sipver = self.sipver, \
                          fr0m = self.getHFBCopy('from'), to = self.getHFBCopy('to'), \
                          via = self.getHFBCopy('via'), callid = self.getHFBCopy('call-id'), \
                          cseq = self.getHFBody('cseq').getCSeqNum(), maxforwards = maxforward, \
                          routes = self.getHFBCopys('route'), target = self.getTarget(), \
                          user_agent = self.user_agent)

    def genRequest(self, method, cseq = None):
        if cseq == None:
            cseq = self.getHFBody('cseq').getCSeqNum()
        maxforwards = self.getHFBodys('max-forwards')
        if len(maxforwards) > 0:
            maxforward = maxforwards[0].getCopy()
        else:
            maxforward = None
        expires = self.getHFBodys('expires')
        if len(expires) > 0:
            expires = expires[0].getCopy()
        else:
            expires = None
        return SipRequest(method = method, ruri = self.ruri.getCopy(), sipver = self.sipver, \
                          fr0m = self.getHFBCopy('from'), to = self.getHFBCopy('to'), \
                          via = self.getHFBCopy('via'), callid = self.getHFBCopy('call-id'), \
                          cseq = cseq, maxforwards = maxforward, \
                          user_agent = self.user_agent, expires = expires)
<|endoftext|>


<commit_before>from cypari import *
import itertools
import re
import time
import tempfile
import os.path

from VolumeUtilities import *
from VolumeFinder import _kickoff_snap, _send_cmd, _drain
from VolumeFinder import RE_INV_TRACE_FIELD_NOT_FOUND, RE_FUNC_REQ_GROUP, RE_ERROR

_RE_NO_BOREL = re.compile('.*Borel Regulator: not computed.*', re.DOTALL)
_RE_BOREL_REGULATOR = re.compile('.*Borel Regulator: \[([-0-9.,]*)\].*', re.DOTALL)

def find_span(elts, n, cutoff = 4096):
    """
    Parameters: elts is a list [a1, a2, a3, ... ], where each ai is a
    (volume (as pari object), name), and n is the upper limit of the
    dimension of the span to be found.

    Returns: returns (s, sn, r, bm, bd), where

    s is determined to be the optimal span that is a subset of elts for
    an n-dimensional lattice,

    sn is the corresponding names for s,

    r is the ratio of det(S) / d, with S the matrix formed out of s, and
    d the gcd of all possible such determinants (it follows that s is a
    basis for elts iff r is 1),

    Optionally, the parameter cutoff may be passed (defaulting to
    4096). This controls the upper bound on absolute values of
    coefficients resulting from pari's lindep that are considered
    relevant.  For example [sqrt(2), 0.00001] gives a linear dependence
    of [-160, 22627417]. In practice, this should probably be ignored.

    Note that if a span that would have greater dimension than n would
    be generated, this method will raise a ValueError. This indicates
    that the input must be invalid.
    """

    # Remove integer multiples, creates something like [2, 5, 7, 9, 11, ... ]
    # This could be done in the next step, but is singled out as a special case
    # of the general operation.
    base_elts = sorted(elts, key = lambda a: -abs(float(a[0])))
    base_elts = [(e[0].replace('-',''), e[1]) for e in base_elts]   # take absolute value
    independent_elts = []
    while base_elts:
        cur_min_elt = base_elts.pop()
        independent_elts.append(cur_min_elt)
        base_elts = [ a for a in base_elts if not _is_int(float(a[0])/float(cur_min_elt[0])) ]

    # Create a rational basis, but be careful of [ 0.5, 0.75, sqrt(2) ], which
    # might select first two as e1 and e2, yet gives pari no representation of
    # sqrt(2) by the other two
    next_pure_basis_element_position = 0 # shall create [1, 0, ..., 0] as next ei
    rational_vectors = None # this will be a list of tuples, (vol, vec, name), with vec in Q^n
    rational_basis = None # shall be { vol where vec is some ei }
    for i in range(0, len(independent_elts)):
        vol, name = independent_elts[i] # gen.pari(independent_elts[i])
        if not rational_basis:
            next_pure_basis_element_position = 1
            rational_basis = [vol]
            rational_vectors = [(vol,gen.pari([1]).mattranspose(),name)]
        else:
            test_vec = rational_basis[:]
            test_vec.insert(0, vol)
            lindep_results = gen.pari(test_vec).lindep(LINDEP_PRECISION)
            if lindep_results[0] == 1:
                # This volume may be expressed as a linear, integral
                # combination of others. It is worthless.
                pass
            elif lindep_results[0] == 0 or max(abs(lindep_results)) > cutoff:
                # This volume is considered by pari to be harder to fit into
                # the current set of results than allowable. Treat it as a new
                # basis element.
                if next_pure_basis_element_position >= n:
                    raise ValueError('The current basis elements are ' + str(rational_basis) +
                                     ', and attempting to fit volume ' + str(vol) +
                                     ' into them determined it should be classed as a new basis element, but there are already ' +
                                     str(n) + ' such!')
                vec = [0] * (next_pure_basis_element_position + 1)
                vec[next_pure_basis_element_position] = 1
                next_pure_basis_element_position += 1
                rational_vectors = [ (c_p,c_v.concat(gen.pari([0])),c_n) for c_p,c_v,c_n in rational_vectors ]
                rational_basis.append(vol)
                rational_vectors.append((vol,gen.pari(vec).mattranspose(),name))
            else:
                # This volume is a rational combination of the selected basis
                # elements. Use it.
                vec = - lindep_results[1:] / lindep_results[0]
                while len(vec) < next_pure_basis_element_position:
                    vec = vec.concat(gen.pari([0]))
                rational_vectors.append((vol,vec.mattranspose(),name))

    # Now rational_vectors is a python list containing pari row vectors representing each elt. Get dets and such
    determined_n = next_pure_basis_element_position # if there is less data than ncp, keep going
    det_gcd = None
    min_det = None
    min_det_elts = None
    min_det_names = None
    for mat_tuple in itertools.combinations(rational_vectors, determined_n):
        mat_elts = [vec for vol,vec,name in list(mat_tuple)]
        names = [name for vol,vec,name in list(mat_tuple)]
        mat = mat_elts[0]
        for v in mat_elts[1:]:
            mat = mat.concat(v)
        cur_det = mat.matdet().abs()

        if cur_det == 0:
            continue

        if det_gcd is None:
            det_gcd = cur_det
        else:
            det_gcd = det_gcd.gcd(cur_det)

        if (min_det is None) or (cur_det.numerator()*min_det.denominator() < min_det.numerator()*cur_det.denominator()):
            min_det = cur_det
            min_det_elts = [vol for vol,vec,name in list(mat_tuple)]
            min_det_names = names

    return min_det_elts, min_det / det_gcd, min_det_names

def find_borel_matrix(manifold_names):
    """Given manifold_names, a list of strings each the name of a manifold
    (i.e., they may be passed as a constructor to SnapPy's Manifold
    object) returns (borel_matrix, borel_matrix_determinant).

    borel_matrix is a list of the Borel regulators (themselves lists) for
    each manifold in sn (not a Pari matrix object)

    borel_matrix_determinant is the determinant (a pari object) of bm if
    it can be formed into a square matrix (all Borel regulators have the
    same dimension, and this dimension is the same as that of
    manifold_names), or None if it cannot.

    Note: This method currently relies on the external program `snap` to
    compute borel regulators. If errors arise, try adjusting the options
    for snap found in VolumeUtilities.
    """
    borel_reg_strings = list()

    next_borel_reg = _borel_regulator(manifold_names[0])
    borel_reg_mat = gen.pari(next_borel_reg).mattranspose()
    borel_reg_strings.append(next_borel_reg)

    for next_name in manifold_names[1:]:
        next_borel_reg = _borel_regulator(next_name)
        borel_reg_mat = borel_reg_mat.concat(gen.pari(next_borel_reg).mattranspose())
        borel_reg_strings.append(next_borel_reg)

    borel_determinant = None
    try:
        borel_determinant = borel_reg_mat.matdet()
    except:
        pass

    return borel_reg_strings, borel_determinant

def _borel_regulator(manifold_name):
    """
    Given a manifold name, return, as a list, the volumes (in string
    form) that make up that volume's Borel regulator.

    Ex: _borel_regulator('m349') returns ['2.721625...', '4.666479...'].
    """
    input_manifold = Manifold(manifold_name)

    trig_file_dir = tempfile.mkdtemp()
    Manifold(manifold_name).save(os.path.join(trig_file_dir, 'tmp.trig'))
    snap_process = _kickoff_snap(trig_file_dir)
    try:
        _send_cmd(snap_process, 'read file tmp.trig')
        _send_cmd(snap_process, 'co inv co sh print ari')
        snap_output = ''
        wait_count = 0
        while True:
            snap_output += _drain(snap_process.stdout)
            borel_match = _RE_BOREL_REGULATOR.match(snap_output)
            if borel_match is not None:
                return re.split(',', borel_match.group(1))
            elif any([RE_INV_TRACE_FIELD_NOT_FOUND.match(snap_output),
                        RE_FUNC_REQ_GROUP.match(snap_output),
                        RE_ERROR.match(snap_output),
                        _RE_NO_BOREL.match(snap_output)]):
                raise Exception('Manifold ' + manifold_name + ' has incalculable Borel regulator')
            else:
                time.sleep(0.05)
                wait_count += 1
                if wait_count > 100:
                    raise Exception('Manifold ' + manifold_name + ' appears to have hung snap')
    finally:
        snap_process.terminate()

def _is_int(r):
    s = r % 1
    return s < EPSILON or s + EPSILON > 1
<commit_msg>Suppress irritating '***   not a square matrix in det'<commit_after>from cypari import *
import itertools
import re
import time
import tempfile
import os.path

from VolumeUtilities import *
from VolumeFinder import _kickoff_snap, _send_cmd, _drain
from VolumeFinder import RE_INV_TRACE_FIELD_NOT_FOUND, RE_FUNC_REQ_GROUP, RE_ERROR

_RE_NO_BOREL = re.compile('.*Borel Regulator: not computed.*', re.DOTALL)
_RE_BOREL_REGULATOR = re.compile('.*Borel Regulator: \[([-0-9.,]*)\].*', re.DOTALL)

def find_span(elts, n, cutoff = 4096):
    """
    Parameters: elts is a list [a1, a2, a3, ... ], where each ai is a
    (volume (as pari object), name), and n is the upper limit of the
    dimension of the span to be found.

    Returns: returns (s, sn, r, bm, bd), where

    s is determined to be the optimal span that is a subset of elts for
    an n-dimensional lattice,

    sn is the corresponding names for s,

    r is the ratio of det(S) / d, with S the matrix formed out of s, and
    d the gcd of all possible such determinants (it follows that s is a
    basis for elts iff r is 1),

    Optionally, the parameter cutoff may be passed (defaulting to
    4096). This controls the upper bound on absolute values of
    coefficients resulting from pari's lindep that are considered
    relevant.  For example [sqrt(2), 0.00001] gives a linear dependence
    of [-160, 22627417]. In practice, this should probably be ignored.

    Note that if a span that would have greater dimension than n would
    be generated, this method will raise a ValueError. This indicates
    that the input must be invalid.
    """

    # Remove integer multiples, creates something like [2, 5, 7, 9, 11, ... ]
    # This could be done in the next step, but is singled out as a special case
    # of the general operation.
    base_elts = sorted(elts, key = lambda a: -abs(float(a[0])))
    base_elts = [(e[0].replace('-',''), e[1]) for e in base_elts]   # take absolute value
    independent_elts = []
    while base_elts:
        cur_min_elt = base_elts.pop()
        independent_elts.append(cur_min_elt)
        base_elts = [ a for a in base_elts if not _is_int(float(a[0])/float(cur_min_elt[0])) ]

    # Create a rational basis, but be careful of [ 0.5, 0.75, sqrt(2) ], which
    # might select first two as e1 and e2, yet gives pari no representation of
    # sqrt(2) by the other two
    next_pure_basis_element_position = 0 # shall create [1, 0, ..., 0] as next ei
    rational_vectors = None # this will be a list of tuples, (vol, vec, name), with vec in Q^n
    rational_basis = None # shall be { vol where vec is some ei }
    for i in range(0, len(independent_elts)):
        vol, name = independent_elts[i] # gen.pari(independent_elts[i])
        if not rational_basis:
            next_pure_basis_element_position = 1
            rational_basis = [vol]
            rational_vectors = [(vol,gen.pari([1]).mattranspose(),name)]
        else:
            test_vec = rational_basis[:]
            test_vec.insert(0, vol)
            lindep_results = gen.pari(test_vec).lindep(LINDEP_PRECISION)
            if lindep_results[0] == 1:
                # This volume may be expressed as a linear, integral
                # combination of others. It is worthless.
                pass
            elif lindep_results[0] == 0 or max(abs(lindep_results)) > cutoff:
                # This volume is considered by pari to be harder to fit into
                # the current set of results than allowable. Treat it as a new
                # basis element.
                if next_pure_basis_element_position >= n:
                    raise ValueError('The current basis elements are ' + str(rational_basis) +
                                     ', and attempting to fit volume ' + str(vol) +
                                     ' into them determined it should be classed as a new basis element, but there are already ' +
                                     str(n) + ' such!')
                vec = [0] * (next_pure_basis_element_position + 1)
                vec[next_pure_basis_element_position] = 1
                next_pure_basis_element_position += 1
                rational_vectors = [ (c_p,c_v.concat(gen.pari([0])),c_n) for c_p,c_v,c_n in rational_vectors ]
                rational_basis.append(vol)
                rational_vectors.append((vol,gen.pari(vec).mattranspose(),name))
            else:
                # This volume is a rational combination of the selected basis
                # elements. Use it.
                vec = - lindep_results[1:] / lindep_results[0]
                while len(vec) < next_pure_basis_element_position:
                    vec = vec.concat(gen.pari([0]))
                rational_vectors.append((vol,vec.mattranspose(),name))

    # Now rational_vectors is a python list containing pari row vectors representing each elt. Get dets and such
    determined_n = next_pure_basis_element_position # if there is less data than ncp, keep going
    det_gcd = None
    min_det = None
    min_det_elts = None
    min_det_names = None
    for mat_tuple in itertools.combinations(rational_vectors, determined_n):
        mat_elts = [vec for vol,vec,name in list(mat_tuple)]
        names = [name for vol,vec,name in list(mat_tuple)]
        mat = mat_elts[0]
        for v in mat_elts[1:]:
            mat = mat.concat(v)
        cur_det = mat.matdet().abs()

        if cur_det == 0:
            continue

        if det_gcd is None:
            det_gcd = cur_det
        else:
            det_gcd = det_gcd.gcd(cur_det)

        if (min_det is None) or (cur_det.numerator()*min_det.denominator() < min_det.numerator()*cur_det.denominator()):
            min_det = cur_det
            min_det_elts = [vol for vol,vec,name in list(mat_tuple)]
            min_det_names = names

    return min_det_elts, min_det / det_gcd, min_det_names

def find_borel_matrix(manifold_names):
    """Given manifold_names, a list of strings each the name of a manifold
    (i.e., they may be passed as a constructor to SnapPy's Manifold
    object) returns (borel_matrix, borel_matrix_determinant).

    borel_matrix is a list of the Borel regulators (themselves lists) for
    each manifold in sn (not a Pari matrix object)

    borel_matrix_determinant is the determinant (a pari object) of bm if
    it can be formed into a square matrix (all Borel regulators have the
    same dimension, and this dimension is the same as that of
    manifold_names), or None if it cannot.

    Note: This method currently relies on the external program `snap` to
    compute borel regulators. If errors arise, try adjusting the options
    for snap found in VolumeUtilities.
    """
    borel_reg_strings = list()

    next_borel_reg = _borel_regulator(manifold_names[0])
    borel_reg_mat = gen.pari(next_borel_reg).mattranspose()
    borel_reg_strings.append(next_borel_reg)

    for next_name in manifold_names[1:]:
        next_borel_reg = _borel_regulator(next_name)
        borel_reg_mat = borel_reg_mat.concat(gen.pari(next_borel_reg).mattranspose())
        borel_reg_strings.append(next_borel_reg)

    borel_determinant = None
    try:
        if len(borel_reg_mat) == len(borel_reg_mat[0]):
            borel_determinant = borel_reg_mat.matdet()
    except:
        pass

    return borel_reg_strings, borel_determinant

def _borel_regulator(manifold_name):
    """
    Given a manifold name, return, as a list, the volumes (in string
    form) that make up that volume's Borel regulator.

    Ex: _borel_regulator('m349') returns ['2.721625...', '4.666479...'].
    """
    input_manifold = Manifold(manifold_name)

    trig_file_dir = tempfile.mkdtemp()
    Manifold(manifold_name).save(os.path.join(trig_file_dir, 'tmp.trig'))
    snap_process = _kickoff_snap(trig_file_dir)
    try:
        _send_cmd(snap_process, 'read file tmp.trig')
        _send_cmd(snap_process, 'co inv co sh print ari')
        snap_output = ''
        wait_count = 0
        while True:
            snap_output += _drain(snap_process.stdout)
            borel_match = _RE_BOREL_REGULATOR.match(snap_output)
            if borel_match is not None:
                return re.split(',', borel_match.group(1))
            elif any([RE_INV_TRACE_FIELD_NOT_FOUND.match(snap_output),
                        RE_FUNC_REQ_GROUP.match(snap_output),
                        RE_ERROR.match(snap_output),
                        _RE_NO_BOREL.match(snap_output)]):
                raise Exception('Manifold ' + manifold_name + ' has incalculable Borel regulator')
            else:
                time.sleep(0.05)
                wait_count += 1
                if wait_count > 100:
                    raise Exception('Manifold ' + manifold_name + ' appears to have hung snap')
    finally:
        snap_process.terminate()

def _is_int(r):
    s = r % 1
    return s < EPSILON or s + EPSILON > 1
<|endoftext|>


<commit_before># ===================
# module from SpicePy
# ===================
import netlist as ntl
import netsolve as slv
import netpost as pst
import netprint as prn

# ===================
# module of SpicePyBot
# ===================
import netprintbot as prn

# ==========================
# python-temegam-bot modules
# ==========================
from telegram.ext import Updater, MessageHandler, Filters
import telegram as telegram
from telegram.ext import CommandHandler

# ==========================
# TOKEN section
# ==========================
# The following function reads the TOKEN from a filename
# it is not incuded in the Repo for obvious reasons
def read_token(filename):
    with open(filename) as f:
        token = f.readline().replace('\n','')
    return token

# set TOKEN and initialization
fname = 'SpicePyBot_token.txt'
updater = Updater(token=read_token(fname))
dispatcher = updater.dispatcher

# ==========================
# standard logging
# ==========================
import logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',level=logging.INFO)

# ==========================
# usefult functions
# ==========================
def sove_dc_network(fname):
    net = ntl.Network(fname)
    slv.dc_solve(net)
    pst.branch_voltage(net)
    pst.branch_current(net)

    mex = prn.print_branch_quantity(net)

    return mex

# ==========================
# welcome message
# ==========================
def start(bot, update):
    bot.send_message(chat_id=update.message.chat_id,
                     text="*Welcome to SpycePyBot*.\n\nIt allows you to solve linear networs\n(So far, only resistive netrork).\n\nRun the code:\n`/tutorial`\n to lean how to use the bot ",
                     parse_mode=telegram.ParseMode.MARKDOWN)

start_handler = CommandHandler('start', start)
dispatcher.add_handler(start_handler)

# =========================================
# catch netlist from a file sent to the bot
# =========================================
def catch_netlist(bot, update):
    file = bot.getFile(update.message.document.file_id)
    fname = str(update.message.chat_id) + '.txt'
    file.download(fname)
    mex = 'This is your netlist:\n\n'
    with open(fname) as f:
        for line in f:
            mex += line
    bot.send_message(chat_id=update.message.chat_id, text=mex)

    mex = sove_dc_network(fname)
    mex = 'This is the circuit solution:\n\n' + mex

    bot.send_message(chat_id=update.message.chat_id, text=mex)

dispatcher.add_handler(MessageHandler(Filters.document, catch_netlist))

# =========================================
# Tutorial
# =========================================
def tutorial(bot, update):
    fname = "./resources/network.net"

    bot.send_message(chat_id=update.message.chat_id, text="Let us assume to solve the following circuit:")
    bot.send_photo(chat_id=update.message.chat_id, photo=open('./resources/circ.png', 'rb'))

    mex = 'You have to write on a file the netlist describing the circuit:\n\nIt should look like the following one:\n\n'
    with open(fname) as f:
        for line in f:
            mex += line
    bot.send_message(chat_id=update.message.chat_id, text=mex)

    mex = 'Instruction to use the bot goes here (TO DO...)'
    bot.send_message(chat_id=update.message.chat_id, text=mex)

    mex = sove_dc_network(fname)
    mex = 'This is the circuit solution:\n\n' + mex
    bot.send_message(chat_id=update.message.chat_id, text=mex)

tutorial_handler = CommandHandler('tutorial', tutorial)
dispatcher.add_handler(tutorial_handler)

# Start the Bot
updater.start_polling()

# Block until you press Ctrl-C or the process receives SIGINT, SIGTERM or
# SIGABRT. This should be used most of the time, since start_polling() is
# non-blocking and will stop the bot gracefully.
updater.idle()
<commit_msg>improved tutorial, added /help<commit_after># ======================
# general python modules
# ======================
import time

# ===================
# module from SpicePy
# ===================
import netlist as ntl
import netsolve as slv
import netpost as pst
import netprint as prn

# ===================
# module of SpicePyBot
# ===================
import netprintbot as prn

# ==========================
# python-temegam-bot modules
# ==========================
from telegram.ext import Updater, MessageHandler, Filters
import telegram as telegram
from telegram.ext import CommandHandler

# ==========================
# TOKEN section
# ==========================
# The following function reads the TOKEN from a filename
# it is not incuded in the Repo for obvious reasons
def read_token(filename):
    with open(filename) as f:
        token = f.readline().replace('\n','')
    return token

# set TOKEN and initialization
fname = 'SpicePyBot_token.txt'
updater = Updater(token=read_token(fname))
dispatcher = updater.dispatcher

# ==========================
# standard logging
# ==========================
import logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',level=logging.INFO)

# ==========================
# usefult functions
# ==========================
def sove_dc_network(fname):
    net = ntl.Network(fname)
    slv.dc_solve(net)
    pst.branch_voltage(net)
    pst.branch_current(net)

    mex = prn.print_branch_quantity(net)

    return mex

# ==========================
# start - welcome message
# ==========================
def start(bot, update):
    bot.send_message(chat_id=update.message.chat_id,
                     text="*Welcome to SpycePyBot*.\n\nIt allows you to solve linear networs\n(So far, only resistive netrork).\n\nRun the code:\n`/tutorial`\n to lean how to use the bot ",
                     parse_mode=telegram.ParseMode.MARKDOWN)

start_handler = CommandHandler('start', start)
dispatcher.add_handler(start_handler)

# =========================================
# catch netlist from a file sent to the bot
# =========================================
def catch_netlist(bot, update):
    file = bot.getFile(update.message.document.file_id)
    fname = str(update.message.chat_id) + '.txt'
    file.download(fname)
    mex = 'This is your netlist:\n\n'
    with open(fname) as f:
        for line in f:
            mex += line
    bot.send_message(chat_id=update.message.chat_id, text=mex)

    mex = sove_dc_network(fname)
    mex = 'This is the circuit solution:\n\n' + mex

    bot.send_message(chat_id=update.message.chat_id, text=mex)

dispatcher.add_handler(MessageHandler(Filters.document, catch_netlist))

# ==========================
# help - short guide
# ==========================
def help(bot, update):
    bot.send_message(chat_id=update.message.chat_id,
                     text="*Very short guide*.\n\n1)upload a file with the netlist (don't know what a netlist is? Run `/tutorial` in the bot)\n2) enjoy\n\n\n*If you need a more detailed guide*\nRun `/tutorial` in the bot",
                     parse_mode=telegram.ParseMode.MARKDOWN)

help_handler = CommandHandler('help', help)
dispatcher.add_handler(help_handler)

# =========================================
# Tutorial - learn to use the bot
# =========================================
def tutorial(bot, update):
    fname = "./resources/network.net"

    bot.send_message(chat_id=update.message.chat_id, text="Let us assume to solve the following circuit:")
    bot.send_photo(chat_id=update.message.chat_id, photo=open('./resources/circ.png', 'rb'))

    mex = 'You have to write on a file the netlist describing the circuit:\n\nIt should look like the following one:\n\n'
    with open(fname) as f:
        for line in f:
            mex += line
    bot.send_message(chat_id=update.message.chat_id, text=mex)

    time.sleep(5)

    mex = 'Here it is your file...'
    bot.send_message(chat_id=update.message.chat_id, text=mex)
    bot.send_photo(chat_id=update.message.chat_id, photo=open('./resources/tutorial1.png', 'rb'))

    mex = 'and its concent...'
    bot.send_message(chat_id=update.message.chat_id, text=mex)
    bot.send_photo(chat_id=update.message.chat_id, photo=open('./resources/tutorial2.png', 'rb'))

    time.sleep(5)

    mex = 'tap on the upload button...'
    bot.send_message(chat_id=update.message.chat_id, text=mex)
    bot.send_photo(chat_id=update.message.chat_id, photo=open('./resources/tutorial3.png', 'rb'))

    time.sleep(5)

    mex = 'tap on your file...'
    bot.send_message(chat_id=update.message.chat_id, text=mex)
    bot.send_photo(chat_id=update.message.chat_id, photo=open('./resources/tutorial4.png', 'rb'))

    mex = 'check the correctenss of the netlist and enjoyt the results...'
    bot.send_message(chat_id=update.message.chat_id, text=mex)
    bot.send_photo(chat_id=update.message.chat_id, photo=open('./resources/tutorial5.png', 'rb'))

    #mex = sove_dc_network(fname)
    #mex = 'This is the circuit solution:\n\n' + mex
    #bot.send_message(chat_id=update.message.chat_id, text=mex)

tutorial_handler = CommandHandler('tutorial', tutorial)
dispatcher.add_handler(tutorial_handler)

# Start the Bot
updater.start_polling()

# Block until you press Ctrl-C or the process receives SIGINT, SIGTERM or
# SIGABRT. This should be used most of the time, since start_polling() is
# non-blocking and will stop the bot gracefully.
updater.idle()
<|endoftext|>


<commit_before># Copyright 2019, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Libraries for the Stackoverflow dataset for federated learning simulation."""

import collections
import json
import os.path
from typing import Optional

import numpy as np
import tensorflow as tf

from tensorflow_federated.python.simulation import from_tensor_slices_client_data
from tensorflow_federated.python.simulation import hdf5_client_data

def load_data(cache_dir=None):
  """Loads the federated Stack Overflow dataset.

  Downloads and caches the dataset locally. If previously downloaded, tries to
  load the dataset from cache.

  This dataset is derived from the Stack Overflow Data hosted by kaggle.com and
  available to query through Kernels using the BigQuery API:
  https://www.kaggle.com/stackoverflow/stackoverflow. The Stack Overflow Data
  is licensed under the Creative Commons Attribution-ShareAlike 3.0 Unported
  License. To view a copy of this license, visit
  http://creativecommons.org/licenses/by-sa/3.0/ or send a letter to
  Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.

  The data consists of the body text of all questions and answers. The bodies
  were parsed into sentences, and any user with fewer than 100 sentences was
  expunged from the data. Minimal preprocessing was performed as follows:

  1. Lowercase the text,
  2. Unescape HTML symbols,
  3. Remove non-ascii symbols,
  4. Separate punctuation as individual tokens (except apostrophes and hyphens),
  5. Removing extraneous whitespace,
  6. Replacing URLS with a special token.

  In addition the following metadata is available:

  1. Creation date
  2. Question title
  3. Question tags
  4. Question score
  5. Type ('question' or 'answer')

  The data is divided into three sets:

    -   Train: Data before 2018-01-01 UTC except the held-out users. 342,477
        unique users with 135,818,730 examples.
    -   Held-out: All examples from users with user_id % 10 == 0 (all dates).
        38,758 unique users with 16,491,230 examples.
    -   Test: All examples after 2018-01-01 UTC except from held-out users.
        204,088 unique users with 16,586,035 examples.

  The `tf.data.Datasets` returned by
  `tff.simulation.ClientData.create_tf_dataset_for_client` will yield
  `collections.OrderedDict` objects at each iteration, with the following keys
  and values:

    -   `'creation_date'`: a `tf.Tensor` with `dtype=tf.string` and shape []
        containing the date/time of the question or answer in UTC format.
    -   `'title'`: a `tf.Tensor` with `dtype=tf.string` and shape [] containing
        the title of the question.
    -   `'score'`: a `tf.Tensor` with `dtype=tf.int64` and shape [] containing
        the score of the question.
    -   `'tags'`: a `tf.Tensor` with `dtype=tf.string` and shape [] containing
        the tags of the question, separated by '|' characters.
    -   `'tokens'`: a `tf.Tensor` with `dtype=tf.string` and shape []
        containing the tokens of the question/answer, separated by space (' ')
        characters.
    -   `'type'`: a `tf.Tensor` with `dtype=tf.string` and shape []
        containing either the string 'question' or 'answer'.

  Args:
    cache_dir: (Optional) directory to cache the downloaded file. If `None`,
      caches in Keras' default cache directory.

  Returns:
    Tuple of (train, held_out, test) where the tuple elements are
    `tff.simulation.ClientData` objects.
  """
  path = tf.keras.utils.get_file(
      'stackoverflow.tar.bz2',
      origin='https://storage.googleapis.com/tff-datasets-public/stackoverflow.tar.bz2',
      file_hash='99eca2f8b8327a09e5fc123979df2d237acbc5e52322f6d86bf523ee47b961a2',
      hash_algorithm='sha256',
      extract=True,
      archive_format='tar',
      cache_dir=cache_dir)

  dir_path = os.path.dirname(path)
  train_client_data = hdf5_client_data.HDF5ClientData(
      os.path.join(dir_path, 'stackoverflow_train.h5'))
  held_out_client_data = hdf5_client_data.HDF5ClientData(
      os.path.join(dir_path, 'stackoverflow_held_out.h5'))
  test_client_data = hdf5_client_data.HDF5ClientData(
      os.path.join(dir_path, 'stackoverflow_test.h5'))

  return train_client_data, held_out_client_data, test_client_data

def load_word_counts(cache_dir=None, vocab_size: Optional[int] = None):
  """Loads the word counts for the Stack Overflow dataset.

  Args:
    cache_dir: (Optional) directory to cache the downloaded file. If `None`,
      caches in Keras' default cache directory.
    vocab_size: (Optional) when specified, only load the first `vocab_size`
      unique words in the vocab file (i.e. the most frequent `vocab_size`
      words).

  Returns:
    A collections.OrderedDict where the keys are string tokens, and the values
    are the counts of unique users who have at least one example in the training
    set containing that token in the body text.

  Raises:
    TypeError if vocab_size is not None or int.
    ValueError if vocab_size is not None but <= 0.
  """
  if vocab_size is not None:
    if not isinstance(vocab_size, int):
      raise TypeError(
          f'vocab_size should be None or int, got {type(vocab_size)}.')
    if vocab_size <= 0:
      raise ValueError(f'vocab_size must be positive, got {vocab_size}.')

  path = tf.keras.utils.get_file(
      'stackoverflow.word_count.tar.bz2',
      origin='https://storage.googleapis.com/tff-datasets-public/stackoverflow.word_count.tar.bz2',
      file_hash='1dc00256d6e527c54b9756d968118378ae14e6692c0b3b6cad470cdd3f0c519c',
      hash_algorithm='sha256',
      extract=True,
      archive_format='tar',
      cache_dir=cache_dir)

  word_counts = collections.OrderedDict()
  dir_path = os.path.dirname(path)
  file_path = os.path.join(dir_path, 'stackoverflow.word_count')
  with open(file_path) as f:
    for line in f:
      word, count = line.split()
      word_counts[word] = int(count)
      if vocab_size is not None and len(word_counts) >= vocab_size:
        break
  return word_counts

def load_tag_counts(cache_dir=None):
  """Loads the tag counts for the Stack Overflow dataset.

  Args:
    cache_dir: (Optional) directory to cache the downloaded file. If `None`,
      caches in Keras' default cache directory.

  Returns:
    A collections.OrderedDict where the keys are string tags, and the values
    are the counts of unique users who have at least one example in the training
    set containing with that tag. The dictionary items are in decreasing order
    of tag frequency.
  """
  path = tf.keras.utils.get_file(
      'stackoverflow.tag_count.tar.bz2',
      origin='https://storage.googleapis.com/tff-datasets-public/stackoverflow.tag_count.tar.bz2',
      file_hash='6fe281cec490d9384a290d560072438e7e2b377bbb823876ce7bd6f82696772d',
      hash_algorithm='sha256',
      extract=True,
      archive_format='tar',
      cache_dir=cache_dir)

  dir_path = os.path.dirname(path)
  file_path = os.path.join(dir_path, 'stackoverflow.tag_count')
  with open(file_path) as f:
    tag_counts = json.load(f)
  return collections.OrderedDict(
      sorted(tag_counts.items(), key=lambda item: item[1], reverse=True))

def get_synthetic():
  """Returns a small synthetic dataset for testing.

  Provides two clients, each client with only 3 examples. The examples are
  derived from a fixed set of examples in the larger dataset, but are not exact
  copies.

  Returns:
     A `tff.simulation.ClientData` object that matches the characteristics
     (other than size) of those provided by
     `tff.simulation.datasets.stackoverflow.load_data`.
  """
  return from_tensor_slices_client_data.FromTensorSlicesClientData(
      _SYNTHETIC_STACKOVERFLOW_DATA)

_SYNTHETIC_STACKOVERFLOW_DATA = {
    'synthetic_1':
        collections.OrderedDict(
            creation_date=[
                b'2010-01-08 09:34:05 UTC',
                b'2008-08-10 08:28:52.1 UTC',
                b'2008-08-10 08:28:52.1 UTC',
            ],
            title=[
                b'function to calculate median in sql server',
                b'creating rounded corners using css',
                b'creating rounded corners using css',
            ],
            score=np.asarray([
                172,
                80,
                80,
            ]).astype(np.int64),
            tags=[
                b'sql|sql-server|aggregate-functions|median',
                b'css|cross-browser|rounded-corners|css3',
                b'css|cross-browser|rounded-corners|css3',
            ],
            tokens=[
                b"if you're using sql 2005 or better this is a nice , simple-ish median calculation for a single column in a table :",
                b'css3 does finally define the',
                b"which is exactly how you'd want it to work .",
            ],
            type=[
                b'answer',
                b'question',
                b'answer',
            ]),
    'synthetic_2':
        collections.OrderedDict(
            creation_date=[
                b'2008-08-05 19:01:55.2 UTC',
                b'2010-07-15 18:15:58.5 UTC',
                b'2010-07-15 18:15:58.5 UTC',
            ],
            title=[
                b'getting started with version control',
                b'writing to / system / framework in emulator',
                b'writing to / system / framework in emulator',
            ],
            score=np.asarray([
                3,
                12,
                -1,
            ]).astype(np.int64),
            tags=[
                b'git|svn|version-control|language-agnostic|dvcs',
                b'android|android-emulator|monkey',
                b'android|android-emulator|monkey',
            ],
            tokens=[
                b'if you are on mac osx , i found <URL> " > versions to be an incredible ( free ) gui front-end to svn .',
                b'edit :',
                b'thanks .',
            ],
            type=[
                b'answer',
                b'question',
                b'question',
            ],
        )
}
<commit_msg>Add one more example for StackOverflow synthetic. More example could be good to test client sampling algorithms, especially the edge case.<commit_after># Copyright 2019, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Libraries for the Stackoverflow dataset for federated learning simulation."""

import collections
import json
import os.path
from typing import Optional

import numpy as np
import tensorflow as tf

from tensorflow_federated.python.simulation import from_tensor_slices_client_data
from tensorflow_federated.python.simulation import hdf5_client_data

def load_data(cache_dir=None):
  """Loads the federated Stack Overflow dataset.

  Downloads and caches the dataset locally. If previously downloaded, tries to
  load the dataset from cache.

  This dataset is derived from the Stack Overflow Data hosted by kaggle.com and
  available to query through Kernels using the BigQuery API:
  https://www.kaggle.com/stackoverflow/stackoverflow. The Stack Overflow Data
  is licensed under the Creative Commons Attribution-ShareAlike 3.0 Unported
  License. To view a copy of this license, visit
  http://creativecommons.org/licenses/by-sa/3.0/ or send a letter to
  Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.

  The data consists of the body text of all questions and answers. The bodies
  were parsed into sentences, and any user with fewer than 100 sentences was
  expunged from the data. Minimal preprocessing was performed as follows:

  1. Lowercase the text,
  2. Unescape HTML symbols,
  3. Remove non-ascii symbols,
  4. Separate punctuation as individual tokens (except apostrophes and hyphens),
  5. Removing extraneous whitespace,
  6. Replacing URLS with a special token.

  In addition the following metadata is available:

  1. Creation date
  2. Question title
  3. Question tags
  4. Question score
  5. Type ('question' or 'answer')

  The data is divided into three sets:

    -   Train: Data before 2018-01-01 UTC except the held-out users. 342,477
        unique users with 135,818,730 examples.
    -   Held-out: All examples from users with user_id % 10 == 0 (all dates).
        38,758 unique users with 16,491,230 examples.
    -   Test: All examples after 2018-01-01 UTC except from held-out users.
        204,088 unique users with 16,586,035 examples.

  The `tf.data.Datasets` returned by
  `tff.simulation.ClientData.create_tf_dataset_for_client` will yield
  `collections.OrderedDict` objects at each iteration, with the following keys
  and values:

    -   `'creation_date'`: a `tf.Tensor` with `dtype=tf.string` and shape []
        containing the date/time of the question or answer in UTC format.
    -   `'title'`: a `tf.Tensor` with `dtype=tf.string` and shape [] containing
        the title of the question.
    -   `'score'`: a `tf.Tensor` with `dtype=tf.int64` and shape [] containing
        the score of the question.
    -   `'tags'`: a `tf.Tensor` with `dtype=tf.string` and shape [] containing
        the tags of the question, separated by '|' characters.
    -   `'tokens'`: a `tf.Tensor` with `dtype=tf.string` and shape []
        containing the tokens of the question/answer, separated by space (' ')
        characters.
    -   `'type'`: a `tf.Tensor` with `dtype=tf.string` and shape []
        containing either the string 'question' or 'answer'.

  Args:
    cache_dir: (Optional) directory to cache the downloaded file. If `None`,
      caches in Keras' default cache directory.

  Returns:
    Tuple of (train, held_out, test) where the tuple elements are
    `tff.simulation.ClientData` objects.
  """
  path = tf.keras.utils.get_file(
      'stackoverflow.tar.bz2',
      origin='https://storage.googleapis.com/tff-datasets-public/stackoverflow.tar.bz2',
      file_hash='99eca2f8b8327a09e5fc123979df2d237acbc5e52322f6d86bf523ee47b961a2',
      hash_algorithm='sha256',
      extract=True,
      archive_format='tar',
      cache_dir=cache_dir)

  dir_path = os.path.dirname(path)
  train_client_data = hdf5_client_data.HDF5ClientData(
      os.path.join(dir_path, 'stackoverflow_train.h5'))
  held_out_client_data = hdf5_client_data.HDF5ClientData(
      os.path.join(dir_path, 'stackoverflow_held_out.h5'))
  test_client_data = hdf5_client_data.HDF5ClientData(
      os.path.join(dir_path, 'stackoverflow_test.h5'))

  return train_client_data, held_out_client_data, test_client_data

def load_word_counts(cache_dir=None, vocab_size: Optional[int] = None):
  """Loads the word counts for the Stack Overflow dataset.

  Args:
    cache_dir: (Optional) directory to cache the downloaded file. If `None`,
      caches in Keras' default cache directory.
    vocab_size: (Optional) when specified, only load the first `vocab_size`
      unique words in the vocab file (i.e. the most frequent `vocab_size`
      words).

  Returns:
    A collections.OrderedDict where the keys are string tokens, and the values
    are the counts of unique users who have at least one example in the training
    set containing that token in the body text.

  Raises:
    TypeError if vocab_size is not None or int.
    ValueError if vocab_size is not None but <= 0.
  """
  if vocab_size is not None:
    if not isinstance(vocab_size, int):
      raise TypeError(
          f'vocab_size should be None or int, got {type(vocab_size)}.')
    if vocab_size <= 0:
      raise ValueError(f'vocab_size must be positive, got {vocab_size}.')

  path = tf.keras.utils.get_file(
      'stackoverflow.word_count.tar.bz2',
      origin='https://storage.googleapis.com/tff-datasets-public/stackoverflow.word_count.tar.bz2',
      file_hash='1dc00256d6e527c54b9756d968118378ae14e6692c0b3b6cad470cdd3f0c519c',
      hash_algorithm='sha256',
      extract=True,
      archive_format='tar',
      cache_dir=cache_dir)

  word_counts = collections.OrderedDict()
  dir_path = os.path.dirname(path)
  file_path = os.path.join(dir_path, 'stackoverflow.word_count')
  with open(file_path) as f:
    for line in f:
      word, count = line.split()
      word_counts[word] = int(count)
      if vocab_size is not None and len(word_counts) >= vocab_size:
        break
  return word_counts

def load_tag_counts(cache_dir=None):
  """Loads the tag counts for the Stack Overflow dataset.

  Args:
    cache_dir: (Optional) directory to cache the downloaded file. If `None`,
      caches in Keras' default cache directory.

  Returns:
    A collections.OrderedDict where the keys are string tags, and the values
    are the counts of unique users who have at least one example in the training
    set containing with that tag. The dictionary items are in decreasing order
    of tag frequency.
  """
  path = tf.keras.utils.get_file(
      'stackoverflow.tag_count.tar.bz2',
      origin='https://storage.googleapis.com/tff-datasets-public/stackoverflow.tag_count.tar.bz2',
      file_hash='6fe281cec490d9384a290d560072438e7e2b377bbb823876ce7bd6f82696772d',
      hash_algorithm='sha256',
      extract=True,
      archive_format='tar',
      cache_dir=cache_dir)

  dir_path = os.path.dirname(path)
  file_path = os.path.join(dir_path, 'stackoverflow.tag_count')
  with open(file_path) as f:
    tag_counts = json.load(f)
  return collections.OrderedDict(
      sorted(tag_counts.items(), key=lambda item: item[1], reverse=True))

def get_synthetic():
  """Returns a small synthetic dataset for testing.

  Provides two clients, each client with only 3 examples. The examples are
  derived from a fixed set of examples in the larger dataset, but are not exact
  copies.

  Returns:
     A `tff.simulation.ClientData` object that matches the characteristics
     (other than size) of those provided by
     `tff.simulation.datasets.stackoverflow.load_data`.
  """
  return from_tensor_slices_client_data.FromTensorSlicesClientData(
      _SYNTHETIC_STACKOVERFLOW_DATA)

_SYNTHETIC_STACKOVERFLOW_DATA = {
    'synthetic_1':
        collections.OrderedDict(
            creation_date=[
                b'2010-01-08 09:34:05 UTC',
                b'2008-08-10 08:28:52.1 UTC',
                b'2008-08-10 08:28:52.1 UTC',
            ],
            title=[
                b'function to calculate median in sql server',
                b'creating rounded corners using css',
                b'creating rounded corners using css',
            ],
            score=np.asarray([
                172,
                80,
                80,
            ]).astype(np.int64),
            tags=[
                b'sql|sql-server|aggregate-functions|median',
                b'css|cross-browser|rounded-corners|css3',
                b'css|cross-browser|rounded-corners|css3',
            ],
            tokens=[
                b"if you're using sql 2005 or better this is a nice , simple-ish median calculation for a single column in a table :",
                b'css3 does finally define the',
                b"which is exactly how you'd want it to work .",
            ],
            type=[
                b'answer',
                b'question',
                b'answer',
            ]),
    'synthetic_2':
        collections.OrderedDict(
            creation_date=[
                b'2008-08-05 19:01:55.2 UTC',
                b'2010-07-15 18:15:58.5 UTC',
                b'2010-07-15 18:15:58.5 UTC',
            ],
            title=[
                b'getting started with version control',
                b'writing to / system / framework in emulator',
                b'writing to / system / framework in emulator',
            ],
            score=np.asarray([
                3,
                12,
                -1,
            ]).astype(np.int64),
            tags=[
                b'git|svn|version-control|language-agnostic|dvcs',
                b'android|android-emulator|monkey',
                b'android|android-emulator|monkey',
            ],
            tokens=[
                b'if you are on mac osx , i found <URL> " > versions to be an incredible ( free ) gui front-end to svn .',
                b'edit :',
                b'thanks .',
            ],
            type=[
                b'answer',
                b'question',
                b'question',
            ],
        ),
    'synthetic_3':
        collections.OrderedDict(
            creation_date=[
                b'2008-10-30 16:49:26.9 UTC',
                b'2008-10-30 16:49:26.9 UTC',
            ],
            title=[
                b'iterator pattern in vb . net ( c # would use yield ! )',
                b'iterator pattern in vb . net ( c # would use yield ! )',
            ],
            score=np.asarray([
                1,
                1,
            ]).astype(np.int64),
            tags=[
                b'vb . net|design-patterns|iterator|yield',
                b'vb . net|design-patterns|iterator|yield',
            ],
            tokens=[
                b'edit :',
                b'the spec is available here .',
            ],
            type=[
                b'answer',
                b'answer',
            ],
        )
}
<|endoftext|>