<commit_before># Copyright 2011,2013 James McCauley
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

enabled = False
try:
  import platform
  import importlib
  _module = 'pox.lib.pxpcap.%s.pxpcap' % (platform.system().lower(),)
  pcapc = importlib.import_module(_module)
  enabled = True
except:
  # Try generic...
  try:
    import pxpcap as pcapc
    enabled = True
  except:
    # We can at least import the rest
    pass

from pox.lib.addresses import IPAddr, EthAddr, IPAddr6
import parser
from threading import Thread, Lock
import pox.lib.packet as pkt
import copy

# pcap's filter compiling function isn't threadsafe, so we use this
# lock when compiling filters.
_compile_lock = Lock()

class PCap (object):
  @staticmethod
  def get_devices ():
    def ip (addr):
      if addr is None: return None
      return IPAddr(addr, networkOrder=True)
    def ip6 (addr):
      if addr is None: return None
      return IPAddr6.from_raw(addr)
    def link (addr):
      if addr is None: return None
      if len(addr) != 6: return None
      return EthAddr(addr)
    devs = pcapc.findalldevs()
    out = {}
    for d in devs:
      addrs = {}
      n = {'desc':d[1],'addrs':addrs}
      out[d[0]] = n
      for a in d[2]:
        if a[0] == 'AF_INET':
          na = {}
          addrs[a[0]] = na
          na['addr'] = ip(a[1])
          na['netmask'] = ip(a[2])
          na['broadaddr'] = ip(a[3])
          na['dstaddr'] = ip(a[4])
        elif a[0] == 'AF_INET6':
          na = {}
          addrs[a[0]] = na
          na['addr'] = ip6(a[1])
          na['netmask'] = ip6(a[2])
          na['broadaddr'] = ip6(a[3])
          na['dstaddr'] = ip6(a[4])
        elif a[0] == 'AF_LINK':
          na = {}
          addrs[a[0]] = na
          na['addr'] = link(a[1])
          na['netmask'] = link(a[2])
          na['broadaddr'] = link(a[3])
          na['dstaddr'] = link(a[4])
        elif a[0] == 'AF_PACKET':
          addrs[a[0]] = {'addr':link(a[1])}
        elif a[0] == 'ethernet':
          addrs[a[0]] = {'addr':link(a[1])}
    return out

  @staticmethod
  def get_device_names ():
    return [d[0] for d in pcapc.findalldevs()]

  def __init__ (self, device = None, promiscuous = True, period = 10,
                start = True, callback = None, filter = None,
                use_bytearray = False, **kw):
    """
    Initialize this instance

    use_bytearray: specifies capturing to bytearray buffers instead of bytes
    """

    if filter is not None:
      self.deferred_filter = (filter,)
    else:
      self.deferred_filter = None
    self.packets_received = 0
    self.packets_dropped = 0
    self._thread = None
    self.pcap = None
    self.promiscuous = promiscuous
    self.device = None
    self.use_bytearray = use_bytearray
    self.period = period
    self.netmask = IPAddr("0.0.0.0")
    self._quitting = False
    self.addresses = {}
    if callback is None:
      self.callback = self.__class__._handle_rx
    else:
      self.callback = callback

    for k,v in kw.items():
      assert not hasattr(self, k)
      setattr(self, k, v)

    if device is not None:
      self.open(device)
    if self.pcap is not None:
      if start:
        self.start()

  def _handle_rx (self, data, sec, usec, length):
    pass

  def open (self, device, promiscuous = None, period = None,
            incoming = True, outgoing = False):
    assert self.device is None
    self.addresses = self.get_devices()[device]['addrs']
    if 'AF_INET' in self.addresses:
      self.netmask = self.addresses['AF_INET'].get('netmask')
      if self.netmask is None: self.netmask = IPAddr("0.0.0.0")
    #print "NM:",self.netmask
    #print self.addresses['AF_LINK']['addr']
    self.device = device
    if period is not None:
      self.period = period
    if promiscuous is not None:
      self.promiscuous = promiscuous
    self.pcap = pcapc.open_live(device, 65535,
                                1 if self.promiscuous else 0, self.period)
    pcapc.setdirection(self.pcap, incoming, outgoing)
    self.packets_received = 0
    self.packets_dropped = 0
    if self.deferred_filter is not None:
      self.set_filter(*self.deferred_filter)
      self.deferred_filter = None

  def set_direction (self, incoming, outgoing):
    pcapc.setdirection(self._pcap, incoming, outgoing)

  def set_nonblocking (self, nonblocking = True):
    pcapc.setnonblock(self._pcap, 1 if nonblocking else 0)

  def set_blocking (self, blocking = True):
    self.set_nonblocking(nonblocking = not blocking)

  @property
  def blocking (self):
    return False if pcapc.getnonblock(self._pcap) else True

  @blocking.setter
  def blocking (self, value):
    self.set_blocking(value)

  def _thread_func (self):
    while not self._quitting:
      pcapc.dispatch(self.pcap,100,self.callback,self,bool(self.use_bytearray))
      self.packets_received,self.packets_dropped = pcapc.stats(self.pcap)

    self._quitting = False
    self._thread = None

  def _handle_GoingDownEvent (self, event):
    self.close()

  def start (self):
    assert self._thread is None
    from pox.core import core
    core.addListeners(self, weak=True)
    self._thread = Thread(target=self._thread_func)
    #self._thread.daemon = True
    self._thread.start()

  def stop (self):
    t = self._thread
    if t is not None:
      self._quitting = True
      pcapc.breakloop(self.pcap)
      t.join()

  def close (self):
    if self.pcap is None: return
    self.stop()
    pcapc.close(self.pcap)
    self.pcap = None

  def __del__ (self):
    self.close()

  @property
  def _pcap (self):
    if self.pcap is None:
      raise RuntimeError("PCap object not open")
    return self.pcap

  def inject (self, data):
    if isinstance(data, pkt.ethernet):
      data = data.pack()
    if not isinstance(data, (bytes,bytearray)):
      data = bytes(data) # Give it a try...
    return pcapc.inject(self.pcap, data)

  def set_filter (self, filter, optimize = True):
    if self.pcap is None:
      self.deferred_filter = (filter, optimize)
      return

    if isinstance(filter, str):
      filter = Filter(filter, optimize, self.netmask.toSignedN(),
                      pcap_obj=self)
    elif isinstance(filter, Filter):
      pass
    else:
      raise RuntimeError("Filter must be string or Filter object")

    pcapc.setfilter(self.pcap, filter._pprogram)

class Filter (object):
  def __init__ (self, filter, optimize = True, netmask = None,
                pcap_obj = None, link_type = 1, snaplen = 65535):
    self._pprogram = None
    if netmask is None:
      netmask = 0
    elif isinstance(netmask, IPAddr):
      netmask = netmask.toSignedN()

    delpc = False
    if pcap_obj is None:
      delpc = True
      pcap_obj = pcapc.open_dead(link_type, snaplen)
    if isinstance(pcap_obj, PCap):
      pcap_obj = pcap_obj.pcap

    with _compile_lock:
      self._pprogram = pcapc.compile(pcap_obj, filter,
                                     1 if optimize else 0, netmask)
    if delpc:
      pcapc.close(pcap_obj)

  def __del__ (self):
    if self._pprogram:
      pcapc.freecode(self._pprogram)

try:
  _link_type_names = {}
  for k,v in copy.copy(pcapc.__dict__).iteritems():
    if k.startswith("DLT_"):
      _link_type_names[v] = k
except:
  pass

def get_link_type_name (dlt):
  return _link_type_names.get(dlt, "<Unknown " + str(dlt) + ">")

def launch (interface = "en1"):
  """ Test function """
  global drop,total,bytes_got,bytes_real,bytes_diff
  drop = 0
  total = 0
  bytes_got = 0
  bytes_real = 0
  bytes_diff = 0
  def cb (obj, data, sec, usec, length):
    global drop,total,bytes_got,bytes_real,bytes_diff
    #print ">>>",data
    t,d = pcapc.stats(obj.pcap)
    bytes_got += len(data)
    bytes_real += length
    nbd = bytes_real - bytes_got
    if nbd != bytes_diff:
      bytes_diff = nbd
      print "lost bytes:",nbd
    if t > total:
      total = t + 500
      print t,"total"
    if d > drop:
      drop = d
      print d, "dropped"
    p = pkt.ethernet(data)
    ip = p.find('ipv4')
    if ip:
      print ip.srcip,"\t",ip.dstip, p

  print "\n".join(["%i. %s" % x for x in
                  enumerate(PCap.get_device_names())])

  if interface.startswith("#"):
    interface = int(interface[1:])
    interface = PCap.get_device_names()[interface]
  print "Interface:",interface

  p = PCap(interface, callback = cb,
           filter = "icmp")
           #[icmptype] != icmp-echoreply")
           #filter = "ip host 74.125.224.148")

  p.set_direction(True, True)

  def ping (eth='00:18:02:6e:ce:55', ip='192.168.0.1'):
    e = pkt.ethernet()
    e.src = p.addresses['ethernet']['addr'] or '02:00:00:11:22:33'
    e.dst = EthAddr(eth)
    e.type = e.IP_TYPE
    ipp = pkt.ipv4()
    ipp.protocol = ipp.ICMP_PROTOCOL
    ipp.srcip = p.addresses['AF_INET']['addr']
    ipp.dstip = IPAddr(ip)
    icmp = pkt.icmp()
    icmp.type = pkt.ICMP.TYPE_ECHO_REQUEST
    icmp.payload = "PingPing" * 6
    ipp.payload = icmp
    e.payload = ipp

    p.inject(e)

  def broadcast ():
    ping('ff:ff:ff:ff:ff:ff','255.255.255.255')

  import code
  code.interact(local=locals())
<commit_msg>pxpcap: Add .next_packet()<commit_after># Copyright 2011,2013 James McCauley
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

enabled = False
try:
  import platform
  import importlib
  _module = 'pox.lib.pxpcap.%s.pxpcap' % (platform.system().lower(),)
  pcapc = importlib.import_module(_module)
  enabled = True
except:
  # Try generic...
  try:
    import pxpcap as pcapc
    enabled = True
  except:
    # We can at least import the rest
    pass

from pox.lib.addresses import IPAddr, EthAddr, IPAddr6
import parser
from threading import Thread, Lock
import pox.lib.packet as pkt
import copy

# pcap's filter compiling function isn't threadsafe, so we use this
# lock when compiling filters.
_compile_lock = Lock()

class PCap (object):
  @staticmethod
  def get_devices ():
    def ip (addr):
      if addr is None: return None
      return IPAddr(addr, networkOrder=True)
    def ip6 (addr):
      if addr is None: return None
      return IPAddr6.from_raw(addr)
    def link (addr):
      if addr is None: return None
      if len(addr) != 6: return None
      return EthAddr(addr)
    devs = pcapc.findalldevs()
    out = {}
    for d in devs:
      addrs = {}
      n = {'desc':d[1],'addrs':addrs}
      out[d[0]] = n
      for a in d[2]:
        if a[0] == 'AF_INET':
          na = {}
          addrs[a[0]] = na
          na['addr'] = ip(a[1])
          na['netmask'] = ip(a[2])
          na['broadaddr'] = ip(a[3])
          na['dstaddr'] = ip(a[4])
        elif a[0] == 'AF_INET6':
          na = {}
          addrs[a[0]] = na
          na['addr'] = ip6(a[1])
          na['netmask'] = ip6(a[2])
          na['broadaddr'] = ip6(a[3])
          na['dstaddr'] = ip6(a[4])
        elif a[0] == 'AF_LINK':
          na = {}
          addrs[a[0]] = na
          na['addr'] = link(a[1])
          na['netmask'] = link(a[2])
          na['broadaddr'] = link(a[3])
          na['dstaddr'] = link(a[4])
        elif a[0] == 'AF_PACKET':
          addrs[a[0]] = {'addr':link(a[1])}
        elif a[0] == 'ethernet':
          addrs[a[0]] = {'addr':link(a[1])}
    return out

  @staticmethod
  def get_device_names ():
    return [d[0] for d in pcapc.findalldevs()]

  def __init__ (self, device = None, promiscuous = True, period = 10,
                start = True, callback = None, filter = None,
                use_bytearray = False, **kw):
    """
    Initialize this instance

    use_bytearray: specifies capturing to bytearray buffers instead of bytes
    """

    if filter is not None:
      self.deferred_filter = (filter,)
    else:
      self.deferred_filter = None
    self.packets_received = 0
    self.packets_dropped = 0
    self._thread = None
    self.pcap = None
    self.promiscuous = promiscuous
    self.device = None
    self.use_bytearray = use_bytearray
    self.period = period
    self.netmask = IPAddr("0.0.0.0")
    self._quitting = False
    self.addresses = {}
    if callback is None:
      self.callback = self.__class__._handle_rx
    else:
      self.callback = callback

    for k,v in kw.items():
      assert not hasattr(self, k)
      setattr(self, k, v)

    if device is not None:
      self.open(device)
    if self.pcap is not None:
      if start:
        self.start()

  def _handle_rx (self, data, sec, usec, length):
    pass

  def open (self, device, promiscuous = None, period = None,
            incoming = True, outgoing = False):
    assert self.device is None
    self.addresses = self.get_devices()[device]['addrs']
    if 'AF_INET' in self.addresses:
      self.netmask = self.addresses['AF_INET'].get('netmask')
      if self.netmask is None: self.netmask = IPAddr("0.0.0.0")
    #print "NM:",self.netmask
    #print self.addresses['AF_LINK']['addr']
    self.device = device
    if period is not None:
      self.period = period
    if promiscuous is not None:
      self.promiscuous = promiscuous
    self.pcap = pcapc.open_live(device, 65535,
                                1 if self.promiscuous else 0, self.period)
    pcapc.setdirection(self.pcap, incoming, outgoing)
    self.packets_received = 0
    self.packets_dropped = 0
    if self.deferred_filter is not None:
      self.set_filter(*self.deferred_filter)
      self.deferred_filter = None

  def set_direction (self, incoming, outgoing):
    pcapc.setdirection(self._pcap, incoming, outgoing)

  def set_nonblocking (self, nonblocking = True):
    pcapc.setnonblock(self._pcap, 1 if nonblocking else 0)

  def set_blocking (self, blocking = True):
    self.set_nonblocking(nonblocking = not blocking)

  @property
  def blocking (self):
    return False if pcapc.getnonblock(self._pcap) else True

  @blocking.setter
  def blocking (self, value):
    self.set_blocking(value)

  def next_packet (self, allow_threads = True):
    """
    Get next packet

    Returns tuple with:
      data, timestamp_seconds, timestamp_useconds, total length, and
      the pcap_next_ex return value -- 1 is success
    """
    return pcapc.next_ex(self._pcap, bool(self.use_bytearray), allow_threads)

  def _thread_func (self):
    while not self._quitting:
      pcapc.dispatch(self.pcap,100,self.callback,self,bool(self.use_bytearray))
      self.packets_received,self.packets_dropped = pcapc.stats(self.pcap)

    self._quitting = False
    self._thread = None

  def _handle_GoingDownEvent (self, event):
    self.close()

  def start (self):
    assert self._thread is None
    from pox.core import core
    core.addListeners(self, weak=True)
    self._thread = Thread(target=self._thread_func)
    #self._thread.daemon = True
    self._thread.start()

  def stop (self):
    t = self._thread
    if t is not None:
      self._quitting = True
      pcapc.breakloop(self.pcap)
      t.join()

  def close (self):
    if self.pcap is None: return
    self.stop()
    pcapc.close(self.pcap)
    self.pcap = None

  def __del__ (self):
    self.close()

  @property
  def _pcap (self):
    if self.pcap is None:
      raise RuntimeError("PCap object not open")
    return self.pcap

  def inject (self, data):
    if isinstance(data, pkt.ethernet):
      data = data.pack()
    if not isinstance(data, (bytes,bytearray)):
      data = bytes(data) # Give it a try...
    return pcapc.inject(self.pcap, data)

  def set_filter (self, filter, optimize = True):
    if self.pcap is None:
      self.deferred_filter = (filter, optimize)
      return

    if isinstance(filter, str):
      filter = Filter(filter, optimize, self.netmask.toSignedN(),
                      pcap_obj=self)
    elif isinstance(filter, Filter):
      pass
    else:
      raise RuntimeError("Filter must be string or Filter object")

    pcapc.setfilter(self.pcap, filter._pprogram)

class Filter (object):
  def __init__ (self, filter, optimize = True, netmask = None,
                pcap_obj = None, link_type = 1, snaplen = 65535):
    self._pprogram = None
    if netmask is None:
      netmask = 0
    elif isinstance(netmask, IPAddr):
      netmask = netmask.toSignedN()

    delpc = False
    if pcap_obj is None:
      delpc = True
      pcap_obj = pcapc.open_dead(link_type, snaplen)
    if isinstance(pcap_obj, PCap):
      pcap_obj = pcap_obj.pcap

    with _compile_lock:
      self._pprogram = pcapc.compile(pcap_obj, filter,
                                     1 if optimize else 0, netmask)
    if delpc:
      pcapc.close(pcap_obj)

  def __del__ (self):
    if self._pprogram:
      pcapc.freecode(self._pprogram)

try:
  _link_type_names = {}
  for k,v in copy.copy(pcapc.__dict__).iteritems():
    if k.startswith("DLT_"):
      _link_type_names[v] = k
except:
  pass

def get_link_type_name (dlt):
  return _link_type_names.get(dlt, "<Unknown " + str(dlt) + ">")

def launch (interface = "en1"):
  """ Test function """
  global drop,total,bytes_got,bytes_real,bytes_diff
  drop = 0
  total = 0
  bytes_got = 0
  bytes_real = 0
  bytes_diff = 0
  def cb (obj, data, sec, usec, length):
    global drop,total,bytes_got,bytes_real,bytes_diff
    #print ">>>",data
    t,d = pcapc.stats(obj.pcap)
    bytes_got += len(data)
    bytes_real += length
    nbd = bytes_real - bytes_got
    if nbd != bytes_diff:
      bytes_diff = nbd
      print "lost bytes:",nbd
    if t > total:
      total = t + 500
      print t,"total"
    if d > drop:
      drop = d
      print d, "dropped"
    p = pkt.ethernet(data)
    ip = p.find('ipv4')
    if ip:
      print ip.srcip,"\t",ip.dstip, p

  print "\n".join(["%i. %s" % x for x in
                  enumerate(PCap.get_device_names())])

  if interface.startswith("#"):
    interface = int(interface[1:])
    interface = PCap.get_device_names()[interface]
  print "Interface:",interface

  p = PCap(interface, callback = cb,
           filter = "icmp")
           #[icmptype] != icmp-echoreply")
           #filter = "ip host 74.125.224.148")

  p.set_direction(True, True)

  def ping (eth='00:18:02:6e:ce:55', ip='192.168.0.1'):
    e = pkt.ethernet()
    e.src = p.addresses['ethernet']['addr'] or '02:00:00:11:22:33'
    e.dst = EthAddr(eth)
    e.type = e.IP_TYPE
    ipp = pkt.ipv4()
    ipp.protocol = ipp.ICMP_PROTOCOL
    ipp.srcip = p.addresses['AF_INET']['addr']
    ipp.dstip = IPAddr(ip)
    icmp = pkt.icmp()
    icmp.type = pkt.ICMP.TYPE_ECHO_REQUEST
    icmp.payload = "PingPing" * 6
    ipp.payload = icmp
    e.payload = ipp

    p.inject(e)

  def broadcast ():
    ping('ff:ff:ff:ff:ff:ff','255.255.255.255')

  import code
  code.interact(local=locals())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#  May 08, 2014 Author: Cornelius Kölbel
#  contact:  http://www.privacyidea.org
#
# This code is free software; you can redistribute it and/or
# modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
# License as published by the Free Software Foundation; either
# version 3 of the License, or any later version.
#
# This code is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU AFFERO GENERAL PUBLIC LICENSE for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
from privacyidea.lib.policy import PolicyClass
from privacyidea.lib.config import get_privacyIDEA_config
from privacyidea.lib.log import log_with
from pylons import config as ini_config
import urllib, httplib2, json
import crypt
ENCODING = "utf-8"
import logging
from pylons import request, config, tmpl_context as c

log = logging.getLogger(__name__)

@log_with(log)
def _get_superuser_realms():
    '''
    return the list of realms, that contain superusers.
    '''
    superuser_realms = [ "admin" ]
    ini_su = ini_config.get("privacyideaSuperuserRealms", "")
    ini_su_list = [w.strip() for w in ini_su.split(",")]
    for r in ini_su_list:
        superuser_realms.append(r.lower())
    return superuser_realms

@log_with(log)
def check_admin_password(user, password, realm="admin"):
    '''
    Check the password of the admin user from the file defined in
    ini: privacyideaSuperuserFile

    :param user: the username used to login
    :param password: the password provided at login
    :return: The identity (user@realm) of the user or None
    :rtype: string
    '''
    success = None

    # check in the admin file
    admin_file = ini_config.get("privacyideaSuperuserFile")
    if admin_file:
        fileHandle = open(admin_file, "r")
        line = fileHandle.readline()
        while line:
            line = line.strip()
            fields = line.split(":", 7)
            if fields[0] == user:
                crypted_pass = fields[1]
                if crypt.crypt(password, crypted_pass) == crypted_pass:
                    success = "%s@%s" % (user, realm)
                    break

            line = fileHandle.readline()
        fileHandle.close()

    # check in LinOTP
    if not success:
        success = authenticate_privacyidea_user(user, realm, password)

    return success

@log_with(log)
def authenticate_privacyidea_user(user, realm, password):
    '''
    this function performs an authentication against the
    privacyidea server.

    :param user: Username of the user
    :type user: string
    :return: In case of success return the username
    :rtype: string
    '''
    res = False
    success = None
    Policy = PolicyClass(request, config, c,
                             get_privacyIDEA_config())
    if Policy.check_user_authorization(user, realm, exception=False):
        '''
        We SHOULD do it this way, but unfortunately we
        only get the complete context in a web request.
        We are missing the client and the HSM!

        (res, _opt) = checkUserPass(User(login=user, realm=realm), password)

        FIXME: THe server is asking himself... :-/
        '''
        # FIXME: we need to pass the client= to cope with client dependent policies.
        data = urllib.urlencode({'user': user,
                                 'realm': realm,
                                 'pass': password})
        url = ini_config.get("privacyideaURL") + "/validate/check"
        disable_ssl = ini_config.get("privacyideaURL.disable_ssl", False)
        headers = {"Content-type": "application/x-www-form-urlencoded",
                   "Accept": "text/plain"}
        try:
            ## is httplib compiled with ssl?
            http = httplib2.Http(disable_ssl_certificate_validation=disable_ssl)
        except TypeError as exx:
            ## not so on squeeze:
            ## TypeError: __init__() got an unexpected keyword argument
            ## 'disable_ssl_certificate_validation'
            log.warning("httplib2 'disable_ssl_certificate_validation' "
                        "attribute error: %r" % exx)
            ## so we run in fallback mode
            http = httplib2.Http()
        (_resp, content) = http.request(url,
                                       method="POST",
                                       body=data,
                                       headers=headers)
        rv = json.loads(content)
        if rv.get("result"):
            # in case of normal json output
            res = rv['result'].get('value', False)

        if res:
            success = "%s@%s" % (user, realm)

    return success

@log_with(log)
def is_admin_identity(identity, exception=True):
    '''
    Check if the repoze identity is an admin, who is allowed to
    use the /admin, /system and all management controllers

    This is checked by verifying the standard realmname "admin" and
    the realms defined in privacyideaSuperuserRealms.

    Fixme: We need to get rid of the standard realm

    :param exception: If set to False the method will return a bool value otherwise it will throw an exception
    '''
    # During selftest the identity can be None!
    if identity == None:
        return False

    res = True
    try:
        # the identity could be a repoze identity object
        user_id = identity.get('repoze.who.userid')
        if type(user_id) == unicode:
            user_id = user_id.encode(ENCODING)
        identity = user_id.decode(ENCODING)
    except:
        pass

    (user, _foo, realm) = identity.rpartition('@')
    if realm.lower() not in _get_superuser_realms():
        if exception:
            path = request.path.lower()
            log.warning("User %s@%s tried to call the admin function %s." %(user, realm, path))
            raise Exception("You are not an admin user and are not allowed to call this function!")
        else:
            res = False
    return res
<commit_msg>Use client information when logging in with OTP<commit_after># -*- coding: utf-8 -*-
#
#  May 08, 2014 Author: Cornelius Kölbel
#  contact:  http://www.privacyidea.org
#
# This code is free software; you can redistribute it and/or
# modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
# License as published by the Free Software Foundation; either
# version 3 of the License, or any later version.
#
# This code is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU AFFERO GENERAL PUBLIC LICENSE for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
from privacyidea.lib.policy import PolicyClass
from privacyidea.lib.config import get_privacyIDEA_config
from privacyidea.lib.log import log_with
from pylons import config as ini_config
import urllib
import httplib2
import json
import crypt
ENCODING = "utf-8"
import logging
from pylons import request, config, tmpl_context as c

log = logging.getLogger(__name__)

@log_with(log)
def _get_superuser_realms():
    '''
    return the list of realms, that contain superusers.
    '''
    superuser_realms = ["admin"]
    ini_su = ini_config.get("privacyideaSuperuserRealms", "")
    ini_su_list = [w.strip() for w in ini_su.split(",")]
    for r in ini_su_list:
        superuser_realms.append(r.lower())
    return superuser_realms

@log_with(log)
def check_admin_password(user, password, realm="admin"):
    '''
    Check the password of the admin user from the file defined in
    ini: privacyideaSuperuserFile

    :param user: the username used to login
    :param password: the password provided at login
    :return: The identity (user@realm) of the user or None
    :rtype: string
    '''
    success = None

    # check in the admin file
    admin_file = ini_config.get("privacyideaSuperuserFile")
    if admin_file:
        fileHandle = open(admin_file, "r")
        line = fileHandle.readline()
        while line:
            line = line.strip()
            fields = line.split(":", 7)
            if fields[0] == user:
                crypted_pass = fields[1]
                if crypt.crypt(password, crypted_pass) == crypted_pass:
                    success = "%s@%s" % (user, realm)
                    break

            line = fileHandle.readline()
        fileHandle.close()

    # check in LinOTP
    if not success:
        success = authenticate_privacyidea_user(user, realm, password)

    return success

@log_with(log)
def authenticate_privacyidea_user(user, realm, password):
    '''
    this function performs an authentication against the
    privacyidea server.

    :param user: Username of the user
    :type user: string
    :return: In case of success return the username
    :rtype: string
    '''
    res = False
    success = None
    Policy = PolicyClass(request,
                         config,
                         c,
                         get_privacyIDEA_config())
    if Policy.check_user_authorization(user, realm, exception=False):
        '''
        We SHOULD do it this way, but unfortunately we
        only get the complete context in a web request.
        We are missing the client and the HSM!

        (res, _opt) = checkUserPass(User(login=user, realm=realm), password)

        Big FIXME: The server is asking himself... :-/
        '''
        # we need to pass the client= to cope with client dependent policies.
        # Otherwise the authentication request will have the client 127.0.0.1
        # as the source.
        client = request.client_addr
        data = urllib.urlencode({'user': user,
                                 'realm': realm,
                                 'pass': password,
                                 'client': client})
        url = ini_config.get("privacyideaURL") + "/validate/check"
        disable_ssl = ini_config.get("privacyideaURL.disable_ssl", False)
        headers = {"Content-type": "application/x-www-form-urlencoded",
                   "Accept": "text/plain"}
        try:
            # is httplib compiled with ssl?
            http = httplib2.\
                Http(disable_ssl_certificate_validation=disable_ssl)
        except TypeError as exx:
            # not so on squeeze:
            # TypeError: __init__() got an unexpected keyword argument
            # 'disable_ssl_certificate_validation'
            log.warning("httplib2 'disable_ssl_certificate_validation' "
                        "attribute error: %r" % exx)
            # so we run in fallback mode
            http = httplib2.Http()
        (_resp, content) = http.request(url,
                                        method="POST",
                                        body=data,
                                        headers=headers)
        rv = json.loads(content)
        if rv.get("result"):
            # in case of normal json output
            res = rv['result'].get('value', False)

        if res:
            success = "%s@%s" % (user, realm)

    return success

@log_with(log)
def is_admin_identity(identity, exception=True):
    '''
    Check if the repoze identity is an admin, who is allowed to
    use the /admin, /system and all management controllers

    This is checked by verifying the standard realmname "admin" and
    the realms defined in privacyideaSuperuserRealms.

    Fixme: We need to get rid of the standard realm

    :param exception: If set to False the method will return a bool
                      value otherwise it will throw an exception
    :type exception: bool
    '''
    # During selftest the identity can be None!
    if identity is None:
        return False

    res = True
    try:
        # the identity could be a repoze identity object
        user_id = identity.get('repoze.who.userid')
        if type(user_id) == unicode:
            user_id = user_id.encode(ENCODING)
        identity = user_id.decode(ENCODING)
    except:
        pass

    (user, _foo, realm) = identity.rpartition('@')
    if realm.lower() not in _get_superuser_realms():
        if exception:
            path = request.path.lower()
            log.warning("User %s@%s tried to call the admin function %s."
                        % (user, realm, path))
            raise Exception("You are not an admin user and are not allowed "
                            "to call this function!")
        else:
            res = False
    return res
<|endoftext|>


<commit_before>from django.contrib import admin
from versions.admin import VersionedAdmin

from .models import SocialUser, UserCategoryRelationship, Category, Profile, Queue, Moderation, TwitterList

class CategoryRelationshipInline(admin.TabularInline):
    model = UserCategoryRelationship
    extra = 10
    fk_name = 'category'
    raw_id_fields = ("social_user", "moderator",)

    def screen_name_tag(self, obj):
        screen_name = obj.social_user.profile.json.get("screen_name", None)
        return screen_name

    screen_name_tag.short_description = 'Screen name'
    readonly_fields = ('screen_name_tag',)

class UserRelationshipInline(admin.TabularInline):
    model = UserCategoryRelationship
    extra = 10
    fk_name = 'social_user'

class SocialUserAdmin(admin.ModelAdmin):
    inlines = (UserRelationshipInline,)
    list_display = ('user_id', 'screen_name_tag', 'mini_image_tag',  'name_tag',  'social_media', )
    fields = ('screen_name_tag', 'normal_image_tag', 'name_tag', 'user_id', 'social_media', 'category',)
    readonly_fields = ( 'screen_name_tag', 'normal_image_tag', 'name_tag', 'user_id', 'social_media', 'category',)
    search_fields = ('user_id', 'profile__json',)

class CategoryAdmin(admin.ModelAdmin):
    list_display = ('name', 'label', 'description', 'relationships_count_tag',
                    'quickreply',)
    inlines = (CategoryRelationshipInline,)
    fields = ('name', 'label', 'quickreply', 'relationships_count_tag', 'description')

    def relationships_count_tag(self, obj):
        return obj.relationships.count()

    relationships_count_tag.short_description = 'Count'
    readonly_fields = ('name', 'relationships_count_tag',)

class QueueAdmin(VersionedAdmin):
    pass
    list_display = ('mini_image_tag', 'screen_name_tag', 'name_tag', 'status_tag', 'user_id',)
    readonly_fields = ('mini_image_tag', 'screen_name_tag', 'name_tag', 'status_tag', 'user_id', 'status_id',)
    list_display_show_identity = False
    list_display_show_end_date = False
    list_display_show_start_date = True

class ProfileAdmin(admin.ModelAdmin):
    list_display = ('mini_image_tag', 'screen_name_tag', 'name_tag', 'socialuser',)
    fields = ('mini_image_tag', 'screen_name_tag', 'name_tag', 'socialuser', 'json', 'updated', 'normalavatar',)
    readonly_fields = ('mini_image_tag', 'screen_name_tag', 'name_tag', 'socialuser', 'json', 'updated', 'normalavatar',)

class ModerationAdmin(VersionedAdmin):
    pass
    list_display = ('moderator_mini_image_tag', 'moderator_screen_name_tag', 'moderated_mini_image_tag', 'moderated_screen_name_tag', 'status_tag',)
    fields = ('moderator_mini_image_tag',
              'moderator_screen_name_tag',
              'moderated_mini_image_tag',
              'moderated_screen_name_tag',
              'status_tag',
              'id',
              'identity',
              )
    readonly_fields = ('moderator_mini_image_tag', 'moderator_screen_name_tag', 'moderated_mini_image_tag', 'moderated_screen_name_tag', 'status_tag',)
    list_display_show_identity = True
    list_display_show_end_date = True
    list_display_show_start_date = True

class TwitterListAdmin(admin.ModelAdmin):
    list_display = (
        'list_id',
        'slug',
        'name',
        'twitter_description',
        'uid',
        'label',
        'local_description',
        'member_count_tag',
    )
    fields = (
        'list_id',
        'slug',
        'name',
        'twitter_description',
        'member_count_tag',
        'uid',
        'label',
        'local_description',
        'json',

    )
    readonly_fields = (
        'list_id',
        'slug',
        'name',
        'twitter_description',
        'json',
        'member_count_tag',
    )
    def member_count_tag(self, obj):
        member_count = obj.json.get("member_count", None)
        return member_count

    member_count_tag.short_description = 'Members'

admin.site.register(Category, CategoryAdmin)
admin.site.register(SocialUser, SocialUserAdmin)

admin.site.register(Profile, ProfileAdmin)

admin.site.register(Queue, QueueAdmin)
admin.site.register(Moderation, ModerationAdmin)
admin.site.register(TwitterList, TwitterListAdmin)<commit_msg>improve moderation category admin<commit_after>from django.contrib import admin
from django.db.models import Count
from versions.admin import VersionedAdmin
from django.utils.safestring import mark_safe
from django.urls import reverse
from .models import SocialUser, UserCategoryRelationship, Category, Profile, Queue, Moderation, TwitterList

class CategoryRelationshipInline(admin.TabularInline):
    model = UserCategoryRelationship
    extra = 10
    fk_name = 'category'
    raw_id_fields = ("social_user", "moderator",)

    def social_user_screen_name_tag(self, obj):
        screen_name = obj.social_user.profile.json.get("screen_name", None)
        return screen_name

    def moderator_screen_name_tag(self, obj):
        screen_name = obj.moderator.profile.json.get("screen_name", None)
        return screen_name

    moderator_screen_name_tag.short_description = 'Moderator screen name'
    social_user_screen_name_tag.short_description = 'Social User screen name'
    readonly_fields = (
        'social_user_screen_name_tag',
        'moderator_screen_name_tag',
    )

class UserRelationshipInline(admin.TabularInline):
    model = UserCategoryRelationship
    extra = 10
    fk_name = 'social_user'

class SocialUserAdmin(admin.ModelAdmin):
    inlines = (UserRelationshipInline,)
    list_display = ('user_id', 'screen_name_tag', 'mini_image_tag',  'name_tag',  'social_media', )
    fields = ('screen_name_tag', 'normal_image_tag', 'name_tag', 'user_id', 'social_media', 'category',)
    readonly_fields = ( 'screen_name_tag', 'normal_image_tag', 'name_tag', 'user_id', 'social_media', 'category',)
    search_fields = ('user_id', 'profile__json',)

class CategoryAdmin(admin.ModelAdmin):
    list_display = ('name', 'label', 'description', 'show_relationships_count',
                    'quickreply',)
    inlines = (CategoryRelationshipInline,)
    fields = ('name', 'label', 'quickreply', 'show_relationships_count', 'description')

    def get_queryset(self, request):
        qs = super(CategoryAdmin, self).get_queryset(request)
        return qs.annotate(relationships_count=Count('relationships'))

    def show_relationships_count(self, inst):
        return inst.relationships_count

    show_relationships_count.admin_order_field = '-relationships_count'
    show_relationships_count.short_description = 'Count'
    readonly_fields = ('name', 'show_relationships_count',)

class QueueAdmin(VersionedAdmin):
    pass
    list_display = ('mini_image_tag', 'screen_name_tag', 'name_tag', 'status_tag', 'user_id',)
    readonly_fields = ('mini_image_tag', 'screen_name_tag', 'name_tag', 'status_tag', 'user_id', 'status_id',)
    list_display_show_identity = False
    list_display_show_end_date = False
    list_display_show_start_date = True

class ProfileAdmin(admin.ModelAdmin):
    list_display = ('mini_image_tag', 'screen_name_tag', 'name_tag', 'socialuser',)
    fields = ('mini_image_tag', 'screen_name_tag', 'name_tag', 'socialuser', 'json', 'updated', 'normalavatar',)
    readonly_fields = ('mini_image_tag', 'screen_name_tag', 'name_tag', 'socialuser', 'json', 'updated', 'normalavatar',)

class ModerationAdmin(VersionedAdmin):
    pass
    list_display = ('moderator_mini_image_tag', 'moderator_screen_name_tag', 'moderated_mini_image_tag', 'moderated_screen_name_tag', 'status_tag',)
    fields = ('moderator_mini_image_tag',
              'moderator_screen_name_tag',
              'moderated_mini_image_tag',
              'moderated_screen_name_tag',
              'status_tag',
              'id',
              'identity',
              )
    readonly_fields = ('moderator_mini_image_tag', 'moderator_screen_name_tag', 'moderated_mini_image_tag', 'moderated_screen_name_tag', 'status_tag',)
    list_display_show_identity = True
    list_display_show_end_date = True
    list_display_show_start_date = True

class TwitterListAdmin(admin.ModelAdmin):
    list_display = (
        'list_id',
        'slug',
        'name',
        'twitter_description',
        'uid',
        'label',
        'local_description',
        'member_count_tag',
    )
    fields = (
        'list_id',
        'slug',
        'name',
        'twitter_description',
        'member_count_tag',
        'uid',
        'label',
        'local_description',
        'json',

    )
    readonly_fields = (
        'list_id',
        'slug',
        'name',
        'twitter_description',
        'json',
        'member_count_tag',
    )
    def member_count_tag(self, obj):
        member_count = obj.json.get("member_count", None)
        return member_count

    member_count_tag.short_description = 'Members'

admin.site.register(Category, CategoryAdmin)
admin.site.register(SocialUser, SocialUserAdmin)

admin.site.register(Profile, ProfileAdmin)

admin.site.register(Queue, QueueAdmin)
admin.site.register(Moderation, ModerationAdmin)
admin.site.register(TwitterList, TwitterListAdmin)<|endoftext|>


<commit_before>from unicodedata import category

from versions.exceptions import DeletionOfNonCurrentVersionError

from doctocnet.celery import app
from bot.tasks import handle_retweet
from .profile import twitterprofile
from moderation.lists.poll import poll_lists_members, create_update_lists
from django.conf import settings
from conversation.utils import screen_name

from django.db import transaction, DatabaseError
from dm.models import DirectMessage
from moderation.models import Moderation, SocialUser, Category, UserCategoryRelationship
import os

from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)

"""
@after_setup_task_logger.connect
def setup_loggers(logger, *args, **kwargs):
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    # SysLogHandler
    slh = logging.handlers.SysLogHandler()
    slh.setFormatter(formatter)
    logger.addHandler(slh)
    # StreamHandler
    str = logging.StreamHandler()
    str.setFormatter(formatter)
    logger.addHandler(str)

    logger.logLevel = settings.LOG_LEVEL
"""

@app.task
def handle_backup_lists():
    from .lists.poll import backup_lists
    backup_lists()

@app.task
def handle_poll_lists_members():
    poll_lists_members()

@app.task
def handle_create_update_lists():
    create_update_lists()

@app.task
def handle_create_update_profile(userid_int):
    from conversation.models import Tweetdj

    try:
        tweetdj_mi = Tweetdj.objects.filter(userid = userid_int).latest()
    except Tweetdj.DoesNotExist:
        tweetdj_mi = None

    if tweetdj_mi is None:
        from bot.bin.user import getuser
        userjson = getuser(userid_int)
    else:
        userjson = tweetdj_mi.json.get("user")

    logger.debug(userjson)
    twitterprofile(userjson)

@app.task
def handle_sendmoderationdm(mod_instance_id):
    from .moderate import quickreply
    from .models import Moderation
    from dm.api import senddm
    import time

    mod_mi = Moderation.objects.get(pk=mod_instance_id)
    qr = quickreply(mod_instance_id)
    logger.info(f"mod_mi.moderator.user_id: {mod_mi.moderator.user_id}")
    #logger.debug(f"moderator profile: {mod_mi.moderator.profile}")
    mod_mi.refresh_from_db()
    logger.info(f"mod_mi.queue.user_id {mod_mi.queue.user_id}")
    handle_create_update_profile.apply_async(args=(mod_mi.queue.user_id,))
    sn = screen_name(mod_mi.queue.status_id)
    status_id = mod_mi.queue.status_id
    dm_txt = ("Please moderate this user: @{screen_name} "
              "Tweet: https://twitter.com/{screen_name}/status/{status_id}"
              .format(screen_name=sn, status_id=status_id))

    senddm(dm_txt,
           user_id=mod_mi.moderator.user_id,
           return_json=True,
           quick_reply=qr)

@app.task
def poll_moderation_dm():
    def delete_moderation_queue(mod_mi):
        with transaction.atomic():
            try:
                mod_mi.queue.delete()
            except DeletionOfNonCurrentVersionError as e:
                logger.info(f"Queue {mod_mi.queue} was already deleted. %s" % e)
            try:
                mod_mi.delete()
            except DeletionOfNonCurrentVersionError as e:
                logger.info(f"Moderation instance {mod_mi} was already moderated. %s" % e)

    bot_id = settings.BOT_ID
    #current_moderations_uuid_str_lst = [str(mod.id) for mod in Moderation.objects.current.all()]
    #logger.info(f"current_moderations_uuid_str_lst: {current_moderations_uuid_str_lst}")
    dms = DirectMessage.objects\
        .filter(recipient_id=bot_id)\
        .filter(jsn__kwargs__message_create__message_data__quick_reply_response__metadata__startswith="moderation_")
    logger.info(f"all moderation direct messages answers: {len(dms)} {[dm.text + os.linesep for dm in dms]}")
    #dmsgs_current = [dmsg for dmsg in dmsgs if (dmsg.jsn['kwargs']['message_create']['message_data']['quick_reply_response']['metadata'].split("_")[1] in current_moderations_uuid_str_lst)]
    """
    dms_current = []
    for dm in dms:
        uid = dm.jsn['kwargs']['message_create']['message_data']['quick_reply_response']['metadata'].split("_")[1]
        logger.info(f"uid: {uid}")
        ok = uid in current_moderations_uuid_str_lst
        if ok:
            dms_current.append(dm)
    """

    #logger.info(f"current moderation direct messages answers: {len(dms_current)} {[dm.text + os.linesep for dm in dms_current]}")
    for dm in dms:
        metadata = dm.jsn["kwargs"]["message_create"]["message_data"]["quick_reply_response"]["metadata"]
        moderation_id, mod_cat_name = metadata[11:].split("_")
        logger.info(f"dm moderation_id: {moderation_id}, cat: {mod_cat_name}")
        #retrieve moderaton instance
        try:
            mod_mi = Moderation.objects.get(pk=moderation_id)
        except:
            Moderation.DoesNotExist as e:
            logger.error(f"Moderation object with id {moderation_id} "
                         f"does not exist. Error message: {e}")
            continue
        logger.info(f"mod_mi:{mod_mi}")
        # if mod_mi is not the current version, current_version() returns None
        # and it means this moderation was already done and we pass
        is_current = bool(Moderation.objects.current_version(mod_mi))
        if not is_current:
            logger.info(f"Moderation instance {mod_mi} was already moderated.")
            continue

        # determine id of moderated user
        su_id_int = mod_mi.queue.user_id

        # moderated SocialUser instance
        try:
            moderated_su_mi = SocialUser.objects.get(user_id=su_id_int)
            logger.info(f"moderated_su_mi:{moderated_su_mi}")
            logger.info(f"moderated_su_mi.profile:{moderated_su_mi.profile}")

        except SocialUser.DoesNotExist:
            moderated_su_mi = None

        # Category instance
        try:
            cat_mi = Category.objects.get(name=mod_cat_name)
            logger.info(f"cat_mi:{cat_mi}")
        except Category.DoesNotExist:
            cat_mi = None

        # moderator SocialUser instance
        try:
            moderator_su_mi = SocialUser.objects.get(user_id=dm.sender_id)
            logger.info(f"moderator_su_mi:{moderator_su_mi}")
        except SocialUser.DoesNotExist:
            moderator_su_mi = None

        #Check if relationship already exists
        is_already = None
        all_cat = moderated_su_mi.category.all()
        if all_cat:
            is_already = cat_mi in all_cat
            if is_already:
                logger.info(f"{moderated_su_mi} is already a {cat_mi}")
                delete_moderation_queue(mod_mi)

        # create UserCategoryRelationship
        ucr = None
        if (is_current and cat_mi and moderated_su_mi and not is_already):
            with transaction.atomic():
                try:
                    ucr = UserCategoryRelationship.objects.create(
                        social_user = moderated_su_mi,
                        category = cat_mi,
                        moderator = moderator_su_mi)
                except DatabaseError as e:
                    logger.error(e)

        # retweet status if user category is among authorized categories
        if ucr:
            logger.info(f"ucr creation successful: {ucr}")
            status_id = mod_mi.queue.status_id
            delete_moderation_queue(mod_mi)
            if cat_mi in Category.authorized.all():
                handle_retweet.apply_async(args=(status_id,))

        # TODO: create a cursor based on DM timestamp to avoid processing all DMs during each polling
        # TODO: check that the moderator is following the bot before attempting to send a DM<commit_msg>add exception handling for wrong Moderation IDs fix<commit_after>from unicodedata import category

from versions.exceptions import DeletionOfNonCurrentVersionError

from doctocnet.celery import app
from bot.tasks import handle_retweet
from .profile import twitterprofile
from moderation.lists.poll import poll_lists_members, create_update_lists
from django.conf import settings
from conversation.utils import screen_name

from django.db import transaction, DatabaseError
from dm.models import DirectMessage
from moderation.models import Moderation, SocialUser, Category, UserCategoryRelationship
import os

from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)

"""
@after_setup_task_logger.connect
def setup_loggers(logger, *args, **kwargs):
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    # SysLogHandler
    slh = logging.handlers.SysLogHandler()
    slh.setFormatter(formatter)
    logger.addHandler(slh)
    # StreamHandler
    str = logging.StreamHandler()
    str.setFormatter(formatter)
    logger.addHandler(str)

    logger.logLevel = settings.LOG_LEVEL
"""

@app.task
def handle_backup_lists():
    from .lists.poll import backup_lists
    backup_lists()

@app.task
def handle_poll_lists_members():
    poll_lists_members()

@app.task
def handle_create_update_lists():
    create_update_lists()

@app.task
def handle_create_update_profile(userid_int):
    from conversation.models import Tweetdj

    try:
        tweetdj_mi = Tweetdj.objects.filter(userid = userid_int).latest()
    except Tweetdj.DoesNotExist:
        tweetdj_mi = None

    if tweetdj_mi is None:
        from bot.bin.user import getuser
        userjson = getuser(userid_int)
    else:
        userjson = tweetdj_mi.json.get("user")

    logger.debug(userjson)
    twitterprofile(userjson)

@app.task
def handle_sendmoderationdm(mod_instance_id):
    from .moderate import quickreply
    from .models import Moderation
    from dm.api import senddm
    import time

    mod_mi = Moderation.objects.get(pk=mod_instance_id)
    qr = quickreply(mod_instance_id)
    logger.info(f"mod_mi.moderator.user_id: {mod_mi.moderator.user_id}")
    #logger.debug(f"moderator profile: {mod_mi.moderator.profile}")
    mod_mi.refresh_from_db()
    logger.info(f"mod_mi.queue.user_id {mod_mi.queue.user_id}")
    handle_create_update_profile.apply_async(args=(mod_mi.queue.user_id,))
    sn = screen_name(mod_mi.queue.status_id)
    status_id = mod_mi.queue.status_id
    dm_txt = ("Please moderate this user: @{screen_name} "
              "Tweet: https://twitter.com/{screen_name}/status/{status_id}"
              .format(screen_name=sn, status_id=status_id))

    senddm(dm_txt,
           user_id=mod_mi.moderator.user_id,
           return_json=True,
           quick_reply=qr)

@app.task
def poll_moderation_dm():
    def delete_moderation_queue(mod_mi):
        with transaction.atomic():
            try:
                mod_mi.queue.delete()
            except DeletionOfNonCurrentVersionError as e:
                logger.info(f"Queue {mod_mi.queue} was already deleted. %s" % e)
            try:
                mod_mi.delete()
            except DeletionOfNonCurrentVersionError as e:
                logger.info(f"Moderation instance {mod_mi} was already moderated. %s" % e)

    bot_id = settings.BOT_ID
    #current_moderations_uuid_str_lst = [str(mod.id) for mod in Moderation.objects.current.all()]
    #logger.info(f"current_moderations_uuid_str_lst: {current_moderations_uuid_str_lst}")
    dms = DirectMessage.objects\
        .filter(recipient_id=bot_id)\
        .filter(jsn__kwargs__message_create__message_data__quick_reply_response__metadata__startswith="moderation_")
    logger.info(f"all moderation direct messages answers: {len(dms)} {[dm.text + os.linesep for dm in dms]}")
    #dmsgs_current = [dmsg for dmsg in dmsgs if (dmsg.jsn['kwargs']['message_create']['message_data']['quick_reply_response']['metadata'].split("_")[1] in current_moderations_uuid_str_lst)]
    """
    dms_current = []
    for dm in dms:
        uid = dm.jsn['kwargs']['message_create']['message_data']['quick_reply_response']['metadata'].split("_")[1]
        logger.info(f"uid: {uid}")
        ok = uid in current_moderations_uuid_str_lst
        if ok:
            dms_current.append(dm)
    """

    #logger.info(f"current moderation direct messages answers: {len(dms_current)} {[dm.text + os.linesep for dm in dms_current]}")
    for dm in dms:
        metadata = dm.jsn["kwargs"]["message_create"]["message_data"]["quick_reply_response"]["metadata"]
        moderation_id, mod_cat_name = metadata[11:].split("_")
        logger.info(f"dm moderation_id: {moderation_id}, cat: {mod_cat_name}")
        #retrieve moderaton instance
        try:
            mod_mi = Moderation.objects.get(pk=moderation_id)
        except Moderation.DoesNotExist as e:
            logger.error(f"Moderation object with id {moderation_id} "
                         f"does not exist. Error message: {e}")
            continue
        logger.info(f"mod_mi:{mod_mi}")
        # if mod_mi is not the current version, current_version() returns None
        # and it means this moderation was already done and we pass
        is_current = bool(Moderation.objects.current_version(mod_mi))
        if not is_current:
            logger.info(f"Moderation instance {mod_mi} was already moderated.")
            continue

        # determine id of moderated user
        su_id_int = mod_mi.queue.user_id

        # moderated SocialUser instance
        try:
            moderated_su_mi = SocialUser.objects.get(user_id=su_id_int)
            logger.info(f"moderated_su_mi:{moderated_su_mi}")
            logger.info(f"moderated_su_mi.profile:{moderated_su_mi.profile}")

        except SocialUser.DoesNotExist:
            moderated_su_mi = None

        # Category instance
        try:
            cat_mi = Category.objects.get(name=mod_cat_name)
            logger.info(f"cat_mi:{cat_mi}")
        except Category.DoesNotExist:
            cat_mi = None

        # moderator SocialUser instance
        try:
            moderator_su_mi = SocialUser.objects.get(user_id=dm.sender_id)
            logger.info(f"moderator_su_mi:{moderator_su_mi}")
        except SocialUser.DoesNotExist:
            moderator_su_mi = None

        #Check if relationship already exists
        is_already = None
        all_cat = moderated_su_mi.category.all()
        if all_cat:
            is_already = cat_mi in all_cat
            if is_already:
                logger.info(f"{moderated_su_mi} is already a {cat_mi}")
                delete_moderation_queue(mod_mi)

        # create UserCategoryRelationship
        ucr = None
        if (is_current and cat_mi and moderated_su_mi and not is_already):
            with transaction.atomic():
                try:
                    ucr = UserCategoryRelationship.objects.create(
                        social_user = moderated_su_mi,
                        category = cat_mi,
                        moderator = moderator_su_mi)
                except DatabaseError as e:
                    logger.error(e)

        # retweet status if user category is among authorized categories
        if ucr:
            logger.info(f"ucr creation successful: {ucr}")
            status_id = mod_mi.queue.status_id
            delete_moderation_queue(mod_mi)
            if cat_mi in Category.authorized.all():
                handle_retweet.apply_async(args=(status_id,))

        # TODO: create a cursor based on DM timestamp to avoid processing all DMs during each polling
        # TODO: check that the moderator is following the bot before attempting to send a DM<|endoftext|>


<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

from hashlib import sha1

from flask import render_template

from indico.core.models.settings import EventSettingsProxy
from indico.modules.events.agreements.models.agreements import Agreement
from indico.util.caching import make_hashable
from indico.util.decorators import cached_classproperty, classproperty
from indico.util.string import return_ascii
from indico.web.flask.templating import get_overridable_template_name, get_template_module
from MaKaC.accessControl import AccessWrapper

class AgreementPersonInfo(object):
    def __init__(self, name=None, email=None, user=None, data=None):
        if user:
            if not name:
                name = user.getStraightFullName()
            if not email:
                email = user.getEmail()
        if not name:
            raise ValueError('name is missing')
        if not email:
            raise ValueError('email is missing')
        self.name = name
        self.email = email
        self.user = user
        self.data = data

    @return_ascii
    def __repr__(self):
        return '<AgreementPersonInfo({}, {}, {})>'.format(self.name, self.email, self.identifier)

    @property
    def identifier(self):
        data_string = None
        if self.data:
            data_string = '-'.join('{}={}'.format(k, make_hashable(v)) for k, v in sorted(self.data.viewitems()))
        identifier = '{}:{}'.format(self.email, data_string or None)
        return sha1(identifier).hexdigest()

class EmailPlaceholderBase(object):
    """Base class for agreement email placeholders"""
    required = False
    description = None

    @classmethod
    def render(cls, agreement):
        """Converts the placeholder to a string

        :param agreement: the `Agreement` object the email is being sent for
        """
        raise NotImplementedError

class AgreementDefinitionBase(object):
    """Base class for agreement definitions"""

    #: unique name of the agreement definition
    name = None
    #: readable name of the agreement definition
    title = None
    #: optional and short description of the agreement definition
    description = None
    #: url to obtain the paper version of the agreement form
    paper_form_url = None
    #: template of the agreement form - agreement definition name by default
    form_template_name = None
    #: template of the email body - emails/agreement_default_body.html by default
    email_body_template_name = None
    #: dict containing custom email placeholders
    email_placeholders = {}
    #: plugin containing this agreement definition - assigned automatically
    plugin = None

    #: default settings for an event
    default_event_settings = {'manager_notifications_enabled': False}

    @classproperty
    @classmethod
    def locator(cls):
        return {'definition': cls.name}

    @cached_classproperty
    @classmethod
    def event_settings(cls):
        return EventSettingsProxy('agreement_{}'.format(cls.name), cls.default_event_settings)

    @classmethod
    def can_access_api(cls, user, event):
        """Checks if a user can list the agreements for an event"""
        return event.canModify(AccessWrapper(user))

    @classmethod
    def extend_api_data(cls, event, person, agreement, data):  # pragma: no cover
        """Extends the data returned in the HTTP API

        :param event: the event
        :param person: the :class:`AgreementPersonInfo`
        :param agreement: the :class:`Agreement` if available
        :param data: a dict containing the default data for the agreement
        """
        pass

    @classmethod
    def get_email_body_template(cls, event, **kwargs):
        """Returns the template of the email body for this agreement definition"""
        template_name = cls.email_body_template_name or 'emails/agreement_default_body.html'
        template_path = get_overridable_template_name(template_name, cls.plugin, 'events/agreements/')
        return get_template_module(template_path, event=event)

    @classmethod
    def render_form(cls, agreement, form, **kwargs):
        template_name = cls.form_template_name or '{}.html'.format(cls.name.replace('-', '_'))
        template_path = get_overridable_template_name(template_name, cls.plugin, 'events/agreements/')
        return render_template(template_path, agreement=agreement, form=form, **kwargs)

    @classmethod
    def get_people(cls, event):
        """Returns a dictionary of :class:`AgreementPersonInfo` required to sign agreements"""
        people = cls.iter_people(event)
        if people is None:
            return {}
        return {p.identifier: p for p in people}

    @classmethod
    def get_people_not_notified(cls, event):
        """Returns a dictionary of :class:`AgreementPersonInfo` yet to be notified"""
        people = cls.get_people(event)
        sent_agreements = {a.identifier for a in Agreement.find(event_id=event.getId(), type=cls.name)}
        return {k: v for k, v in people.items() if v.identifier not in sent_agreements}

    @classmethod
    def get_stats_for_signed_agreements(cls, event):
        """Returns a digest of signed agreements on an event

        :param event: the event
        :return: (everybody_signed, num_accepted, num_rejected)
        """
        people = cls.get_people(event)
        identifiers = [p.identifier for p in people.itervalues()]
        query = Agreement.find(Agreement.event_id == event.getId(),
                               Agreement.type == cls.name,
                               Agreement.identifier.in_(identifiers))
        num_accepted = query.filter(Agreement.accepted).count()
        num_rejected = query.filter(Agreement.rejected).count()
        everybody_signed = len(people) == (num_accepted + num_rejected)
        return (everybody_signed, num_accepted, num_rejected)

    @classmethod
    def render_data(cls, event, data):  # pragma: no cover
        """Returns extra data to display in the agreement list

        If you want a column to be rendered as HTML, use a :class:`~markupsafe.Markup`
        object instead of a plain string.

        :param event: The event containing the agreements
        :param data: The data from the :class:`AgreementPersonInfo`
        :return: List of extra columns for a row
        """
        return None

    @classmethod
    def handle_accepted(cls, agreement):  # pragma: no cover
        """Handles logic on agreement accepted"""
        pass

    @classmethod
    def handle_rejected(cls, agreement):  # pragma: no cover
        """Handles logic on agreement rejected"""
        pass

    @classmethod
    def handle_reset(cls, agreement):  # pragma: no cover
        """Handles logic on agreement reset"""
        pass

    @classmethod
    def iter_people(cls, event):  # pragma: no cover
        """Yields :class:`AgreementPersonInfo` required to sign agreements"""
        raise NotImplementedError

    @classmethod
    def get_email_placeholders(cls):
        """Returns all available email placeholders"""
        from indico.modules.events.agreements.placeholders import PersonNamePlaceholder, AgreementLinkPlaceholder
        placeholders = {'person_name': PersonNamePlaceholder,
                        'agreement_link': AgreementLinkPlaceholder}
        placeholders.update(cls.email_placeholders or {})
        return placeholders
<commit_msg>Reorder methods<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

from hashlib import sha1

from flask import render_template

from indico.core.models.settings import EventSettingsProxy
from indico.modules.events.agreements.models.agreements import Agreement
from indico.util.caching import make_hashable
from indico.util.decorators import cached_classproperty, classproperty
from indico.util.string import return_ascii
from indico.web.flask.templating import get_overridable_template_name, get_template_module
from MaKaC.accessControl import AccessWrapper

class AgreementPersonInfo(object):
    def __init__(self, name=None, email=None, user=None, data=None):
        if user:
            if not name:
                name = user.getStraightFullName()
            if not email:
                email = user.getEmail()
        if not name:
            raise ValueError('name is missing')
        if not email:
            raise ValueError('email is missing')
        self.name = name
        self.email = email
        self.user = user
        self.data = data

    @return_ascii
    def __repr__(self):
        return '<AgreementPersonInfo({}, {}, {})>'.format(self.name, self.email, self.identifier)

    @property
    def identifier(self):
        data_string = None
        if self.data:
            data_string = '-'.join('{}={}'.format(k, make_hashable(v)) for k, v in sorted(self.data.viewitems()))
        identifier = '{}:{}'.format(self.email, data_string or None)
        return sha1(identifier).hexdigest()

class EmailPlaceholderBase(object):
    """Base class for agreement email placeholders"""
    required = False
    description = None

    @classmethod
    def render(cls, agreement):
        """Converts the placeholder to a string

        :param agreement: the `Agreement` object the email is being sent for
        """
        raise NotImplementedError

class AgreementDefinitionBase(object):
    """Base class for agreement definitions"""

    #: unique name of the agreement definition
    name = None
    #: readable name of the agreement definition
    title = None
    #: optional and short description of the agreement definition
    description = None
    #: url to obtain the paper version of the agreement form
    paper_form_url = None
    #: template of the agreement form - agreement definition name by default
    form_template_name = None
    #: template of the email body - emails/agreement_default_body.html by default
    email_body_template_name = None
    #: dict containing custom email placeholders
    email_placeholders = {}
    #: plugin containing this agreement definition - assigned automatically
    plugin = None

    #: default settings for an event
    default_event_settings = {'manager_notifications_enabled': False}

    @classproperty
    @classmethod
    def locator(cls):
        return {'definition': cls.name}

    @cached_classproperty
    @classmethod
    def event_settings(cls):
        return EventSettingsProxy('agreement_{}'.format(cls.name), cls.default_event_settings)

    @classmethod
    def can_access_api(cls, user, event):
        """Checks if a user can list the agreements for an event"""
        return event.canModify(AccessWrapper(user))

    @classmethod
    def extend_api_data(cls, event, person, agreement, data):  # pragma: no cover
        """Extends the data returned in the HTTP API

        :param event: the event
        :param person: the :class:`AgreementPersonInfo`
        :param agreement: the :class:`Agreement` if available
        :param data: a dict containing the default data for the agreement
        """
        pass

    @classmethod
    def get_email_body_template(cls, event, **kwargs):
        """Returns the template of the email body for this agreement definition"""
        template_name = cls.email_body_template_name or 'emails/agreement_default_body.html'
        template_path = get_overridable_template_name(template_name, cls.plugin, 'events/agreements/')
        return get_template_module(template_path, event=event)

    @classmethod
    def get_email_placeholders(cls):
        """Returns all available email placeholders"""
        from indico.modules.events.agreements.placeholders import PersonNamePlaceholder, AgreementLinkPlaceholder
        placeholders = {'person_name': PersonNamePlaceholder,
                        'agreement_link': AgreementLinkPlaceholder}
        placeholders.update(cls.email_placeholders or {})
        return placeholders

    @classmethod
    def get_people(cls, event):
        """Returns a dictionary of :class:`AgreementPersonInfo` required to sign agreements"""
        people = cls.iter_people(event)
        if people is None:
            return {}
        return {p.identifier: p for p in people}

    @classmethod
    def get_people_not_notified(cls, event):
        """Returns a dictionary of :class:`AgreementPersonInfo` yet to be notified"""
        people = cls.get_people(event)
        sent_agreements = {a.identifier for a in Agreement.find(event_id=event.getId(), type=cls.name)}
        return {k: v for k, v in people.items() if v.identifier not in sent_agreements}

    @classmethod
    def get_stats_for_signed_agreements(cls, event):
        """Returns a digest of signed agreements on an event

        :param event: the event
        :return: (everybody_signed, num_accepted, num_rejected)
        """
        people = cls.get_people(event)
        identifiers = [p.identifier for p in people.itervalues()]
        query = Agreement.find(Agreement.event_id == event.getId(),
                               Agreement.type == cls.name,
                               Agreement.identifier.in_(identifiers))
        num_accepted = query.filter(Agreement.accepted).count()
        num_rejected = query.filter(Agreement.rejected).count()
        everybody_signed = len(people) == (num_accepted + num_rejected)
        return (everybody_signed, num_accepted, num_rejected)

    @classmethod
    def render_form(cls, agreement, form, **kwargs):
        template_name = cls.form_template_name or '{}.html'.format(cls.name.replace('-', '_'))
        template_path = get_overridable_template_name(template_name, cls.plugin, 'events/agreements/')
        return render_template(template_path, agreement=agreement, form=form, **kwargs)

    @classmethod
    def render_data(cls, event, data):  # pragma: no cover
        """Returns extra data to display in the agreement list

        If you want a column to be rendered as HTML, use a :class:`~markupsafe.Markup`
        object instead of a plain string.

        :param event: The event containing the agreements
        :param data: The data from the :class:`AgreementPersonInfo`
        :return: List of extra columns for a row
        """
        return None

    @classmethod
    def handle_accepted(cls, agreement):  # pragma: no cover
        """Handles logic on agreement accepted"""
        pass

    @classmethod
    def handle_rejected(cls, agreement):  # pragma: no cover
        """Handles logic on agreement rejected"""
        pass

    @classmethod
    def handle_reset(cls, agreement):  # pragma: no cover
        """Handles logic on agreement reset"""
        pass

    @classmethod
    def iter_people(cls, event):  # pragma: no cover
        """Yields :class:`AgreementPersonInfo` required to sign agreements"""
        raise NotImplementedError
<|endoftext|>


<commit_before>import logging
from . import ModelChecker
from indra.statements import *

logger = logging.getLogger(__name__)

class PybelModelChecker(ModelChecker):
    """Check a PyBEL model against a set of INDRA statements.

    Parameters
    ----------
    model : pybel.BELGraph
        A Pybel model to check.
    statements : Optional[list[indra.statements.Statement]]
        A list of INDRA Statements to check the model against.
    do_sampling : bool
        Whether to use breadth-first search or weighted sampling to
        generate paths. Default is False (breadth-first search).
    seed : int
        Random seed for sampling (optional, default is None).
    """
    def __init__(self, model, statements=None, do_sampling=False, seed=None):
        super().__init__(model, statements, do_sampling, seed)

    def get_graph(self):
        """Convert a PyBELGraph to a graph with signed nodes."""
        # This import is done here rather than at the top level to avoid
        # making pybel an implicit dependency of the model checker
        from indra.assemblers.pybel.assembler import belgraph_to_signed_graph
        if self.graph:
            return self.graph
        signed_edges = belgraph_to_signed_graph(self.model)
        self.graph = self.signed_edges_to_signed_nodes(signed_edges)
        return self.graph

    def process_statement(self, stmt):
        # This import is done here rather than at the top level to avoid
        # making pybel an implicit dependency of the model checker
        from indra.assemblers.pybel.assembler import _get_agent_node
        # Check if this is one of the statement types that we can check
        if not isinstance(stmt, (Modification, RegulateAmount,
                                 RegulateActivity, Influence)):
            logger.info('Statement type %s not handled' %
                        stmt.__class__.__name__)
            return (None, None, 'STATEMENT_TYPE_NOT_HANDLED')
        # Get the polarity for the statement
        if isinstance(stmt, Modification):
            target_polarity = 1 if isinstance(stmt, RemoveModification) else 0
        elif isinstance(stmt, RegulateActivity):
            target_polarity = 0 if stmt.is_activation else 1
        elif isinstance(stmt, RegulateAmount):
            target_polarity = 1 if isinstance(stmt, DecreaseAmount) else 0
        elif isinstance(stmt, Influence):
            target_polarity = 1 if stmt.overall_polarity() == -1 else 0
        subj, obj = stmt.agent_list()
        if subj is not None:
            subj_node = (_get_agent_node(subj)[0], 0)
        if subj is None or subj_node not in self.graph.nodes:
            return (None, None, 'SUBJECT_NOT_FOUND')
        if obj is not None:
            obj_node = (_get_agent_node(obj)[0], target_polarity)
        if obj is None or obj_node not in self.graph.nodes:
            return (None, None, 'OBJECT_NOT_FOUND')
        return ([subj_node], [obj_node], None)

    def process_subject(self, subj):
        return [subj], None
<commit_msg>Preprocess pybel nodes<commit_after>import logging
from . import ModelChecker
from indra.statements import *
from copy import deepcopy

logger = logging.getLogger(__name__)

class PybelModelChecker(ModelChecker):
    """Check a PyBEL model against a set of INDRA statements.

    Parameters
    ----------
    model : pybel.BELGraph
        A Pybel model to check.
    statements : Optional[list[indra.statements.Statement]]
        A list of INDRA Statements to check the model against.
    do_sampling : bool
        Whether to use breadth-first search or weighted sampling to
        generate paths. Default is False (breadth-first search).
    seed : int
        Random seed for sampling (optional, default is None).
    """
    def __init__(self, model, statements=None, do_sampling=False, seed=None):
        super().__init__(model, statements, do_sampling, seed)

    def get_graph(self):
        """Convert a PyBELGraph to a graph with signed nodes."""
        # This import is done here rather than at the top level to avoid
        # making pybel an implicit dependency of the model checker
        from indra.assemblers.pybel.assembler import belgraph_to_signed_graph
        if self.graph:
            return self.graph
        signed_edges = belgraph_to_signed_graph(self.model)
        self.graph = self.signed_edges_to_signed_nodes(signed_edges)
        return self.graph

    def process_statement(self, stmt):
        # This import is done here rather than at the top level to avoid
        # making pybel an implicit dependency of the model checker
        from indra.assemblers.pybel.assembler import _get_agent_node
        # Check if this is one of the statement types that we can check
        if not isinstance(stmt, (Modification, RegulateAmount,
                                 RegulateActivity, Influence)):
            logger.info('Statement type %s not handled' %
                        stmt.__class__.__name__)
            return (None, None, 'STATEMENT_TYPE_NOT_HANDLED')
        subj, obj = stmt.agent_list()
        # Get the polarity for the statement
        if isinstance(stmt, Modification):
            target_polarity = 1 if isinstance(stmt, RemoveModification) else 0
            obj_agent = deepcopy(obj)
            obj_agent.mods.append(stmt._get_mod_condition())
            obj = obj_agent
        elif isinstance(stmt, RegulateActivity):
            target_polarity = 0 if stmt.is_activation else 1
            obj_agent = deepcopy(obj)
            obj_agent.activity = stmt._get_activity_condition()
            obj_agent.activity.is_active = True
            obj = obj_agent
        elif isinstance(stmt, RegulateAmount):
            target_polarity = 1 if isinstance(stmt, DecreaseAmount) else 0
        elif isinstance(stmt, Influence):
            target_polarity = 1 if stmt.overall_polarity() == -1 else 0
        if subj is not None:
            subj_node = (_get_agent_node(subj)[0], 0)
        if subj is None or subj_node not in self.graph.nodes:
            return (None, None, 'SUBJECT_NOT_FOUND')
        if obj is not None:
            obj_node = (_get_agent_node(obj)[0], target_polarity)
        if obj is None or obj_node not in self.graph.nodes:
            return (None, None, 'OBJECT_NOT_FOUND')
        return ([subj_node], [obj_node], None)

    def process_subject(self, subj):
        return [subj], None
<|endoftext|>


<commit_before>import csv
from indra.statements import Phosphorylation
from indra.sources.phosphoelm.api import PhosphoElmProcessor, \
    _get_json_from_entry_rows, _get_s3_client

columns = ['acc', 'sequence', 'position', 'code', 'pmids', 'kinases',
           'source', 'species', 'entry_date']
non_human_no_kinase = ['O08539',
                       'MAEMGSKG', '6', 'S', '17114649', '', 'HTP',
                       'Mus musculus', '2005-03-14 12:16:11.108314+01']
human_no_kinase = ['O14543',
                   'MVTHSKFPAAGMSRPLDTSLRLKTFSSKSEYQL', '31', 'Y',
                   '12783885', '', 'LTP', 'Homo sapiens',
                   '2006-10-17 12:06:48.271076+02']
human_kinase1 = ['O14543',
                 'FAKESEQUENCESRPLDTSLRLKTFSSKSEYQL',
                 '31', 'Y', '12783885', 'Lck', 'LTP', 'Homo sapiens',
                 '2006-10-17 12:06:48.16767+02']
human_kinase2 = ['O14746', 'FAKEPRCRAVRSLL', '12', 'S', '10224060',
                 'PKB_group', 'LTP', 'Homo sapiens',
                 '2004-12-31 00:00:00+01']

raw_data = [columns, non_human_no_kinase, human_no_kinase, human_kinase1,
            human_kinase2]

def test_download_from_s3():
    s3 = _get_s3_client()
    s3_bucket = 'bigmech'
    ppelm_s3_key = 'phosphoELM_data/phosphoELM_all_2015-04.dump'
    s3_obj = s3.get_object(Bucket=s3_bucket, Key=ppelm_s3_key)
    csv_reader = csv.reader(
        s3_obj['Body'].read().decode('utf8').splitlines(True),
        delimiter='\t'
    )
    ppelm_json = _get_json_from_entry_rows(csv_reader)
    assert ppelm_json

def test_json_processing():
    test_json = _get_json_from_entry_rows(iter(raw_data))
    assert len(test_json) == len(raw_data) - 1, len(test_json)
    assert set(columns) == set(test_json[0].keys())

def test_keep_empty():
    stmts = PhosphoElmProcessor(
        file_dump_json=_get_json_from_entry_rows(iter(raw_data)),
        keep_empty=True
    ).statements
    assert len(stmts) == 3
    assert all(isinstance(st, Phosphorylation) for st in stmts)
    assert all(st.evidence[0].source_api == 'phospho.ELM' for st in stmts)
    assert all(len(st.evidence[0].annotations['sequence']) > 0
               for st in stmts)

def test_not_empty():
    stmts = PhosphoElmProcessor(
        file_dump_json=_get_json_from_entry_rows(iter(raw_data))
    ).statements
    assert len(stmts) == 2

<commit_msg>Update tests<commit_after>import csv
from indra.statements import Phosphorylation
from indra.sources.phosphoelm.api import PhosphoElmProcessor, \
    _get_json_from_entry_rows, _get_s3_client

columns = ['acc', 'sequence', 'position', 'code', 'pmids', 'kinases',
           'source', 'species', 'entry_date']
non_human_no_kinase = ['O08539',
                       'MAEMGSKG', '6', 'S', '17114649', '', 'HTP',
                       'Mus musculus', '2005-03-14 12:16:11.108314+01']
human_no_kinase = ['O14543',
                   'MVTHSKFPAAGMSRPLDTSLRLKTFSSKSEYQL', '31', 'Y',
                   '12783885', '', 'LTP', 'Homo sapiens',
                   '2006-10-17 12:06:48.271076+02']
human_kinase1 = ['O14543',
                 'FAKESEQUENCESRPLDTSLRLKTFSSKSEYQL',
                 '31', 'Y', '12783885', 'Lck', 'LTP', 'Homo sapiens',
                 '2006-10-17 12:06:48.16767+02']
human_kinase2 = ['O14746', 'FAKEPRCRAVRSLL', '12', 'S', '10224060',
                 'PKB_group', 'LTP', 'Homo sapiens',
                 '2004-12-31 00:00:00+01']

raw_data = [columns, non_human_no_kinase, human_no_kinase, human_kinase1,
            human_kinase2]

def test_download_from_s3():
    s3 = _get_s3_client()
    s3_bucket = 'bigmech'
    ppelm_s3_key = 'phosphoELM_data/phosphoELM_all_2015-04.dump'
    s3_obj = s3.get_object(Bucket=s3_bucket, Key=ppelm_s3_key)
    csv_reader = csv.reader(
        s3_obj['Body'].read().decode('utf8').splitlines(True),
        delimiter='\t'
    )
    ppelm_json = _get_json_from_entry_rows(csv_reader)
    assert ppelm_json

def test_json_processing():
    test_json = _get_json_from_entry_rows(iter(raw_data))
    assert len(test_json) == len(raw_data) - 1, len(test_json)
    assert set(columns) == set(test_json[0].keys())

def test_keep_empty():
    pep = PhosphoElmProcessor(
        phosphoelm_data=_get_json_from_entry_rows(iter(raw_data))
    )
    pep.process_phosphorylations(True)
    stmts = pep.statements
    assert len(stmts) == 3
    assert all(isinstance(st, Phosphorylation) for st in stmts)
    assert all(st.evidence[0].source_api == 'phospho.ELM' for st in stmts)
    assert all(len(st.evidence[0].annotations['sequence']) > 0
               for st in stmts)

def test_not_empty():
    pep = PhosphoElmProcessor(
        phosphoelm_data=_get_json_from_entry_rows(iter(raw_data))
    )
    pep.process_phosphorylations()
    stmts = pep.statements
    assert len(stmts) == 2

<|endoftext|>


<commit_before># Copyright (c) 2015 - present Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

import os
import logging
import re
import util

MODULE_NAME = __name__
MODULE_DESCRIPTION = '''Run analysis of code built with a command like:
mvn [options] [task]

Analysis examples:
infer -- mvn build'''

def gen_instance(*args):
    return MavenCapture(*args)

# This creates an empty argparser for the module, which provides only
# description/usage information and no arguments.
create_argparser = util.base_argparser(MODULE_DESCRIPTION, MODULE_NAME)

class MavenCapture:
    def __init__(self, args, cmd):
        self.args = args
        logging.info(util.run_cmd_ignore_fail(['mvn', '-version']))
        # TODO: make the extraction of targets smarter
        self.build_cmd = ['mvn', '-X'] + cmd[1:]

    def get_infer_commands(self, verbose_output):
        file_pattern = r'\[DEBUG\] Stale source detected: ([^ ]*\.java)'
        options_pattern = '[DEBUG] Command line options:'
        source_roots_pattern = '[DEBUG] Source roots:'

        files_to_compile = []
        calls = []
        options_next = False
        source_roots_next = False
        for line in verbose_output:
            if options_next:
                #  line has format [Debug] <space separated options>
                javac_args = line.split(' ')[1:] + files_to_compile
                capture = jwlib.create_infer_command(self.args, javac_args)
                calls.append(capture)
                options_next = False
                files_to_compile = []

            elif source_roots_next:
                # line has format [Debug] <space separated directories>
                src_roots = line.split(' ')[1:]
                for src_root in src_roots:
                    for root, dirs, files in os.walk(src_root):
                        for name in files:
                            files_to_compile.append(os.path.join(root, name))
                source_roots_next = False

            elif options_pattern in line:
                #  Next line will have javac options to run
                options_next = True

            elif source_roots_pattern in line:
                # Next line will have directory containing files to compile
                source_roots_next = True

            else:
                found = re.match(file_pattern, line)
                if found:
                    files_to_compile.append(found.group(1))

        return calls

    def capture(self):
        cmds = self.get_infer_commands(util.get_build_output(self.build_cmd))
        clean_cmd = '%s clean' % self.build_cmd[0]
        return util.run_compilation_commands(cmds, clean_cmd)
<commit_msg>Fix missing import statement bug.<commit_after># Copyright (c) 2015 - present Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

import os
import logging
import re
import util

from inferlib import jwlib

MODULE_NAME = __name__
MODULE_DESCRIPTION = '''Run analysis of code built with a command like:
mvn [options] [task]

Analysis examples:
infer -- mvn build'''

def gen_instance(*args):
    return MavenCapture(*args)

# This creates an empty argparser for the module, which provides only
# description/usage information and no arguments.
create_argparser = util.base_argparser(MODULE_DESCRIPTION, MODULE_NAME)

class MavenCapture:
    def __init__(self, args, cmd):
        self.args = args
        logging.info(util.run_cmd_ignore_fail(['mvn', '-version']))
        # TODO: make the extraction of targets smarter
        self.build_cmd = ['mvn', '-X'] + cmd[1:]

    def get_infer_commands(self, verbose_output):
        file_pattern = r'\[DEBUG\] Stale source detected: ([^ ]*\.java)'
        options_pattern = '[DEBUG] Command line options:'
        source_roots_pattern = '[DEBUG] Source roots:'

        files_to_compile = []
        calls = []
        options_next = False
        source_roots_next = False
        for line in verbose_output:
            if options_next:
                #  line has format [Debug] <space separated options>
                javac_args = line.split(' ')[1:] + files_to_compile
                capture = jwlib.create_infer_command(self.args, javac_args)
                calls.append(capture)
                options_next = False
                files_to_compile = []

            elif source_roots_next:
                # line has format [Debug] <space separated directories>
                src_roots = line.split(' ')[1:]
                for src_root in src_roots:
                    for root, dirs, files in os.walk(src_root):
                        for name in files:
                            files_to_compile.append(os.path.join(root, name))
                source_roots_next = False

            elif options_pattern in line:
                #  Next line will have javac options to run
                options_next = True

            elif source_roots_pattern in line:
                # Next line will have directory containing files to compile
                source_roots_next = True

            else:
                found = re.match(file_pattern, line)
                if found:
                    files_to_compile.append(found.group(1))

        return calls

    def capture(self):
        cmds = self.get_infer_commands(util.get_build_output(self.build_cmd))
        clean_cmd = '%s clean' % self.build_cmd[0]
        return util.run_compilation_commands(cmds, clean_cmd)
<|endoftext|>