<commit_before>import ast
import logging
import re
import subprocess
import time
import types

import monasca_agent.collector.checks as checks

log = logging.getLogger(__name__)

class SwiftRecon(checks.AgentCheck):

    GAUGES = [
        "storage.used",
        "storage.free",
        "storage.capacity",
        "md5.ring.matched",
        "md5.ring.not_matched",
        "md5.ring.errors",
        "md5.ring.all",
        "md5.swiftconf.matched",
        "md5.swiftconf.not_matched",
        "md5.swiftconf.errors",
        "md5.swiftconf.all",
        "containers.updater_sweep_time",
        "objects.updater_sweep_time",
        "objects.replication.duration",
        "objects.replication.age",
        "containers.replication.duration",
        "containers.replication.age",
        "accounts.replication.duration",
        "accounts.replication.age",
        "drives.audit.errors",
        "drives.unmounted",
        "objects.quarantined",
        "containers.quarantined",
        "accounts.quarantined",
    ]

    def prepare(self):
        self.diskusage = {}
        self.md5 = {}
        self.replication_times = {}
        self.quarantined_things = {}

    def swift_recon(self, *params):
        executable = 'swift-recon'
        if 'swift_recon' in self.init_config:
            executable = self.init_config['swift_recon']

        cmd = " ".join((executable, " ".join(params)))
        pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
                                universal_newlines=True)
        out = "".join(pipe.stdout.readlines())
        return out

    def swift_recon_json(self, *params):
        # call swift-recon in verbose mode
        output = self.swift_recon("-v", *params)

        # look for verbose output containing the raw JSON documents received
        # from the storage nodes
        result = {}
        for line in output.splitlines():
            m = re.match(r'^-> https?://([a-zA-Z0-9-.]+)\S*\s(.*)', line)
            if m:
                log.debug("Output from swift-recon {0}: {1}".format(" ".join(params), line))
                hostname, data_str = m.group(1), m.group(2)
                try:
                    result[hostname] = ast.literal_eval(data_str)
                except (ValueError, SyntaxError):
                    log.error("swift-recon {0} erroneous for node {1}: {2}".format(params, hostname, data_str))
                    continue
        return result

    ############################################################################
    # helpers for parsing output of swift-recon

    def get_diskusage(self):
        if not self.diskusage:
            result = self.swift_recon("--diskusage")
            for line in result.splitlines():
                m = re.match(r'Disk usage: space (\w+): (\d+) of (\d+)', line)
                if m:
                    if m.group(1) == 'used':
                        self.diskusage['used'] = long(m.group(2))
                        self.diskusage['capacity'] = long(m.group(3))
                    elif m.group(1) == 'free':
                        self.diskusage['free'] = long(m.group(2))
                else:
                    continue
        return self.diskusage

    def get_md5(self):
        if not self.md5:
            result = self.swift_recon("--md5")
            kind = 'undef'
            for line in result.splitlines():
                m = re.match(r'.* Checking ([\.a-zA-Z0-9_]+) md5sum', line)
                if m:
                    kind = m.group(1).replace(".", "")
                    self.md5[kind] = {}
                    continue
                pattern = (r"(\d+)/(\d+) hosts matched, (\d+) error\[s\] "
                           "while checking hosts")
                m = re.match(pattern, line)
                if m:
                    self.md5[kind]['matched'] = int(m.group(1))
                    self.md5[kind]['not_matched'] = (int(m.group(2)) -
                                                     int(m.group(1)))
                    self.md5[kind]['errors'] = int(m.group(3))
                    self.md5[kind]['all'] = (self.md5[kind]['matched'] +
                                             self.md5[kind]['not_matched'] +
                                             self.md5[kind]['errors'])
                else:
                    continue
        return self.md5

    def get_updater_sweeps(self, server_type):
        data = self.swift_recon_json(server_type, "--updater")
        key = server_type + "_updater_sweep"
        return {hostname: data[hostname][key] for hostname in data}

    def get_replication(self, server_type):
        if server_type in self.replication_times:
            return self.replication_times[server_type]

        # https://twitter.com/stefanmajewsky/status/654660805607096322
        if server_type == "object":
            duration_key, last_key = "object_replication_time", "object_replication_last"
        else:
            duration_key, last_key = "replication_time", "replication_last"

        current_timestamp = time.time()
        data = self.swift_recon_json(server_type, "--replication")
        result = {"duration": {}, "age": {}}
        for hostname in data:
            result["duration"][hostname] = data[hostname][duration_key]
            # convert timestamp of last completion into an age
            result["age"][hostname] = current_timestamp - data[hostname][last_key]

        self.replication_times[server_type] = result
        return result

    def get_unmounted_drives(self):
        data = self.swift_recon_json("--unmounted")
        return {hostname: len(data[hostname]) for hostname in data}

    def get_drive_audit_errors(self):
        data = self.swift_recon_json("--driveaudit")
        return {hostname: data[hostname]['drive_audit_errors'] for hostname in data}

    def get_quarantined(self):
        if not self.quarantined_things:
            data = self.swift_recon_json("--quarantined")
            for hostname in data:
                for key in data[hostname]:
                    self.quarantined_things.setdefault(key, {})[hostname] = data[hostname][key]
        return self.quarantined_things

    ############################################################################
    # helpers for accessing parsed output of swift-recon

    def storage(self, value):
        self.get_diskusage()
        if value in self.diskusage:
            return self.diskusage[value]

    def consistency(self, kind, value):
        self.get_md5()
        if kind in self.md5:
            if value in self.md5[kind]:
                return self.md5[kind][value]

    def replication(self, server_type, value):
        data = self.get_replication(server_type)
        if value in data:
            return data[value]

    def quarantined(self, value):
        data = self.get_quarantined()
        if value in data:
            return data[value]

    ############################################################################
    # one method for each exposed metric

    # storage capacity

    def storage_free(self):
        return self.storage('free')

    def storage_used(self):
        return self.storage('used')

    def storage_capacity(self):
        return self.storage('capacity')

    # configuration consistency

    def md5_ring_matched(self):
        return self.consistency('ring', 'matched')

    def md5_ring_not_matched(self):
        return self.consistency('ring', 'not_matched')

    def md5_ring_errors(self):
        return self.consistency('ring', 'errors')

    def md5_ring_all(self):
        return self.consistency('ring', 'all')

    def md5_swiftconf_matched(self):
        return self.consistency('swiftconf', 'matched')

    def md5_swiftconf_not_matched(self):
        return self.consistency('swiftconf', 'not_matched')

    def md5_swiftconf_errors(self):
        return self.consistency('swiftconf', 'errors')

    def md5_swiftconf_all(self):
        return self.consistency('swiftconf', 'all')

    # eventual consistency 1: updater sweep timings

    def containers_updater_sweep_time(self):
        return self.get_updater_sweeps('container')

    def objects_updater_sweep_time(self):
        return self.get_updater_sweeps('object')

    # eventual consistency 2: replication timings

    def objects_replication_duration(self):
        return self.replication("object", "duration")

    def objects_replication_age(self):
        return self.replication("object", "age")

    def containers_replication_duration(self):
        return self.replication("container", "duration")

    def containers_replication_age(self):
        return self.replication("container", "age")

    def accounts_replication_duration(self):
        return self.replication("account", "duration")

    def accounts_replication_age(self):
        return self.replication("account", "age")

    # cluster health

    def drives_unmounted(self):
        return self.get_unmounted_drives()

    def drives_audit_errors(self):
        return self.get_drive_audit_errors()

    def objects_quarantined(self):
        return self.quarantined("objects")

    def containers_quarantined(self):
        return self.quarantined("containers")

    def accounts_quarantined(self):
        return self.quarantined("accounts")

    ############################################################################
    # collect metrics

    def check(self, instance):
        self.prepare()
        dimensions = self._set_dimensions(None, instance)

        for metric in self.GAUGES:
            log.debug("Checking metric {0}".format(metric))

            value = eval("self." + metric.replace(".", "_") + "()")

            if metric.startswith('storage'):
                metric = metric + '_bytes'
            metric = self.normalize(metric.lower(), 'swift.cluster')

            # value may be a dictionary with values by storage node...
            if isinstance(value, dict):
                for hostname in value:
                    self.submit_gauge(metric, hostname, value[hostname], dimensions)
            else:
                self.submit_gauge(metric, None, value, dimensions)

    def submit_gauge(self, metric, hostname, value, dimensions):
        if value is None:
            value = 0 ## fail-safe
        assert(type(value) in (types.IntType, types.LongType,
                               types.FloatType))

        if hostname:
            dim = dimensions.copy()
            dim["storage_node"] = hostname
            log.debug("Sending {0}={1} for storage node {2}".format(metric, value, hostname))
        else:
            dim = dimensions
            log.debug("Sending {0}={1}".format(metric, value))

        self.gauge(metric, value, dimensions=dim)
<commit_msg>Update swift_recon.py<commit_after>import ast
import logging
import re
import subprocess
import time
import types

import monasca_agent.collector.checks as checks

log = logging.getLogger(__name__)

class SwiftRecon(checks.AgentCheck):

    GAUGES = [
        "storage.used",
        "storage.free",
        "storage.capacity",
        "storage.used_percent",
        "md5.ring.matched",
        "md5.ring.not_matched",
        "md5.ring.errors",
        "md5.ring.all",
        "md5.swiftconf.matched",
        "md5.swiftconf.not_matched",
        "md5.swiftconf.errors",
        "md5.swiftconf.all",
        "containers.updater_sweep_time",
        "objects.updater_sweep_time",
        "objects.replication.duration",
        "objects.replication.age",
        "containers.replication.duration",
        "containers.replication.age",
        "accounts.replication.duration",
        "accounts.replication.age",
        "drives.audit.errors",
        "drives.unmounted",
        "objects.quarantined",
        "containers.quarantined",
        "accounts.quarantined",
    ]

    def prepare(self):
        self.diskusage = {}
        self.md5 = {}
        self.replication_times = {}
        self.quarantined_things = {}

    def swift_recon(self, *params):
        executable = 'swift-recon'
        if 'swift_recon' in self.init_config:
            executable = self.init_config['swift_recon']

        cmd = " ".join((executable, " ".join(params)))
        pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
                                universal_newlines=True)
        out = "".join(pipe.stdout.readlines())
        return out

    def swift_recon_json(self, *params):
        # call swift-recon in verbose mode
        output = self.swift_recon("-v", *params)

        # look for verbose output containing the raw JSON documents received
        # from the storage nodes
        result = {}
        for line in output.splitlines():
            m = re.match(r'^-> https?://([a-zA-Z0-9-.]+)\S*\s(.*)', line)
            if m:
                log.debug("Output from swift-recon {0}: {1}".format(" ".join(params), line))
                hostname, data_str = m.group(1), m.group(2)
                try:
                    result[hostname] = ast.literal_eval(data_str)
                except (ValueError, SyntaxError):
                    log.error("swift-recon {0} erroneous for node {1}: {2}".format(params, hostname, data_str))
                    continue
        return result

    ############################################################################
    # helpers for parsing output of swift-recon

    def get_diskusage(self):
        if not self.diskusage:
            result = self.swift_recon("--diskusage")
            for line in result.splitlines():
                m = re.match(r'Disk usage: space (\w+): (\d+) of (\d+)', line)
                if m:
                    if m.group(1) == 'used':
                        self.diskusage['used'] = long(m.group(2))
                        self.diskusage['capacity'] = long(m.group(3))
                    elif m.group(1) == 'free':
                        self.diskusage['free'] = long(m.group(2))
                else:
                    continue
        return self.diskusage

    def get_md5(self):
        if not self.md5:
            result = self.swift_recon("--md5")
            kind = 'undef'
            for line in result.splitlines():
                m = re.match(r'.* Checking ([\.a-zA-Z0-9_]+) md5sum', line)
                if m:
                    kind = m.group(1).replace(".", "")
                    self.md5[kind] = {}
                    continue
                pattern = (r"(\d+)/(\d+) hosts matched, (\d+) error\[s\] "
                           "while checking hosts")
                m = re.match(pattern, line)
                if m:
                    self.md5[kind]['matched'] = int(m.group(1))
                    self.md5[kind]['not_matched'] = (int(m.group(2)) -
                                                     int(m.group(1)))
                    self.md5[kind]['errors'] = int(m.group(3))
                    self.md5[kind]['all'] = (self.md5[kind]['matched'] +
                                             self.md5[kind]['not_matched'] +
                                             self.md5[kind]['errors'])
                else:
                    continue
        return self.md5

    def get_updater_sweeps(self, server_type):
        data = self.swift_recon_json(server_type, "--updater")
        key = server_type + "_updater_sweep"
        return {hostname: data[hostname][key] for hostname in data}

    def get_replication(self, server_type):
        if server_type in self.replication_times:
            return self.replication_times[server_type]

        # https://twitter.com/stefanmajewsky/status/654660805607096322
        if server_type == "object":
            duration_key, last_key = "object_replication_time", "object_replication_last"
        else:
            duration_key, last_key = "replication_time", "replication_last"

        current_timestamp = time.time()
        data = self.swift_recon_json(server_type, "--replication")
        result = {"duration": {}, "age": {}}
        for hostname in data:
            result["duration"][hostname] = data[hostname][duration_key]
            # convert timestamp of last completion into an age
            result["age"][hostname] = current_timestamp - data[hostname][last_key]

        self.replication_times[server_type] = result
        return result

    def get_unmounted_drives(self):
        data = self.swift_recon_json("--unmounted")
        return {hostname: len(data[hostname]) for hostname in data}

    def get_drive_audit_errors(self):
        data = self.swift_recon_json("--driveaudit")
        return {hostname: data[hostname]['drive_audit_errors'] for hostname in data}

    def get_quarantined(self):
        if not self.quarantined_things:
            data = self.swift_recon_json("--quarantined")
            for hostname in data:
                for key in data[hostname]:
                    self.quarantined_things.setdefault(key, {})[hostname] = data[hostname][key]
        return self.quarantined_things

    ############################################################################
    # helpers for accessing parsed output of swift-recon

    def storage(self, value):
        self.get_diskusage()
        if value in self.diskusage:
            return self.diskusage[value]

    def consistency(self, kind, value):
        self.get_md5()
        if kind in self.md5:
            if value in self.md5[kind]:
                return self.md5[kind][value]

    def replication(self, server_type, value):
        data = self.get_replication(server_type)
        if value in data:
            return data[value]

    def quarantined(self, value):
        data = self.get_quarantined()
        if value in data:
            return data[value]

    ############################################################################
    # one method for each exposed metric

    # storage capacity

    def storage_free(self):
        return self.storage('free')

    def storage_used(self):
        return self.storage('used')

    def storage_capacity(self):
        return self.storage('capacity')

    def storage_used_percent:
        return self.storage['used'] / self.storage['capacity']

    # configuration consistency

    def md5_ring_matched(self):
        return self.consistency('ring', 'matched')

    def md5_ring_not_matched(self):
        return self.consistency('ring', 'not_matched')

    def md5_ring_errors(self):
        return self.consistency('ring', 'errors')

    def md5_ring_all(self):
        return self.consistency('ring', 'all')

    def md5_swiftconf_matched(self):
        return self.consistency('swiftconf', 'matched')

    def md5_swiftconf_not_matched(self):
        return self.consistency('swiftconf', 'not_matched')

    def md5_swiftconf_errors(self):
        return self.consistency('swiftconf', 'errors')

    def md5_swiftconf_all(self):
        return self.consistency('swiftconf', 'all')

    # eventual consistency 1: updater sweep timings

    def containers_updater_sweep_time(self):
        return self.get_updater_sweeps('container')

    def objects_updater_sweep_time(self):
        return self.get_updater_sweeps('object')

    # eventual consistency 2: replication timings

    def objects_replication_duration(self):
        return self.replication("object", "duration")

    def objects_replication_age(self):
        return self.replication("object", "age")

    def containers_replication_duration(self):
        return self.replication("container", "duration")

    def containers_replication_age(self):
        return self.replication("container", "age")

    def accounts_replication_duration(self):
        return self.replication("account", "duration")

    def accounts_replication_age(self):
        return self.replication("account", "age")

    # cluster health

    def drives_unmounted(self):
        return self.get_unmounted_drives()

    def drives_audit_errors(self):
        return self.get_drive_audit_errors()

    def objects_quarantined(self):
        return self.quarantined("objects")

    def containers_quarantined(self):
        return self.quarantined("containers")

    def accounts_quarantined(self):
        return self.quarantined("accounts")

    ############################################################################
    # collect metrics

    def check(self, instance):
        self.prepare()
        dimensions = self._set_dimensions(None, instance)

        for metric in self.GAUGES:
            log.debug("Checking metric {0}".format(metric))

            value = eval("self." + metric.replace(".", "_") + "()")

            if metric.startswith('storage') and not metric.endswith('percent'):
                metric = metric + '_bytes'

            metric = self.normalize(metric.lower(), 'swift.cluster')

            # value may be a dictionary with values by storage node...
            if isinstance(value, dict):
                for hostname in value:
                    self.submit_gauge(metric, hostname, value[hostname], dimensions)
            else:
                self.submit_gauge(metric, None, value, dimensions)

    def submit_gauge(self, metric, hostname, value, dimensions):
        if value is None:
            value = 0 ## fail-safe
        assert(type(value) in (types.IntType, types.LongType,
                               types.FloatType))

        if hostname:
            dim = dimensions.copy()
            dim["storage_node"] = hostname
            log.debug("Sending {0}={1} for storage node {2}".format(metric, value, hostname))
        else:
            dim = dimensions
            log.debug("Sending {0}={1}".format(metric, value))

        self.gauge(metric, value, dimensions=dim)
<|endoftext|>


<commit_before>#-------------------------------------------------------------------------------
# Name:         sfp_portscan_basic
# Purpose:      SpiderFoot plug-in for performing a basic port scan of IP
#               addresses identified.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     20/02/2013
# Copyright:   (c) Steve Micallef 2013
# Licence:     GPL
#-------------------------------------------------------------------------------

import sys
import re
import socket
import random
import threading
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

# SpiderFoot standard lib (must be initialized in setup)
sf = None

class sfp_portscan_basic(SpiderFootPlugin):
    """Port Scanner:Scans for commonly open TCP ports on Internet-facing systems."""

    # Default options
    opts = {
                            # Commonly used ports on external-facing systems
        'ports':            [ '21', '22', '23', '25', '53', '79', '80', '81', '88', '110','111',
                            '113', '119', '123', '137', '138', '139', '143', '161', '179',
                            '389', '443', '445', '465', '512', '513', '514', '515', '631', '636',
                            '990', '992', '993', '995', '1080', '8080', '8888', '9000' ],
        'timeout':          15,
        'maxthreads':       10,
        'randomize':        True
    }

    # Option descriptions
    optdescs = {
        'maxthreads':   "Number of ports to try to open simultaneously (number of threads to spawn at once.)",
        'ports':    "The TCP ports to scan. Prefix with an '@' to iterate through a file containing ports to try (one per line), e.g. @C:\ports.txt or @/home/bob/ports.txt. Or supply a URL to load the list from there.",
        'timeout':  "Seconds before giving up on a port.",
        'randomize':    "Randomize the order of ports scanned."
    }

    # Target
    baseDomain = None
    results = dict()
    portlist = list()
    portResults = dict()

    def setup(self, sfc, target, userOpts=dict()):
        global sf

        sf = sfc
        self.baseDomain = target
        self.results = dict()

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

        if self.opts['ports'][0].startswith("http://") or \
            self.opts['ports'][0].startswith("https://") or \
            self.opts['ports'][0].startswith("@"):
            self.portlist = sf.optValueToData(self.opts['ports'][0])
        else:
            self.portlist = self.opts['ports']

        # Convert to integers
        self.portlist = [int(x) for x in self.portlist]

        if self.opts['randomize']:
            random.shuffle(self.portlist)

    # What events is this module interested in for input
    def watchedEvents(self):
        return ['IP_ADDRESS']

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return [ "TCP_PORT_OPEN", "TCP_PORT_OPEN_BANNER" ]

    def tryPort(self, ip, port):
        try:
            sock = socket.create_connection((ip, port), self.opts['timeout'])
            self.portResults[ip + ":" + str(port)] = True
        except Exception as e:
            self.portResults[ip + ":" + str(port)] = False
            return

        # If the port was open, see what we can read
        try:
            self.portResults[ip + ":" + str(port)] = sock.recv(4096)
        except Exception as e:
            sock.close()
            return

    def tryPortWrapper(self, ip, portList):
        self.portResults = dict()
        running = True
        i = 0
        t = []

        # Spawn threads for scanning
        while i < len(portList):
            sf.info("Spawning thread to check port: " + str(portList[i]) + " on " + ip)
            t.append(threading.Thread(name='sfp_portscan_basic_' + str(portList[i]),
                target=self.tryPort, args=(ip, portList[i])))
            t[i].start()
            i += 1

        # Block until all threads are finished
        while running:
            found = False
            for rt in threading.enumerate():
                if rt.name.startswith("sfp_portscan_basic_"):
                    found = True

            if not found:
                running = False

        return self.portResults

    # Generate TCP_PORT_OPEN_BANNER event
    def sendEvent(self, resArray, srcEvent):
        for cp in resArray:
            if resArray[cp]:
                sf.info("TCP Port " + cp + " found to be OPEN.")
                (addr, port) = cp.split(":")
                evt = SpiderFootEvent("TCP_PORT_OPEN", port, self.__name__, srcEvent)
                self.notifyListeners(evt)
                if resArray[cp] != "" and resArray[cp] != True:
                    bevt = SpiderFootEvent("TCP_PORT_OPEN_BANNER", resArray[cp],
                        self.__name__, evt)
                    self.notifyListeners(bevt)

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        # Don't look up stuff twice
        if self.results.has_key(eventData):
            sf.debug("Skipping " + eventData + " as already scanned.")
            return None
        else:
            self.results[eventData] = True

        i = 0
        portArr = []
        for port in self.portlist:
            if self.checkForStop():
                return None

            if i < self.opts['maxthreads']:
                portArr.append(port)
                i += 1
            else:
                self.sendEvent(self.tryPortWrapper(eventData, portArr), event)
                i = 1
                portArr = []
                portArr.append(port)

        # Scan whatever is remaining
        self.sendEvent(self.tryPortWrapper(eventData, portArr), event)

        return None

# End of sfp_portscan_basic class
<commit_msg>Port scanning timeouts.<commit_after>#-------------------------------------------------------------------------------
# Name:         sfp_portscan_basic
# Purpose:      SpiderFoot plug-in for performing a basic port scan of IP
#               addresses identified.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     20/02/2013
# Copyright:   (c) Steve Micallef 2013
# Licence:     GPL
#-------------------------------------------------------------------------------

import sys
import re
import socket
import random
import threading
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

# SpiderFoot standard lib (must be initialized in setup)
sf = None

class sfp_portscan_basic(SpiderFootPlugin):
    """Port Scanner:Scans for commonly open TCP ports on Internet-facing systems."""

    # Default options
    opts = {
                            # Commonly used ports on external-facing systems
        'ports':            [ '21', '22', '23', '25', '53', '79', '80', '81', '88', '110','111',
                            '113', '119', '123', '137', '138', '139', '143', '161', '179',
                            '389', '443', '445', '465', '512', '513', '514', '515', '631', '636',
                            '990', '992', '993', '995', '1080', '8080', '8888', '9000' ],
        'timeout':          15,
        'maxthreads':       10,
        'randomize':        True
    }

    # Option descriptions
    optdescs = {
        'maxthreads':   "Number of ports to try to open simultaneously (number of threads to spawn at once.)",
        'ports':    "The TCP ports to scan. Prefix with an '@' to iterate through a file containing ports to try (one per line), e.g. @C:\ports.txt or @/home/bob/ports.txt. Or supply a URL to load the list from there.",
        'timeout':  "Seconds before giving up on a port.",
        'randomize':    "Randomize the order of ports scanned."
    }

    # Target
    baseDomain = None
    results = dict()
    portlist = list()
    portResults = dict()

    def setup(self, sfc, target, userOpts=dict()):
        global sf

        sf = sfc
        self.baseDomain = target
        self.results = dict()

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

        if self.opts['ports'][0].startswith("http://") or \
            self.opts['ports'][0].startswith("https://") or \
            self.opts['ports'][0].startswith("@"):
            self.portlist = sf.optValueToData(self.opts['ports'][0])
        else:
            self.portlist = self.opts['ports']

        # Convert to integers
        self.portlist = [int(x) for x in self.portlist]

        if self.opts['randomize']:
            random.shuffle(self.portlist)

    # What events is this module interested in for input
    def watchedEvents(self):
        return ['IP_ADDRESS']

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return [ "TCP_PORT_OPEN", "TCP_PORT_OPEN_BANNER" ]

    def tryPort(self, ip, port):
        try:
            sock = socket.create_connection((ip, port), self.opts['timeout'])
            sock.settimeout(self.opts['timeout'])
            self.portResults[ip + ":" + str(port)] = True
        except Exception as e:
            self.portResults[ip + ":" + str(port)] = False
            return

        # If the port was open, see what we can read
        try:
            self.portResults[ip + ":" + str(port)] = sock.recv(4096)
        except Exception as e:
            sock.close()
            return

        sock.close()

    def tryPortWrapper(self, ip, portList):
        self.portResults = dict()
        running = True
        i = 0
        t = []

        # Spawn threads for scanning
        while i < len(portList):
            sf.info("Spawning thread to check port: " + str(portList[i]) + " on " + ip)
            t.append(threading.Thread(name='sfp_portscan_basic_' + str(portList[i]),
                target=self.tryPort, args=(ip, portList[i])))
            t[i].start()
            i += 1

        # Block until all threads are finished
        while running:
            found = False
            for rt in threading.enumerate():
                if rt.name.startswith("sfp_portscan_basic_"):
                    found = True

            if not found:
                running = False

        return self.portResults

    # Generate TCP_PORT_OPEN_BANNER event
    def sendEvent(self, resArray, srcEvent):
        for cp in resArray:
            if resArray[cp]:
                sf.info("TCP Port " + cp + " found to be OPEN.")
                (addr, port) = cp.split(":")
                evt = SpiderFootEvent("TCP_PORT_OPEN", port, self.__name__, srcEvent)
                self.notifyListeners(evt)
                if resArray[cp] != "" and resArray[cp] != True:
                    bevt = SpiderFootEvent("TCP_PORT_OPEN_BANNER", resArray[cp],
                        self.__name__, evt)
                    self.notifyListeners(bevt)

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        # Don't look up stuff twice
        if self.results.has_key(eventData):
            sf.debug("Skipping " + eventData + " as already scanned.")
            return None
        else:
            self.results[eventData] = True

        i = 0
        portArr = []
        for port in self.portlist:
            if self.checkForStop():
                return None

            if i < self.opts['maxthreads']:
                portArr.append(port)
                i += 1
            else:
                self.sendEvent(self.tryPortWrapper(eventData, portArr), event)
                i = 1
                portArr = []
                portArr.append(port)

        # Scan whatever is remaining
        self.sendEvent(self.tryPortWrapper(eventData, portArr), event)

        return None

# End of sfp_portscan_basic class
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Basic utilities for the Flask app

These are basic utilities necessary for the Flask app which are
disjoint from any blueprint.
"""
from babel import Locale
from flask import request, session
from flask_login import AnonymousUserMixin, LoginManager
from werkzeug.routing import IntegerConverter as BaseIntegerConverter

from sipa.babel import possible_locales
from sipa.model import backends

login_manager = LoginManager()

class IntegerConverter(BaseIntegerConverter):
    """IntegerConverter supporting negative values

    This is a Modification of the standard IntegerConverter which does
    not support negative values.  See the corresponding `werkzeug
    documentation
    <http://werkzeug.pocoo.org/docs/0.10/routing/#werkzeug.routing.IntegerConverter>`_.
    """
    regex = r'-?\d+'

@login_manager.user_loader
def load_user(username):
    """Loads a User object from/into the session at every request
    """
    if request.blueprint == "documents" or request.endpoint == "static":
        return AnonymousUserMixin()

    dormitory = backends.get_dormitory(session.get('dormitory', None))
    if dormitory:
        return dormitory.datasource.user_class.get(username)
    else:
        return AnonymousUserMixin()

def babel_selector():
    """Select a suitable locale

    Try to pick a locale from the following ordered sources:

        1. The ``lang`` query argument, if available.

        2. The current ``session`` cookie

        3. The best match to the ``accept-language`` HTTP header

    In any case, the session will be updated to the determined value.

    :returns: The determined locale

    :rtype: str
    """
    if 'locale' in request.args and Locale(
            request.args['locale']) in possible_locales():
        session['locale'] = request.args['locale']
    elif not session.get('locale'):
        langs = []
        for lang in possible_locales():
            langs.append(lang.language)
        session['locale'] = request.accept_languages.best_match(langs)

    return session.get('locale')
<commit_msg>Use Babel's negotiate_locale to pick language<commit_after># -*- coding: utf-8 -*-
"""
Basic utilities for the Flask app

These are basic utilities necessary for the Flask app which are
disjoint from any blueprint.
"""
from babel import Locale, negotiate_locale
from flask import request, session
from flask_login import AnonymousUserMixin, LoginManager
from werkzeug.routing import IntegerConverter as BaseIntegerConverter

from sipa.babel import possible_locales
from sipa.model import backends

login_manager = LoginManager()

class IntegerConverter(BaseIntegerConverter):
    """IntegerConverter supporting negative values

    This is a Modification of the standard IntegerConverter which does
    not support negative values.  See the corresponding `werkzeug
    documentation
    <http://werkzeug.pocoo.org/docs/0.10/routing/#werkzeug.routing.IntegerConverter>`_.
    """
    regex = r'-?\d+'

@login_manager.user_loader
def load_user(username):
    """Loads a User object from/into the session at every request
    """
    if request.blueprint == "documents" or request.endpoint == "static":
        return AnonymousUserMixin()

    dormitory = backends.get_dormitory(session.get('dormitory', None))
    if dormitory:
        return dormitory.datasource.user_class.get(username)
    else:
        return AnonymousUserMixin()

def babel_selector():
    """Select a suitable locale

    Try to pick a locale from the following ordered sources:

        1. The ``lang`` query argument, if available.

        2. The current ``session`` cookie

        3. The best match to the ``accept-language`` HTTP header

    In any case, the session will be updated to the determined value.

    :returns: The determined locale

    :rtype: str
    """
    if 'locale' in request.args and Locale(
            request.args['locale']) in possible_locales():
        session['locale'] = request.args['locale']
    elif not session.get('locale'):
        session['locale'] = negotiate_locale(
            request.accept_languages.values(),
            list(map(str, possible_locales())), sep='-')

    return session.get('locale')
<|endoftext|>


<commit_before>
"""
Demonstration text for program's output:
Do you wish to encrypt or decrypt a message?  encrypt
Enter your message:  The sky above the port was the color of television, tuned to a dead channel.
Enter the key number (1-26)  13
Your translated text is:  Gur fxl nobir gur cbeg jnf gur pbybe bs gryrivfvba, gharq gb n qrnq punaary.
"""

def main():
	#TODO: test that program cycles correctly

	while(True):
		en_de_crypt = raw_input('Do you wish to "encrypt" or "decrypt" a message? ')
		if (en_de_crypt == 'encrypt') or (en_de_crypt =='Encrypt') or (en_de_crypt =='decrypt') or (en_de_crypt =='Decrypt'):
		    break
		else:
			print "you need to write 'encrypt' or 'decrypt'"
			continue

	while(True):
		inputMessage = raw_input('Enter your message: ')
		if inputMessage is None:
			print "you need to enter a message"
			continue
		else:
			break

	while(True):
		keySize = raw_input('Enter the key number (1-26): ')
		keySize = int(keySize)
		if keySize is None:
			print "you need to enter a key number"
			continue
		elif type(keySize) is int:
			if keySize > 26 or keySize < 1:
				print "enter a value between 1 and 26"
				continue
			else:
			    logic = crypt(en_de_crypt)
			    break

		else:
			print "Enter an integer key number (1-26)"
			pass

	if logic is True:
	    cipher= encryption(inputMessage, keySize)
	    print cipher
	elif logic is False:
	    plain= decryption(inputMessage, keySize)
	    print plain
	else:
	    pass
        #throw exception

def crypt(cryptInput):
	#returns boolean value, true if encrypt, false if decrypt

	truth = False

	if (cryptInput == 'encrypt') or (cryptInput =='Encrypt'):
	    truth = True
	    return truth
	elif (cryptInput == 'decrypt') or (cryptInput =='Decrypt'):
	    truth = False
	    return truth
	else:
		pass
		#throw exception

def encryption(plaintext, size):
	#takes in message string and the key size, and runs caesar cipher

	cipher = list(plaintext)

	for i in range(len(cipher)):
		cipher[i] = chr(ord(cipher[i]) + size)

	output= ''.join(cipher)

	return output

def decryption(ciphertext, size):
	#takes in the ciphertext and key size, performs reverse caeasar cipher

	plaintext = list(ciphertext)

	for i in range(len(plaintext)):
		plaintext[i] = chr(ord(plaintext[i]) - size)

	output= ''.join(plaintext)

	return output

if __name__ == '__main__':
    main()
<commit_msg>updated TODO<commit_after>
"""
Demonstration text for program's output:
Do you wish to encrypt or decrypt a message?  encrypt
Enter your message:  The sky above the port was the color of television, tuned to a dead channel.
Enter the key number (1-26)  13
Your translated text is:  Gur fxl nobir gur cbeg jnf gur pbybe bs gryrivfvba, gharq gb n qrnq punaary.
"""

def main():

	while(True):
		en_de_crypt = raw_input('Do you wish to "encrypt" or "decrypt" a message? ')
		if (en_de_crypt == 'encrypt') or (en_de_crypt =='Encrypt') or (en_de_crypt =='decrypt') or (en_de_crypt =='Decrypt'):
		    break
		else:
			print "you need to write 'encrypt' or 'decrypt'"
			continue

	while(True):
		inputMessage = raw_input('Enter your message: ')
		if inputMessage is None:
			print "you need to enter a message"
			continue
		else:
			break

	while(True):
		keySize = raw_input('Enter the key number (1-26): ')
		keySize = int(keySize)
		if keySize is None:
			print "you need to enter a key number"
			continue
		elif type(keySize) is int:
			if keySize > 26 or keySize < 1:
				print "enter a value between 1 and 26"
				continue
			else:
			    logic = crypt(en_de_crypt)
			    break

		else:
			print "Enter an integer key number (1-26)"
			pass

	if logic is True:
	    cipher= encryption(inputMessage, keySize)
	    print cipher
	elif logic is False:
	    plain= decryption(inputMessage, keySize)
	    print plain
	else:
	    pass
        #throw exception

def crypt(cryptInput):
	#returns boolean value, true if encrypt, false if decrypt

	truth = False

	if (cryptInput == 'encrypt') or (cryptInput =='Encrypt'):
	    truth = True
	    return truth
	elif (cryptInput == 'decrypt') or (cryptInput =='Decrypt'):
	    truth = False
	    return truth
	else:
		pass
		#throw exception

def encryption(plaintext, size):
	#takes in message string and the key size, and runs caesar cipher
	#TODO: keeps spaces as spaces
	#TODO: only phase shifts within the 26 letter alphabet

	cipher = list(plaintext)

	for i in range(len(cipher)):
		cipher[i] = chr(ord(cipher[i]) + size)

	output= ''.join(cipher)

	return output

def decryption(ciphertext, size):
	#takes in the ciphertext and key size, performs reverse caeasar cipher
	#TODO: keeps spaces as spaces
	#TODO: only shifts within 26 letter alphabet

	plaintext = list(ciphertext)

	for i in range(len(plaintext)):
		plaintext[i] = chr(ord(plaintext[i]) - size)

	output= ''.join(plaintext)

	return output

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from . import Store, Field, BooleanField, DateField, IntField, EmbeddedStoreField
from rpc import RpcMixin, rpc_call
from models import Price
from document import Document
from voucher import Voucher
from django.utils.translation import ugettext as _
from money import Money

class Item(Store):
    """Base item class, to be extended for specific purposes."""
    _total = EmbeddedStoreField(target='positionTotal', store_class=Price)
    _net_total = EmbeddedStoreField(target='positionNetTotal', store_class=Price)
    _tax_total = EmbeddedStoreField(target='positionTaxTotal', store_class=Price)
    _undiscounted_total = EmbeddedStoreField(target='undiscountedPositionTotal', store_class=Price)

    @property
    def total(self):
        return Money(amount=self._total.amount, currency=self._total.currency)

    @property
    def net_total(self):
        return Money(amount=self._net_total.amount, currency=self._net_total.currency)

    @property
    def tax_total(self):
        return Money(amount=self._tax_total.amount, currency=self._tax_total.currency)

    @property
    def undiscounted_total(self):
        return Money(amount=self._undiscounted_total.amount, currency=self._undiscounted_total.currency)

    @classmethod
    def add_to_basket(cls, token, basket_id, item_id):
        """A convenience shortcut to provide nicer API."""
        return cls.set_basket_quantity(token, basket_id, item_id, 1)

    @classmethod
    def remove_from_basket(cls, token, basket_id, item_id):
        """A convenience shortcut to provide nicer API."""
        return cls.set_basket_quantity(token, basket_id, item_id, 0)

class DocumentItem(Item, RpcMixin):
    interface = 'WSDocMgmt'
    document = EmbeddedStoreField(target='item', store_class=Document)

    @classmethod
    @rpc_call
    def set_basket_quantity(cls, token, basket_id, doc_id, quantity):
        """Reaktor `WSDocMgmt.changeDocumentBasketPosition` call.
        If `quantity` is 0, then removes the document from the basket.
        If `quantity` is not 0, then adds the document into the basket.
        Returns None.
        """
        return cls.signature(method='changeDocumentBasketPosition', args=[token, basket_id, doc_id, quantity])

class VoucherItem(Item, RpcMixin):
    voucher = EmbeddedStoreField(target='item', store_class=Voucher)

    @classmethod
    @rpc_call
    def set_basket_quantity(cls, token, basket_id, voucher_code, quantity):
        """This method should call the similar to `changeDocumentBasketPosition` reaktor method, but for Voucher.
        """
        raise NotImplemented

def item_factory(data=None):
    """Item factory to get properly typed basket items."""
    if data is None:
        return Item()
    item_type = data.get('itemType', 'NONE')
    if item_type == 'DOCUMENT':
        return DocumentItem(data)
    elif item_type == 'VOUCHER':
        return VoucherItem(data)
    else:
        raise ValueError('Basket item type not supported: %s' % item_type)

class CheckoutProperties(Store):
    clear_failed_preauth = BooleanField(target='clearFailedAuthorization')
    clear_preauth = BooleanField(target='clearPreAuthorization')
    use_preauth = BooleanField(target='usePreAuthorization')
    recurring_payment = BooleanField(target='requestedRecurringPayment')
    affiliate_id = Field(target='affiliateID')
    external_transaction_id = Field(target='externalTransactionID')

class Basket(Store, RpcMixin):
    interface = 'WSShopMgmt'

    # txtr to adyen mapping of payment methods;
    # see Enum com.bookpac.server.shop.payment.PaymentMethod and adyen's Integration Manual pp 12+13 for the names
    PAYMENT_METHODS = { "CREDITCARD" : ["visa", "mc"], "ELV":["elv"] }

    PAYMENT_TRANSLATIONS = {
        # See: SKINSTXTRCOM-2165
        'CREDITCARD': _('Credit Card'), # "CREDITCARD should stay to be backward compatible, my son." --Stephan Noske
        'AMEX-SSL': _('American Express'),
        'VISA-SSL': _('Visa'),
        'VISA_COMMERCIAL_CREDIT-SSL': _('Visa'),
        'VISA_CREDIT-SSL': _('Visa'),
        'visa': _('Visa'), # lowercase key, comes from Adyen
        'ECMC-SSL': _('MasterCard'),
        'ECMC_COMMERCIAL_CREDIT-SSL': _('MasterCard'),
        'ECMC_CREDIT-SSL': _('MasterCard'),
        'mc': _('MasterCard'), # lowercase key, comes from Adyen
        'PAYPAL_EXPRESS': _('PayPal'),
        'ELV': _('Direct Debit'),
        'ELV-SSL': _('Direct Debit'),
        'DINERS': _("Diner's Club"),
        'DINERS-SSL': _("Diner's Club"),
    }

    # not sure if this is used
    # NOTE (Iurii Kudriavtsev): this is not a complete fields definition
    # class PaymentProperty(Store):
    #     merchant_account = Field(target='merchantAccount')
    #     merchant_ref = Field(target='merchantReference')

    class AppliedVoucherItem(Store):
        voucher = EmbeddedStoreField(target='voucher', store_class=Voucher)
        discount = EmbeddedStoreField(target='discountAmount', store_class=Price)

    class PaymentForm(Store):
        form = Field(target='com.bookpac.server.shop.payment.paymentform')
        recurring = Field(target='com.bookpac.server.shop.payment.paymentform.recurring')
        worecurring = Field(target='com.bookpac.server.shop.payment.paymentform.worecurring')

    id = Field(target='ID')
    checked_out = BooleanField(target='checkedOut')
    creation_date = DateField(target='creationTime')
    modification_date = DateField(target='modificationTime')
    country = Field(target='country')
    _total = EmbeddedStoreField(target='total', store_class=Price)
    _net_total = EmbeddedStoreField(target='netTotal', store_class=Price)
    _tax_total = EmbeddedStoreField(target='taxTotal', store_class=Price)
    _undiscounted_total = EmbeddedStoreField(target='undiscountedTotal', store_class=Price)
    # payment_props = EmbeddedStoreField(target='paymentProperties', store_class=PaymentProperty)
    payment_forms = EmbeddedStoreField(target='paymentForms', store_class=PaymentForm)
    items = EmbeddedStoreField(target='positions', store_class=item_factory, is_array=True)
    authorized_payment_methods = Field(target='authorizedPaymentMethods')
    applied_vouchers = EmbeddedStoreField(target='voucherApplications', store_class=AppliedVoucherItem, is_array=True)

    @property
    def total(self):
        return Money(amount=self._total.amount, currency=self._total.currency)

    @property
    def net_total(self):
        return Money(amount=self._net_total.amount, currency=self._net_total.currency)

    @property
    def tax_total(self):
        return Money(amount=self._tax_total.amount, currency=self._tax_total.currency)

    @property
    def undiscounted_total(self):
        return Money(amount=self._undiscounted_total.amount, currency=self._undiscounted_total.currency)

    @property
    def document_items(self):
        """A property that allows to iterate over the document items.
        Returns generator.
        """
        for item in self.items:
            if isinstance(item, DocumentItem):
                yield item

    @property
    def voucher_items(self):
        """A property that allows to iterate over the voucher items.
        Returns generator.
        """
        for item in self.items:
            if isinstance(item, VoucherItem):
                yield item

    @property
    def documents(self):
        """A property that allows to iterate over the documents.
        Returns generator.
        """
        for item in self.document_items:
            yield item.document

    @property
    def vouchers(self):
        """A property that allows to iterate over the vouchers.
        Returns generator.
        """
        for item in self.voucher_items:
            yield item.voucher

    def is_authorized_for(self, payment_method):
        """Check whether the basket is authorized for the given payment_method.
        """
        if payment_method in self.PAYMENT_METHODS.get("CREDITCARD") and hasattr(self, 'authorized_payment_methods'):
            return "CREDITCARD" in self.authorized_payment_methods
        elif payment_method in self.PAYMENT_METHODS.get("ELV") and hasattr(self, 'authorized_payment_methods'):
            return "ELV" in self.authorized_payment_methods
        else:
            return False

    def translate_authorized_payment_methods(self):
        """Returns a list of human-readable authorized payment methods.
        """
        payment_methods = []
        if hasattr(self, 'authorized_payment_methods'):
            for payment_method in self.authorized_payment_methods:
                if payment_method in self.PAYMENT_TRANSLATIONS:
                    payment_methods.append(self.PAYMENT_TRANSLATIONS[payment_method])
                else:
                    payment_methods.append(payment_method)
        return payment_methods

    @classmethod
    @rpc_call
    def get_by_id(cls, token, basket_id):
        return cls.signature(method='getBasket', args=[token, basket_id])

    @classmethod
    @rpc_call
    def checkout(cls, token, basket_id, method_preference, checkout_props):
        # checkoutBasket is deprecated
        return cls.signature(method='checkoutBasket', data_converter=CheckoutResult, args=[token, basket_id, method_preference, checkout_props.data])

    @classmethod
    @rpc_call
    def create(cls, token, marker=None):
        return cls.signature(method='getNewBasket', args=[token, marker])

class CheckoutResult(Store):

    class ItemProperty(Store):
        document_id = Field(target='com.bookpac.server.shop.fulfillment.document.newid')

    basket = EmbeddedStoreField(target='checkedOutBasket', store_class=Basket)
    code = Field(target='resultCode')
    failed_item_index = IntField(target='failingPosition')
    item_properties = EmbeddedStoreField(target='positionResultProperties', store_class=ItemProperty, is_array=True)
    receipt_id = Field(target='receiptIdentifier')
    transaction_id = Field(target='transactionID')
<commit_msg>SKINSTXTRCOM-4226 | Improved usage of `CheckoutProperties` class.<commit_after>from . import Store, Field, BooleanField, DateField, IntField, EmbeddedStoreField
from rpc import RpcMixin, rpc_call
from models import Price
from document import Document
from voucher import Voucher
from django.utils.translation import ugettext as _
from money import Money

class Item(Store):
    """Base item class, to be extended for specific purposes."""
    _total = EmbeddedStoreField(target='positionTotal', store_class=Price)
    _net_total = EmbeddedStoreField(target='positionNetTotal', store_class=Price)
    _tax_total = EmbeddedStoreField(target='positionTaxTotal', store_class=Price)
    _undiscounted_total = EmbeddedStoreField(target='undiscountedPositionTotal', store_class=Price)

    @property
    def total(self):
        return Money(amount=self._total.amount, currency=self._total.currency)

    @property
    def net_total(self):
        return Money(amount=self._net_total.amount, currency=self._net_total.currency)

    @property
    def tax_total(self):
        return Money(amount=self._tax_total.amount, currency=self._tax_total.currency)

    @property
    def undiscounted_total(self):
        return Money(amount=self._undiscounted_total.amount, currency=self._undiscounted_total.currency)

    @classmethod
    def add_to_basket(cls, token, basket_id, item_id):
        """A convenience shortcut to provide nicer API."""
        return cls.set_basket_quantity(token, basket_id, item_id, 1)

    @classmethod
    def remove_from_basket(cls, token, basket_id, item_id):
        """A convenience shortcut to provide nicer API."""
        return cls.set_basket_quantity(token, basket_id, item_id, 0)

class DocumentItem(Item, RpcMixin):
    interface = 'WSDocMgmt'
    document = EmbeddedStoreField(target='item', store_class=Document)

    @classmethod
    @rpc_call
    def set_basket_quantity(cls, token, basket_id, doc_id, quantity):
        """Reaktor `WSDocMgmt.changeDocumentBasketPosition` call.
        If `quantity` is 0, then removes the document from the basket.
        If `quantity` is not 0, then adds the document into the basket.
        Returns None.
        """
        return cls.signature(method='changeDocumentBasketPosition', args=[token, basket_id, doc_id, quantity])

class VoucherItem(Item, RpcMixin):
    voucher = EmbeddedStoreField(target='item', store_class=Voucher)

    @classmethod
    @rpc_call
    def set_basket_quantity(cls, token, basket_id, voucher_code, quantity):
        """This method should call the similar to `changeDocumentBasketPosition` reaktor method, but for Voucher.
        """
        raise NotImplemented

def item_factory(data=None):
    """Item factory to get properly typed basket items."""
    if data is None:
        return Item()
    item_type = data.get('itemType', 'NONE')
    if item_type == 'DOCUMENT':
        return DocumentItem(data)
    elif item_type == 'VOUCHER':
        return VoucherItem(data)
    else:
        raise ValueError('Basket item type not supported: %s' % item_type)

class CheckoutProperties(Store):
    clear_failed_preauth = BooleanField(target='clearFailedAuthorization')
    clear_preauth = BooleanField(target='clearPreAuthorization')
    use_preauth = BooleanField(target='usePreAuthorization')
    recurring_payment = BooleanField(target='requestedRecurringPayment')
    affiliate_id = Field(target='affiliateID')
    external_transaction_id = Field(target='externalTransactionID')

class Basket(Store, RpcMixin):
    interface = 'WSShopMgmt'

    # txtr to adyen mapping of payment methods;
    # see Enum com.bookpac.server.shop.payment.PaymentMethod and adyen's Integration Manual pp 12+13 for the names
    PAYMENT_METHODS = { "CREDITCARD" : ["visa", "mc"], "ELV":["elv"] }

    PAYMENT_TRANSLATIONS = {
        # See: SKINSTXTRCOM-2165
        'CREDITCARD': _('Credit Card'), # "CREDITCARD should stay to be backward compatible, my son." --Stephan Noske
        'AMEX-SSL': _('American Express'),
        'VISA-SSL': _('Visa'),
        'VISA_COMMERCIAL_CREDIT-SSL': _('Visa'),
        'VISA_CREDIT-SSL': _('Visa'),
        'visa': _('Visa'), # lowercase key, comes from Adyen
        'ECMC-SSL': _('MasterCard'),
        'ECMC_COMMERCIAL_CREDIT-SSL': _('MasterCard'),
        'ECMC_CREDIT-SSL': _('MasterCard'),
        'mc': _('MasterCard'), # lowercase key, comes from Adyen
        'PAYPAL_EXPRESS': _('PayPal'),
        'ELV': _('Direct Debit'),
        'ELV-SSL': _('Direct Debit'),
        'DINERS': _("Diner's Club"),
        'DINERS-SSL': _("Diner's Club"),
    }

    # not sure if this is used
    # NOTE (Iurii Kudriavtsev): this is not a complete fields definition
    # class PaymentProperty(Store):
    #     merchant_account = Field(target='merchantAccount')
    #     merchant_ref = Field(target='merchantReference')

    class AppliedVoucherItem(Store):
        voucher = EmbeddedStoreField(target='voucher', store_class=Voucher)
        discount = EmbeddedStoreField(target='discountAmount', store_class=Price)

    class PaymentForm(Store):
        form = Field(target='com.bookpac.server.shop.payment.paymentform')
        recurring = Field(target='com.bookpac.server.shop.payment.paymentform.recurring')
        worecurring = Field(target='com.bookpac.server.shop.payment.paymentform.worecurring')

    id = Field(target='ID')
    checked_out = BooleanField(target='checkedOut')
    creation_date = DateField(target='creationTime')
    modification_date = DateField(target='modificationTime')
    country = Field(target='country')
    _total = EmbeddedStoreField(target='total', store_class=Price)
    _net_total = EmbeddedStoreField(target='netTotal', store_class=Price)
    _tax_total = EmbeddedStoreField(target='taxTotal', store_class=Price)
    _undiscounted_total = EmbeddedStoreField(target='undiscountedTotal', store_class=Price)
    # payment_props = EmbeddedStoreField(target='paymentProperties', store_class=PaymentProperty)
    payment_forms = EmbeddedStoreField(target='paymentForms', store_class=PaymentForm)
    items = EmbeddedStoreField(target='positions', store_class=item_factory, is_array=True)
    authorized_payment_methods = Field(target='authorizedPaymentMethods')
    applied_vouchers = EmbeddedStoreField(target='voucherApplications', store_class=AppliedVoucherItem, is_array=True)

    @property
    def total(self):
        return Money(amount=self._total.amount, currency=self._total.currency)

    @property
    def net_total(self):
        return Money(amount=self._net_total.amount, currency=self._net_total.currency)

    @property
    def tax_total(self):
        return Money(amount=self._tax_total.amount, currency=self._tax_total.currency)

    @property
    def undiscounted_total(self):
        return Money(amount=self._undiscounted_total.amount, currency=self._undiscounted_total.currency)

    @property
    def document_items(self):
        """A property that allows to iterate over the document items.
        Returns generator.
        """
        for item in self.items:
            if isinstance(item, DocumentItem):
                yield item

    @property
    def voucher_items(self):
        """A property that allows to iterate over the voucher items.
        Returns generator.
        """
        for item in self.items:
            if isinstance(item, VoucherItem):
                yield item

    @property
    def documents(self):
        """A property that allows to iterate over the documents.
        Returns generator.
        """
        for item in self.document_items:
            yield item.document

    @property
    def vouchers(self):
        """A property that allows to iterate over the vouchers.
        Returns generator.
        """
        for item in self.voucher_items:
            yield item.voucher

    def is_authorized_for(self, payment_method):
        """Check whether the basket is authorized for the given payment_method.
        """
        if payment_method in self.PAYMENT_METHODS.get("CREDITCARD") and hasattr(self, 'authorized_payment_methods'):
            return "CREDITCARD" in self.authorized_payment_methods
        elif payment_method in self.PAYMENT_METHODS.get("ELV") and hasattr(self, 'authorized_payment_methods'):
            return "ELV" in self.authorized_payment_methods
        else:
            return False

    def translate_authorized_payment_methods(self):
        """Returns a list of human-readable authorized payment methods.
        """
        payment_methods = []
        if hasattr(self, 'authorized_payment_methods'):
            for payment_method in self.authorized_payment_methods:
                if payment_method in self.PAYMENT_TRANSLATIONS:
                    payment_methods.append(self.PAYMENT_TRANSLATIONS[payment_method])
                else:
                    payment_methods.append(payment_method)
        return payment_methods

    @classmethod
    @rpc_call
    def get_by_id(cls, token, basket_id):
        return cls.signature(method='getBasket', args=[token, basket_id])

    @classmethod
    @rpc_call
    def checkout(cls, token, basket_id, method_preference, checkout_props):
        # checkoutBasket is deprecated
        return cls.signature(method='checkoutBasket', data_converter=CheckoutResult, args=[token, basket_id, method_preference, checkout_props])

    @classmethod
    @rpc_call
    def create(cls, token, marker=None):
        return cls.signature(method='getNewBasket', args=[token, marker])

class CheckoutResult(Store):

    class ItemProperty(Store):
        document_id = Field(target='com.bookpac.server.shop.fulfillment.document.newid')

    basket = EmbeddedStoreField(target='checkedOutBasket', store_class=Basket)
    code = Field(target='resultCode')
    failed_item_index = IntField(target='failingPosition')
    item_properties = EmbeddedStoreField(target='positionResultProperties', store_class=ItemProperty, is_array=True)
    receipt_id = Field(target='receiptIdentifier')
    transaction_id = Field(target='transactionID')
<|endoftext|>


<commit_before>#Copyright (C) 2017 Paolo Galeone <nessuno@nerdz.eu>
#
#This Source Code Form is subject to the terms of the Mozilla Public
#License, v. 2.0. If a copy of the MPL was not distributed with this
#file, you can obtain one at http://mozilla.org/MPL/2.0/.
#Exhibit B is not attached; this software is compatible with the
#licenses expressed under Section 1.12 of the MPL v2.
"""Layers to easy create models and nice visualizations"""

import tensorflow as tf
import math
import numbers
from .utils import legalize_name, tf_log
from .visualization import on_grid
from .collections import LOSSES, REQUIRED_NON_TRAINABLES

def weight(name,
           shape,
           initializer=tf.contrib.layers.variance_scaling_initializer(
               factor=2.0, mode='FAN_IN', uniform=False, dtype=tf.float32),
           train_phase=False,
           wd=0.0):
    """Returns a tensor with the requested shape, initialized
      using the provided intitializer (default: He init).
      Applies L2 weight decay penalyt using wd term.
      Enables visualizations when in train_phase"""
    weights = tf.get_variable(
        name, shape=shape, initializer=initializer, dtype=tf.float32)

    # show weights of the first layer
    first = len(shape) == 4 and shape[2] in (1, 3, 4)
    if first and train_phase:
        num_kernels = shape[3]
        # check if is a perfect square
        grid_side = math.floor(math.sqrt(num_kernels))
        tf_log(
            tf.summary.image(
                legalize_name(name),
                on_grid(weights[:, :, :, 0:grid_side**2], grid_side,
                        grid_side)))

    if train_phase:
        # Add weight decay to W
        tf.add_to_collection(LOSSES, tf.multiply(tf.nn.l2_loss(weights), wd))
        tf_log(tf.summary.histogram(legalize_name(name), weights))
    return weights

def bias(name,
         shape,
         initializer=tf.constant_initializer(value=0.0),
         train_phase=False):
    """Returns a bias variabile initializeted wuth the provided initializer.
    No weight decay applied to the bias terms.
    Enables visualization when in train_phase"""
    return weight(name, shape, initializer, wd=0.0, train_phase=train_phase)

def atrous_conv(input_x,
                shape,
                rate,
                padding,
                bias_term=True,
                activation=tf.identity,
                train_phase=False,
                wd=0.0):
    """ Define an atrous conv layer.
    Args:
         input_x: a 4D tensor
         shape: weight shape
         rate: : A positive int32. The stride with which we sample input values
            cross the height and width dimensions. Equivalently, the rate by which
            we upsample the filter values by inserting zeros
            across the height and width dimensions. In the literature, the same
            parameter is sometimes called input stride or dilation
         padding: 'VALID' or 'SAME'
         bias_term: a boolean to add (if True) the bias term. Usually disable when
             the layer is wrapped in a batch norm layer
         activation: activation function. Default linear
         train_phase: boolean that enables/diables visualizations and train-only specific ops
         wd: weight decay
    Rerturns the conv2d op"""

    W = weight("W", shape, wd=wd, train_phase=train_phase)
    result = tf.nn.atrous_conv2d(input_x, W, rate, padding)
    if bias_term:
        b = bias("b", [shape[3]])
        result = tf.nn.bias_add(result, b)

    # apply nonlinearity
    out = activation(result)

    if train_phase:
        # log convolution result pre-activation function
        # on a single image, the first of the batch
        conv_results = tf.split(
            value=result[0], num_or_size_splits=shape[3], axis=2)
        grid_side = math.floor(math.sqrt(shape[3]))

        pre_activation = on_grid(
            tf.transpose(conv_results, perm=(
                1, 2, 3, 0))[:, :, :, 0:grid_side**2], grid_side, grid_side)

        # log post-activation
        conv_results = tf.split(
            value=out[0], num_or_size_splits=shape[3], axis=2)
        post_activation = on_grid(
            tf.transpose(conv_results, perm=(
                1, 2, 3, 0))[:, :, :, 0:grid_side**2], grid_side, grid_side)

        tf_log(
            tf.summary.image(
                legalize_name(result.name + '/pre_post_activation'),
                tf.concat([pre_activation, post_activation], axis=2),
                max_outputs=1))
    return out

def conv(input_x,
         shape,
         stride,
         padding,
         bias_term=True,
         activation=tf.identity,
         train_phase=False,
         wd=0.0):
    """ Define a conv layer.
    Args:
         input_x: a 4D tensor
         shape: weight shape
         stride: a single value supposing equal stride along X and Y
         padding: 'VALID' or 'SAME'
         bias_term: a boolean to add (if True) the bias term. Usually disable when
             the layer is wrapped in a batch norm layer
         activation: activation function. Default linear
         train_phase: boolean that enables/diables visualizations and train-only specific ops
         wd: weight decay
    Rerturns the conv2d op"""

    W = weight("W", shape, wd=wd, train_phase=train_phase)
    result = tf.nn.conv2d(input_x, W, [1, stride, stride, 1], padding)
    if bias_term:
        b = bias("b", [shape[3]])
        result = tf.nn.bias_add(result, b)

    # apply nonlinearity
    out = activation(result)

    if train_phase:
        # log convolution result pre-activation function
        # on a single image, the first of the batch
        conv_results = tf.split(
            value=result[0], num_or_size_splits=shape[3], axis=2)
        grid_side = math.floor(math.sqrt(shape[3]))

        pre_activation = on_grid(
            tf.transpose(conv_results, perm=(
                1, 2, 3, 0))[:, :, :, 0:grid_side**2], grid_side, grid_side)

        # log post-activation
        conv_results = tf.split(
            value=out[0], num_or_size_splits=shape[3], axis=2)
        post_activation = on_grid(
            tf.transpose(conv_results, perm=(
                1, 2, 3, 0))[:, :, :, 0:grid_side**2], grid_side, grid_side)

        tf_log(
            tf.summary.image(
                legalize_name(result.name + '/pre_post_activation'),
                tf.concat([pre_activation, post_activation], axis=2),
                max_outputs=1))
    return out

def fc(input_x,
       shape,
       bias_term=True,
       activation=tf.identity,
       train_phase=False,
       wd=0.0):
    """ Define a fully connected layer.
    Args:
        input_x: a 4d tensor
        shape: weight shape
        bias_term: a boolean to add (if True) the bias term. Usually disable when
             the layer is wrapped in a batch norm layer
        activation: activation function. Default linear
        train_phase: boolean that enables/diables visualizations and train-only specific ops
        wd: weight decay
    Returns the fc layer"""

    W = weight("W", shape, wd=wd, train_phase=train_phase)
    result = tf.matmul(input_x, W)
    if bias_term:
        b = bias("b", [shape[1]])
        result = tf.nn.bias_add(result, b)

    return activation(result)

def batch_norm(layer_output, is_training_, decay=0.9):
    """Applies batch normalization to the layer output.
    Args:
        layer_output: 4-d tensor, output of a FC/convolutional layer
        is_training_: placeholder or boolean variable to set to True when training
        decay:        decay for the moving average.
    """
    return tf.contrib.layers.batch_norm(
        inputs=layer_output,
        decay=decay,
        center=True,
        scale=True,
        epsilon=1e-3,
        activation_fn=None,
        # update moving mean and variance in place
        updates_collections=None,
        is_training=is_training_,
        reuse=None,
        # create a collections of varialbes to save
        # (moving mean and moving variance)
        variables_collections=[REQUIRED_NON_TRAINABLES],
        outputs_collections=None,
        trainable=True,
        batch_weights=None,
        fused=True,
        scope=None)

def direct_dropout(x, keep_prob, noise_shape=None, seed=None, name=None):
    """Computes dropout.
    The original dropout as described in the paper, not the inverted version.
    Thus it requires to scale the activation AT TEST TIME.
    Args:
        x: A tensor.
        keep_prob: A scalar `Tensor` with the same type as x. The probability
        that each element is kept.
        noise_shape: A 1-D `Tensor` of type `int32`, representing the
          shape for randomly generated keep/drop flags.
        seed: A Python integer. Used to create random seeds.
        name: A name for this operation (optional).
    Returns:
        A Tensor of the same shape of `x`.
    Raises:
        ValueError: If `keep_prob` is not in `(0, 1]`.
    """
    with tf.name_scope(name, "direct_dropout", [x]) as name:
        x = tf.convert_to_tensor(x, name="x")
        if isinstance(keep_prob, numbers.Real) and not 0 < keep_prob <= 1:
            raise ValueError(
                "keep_prob must be a scalar tensor or a float in the "
                "range (0, 1], got %g" % keep_prob)
        keep_prob = tf.convert_to_tensor(
            keep_prob, dtype=x.dtype, name="keep_prob")
        keep_prob.get_shape().assert_is_compatible_with(tf.TensorShape([]))

        # Do nothing if we know keep_prob == 1
        if tf.contrib.util.constant_value(keep_prob) == 1:
            return x

        noise_shape = noise_shape if noise_shape is not None else tf.shape(x)
        # uniform [keep_prob, 1.0 + keep_prob)
        random_tensor = keep_prob
        random_tensor += tf.random_uniform(
            noise_shape, seed=seed, dtype=x.dtype)
        # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
        binary_tensor = tf.floor(random_tensor)
        # Do not scale the activation in train time
        ret = tf.multiply(x, binary_tensor)
        ret.set_shape(x.get_shape())
        return ret
<commit_msg>models/layers: made train_phase always required. Add doc<commit_after>#Copyright (C) 2017 Paolo Galeone <nessuno@nerdz.eu>
#
#This Source Code Form is subject to the terms of the Mozilla Public
#License, v. 2.0. If a copy of the MPL was not distributed with this
#file, you can obtain one at http://mozilla.org/MPL/2.0/.
#Exhibit B is not attached; this software is compatible with the
#licenses expressed under Section 1.12 of the MPL v2.
"""Layers to easy create models and nice visualizations"""

import tensorflow as tf
import math
import numbers
from .utils import legalize_name, tf_log
from .visualization import on_grid
from .collections import LOSSES, REQUIRED_NON_TRAINABLES

def weight(name,
           shape,
           train_phase,
           initializer=tf.contrib.layers.variance_scaling_initializer(
               factor=2.0, mode='FAN_IN', uniform=False, dtype=tf.float32),
           wd=0.0):
    """Returns a tensor with the requested shape, initialized
      using the provided intitializer (default: He init).
      Applies L2 weight decay penalyt using wd term.
      Enables visualizations when in train_phase=True.
      Args:
          name: the name of the weight
          shape: the shape of the tensor, a python list or tuple
          train_phase: boolean, to enable/disable visualization and L2 decay
          initializer: the initializer to use
          wd: when train_phase=True uses `wd` as decay penalty
      Returns:
        weights: the weight tensor initialized.
    """
    weights = tf.get_variable(
        name, shape=shape, initializer=initializer, dtype=tf.float32)

    # show weights of the first layer
    first = len(shape) == 4 and shape[2] in (1, 3, 4)
    if first and train_phase:
        num_kernels = shape[3]
        # check if is a perfect square
        grid_side = math.floor(math.sqrt(num_kernels))
        tf_log(
            tf.summary.image(
                legalize_name(name),
                on_grid(weights[:, :, :, 0:grid_side**2], grid_side,
                        grid_side)))

    if train_phase:
        # Add weight decay to W
        tf.add_to_collection(LOSSES, tf.multiply(tf.nn.l2_loss(weights), wd))
        tf_log(tf.summary.histogram(legalize_name(name), weights))
    return weights

def bias(name,
         shape,
         train_phase,
         initializer=tf.constant_initializer(value=0.0)):
    """Returns a bias variabile initializeted wuth the provided initializer.
    No weight decay applied to the bias terms.
    Enables visualization when in train_phase.
    Args:
        name: name for the bias variable
        shape: shape of the variabile
        train_phase: boolean, to enable/disable visualization
        initializer: the initializer to use
    Returns:
        bias: the vias variable correctly initialized
    """
    return weight(name, shape, initializer, wd=0.0, train_phase=train_phase)

def atrous_conv(input_x,
                shape,
                rate,
                padding,
                train_phase,
                bias_term=True,
                activation=tf.identity,
                wd=0.0):
    """ Define an atrous conv layer.
    Args:
         input_x: a 4D tensor
         shape: weight shape
         rate: : A positive int32. The stride with which we sample input values
            cross the height and width dimensions. Equivalently, the rate by which
            we upsample the filter values by inserting zeros
            across the height and width dimensions. In the literature, the same
            parameter is sometimes called input stride or dilation
         padding: 'VALID' or 'SAME'
         train_phase: boolean that enables/diables visualizations and train-only specific ops
         bias_term: a boolean to add (if True) the bias term. Usually disable when
             the layer is wrapped in a batch norm layer
         activation: activation function. Default linear
         wd: weight decay
    Rerturns:
        op: the conv2d op"""

    W = weight("W", shape, wd=wd, train_phase=train_phase)
    result = tf.nn.atrous_conv2d(input_x, W, rate, padding)
    if bias_term:
        b = bias("b", [shape[3]])
        result = tf.nn.bias_add(result, b)

    # apply nonlinearity
    out = activation(result)

    if train_phase:
        # log convolution result pre-activation function
        # on a single image, the first of the batch
        conv_results = tf.split(
            value=result[0], num_or_size_splits=shape[3], axis=2)
        grid_side = math.floor(math.sqrt(shape[3]))

        pre_activation = on_grid(
            tf.transpose(conv_results, perm=(
                1, 2, 3, 0))[:, :, :, 0:grid_side**2], grid_side, grid_side)

        # log post-activation
        conv_results = tf.split(
            value=out[0], num_or_size_splits=shape[3], axis=2)
        post_activation = on_grid(
            tf.transpose(conv_results, perm=(
                1, 2, 3, 0))[:, :, :, 0:grid_side**2], grid_side, grid_side)

        tf_log(
            tf.summary.image(
                legalize_name(result.name + '/pre_post_activation'),
                tf.concat([pre_activation, post_activation], axis=2),
                max_outputs=1))
    return out

def conv(input_x,
         shape,
         stride,
         padding,
         train_phase,
         bias_term=True,
         activation=tf.identity,
         wd=0.0):
    """ Define a conv layer.
    Args:
        input_x: a 4D tensor
        shape: weight shape
        stride: a single value supposing equal stride along X and Y
        padding: 'VALID' or 'SAME'
        train_phase: boolean that enables/diables visualizations and train-only specific ops
        bias_term: a boolean to add (if True) the bias term. Usually disable when
                   the layer is wrapped in a batch norm layer
        activation: activation function. Default linear
        train_phase: boolean that enables/diables visualizations and train-only specific ops
        wd: weight decay
    Rerturns:
        op: the conv2d op
    """

    W = weight("W", shape, wd=wd, train_phase=train_phase)
    result = tf.nn.conv2d(input_x, W, [1, stride, stride, 1], padding)
    if bias_term:
        b = bias("b", [shape[3]])
        result = tf.nn.bias_add(result, b)

    # apply nonlinearity
    out = activation(result)

    if train_phase:
        # log convolution result pre-activation function
        # on a single image, the first of the batch
        conv_results = tf.split(
            value=result[0], num_or_size_splits=shape[3], axis=2)
        grid_side = math.floor(math.sqrt(shape[3]))

        pre_activation = on_grid(
            tf.transpose(conv_results, perm=(
                1, 2, 3, 0))[:, :, :, 0:grid_side**2], grid_side, grid_side)

        # log post-activation
        conv_results = tf.split(
            value=out[0], num_or_size_splits=shape[3], axis=2)
        post_activation = on_grid(
            tf.transpose(conv_results, perm=(
                1, 2, 3, 0))[:, :, :, 0:grid_side**2], grid_side, grid_side)

        tf_log(
            tf.summary.image(
                legalize_name(result.name + '/pre_post_activation'),
                tf.concat([pre_activation, post_activation], axis=2),
                max_outputs=1))
    return out

def fc(input_x,
       shape,
       train_phase,
       bias_term=True,
       activation=tf.identity,
       wd=0.0):
    """ Define a fully connected layer.
    Args:
        input_x: a 4d tensor
        shape: weight shape
        train_phase: boolean that enables/diables visualizations and train-only specific ops
        bias_term: a boolean to add (if True) the bias term. Usually disable when
             the layer is wrapped in a batch norm layer
        activation: activation function. Default linear
        wd: weight decay
    Returns:
        fc: the fc layer
    """

    W = weight("W", shape, wd=wd, train_phase=train_phase)
    result = tf.matmul(input_x, W)
    if bias_term:
        b = bias("b", [shape[1]])
        result = tf.nn.bias_add(result, b)

    return activation(result)

def batch_norm(layer_output, is_training_, decay=0.9):
    """Applies batch normalization to the layer output.
    Args:
        layer_output: 4-d tensor, output of a FC/convolutional layer
        is_training_: placeholder or boolean variable to set to True when training
        decay:        decay for the moving average.
    Returns:
        bn: the batch normalization layer
    """
    return tf.contrib.layers.batch_norm(
        inputs=layer_output,
        decay=decay,
        center=True,
        scale=True,
        epsilon=1e-3,
        activation_fn=None,
        # update moving mean and variance in place
        updates_collections=None,
        is_training=is_training_,
        reuse=None,
        # create a collections of varialbes to save
        # (moving mean and moving variance)
        variables_collections=[REQUIRED_NON_TRAINABLES],
        outputs_collections=None,
        trainable=True,
        batch_weights=None,
        fused=True,
        scope=None)

def direct_dropout(x, keep_prob, noise_shape=None, seed=None, name=None):
    """Computes dropout.
    The original dropout as described in the paper, not the inverted version.
    Thus it requires to scale the activation AT TEST TIME.
    Args:
        x: A tensor.
        keep_prob: A scalar `Tensor` with the same type as x. The probability
        that each element is kept.
        noise_shape: A 1-D `Tensor` of type `int32`, representing the
          shape for randomly generated keep/drop flags.
        seed: A Python integer. Used to create random seeds.
        name: A name for this operation (optional).
    Returns:
        A Tensor of the same shape of `x`.
    Raises:
        ValueError: If `keep_prob` is not in `(0, 1]`.
    """
    with tf.name_scope(name, "direct_dropout", [x]) as name:
        x = tf.convert_to_tensor(x, name="x")
        if isinstance(keep_prob, numbers.Real) and not 0 < keep_prob <= 1:
            raise ValueError(
                "keep_prob must be a scalar tensor or a float in the "
                "range (0, 1], got %g" % keep_prob)
        keep_prob = tf.convert_to_tensor(
            keep_prob, dtype=x.dtype, name="keep_prob")
        keep_prob.get_shape().assert_is_compatible_with(tf.TensorShape([]))

        # Do nothing if we know keep_prob == 1
        if tf.contrib.util.constant_value(keep_prob) == 1:
            return x

        noise_shape = noise_shape if noise_shape is not None else tf.shape(x)
        # uniform [keep_prob, 1.0 + keep_prob)
        random_tensor = keep_prob
        random_tensor += tf.random_uniform(
            noise_shape, seed=seed, dtype=x.dtype)
        # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
        binary_tensor = tf.floor(random_tensor)
        # Do not scale the activation in train time
        ret = tf.multiply(x, binary_tensor)
        ret.set_shape(x.get_shape())
        return ret
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys
import datetime
import time
import urllib
import json
import socket
import re
import random
from slackbot.bot import Bot, listen_to, respond_to
from tinydb import TinyDB

db = TinyDB('quotes.json')

def get_food(day):
    # Get JSON
    URL = 'http://www.hanssonohammar.se/veckansmeny.json'
    response = urllib.urlopen(URL)
    data = json.loads(response.read().decode('utf-8'))

    if day not in data:
        return "(no mat " + str(day) + ")"

    mat_today = data[day][0]

    if 'IKSU' not in mat_today:
        return "(no IKSU today)"

    return "\n".join(mat_today['IKSU'])

@listen_to(r'^compile (.*)$')
def compile(message, code):
    message.reply(u"Jag klarar inte av sånt längre :'(")

@listen_to(r'^run (.*)$')
def run(message, code):
    message.reply(u"Jag klarar inte av sånt längre :'(")

@listen_to(r'^mat(\+*)$')
def mat(message, plus):
    date = datetime.date.fromtimestamp(time.time() + (86400 * len(plus)))
    try:
        message.reply(u"```IKSU - {}\n{}```".format(str(date), get_food(str(date))))
    except Exception as e:
        message.reply(u"Kom inte åt maten 😞 ({what})".format(what=e.message))

@listen_to(ur'^[e\u00E4\u00C4]r.*fredag.*\?', re.IGNORECASE)
def fredag(message):
    if datetime.datetime.today().weekday() == 4:
        message.reply(u"Japp, idag är det fredag! :kreygasm:")
    else:
        message.reply(u"Nej, idag är det INTE fredag! :qq::gun:")

@listen_to(r'^temp(\+*)$')
def temp(message, plus):
    if len(plus) > 0:
        message.reply(u"Jag kan inte se in i framtiden... :qq::gun:")
    else:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect(('temp.acc.umu.se', 2345))
        tmp = s.recv(1024)
        s.close()
        time, temp = tmp[:len(tmp) - 1].split('=')
        message.reply(u"{} C klockan {}".format(temp, time))

@listen_to(r'^quote add (.*)$')
def quote_add(message, quote):
    db.insert({'quote': quote})
    message.reply(u"Quote inlagd!")

@listen_to(r'^quote$')
def quote(message):
    quotes = db.all()
    if len(quotes) == 0:
        message.reply(u"Inga quotes inlagda...")
    else:
        quote = random.choice(quotes)
        message.reply(u">{}".format(quote['quote']))

def main():
    bot = Bot()
    bot.run()

if __name__ == '__main__':
    main()
<commit_msg>Fix quote.json path<commit_after># -*- coding: utf-8 -*-

import sys
import datetime
import time
import urllib
import json
import socket
import re
import random
from slackbot.bot import Bot, listen_to, respond_to
from tinydb import TinyDB

db = TinyDB('/home/simon/bertil/quotes.json')

def get_food(day):
    # Get JSON
    URL = 'http://www.hanssonohammar.se/veckansmeny.json'
    response = urllib.urlopen(URL)
    data = json.loads(response.read().decode('utf-8'))

    if day not in data:
        return "(no mat " + str(day) + ")"

    mat_today = data[day][0]

    if 'IKSU' not in mat_today:
        return "(no IKSU today)"

    return "\n".join(mat_today['IKSU'])

@listen_to(r'^compile (.*)$')
def compile(message, code):
    message.reply(u"Jag klarar inte av sånt längre :'(")

@listen_to(r'^run (.*)$')
def run(message, code):
    message.reply(u"Jag klarar inte av sånt längre :'(")

@listen_to(r'^mat(\+*)$')
def mat(message, plus):
    date = datetime.date.fromtimestamp(time.time() + (86400 * len(plus)))
    try:
        message.reply(u"```IKSU - {}\n{}```".format(str(date), get_food(str(date))))
    except Exception as e:
        message.reply(u"Kom inte åt maten 😞 ({what})".format(what=e.message))

@listen_to(ur'^[e\u00E4\u00C4]r.*fredag.*\?', re.IGNORECASE)
def fredag(message):
    if datetime.datetime.today().weekday() == 4:
        message.reply(u"Japp, idag är det fredag! :kreygasm:")
    else:
        message.reply(u"Nej, idag är det INTE fredag! :qq::gun:")

@listen_to(r'^temp(\+*)$')
def temp(message, plus):
    if len(plus) > 0:
        message.reply(u"Jag kan inte se in i framtiden... :qq::gun:")
    else:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect(('temp.acc.umu.se', 2345))
        tmp = s.recv(1024)
        s.close()
        time, temp = tmp[:len(tmp) - 1].split('=')
        message.reply(u"{} C klockan {}".format(temp, time))

@listen_to(r'^quote add (.*)$')
def quote_add(message, quote):
    db.insert({'quote': quote})
    message.reply(u"Quote inlagd!")

@listen_to(r'^quote$')
def quote(message):
    quotes = db.all()
    if len(quotes) == 0:
        message.reply(u"Inga quotes inlagda...")
    else:
        quote = random.choice(quotes)
        message.reply(u">{}".format(quote['quote']))

def main():
    bot = Bot()
    bot.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from pyramid.response import Response
from pyramid.view import view_config

from .interfaces import IFlatPagesRoot

@view_config(request_method='GET', context=IFlatPagesRoot)
def hello_world(request):
    return Response('Hello, world.')
<commit_msg>Added views that are actually useful... Somewhat.<commit_after>from pyramid.response import Response
from pyramid.view import view_config
from pyramid.exceptions import NotFound
import os

from .interfaces import IFlatPagesRoot

# Temporarily, we will import git directly for testing.
from .data_sources.git import GitDataSource

def handle_flatpage(request):
    lookup_string = os.sep.join(request.subpath)

    if len(request.view_name) > 0:
        lookup_string = os.sep.join(request.view_name, request.subpath)

    if lookup_string[-1] == os.sep:
        lookup_string = lookup_string[:-1]

    g = GitDataSource()
    file_contents = g.read(lookup_string)

    print 'Looking for: {0}'.format(lookup_string)

    # If file_info is None, this file doesn't exist. So, a 404 is the proper
    # response in this situation.
    if file_contents is None:
        raise NotFound

    return Response(file_contents)
<|endoftext|>