<commit_before># -*- coding: utf-8 -*-
from datetime import datetime
from requests import Session
from requests.adapters import HTTPAdapter
from odoo.exceptions import ValidationError
from requests.packages.urllib3.util.retry import Retry as httpRetry
from json import loads as json_load, dumps as json_dump
from unidecode import unidecode
from timeit import default_timer
from time import sleep
from odoo import api
from odoo.modules import registry
from concurrent.futures import ThreadPoolExecutor
import logging

_logger = logging.getLogger('B2B-OUTGOING')
_pool = ThreadPoolExecutor(max_workers=10)
_running = dict()

class Subprocess(object):

	__slots__ = ['_target', 'name']

	def __init__(self, target):
		self._target = target

	def add(self, *args):
		self.name = '%s-%s' % (self._target._name, self._target.id)
		# If we are processing this record on other thread cancel
		# it first, if it's running can not be cancelled
		if self.name in _running:
			_running[self.name].cancel()
		# Schedule task
		_running[self.name] = _pool.submit(self.background_run, *args)

	def background_run(self, method, record, mode):
		"""
		Method to be called in a separate thread

		:param method: Object, method to call
		:param record: Object, record to notify
		:param mode: Str, CRUD mode
		"""
		while not self._target.env.cr.closed:
			# Wait while parent process ends
			sleep(1)

		# Now create a new cursor
		with api.Environment.manage():
			with self._target.pool.cursor() as new_cr:
				# Autocommit ON
				new_cr.autocommit(True)
				# Issolated record
				record_issolated = record.with_env(record.env(cr=new_cr))
				# Call the method
				method(record_issolated, mode)

class Chrono(object):

	__slots__ = ['name', 'start', 'end', 'elapsed']

	def __init__(self, name=None):
		self.name = name
		self.start = None
		self.end = None
		self.elapsed = None

	def __enter__(self):
		self.start = default_timer()
		return self

	def __exit__(self,ty,val,tb):
		self.end = default_timer()
		self.elapsed = self.end - self.start

	def to_str(self):
		if self.elapsed:
			return "%.2fms" % (self.elapsed)
		return None

class JSync(object):

	__slots__ = ['id', 'env', 'model', 'mode', 'name', 'data', 'part', 'related', 'session', 'settings']

	def __init__(self, odoo_env, settings=None, retries=3):
		retry = httpRetry(total=retries, connect=retries, backoff_factor=0.3, status_forcelist=(500, 502, 504))
		adapter = HTTPAdapter(max_retries=retry)

		self.id = None # Record ID
		self.model = None # Record model name
		self.mode = 'create' # CRUD mode
		self.name = None # Data item name
		self.data = {} # Item data dict
		self.part = None # Multi-packet part
		self.related = None # Odoo related model & record
		self.env = odoo_env # Odoo environment
		self.session = Session() # HTTP Session
		self.session.mount('http://', adapter)
		self.session.mount('https://', adapter)

		# If settings passed use it to avoid multiple calls
		self.settings = settings or self.env['b2b.settings'].get_default_params(fields=[
			'url',
			'conexion_error',
			'response_error',
			'packet_size'
		])

	def filter_data(self, vals=None):
		"""
		Filter and normalizes item data (data)

		:param vals: Item data to update (from model)
		:return: dict

		data key modifiers:
			fixed:xxx -> Sends xxx always
			field_xxx_name:xxx -> Sends xxx field if field_xxx_name has changed
			xxx: -> Sends xxx field if has changed (no modifier)
		"""
		if self.data and type(self.data) is dict:
			for field, value in self.data.items():
				obj_old = obj_new = field
				# If field have :
				if ':' in field:
					# Before :
					obj_old = field[:field.index(':')]
					# After :
					obj_new = field[field.index(':') + 1:]
					# Replace key
					self.data[obj_new] = self.data.pop(field)
				if obj_old != 'fixed' and (vals is False or (type(vals) is dict and obj_old not in vals)):
					# Remove field because is not found in vals
					del self.data[obj_new]
				elif type(value) is list:
					# Convert lits to tuples
					self.data[obj_new] = tuple(value)
				elif type(value) is unicode:
					# Decode unicode str's to utf-8
					self.data[obj_new] = value.strip().decode('utf-8', 'replace')

		return self.data

	def can_send(self):
		"""
		Sends data to JSync server and prints on screen

		:return [bool (Send flag), str (Record model and ID string), object ()]
		"""

		RECORD_SEND = False
		RES_ID = '%s,%s' % (self.model, self.id)

		if self.name and self.data and self.mode:
			# Check with new cursor
			with api.Environment.manage():
				with registry.RegistryManager.get(self.env.cr.dbname).cursor() as new_cr:
					env = api.Environment(new_cr, self.env.uid, self.env.context)

					# Check if record is synced with JSync
					EXPORT_RECORD = self.env['b2b.export'].search([('res_id', '=', RES_ID)], limit=1)

					# Send the record?
					RECORD_SEND = (not EXPORT_RECORD or self.mode != 'create')

					# Is related record notifiable?
					if RECORD_SEND and self.related:
						record_model, record_id = self.related.split(',')
						# Do not send the record if the related item is no longer notifiable
						if env[record_model].browse(int(record_id)).is_notifiable_check():
							RECORD_SEND = True

		return RECORD_SEND, RES_ID, EXPORT_RECORD

	def send(self, timeout_sec=10, notify=True, **kwargs):
		"""
		Sends data to JSync server and prints on screen

		:param action: CRUD action
		:param timeout_sec: POST Request timeout

		"""

		# Check record and get required params
		_RECORD_SEND, _RES_ID, _EXPORT_RECORD = self.can_send()

		if _RECORD_SEND:

			jsync_post = None

			# Header
			header_dict = { 'Content-Type': 'application/json' }

			# Content
			json_data = json_dump({
				'object': self.name,
				'operation': self.mode,
				'data': self.data,
				'part': self.part
			})

			# Debug
			debug_msg = "JSync Response: {}" \
						"\n    - object: {}" \
						"\n    - operation: {}" \
						"\n    - data: {}" \
						"\n    - part: {}"

			# Data indented (pretified)
			debug_data = json_dump(self.data, indent=8, sort_keys=True)

			try:

				# Send HTTP POST data
				jsync_post = self.session.post(self.settings['url'], timeout=timeout_sec, headers=header_dict, data=json_data)

			except Exception as e:

				_logger.error(debug_msg.format('CONNECTION ERROR!', self.name, self.mode, debug_data, self.part))

				if self.settings['conexion_error']:
					if type(e) is not ValidationError:
						raise ValidationError("JSync Server Connection Error\n%s" % (e))
					else:
						raise e

			# Si la respuesta es OK
			if jsync_post and jsync_post.status_code is 200:

				print(debug_msg.format(jsync_post.text, self.name, self.mode, debug_data, self.part))

				# En los paquetes múltiples no se establecen estos parámetros
				# por lo que no se notifican al usuario ni se registran en el sistema
				if self.name and self.id and self.model:

					# Mostrar notificación no invasiva al usuario en Odoo
					if notify:
						self.env.user.notify_info('[B2B] %s %s %s' % (self.mode.capitalize(), self.name, self.id))

					# Guardar el estado en Odoo
					# con un nuevo cursor para que se
					# haga un commit de forma inmediata
					with api.Environment.manage():
						with registry.RegistryManager.get(self.env.cr.dbname).cursor() as new_cr:
							env = api.Environment(new_cr, self.env.uid, self.env.context)
							env.cr.autocommit(True)
							if self.mode == 'create':
								env['b2b.export'].create({ 'name': self.name, 'res_id': _RES_ID, 'rel_id': self.related })
							elif _EXPORT_RECORD and self.mode == 'update':
								env['b2b.export'].browse(_EXPORT_RECORD.id).write({ 'name': self.name, 'res_id': _RES_ID, 'rel_id': self.related })
							elif _EXPORT_RECORD and self.mode == 'delete':
								env['b2b.export'].browse(_EXPORT_RECORD.id).unlink()

				try:
					return json_load(jsync_post.text)
				except:
					return jsync_post.text

			elif jsync_post:

				_logger.error(debug_msg.format('RESPONSE ERROR!', self.name, self.mode, debug_data, self.part))

				if self.settings['conexion_error'] and self.settings['response_error']:
					raise ValidationError("JSync Server Response Error\n%s" % (jsync_post.text.encode('latin1').capitalize()))

		return False<commit_msg>Update helper.py<commit_after># -*- coding: utf-8 -*-
from datetime import datetime
from requests import Session
from requests.adapters import HTTPAdapter
from odoo.exceptions import ValidationError
from requests.packages.urllib3.util.retry import Retry as httpRetry
from json import loads as json_load, dumps as json_dump
from unidecode import unidecode
from timeit import default_timer
from time import sleep
from odoo import api
from odoo.modules import registry
from concurrent.futures import ThreadPoolExecutor
import logging

_logger = logging.getLogger('B2B-OUTGOING')
_pool = ThreadPoolExecutor(max_workers=10)
_running = dict()

class Subprocess(object):

	__slots__ = ['_target', 'name']

	def __init__(self, target):
		self._target = target

	def add(self, *args):
		self.name = '%s-%s' % (self._target._name, self._target.id)
		# If we are processing this record on other thread cancel
		# it first, if it's running can not be cancelled
		if self.name in _running:
			_running[self.name].cancel()
		# Schedule task
		_running[self.name] = _pool.submit(self.background_run, *args)

	def background_run(self, method, record, mode):
		"""
		Method to be called in a separate thread

		:param method: Object, method to call
		:param record: Object, record to notify
		:param mode: Str, CRUD mode
		"""
		while not self._target.env.cr.closed:
			# Wait while parent process ends
			sleep(1)

		# Now create a new cursor
		with api.Environment.manage():
			with self._target.pool.cursor() as new_cr:
				# Autocommit ON
				new_cr.autocommit(True)
				# Issolated record
				record_issolated = record.with_env(record.env(cr=new_cr))
				# Call the method
				method(record_issolated, mode)

class Chrono(object):

	__slots__ = ['name', 'start', 'end', 'elapsed']

	def __init__(self, name=None):
		self.name = name
		self.start = None
		self.end = None
		self.elapsed = None

	def __enter__(self):
		self.start = default_timer()
		return self

	def __exit__(self,ty,val,tb):
		self.end = default_timer()
		self.elapsed = self.end - self.start

	def to_str(self):
		if self.elapsed:
			return "%.2fms" % (self.elapsed)
		return None

class JSync(object):

	__slots__ = ['id', 'env', 'model', 'mode', 'name', 'data', 'part', 'related', 'session', 'settings']

	def __init__(self, odoo_env, settings=None, retries=3):
		retry = httpRetry(total=retries, connect=retries, backoff_factor=0.3, status_forcelist=(500, 502, 504))
		adapter = HTTPAdapter(max_retries=retry)

		self.id = None # Record ID
		self.model = None # Record model name
		self.mode = 'create' # CRUD mode
		self.name = None # Data item name
		self.data = {} # Item data dict
		self.part = None # Multi-packet part
		self.related = None # Odoo related model & record
		self.env = odoo_env # Odoo environment
		self.session = Session() # HTTP Session
		self.session.mount('http://', adapter)
		self.session.mount('https://', adapter)

		# If settings passed use it to avoid multiple calls
		self.settings = settings or self.env['b2b.settings'].get_default_params(fields=[
			'url',
			'conexion_error',
			'response_error',
			'packet_size'
		])

	def filter_data(self, vals=None):
		"""
		Filter and normalizes item data (data)

		:param vals: Item data to update (from model)
		:return: dict

		data key modifiers:
			fixed:xxx -> Sends xxx always
			field_xxx_name:xxx -> Sends xxx field if field_xxx_name has changed
			xxx: -> Sends xxx field if has changed (no modifier)
		"""
		if self.data and type(self.data) is dict:
			for field, value in self.data.items():
				obj_old = obj_new = field
				# If field have :
				if ':' in field:
					# Before :
					obj_old = field[:field.index(':')]
					# After :
					obj_new = field[field.index(':') + 1:]
					# Replace key
					self.data[obj_new] = self.data.pop(field)
				if obj_old != 'fixed' and (vals is False or (type(vals) is dict and obj_old not in vals)):
					# Remove field because is not found in vals
					del self.data[obj_new]
				elif type(value) is list:
					# Convert lits to tuples
					self.data[obj_new] = tuple(value)
				elif type(value) is unicode:
					# Decode unicode str's to utf-8
					self.data[obj_new] = value.strip().decode('utf-8', 'replace')

		return self.data

	def can_send(self):
		"""
		Sends data to JSync server and prints on screen

		:return [bool (Send flag), str (Record model and ID string), object ()]
		"""

		RECORD_SEND = False
		EXPORT_RECORD = None
		RES_ID = '%s,%s' % (self.model, self.id)

		if self.name and self.data and self.mode:
			# Check with new cursor
			with api.Environment.manage():
				with registry.RegistryManager.get(self.env.cr.dbname).cursor() as new_cr:
					env = api.Environment(new_cr, self.env.uid, self.env.context)

					# Check if record is synced with JSync
					EXPORT_RECORD = env['b2b.export'].search([('res_id', '=', RES_ID)], limit=1)

					# Send the record?
					RECORD_SEND = (not EXPORT_RECORD or self.mode != 'create')

					# Is related record notifiable?
					if RECORD_SEND and self.related:
						record_model, record_id = self.related.split(',')
						# Do not send the record if the related item is no longer notifiable
						if env[record_model].browse(int(record_id)).is_notifiable_check():
							RECORD_SEND = True

		return RECORD_SEND, RES_ID, EXPORT_RECORD

	def send(self, timeout_sec=10, notify=True, **kwargs):
		"""
		Sends data to JSync server and prints on screen

		:param action: CRUD action
		:param timeout_sec: POST Request timeout

		"""

		# Check record and get required params
		_RECORD_SEND, _RES_ID, _EXPORT_RECORD = self.can_send()

		if _RECORD_SEND:

			jsync_post = None

			# Header
			header_dict = { 'Content-Type': 'application/json' }

			# Content
			json_data = json_dump({
				'object': self.name,
				'operation': self.mode,
				'data': self.data,
				'part': self.part
			})

			# Debug
			debug_msg = "JSync Response: {}" \
						"\n    - object: {}" \
						"\n    - operation: {}" \
						"\n    - data: {}" \
						"\n    - part: {}"

			# Data indented (pretified)
			debug_data = json_dump(self.data, indent=8, sort_keys=True)

			try:

				# Send HTTP POST data
				jsync_post = self.session.post(self.settings['url'], timeout=timeout_sec, headers=header_dict, data=json_data)

			except Exception as e:

				_logger.error(debug_msg.format('CONNECTION ERROR!', self.name, self.mode, debug_data, self.part))

				if self.settings['conexion_error']:
					if type(e) is not ValidationError:
						raise ValidationError("JSync Server Connection Error\n%s" % (e))
					else:
						raise e

			# Si la respuesta es OK
			if jsync_post and jsync_post.status_code is 200:

				print(debug_msg.format(jsync_post.text, self.name, self.mode, debug_data, self.part))

				# En los paquetes múltiples no se establecen estos parámetros
				# por lo que no se notifican al usuario ni se registran en el sistema
				if self.name and self.id and self.model:

					# Mostrar notificación no invasiva al usuario en Odoo
					if notify:
						self.env.user.notify_info('[B2B] %s %s %s' % (self.mode.capitalize(), self.name, self.id))

					# Guardar el estado en Odoo
					# con un nuevo cursor para que se
					# haga un commit de forma inmediata
					with api.Environment.manage():
						with registry.RegistryManager.get(self.env.cr.dbname).cursor() as new_cr:
							env = api.Environment(new_cr, self.env.uid, self.env.context)
							env.cr.autocommit(True)
							if self.mode == 'create':
								env['b2b.export'].create({ 'name': self.name, 'res_id': _RES_ID, 'rel_id': self.related })
							elif _EXPORT_RECORD and self.mode == 'update':
								env['b2b.export'].browse(_EXPORT_RECORD.id).write({ 'name': self.name, 'res_id': _RES_ID, 'rel_id': self.related })
							elif _EXPORT_RECORD and self.mode == 'delete':
								env['b2b.export'].browse(_EXPORT_RECORD.id).unlink()

				try:
					return json_load(jsync_post.text)
				except:
					return jsync_post.text

			elif jsync_post:

				_logger.error(debug_msg.format('RESPONSE ERROR!', self.name, self.mode, debug_data, self.part))

				if self.settings['conexion_error'] and self.settings['response_error']:
					raise ValidationError("JSync Server Response Error\n%s" % (jsync_post.text.encode('latin1').capitalize()))

		return False<|endoftext|>


<commit_before>"""
Modularized detection suite containing the phase_0 classes.
    Authors: Johan Mickos   jmickos@bu.edu
             Josh Navon     navonj@bu.edu
"""

# Standard library
import copy
import itertools

# External dependencies
import numpy as np
import cv2 as cv

# Project-specific
import utility
from generic import Detector

from config import DEBUG_LEVEL

class BoxExtractor(Detector):
    """This class locates and stores box coordinates in frames."""
    def __init__(self,):
        """"Class constructor. Overrides superclass method."""
        self.set = False

    def detect(self, cur_frame, frame_cnt):
        """Perform box extraction. Overrides superclass method."""
        OFFSET = 10
        # Force black frame to ensure first coord is top left of frame
        border_frame = cv.copyMakeBorder(cur_frame, OFFSET, OFFSET, OFFSET, OFFSET, cv.BORDER_CONSTANT, (0, 0, 0))
        # Treshold + grayscale for binary image
        gray = cv.cvtColor(border_frame, cv.COLOR_BGR2GRAY)
        _, gray = cv.threshold(gray, 30, 255, cv.THRESH_BINARY)
        points = [None] * 2
        # Evaluate black lines & extract box coordinates
        for axis in (0, 1):
            projection = gray.sum(axis=axis)
            projection *= 255/(projection.max() + 1)
            black_lines = np.where(projection <= 80)[0] - OFFSET + 1
            clumps =  np.split(black_lines, np.where(np.diff(black_lines) != 1)[0]+1)
            black_lines = np.where(projection <= 25)[0] - OFFSET + 1
            clumps =  np.split(black_lines, np.where(np.diff(black_lines) != 1)[0] + 1)
            coords = [(clumps[i][-1], clumps[i+1][0]) for i in range(len(clumps) - 1)]
            filtered_coords = [coords[i] for i in np.where(np.diff(coords) > 125)[0]]
            points[axis] = filtered_coords

        if utility.in_range(len(points[0]), 0, 2) and \
           utility.in_range(len(points[1]), 0, 2):
            self.variables['player_regions'][:] = []
            local = list()
            for coord in itertools.product(points[0], points[1]):
                local.append([(int(str(coord[0][0]), 10), int(str(coord[0][1]), 10)),
                              (int(str(coord[1][0]), 10), int(str(coord[1][1]), 10))])
            self.variables['player_regions'] = self.sort_boxes(local, cur_frame)
            if not self.set and self.variables['is_started']:
                self.variables['locked_regions'] = copy.deepcopy(self.variables['player_regions'][0:self.variables['num_players']])
                self.set = True
        else:
            # Completely black frame
            self.variables['player_regions'] = [[(0, cur_frame.shape[1]), (0, cur_frame.shape[0])]]

    def sort_boxes(self, boxes, cur_frame):
        """Sorting algorithm that places priority on "top left" boxes"""
        if len(boxes) != 0:
            ordered = list()
            upper = np.max(boxes).astype(float)
            for box in boxes:
                box_normed = np.divide(box, upper)
                rank = np.sum(100 * box_normed[1]) + np.sum(10 * box_normed[0])
                ordered.append((box, rank))
            ordered = sorted(ordered, key=lambda x:x[1])
            result = [el[0] for el in ordered]
            return result
        else:
            return [[(0, cur_frame.shape[1]), (0, cur_frame.shape[0])]]

class Characters(Detector):
    """Detector for detecting characters in MK64."""
    def __init__(self, masks_dir, freq, threshold, default_shape, buf_len=None):
        """
        Instantiates necessary variables and passes control off to superclass constructor.
        Overrides superclass method.
        """
        self.waiting_black = False
        super(Characters, self).__init__(masks_dir, freq, threshold, default_shape, buf_len)

    def detect(self, frame, cur_count, player):
        """Determines whether and how to process current frame. Overrides superclass method."""
        height, width, _ = frame.shape
        focus_region = frame[np.ceil(height * 0.25) : np.ceil(height * 0.95),
                        np.ceil(width * 0.25) : np.ceil(width * 0.75)]
        if self.variables['is_black']:
            if self.waiting_black:
                self.store_players(focus_region)
            else:
                return
        if not self.variables['is_started'] and (cur_count % self.freq == 0):
            self.process(focus_region, cur_count, player)
            if DEBUG_LEVEL > 2:
                cv.imshow(self.name(), focus_region)
                cv.waitKey(1)

    def process(self, frame, cur_count, player):
        """ Compares pre-loaded masks to current frame. Overrides superclass method."""
        player = 0
        if len(self.default_shape) != 1:
            for mask, shape in zip(self.masks, self.default_shape):
                if frame.shape != shape:
                    scaled_mask = (utility.scaleImage(frame,mask[0], shape), mask[1])
                else:
                    scaled_mask = mask
                distances = cv.matchTemplate(frame, scaled_mask[0], cv.TM_SQDIFF_NORMED)
                minval, _, minloc, _ = cv.minMaxLoc(distances)
                if minval <= self.threshold:
                    self.handle(frame, player, mask, cur_count, minloc)
                    if DEBUG_LEVEL > 0:
                        print "[%s]: Found %s :-) ------> %s" % (self.name(), mask[1], minval)
        else:
            for mask in self.masks:
                if frame.shape != self.default_shape[0]:
                    scaled_mask = (utility.scaleImage(frame,mask[0], self.default_shape[0]), mask[1])
                else:
                    scaled_mask = mask
                distances = cv.matchTemplate(frame, scaled_mask[0], cv.TM_SQDIFF_NORMED)
                minval, _, minloc, _ = cv.minMaxLoc(distances)
                if minval <= self.threshold:
                    self.handle(frame, player, mask, cur_count, minloc)
                    if DEBUG_LEVEL > 0:
                        print "[%s]: Found %s :-) ------> %s" % (self.name(), mask[1], minval)

    def handle(self, frame, player, mask, cur_count, location):
        """Perform checks and store variables. Overrides superclass method."""
        self.waiting_black = True
        timestamp = cur_count / self.variables['frame_rate']
        idx = self.buffer.exists(mask[1])
        if idx != -1:
            self.buffer[idx] = (mask[1], location, timestamp)
        else:
            self.buffer.append((mask[1], location, timestamp))

    def store_players(self, frame):
        """Stores players in the state variable."""
        self.waiting_black = False
        self.deactivate()
        ordered = list()
        max_place = 0
        characters = utility.find_unique(self.buffer, 0)

        semi_ordered = self.sort_characters(characters, frame)
        # Find the largest rank in the list.
        for element in semi_ordered:
            if element[1] > max_place:
                max_place = element[1]

        # For every possible player number, find the most recent match
        for ii in xrange(1, (max_place + 1)):
            timestamp = 0
            for element in semi_ordered:
                # Check if the char is player ii and if it was found after <timestamp>
                if element[1] == ii and element[2] > timestamp:
                    timestamp = element[2]
                    temp = (element[0], element[1])
            ordered.append(temp)
        chars = [image[0].split(".", 1)[0] for image in ordered]
        if DEBUG_LEVEL > 0:
            for p, ch in enumerate(chars):
                print "Player %i is %s" % ((p+1), ch)
        self.variables['characters'] = chars
        self.variables['num_players'] = len(ordered)

    def sort_characters(self, characters, frame):
        """Sorting algorithm that ranks based on quadrant."""
        ordered = list()
        center_y, center_x, _ = frame.shape
        center_x /= 2
        center_y /= 2
        for char in characters:
            # Quadrant 2 (Player 1)
            if characters[char][0][1] < (center_x - 10) and characters[char][0][0] < (center_y - 10):
                ordered.append((char, 1, characters[char][1]))
            elif characters[char][0][1] > (center_x + 10) and characters[char][0][0] < (center_y - 10):
                ordered.append((char, 2, characters[char][1]))
            elif characters[char][0][1] < (center_x - 10) and characters[char][0][0] > (center_y + 10):
                ordered.append((char, 3, characters[char][1]))
            else:
                ordered.append((char, 4, characters[char][1]))
        ordered = sorted(ordered, key=lambda x: x[1])
        return ordered

class Map(Detector):
    """Determines which map is being played."""
    def __init__(self, masks_dir, freq, threshold, default_shape, buf_len=None):
        """
        Instantiates necessary variables and passes control off to superclass constructor.
        Overrides superclass method.
        """
        self.waiting_black = False
        self.map = ''
        super(Map, self).__init__(masks_dir, freq, threshold, default_shape, buf_len)

    def detect(self, frame, cur_count, player):
        """Determines whether and how to process current frame. Overrides superclass method."""
        if self.variables['is_black']:
            if self.waiting_black:
                self.variables['map'] = self.map.split('.')[0]
                self.waiting_black = False
                if DEBUG_LEVEL > 0:
                    print 'Locked in map as %s' % (self.map.split('.')[0])
            else:
                return
        if (cur_count % self.freq) == 0:
            self.process(frame, cur_count, player)

    def process(self, frame, cur_count, player):
        """ Compares pre-loaded masks to current frame. Overrides superclass method."""
        best_val = 1
        best_mask = None

        if frame != None:
            # Convert to grayscale and threshold
            binary = cv.cvtColor(frame , cv.COLOR_BGR2GRAY)
            _, binary = cv.threshold(binary, 120, 255, cv.THRESH_BINARY)
            binary = cv.GaussianBlur(binary, (3, 3), 1)
            # Scale and grayscale the masks
            if len(self.default_shape) != 1:
                binary_roi = self.constrain_roi(binary)
                for mask, shape in zip(self.masks, self.default_shape):
                    if frame.shape != shape:
                        scaled_mask = (cv.cvtColor(utility.scaleImage(frame, mask[0], shape),
                            cv.COLOR_BGR2GRAY), mask[1])
                    else:
                        scaled_mask = (cv.cvtColor(mask[0], cv.COLOR_BGR2GRAY), mask[1])
                    distances = cv.matchTemplate(binary_roi, scaled_mask[0], cv.TM_SQDIFF_NORMED)
                    minval, _, minloc, _ = cv.minMaxLoc(distances)
                    if minval <= self.threshold and minval < best_val:
                        best_val = minval
                        best_mask = scaled_mask
                    if DEBUG_LEVEL > 2:
                        cv.imshow('Map Thresh', binary_roi)
                        cv.waitKey(1)
                if best_mask is not None:
                    self.handle(frame, player, best_mask, cur_count, minloc)
                    if DEBUG_LEVEL > 0:
                        print "[%s]: Found %s :-) ------> %s" % (self.name(), best_mask[1], best_val)
            else:
                binary_roi = self.constrain_roi(binary)
                for mask in self.masks:
                    if frame.shape != self.default_shape[0]:
                            scaled_mask = (cv.cvtColor(utility.scaleImage(frame,mask[0], self.default_shape[0]),
                                cv.COLOR_BGR2GRAY), mask[1])
                    else:
                        scaled_mask = (cv.cvtColor(mask[0], cv.COLOR_BGR2GRAY), mask[1])
                    distances = cv.matchTemplate(binary_roi, scaled_mask[0], cv.TM_SQDIFF_NORMED)
                    minval, _, minloc, _ = cv.minMaxLoc(distances)
                    if minval <= self.threshold and minval < best_val:
                        best_val = minval
                        best_mask = scaled_mask
                    if DEBUG_LEVEL > 2:
                        cv.imshow('Map Thresh', binary_roi)
                        cv.waitKey(1)
                if best_mask is not None:
                    self.handle(frame, player, best_mask, cur_count, minloc)
                    if DEBUG_LEVEL > 0:
                        print "[%s]: Found %s :-) ------> %s" % (self.name(), best_mask[1], best_val)

    def handle(self, frame, player, mask, cur_count, location):
        """Set variables. Overrides superclass method."""
        self.map = mask[1]
        self.waiting_black = True
        if DEBUG_LEVEL > 0:
            print '[%s]: Map is: %s' % (self.name(), mask[1].split('.')[0])

    def constrain_roi(self, frame):
        """"Constrains frame w.r.t. Maps. Overrides superclass method."""
        h, w = frame.shape
        frame = frame[np.ceil(h * 0.4):, np.ceil(w * 0.45):np.ceil(w * 0.92)]
        return frame

class StartRace(Detector):
    """Handles the beginning of a race in phase_0"""
    def handle(self, frame, player, mask, cur_count, location):
        """Store variables and toggle detectors. Overrides superclass method."""
        self.variables['is_started'] = True
        self.deactivate()
        self.activate('EndRace')
        self.deactivate('Characters')
        self.deactivate('Map')
        # Lock in player boxes (should be sorted alreadY)
        self.variables['player_regions'] = self.variables['player_regions'][0:self.variables['num_players']]
        # Populate dictionary with start time
        self.variables['start_time'] = np.floor(cur_count / self.variables['frame_rate']) - 2
        if DEBUG_LEVEL > 0:
            print '[%s]: Race started at %d seconds' % (self.name(), self.variables['start_time'])
            cv.waitKey(1)
    def constrain_roi(self, frame):
        """Constrains frame w.r.t. StartRace/BeginRace. Overrides superclass method."""
        h, w, _ = frame.shape
        frame = frame[0:np.ceil(h * 0.5), np.ceil(w * 0.4):np.ceil(w * 0.75)]
        return frame

class EndRace(Detector):
    """Handles the end of a race (phase_0)"""
    def __init__(self):
        """Class constructor. Overrides superclass method."""
        pass

    def detect(self, frame, cur_count, player):
        """Determines whether and how to process current frame. Overrides superclass method."""
        if self.variables['is_started']:
            if self.variables['is_black']:
                # Either rage-quit or clean race finish (we'll handle rage quits later)
                self.handle(cur_count)
        else:
            self.deactivate()

    def handle(self, cur_count):
        """Store variables and toggle detectors. Overrides superclass method."""
        self.variables['is_started'] = False
        # Populate dictionary with race duration
        self.variables['duration'] = np.ceil((cur_count / self.variables['frame_rate']) - self.variables['start_time'])
        self.variables['events'].append([(self.variables['start_time'], self.variables['duration'])])
        # On end race, deactivate EndRaceDetector, activate StartRaceDetector, BoxExtractor, and CharDetector
        self.deactivate()
        self.activate('StartRace')
        self.activate('Characters')
        self.activate('Map')
        self.activate('BoxExtractor')
        # Store as event for splitting
        self.create_event(start_time=self.variables['start_time'],
                          duration=self.variables['duration'])
        if DEBUG_LEVEL == 0:
            try:
                database.put_race(self.variables)
            except: #TODO Figure out exact exceptions
                # UH OH!
                pass
        else:
            print "[%s] End of race detected at t=%2.2f seconds" % (self.name(), self.variables['duration'])
<commit_msg>Removed stale phase0 code for DB handling<commit_after>"""
Modularized detection suite containing the phase_0 classes.
    Authors: Johan Mickos   jmickos@bu.edu
             Josh Navon     navonj@bu.edu
"""

# Standard library
import copy
import itertools

# External dependencies
import numpy as np
import cv2 as cv

# Project-specific
import utility
from generic import Detector

from config import DEBUG_LEVEL

class BoxExtractor(Detector):
    """This class locates and stores box coordinates in frames."""
    def __init__(self,):
        """"Class constructor. Overrides superclass method."""
        self.set = False

    def detect(self, cur_frame, frame_cnt):
        """Perform box extraction. Overrides superclass method."""
        OFFSET = 10
        # Force black frame to ensure first coord is top left of frame
        border_frame = cv.copyMakeBorder(cur_frame, OFFSET, OFFSET, OFFSET, OFFSET, cv.BORDER_CONSTANT, (0, 0, 0))
        # Treshold + grayscale for binary image
        gray = cv.cvtColor(border_frame, cv.COLOR_BGR2GRAY)
        _, gray = cv.threshold(gray, 30, 255, cv.THRESH_BINARY)
        points = [None] * 2
        # Evaluate black lines & extract box coordinates
        for axis in (0, 1):
            projection = gray.sum(axis=axis)
            projection *= 255/(projection.max() + 1)
            black_lines = np.where(projection <= 80)[0] - OFFSET + 1
            clumps =  np.split(black_lines, np.where(np.diff(black_lines) != 1)[0]+1)
            black_lines = np.where(projection <= 25)[0] - OFFSET + 1
            clumps =  np.split(black_lines, np.where(np.diff(black_lines) != 1)[0] + 1)
            coords = [(clumps[i][-1], clumps[i+1][0]) for i in range(len(clumps) - 1)]
            filtered_coords = [coords[i] for i in np.where(np.diff(coords) > 125)[0]]
            points[axis] = filtered_coords

        if utility.in_range(len(points[0]), 0, 2) and \
           utility.in_range(len(points[1]), 0, 2):
            self.variables['player_regions'][:] = []
            local = list()
            for coord in itertools.product(points[0], points[1]):
                local.append([(int(str(coord[0][0]), 10), int(str(coord[0][1]), 10)),
                              (int(str(coord[1][0]), 10), int(str(coord[1][1]), 10))])
            self.variables['player_regions'] = self.sort_boxes(local, cur_frame)
            if not self.set and self.variables['is_started']:
                self.variables['locked_regions'] = copy.deepcopy(self.variables['player_regions'][0:self.variables['num_players']])
                self.set = True
        else:
            # Completely black frame
            self.variables['player_regions'] = [[(0, cur_frame.shape[1]), (0, cur_frame.shape[0])]]

    def sort_boxes(self, boxes, cur_frame):
        """Sorting algorithm that places priority on "top left" boxes"""
        if len(boxes) != 0:
            ordered = list()
            upper = np.max(boxes).astype(float)
            for box in boxes:
                box_normed = np.divide(box, upper)
                rank = np.sum(100 * box_normed[1]) + np.sum(10 * box_normed[0])
                ordered.append((box, rank))
            ordered = sorted(ordered, key=lambda x:x[1])
            result = [el[0] for el in ordered]
            return result
        else:
            return [[(0, cur_frame.shape[1]), (0, cur_frame.shape[0])]]

class Characters(Detector):
    """Detector for detecting characters in MK64."""
    def __init__(self, masks_dir, freq, threshold, default_shape, buf_len=None):
        """
        Instantiates necessary variables and passes control off to superclass constructor.
        Overrides superclass method.
        """
        self.waiting_black = False
        super(Characters, self).__init__(masks_dir, freq, threshold, default_shape, buf_len)

    def detect(self, frame, cur_count, player):
        """Determines whether and how to process current frame. Overrides superclass method."""
        height, width, _ = frame.shape
        focus_region = frame[np.ceil(height * 0.25) : np.ceil(height * 0.95),
                        np.ceil(width * 0.25) : np.ceil(width * 0.75)]
        if self.variables['is_black']:
            if self.waiting_black:
                self.store_players(focus_region)
            else:
                return
        if not self.variables['is_started'] and (cur_count % self.freq == 0):
            self.process(focus_region, cur_count, player)
            if DEBUG_LEVEL > 2:
                cv.imshow(self.name(), focus_region)
                cv.waitKey(1)

    def process(self, frame, cur_count, player):
        """ Compares pre-loaded masks to current frame. Overrides superclass method."""
        player = 0
        if len(self.default_shape) != 1:
            for mask, shape in zip(self.masks, self.default_shape):
                if frame.shape != shape:
                    scaled_mask = (utility.scaleImage(frame,mask[0], shape), mask[1])
                else:
                    scaled_mask = mask
                distances = cv.matchTemplate(frame, scaled_mask[0], cv.TM_SQDIFF_NORMED)
                minval, _, minloc, _ = cv.minMaxLoc(distances)
                if minval <= self.threshold:
                    self.handle(frame, player, mask, cur_count, minloc)
                    if DEBUG_LEVEL > 0:
                        print "[%s]: Found %s :-) ------> %s" % (self.name(), mask[1], minval)
        else:
            for mask in self.masks:
                if frame.shape != self.default_shape[0]:
                    scaled_mask = (utility.scaleImage(frame,mask[0], self.default_shape[0]), mask[1])
                else:
                    scaled_mask = mask
                distances = cv.matchTemplate(frame, scaled_mask[0], cv.TM_SQDIFF_NORMED)
                minval, _, minloc, _ = cv.minMaxLoc(distances)
                if minval <= self.threshold:
                    self.handle(frame, player, mask, cur_count, minloc)
                    if DEBUG_LEVEL > 0:
                        print "[%s]: Found %s :-) ------> %s" % (self.name(), mask[1], minval)

    def handle(self, frame, player, mask, cur_count, location):
        """Perform checks and store variables. Overrides superclass method."""
        self.waiting_black = True
        timestamp = cur_count / self.variables['frame_rate']
        idx = self.buffer.exists(mask[1])
        if idx != -1:
            self.buffer[idx] = (mask[1], location, timestamp)
        else:
            self.buffer.append((mask[1], location, timestamp))

    def store_players(self, frame):
        """Stores players in the state variable."""
        self.waiting_black = False
        self.deactivate()
        ordered = list()
        max_place = 0
        characters = utility.find_unique(self.buffer, 0)

        semi_ordered = self.sort_characters(characters, frame)
        # Find the largest rank in the list.
        for element in semi_ordered:
            if element[1] > max_place:
                max_place = element[1]

        # For every possible player number, find the most recent match
        for ii in xrange(1, (max_place + 1)):
            timestamp = 0
            for element in semi_ordered:
                # Check if the char is player ii and if it was found after <timestamp>
                if element[1] == ii and element[2] > timestamp:
                    timestamp = element[2]
                    temp = (element[0], element[1])
            ordered.append(temp)
        chars = [image[0].split(".", 1)[0] for image in ordered]
        if DEBUG_LEVEL > 0:
            for p, ch in enumerate(chars):
                print "Player %i is %s" % ((p+1), ch)
        self.variables['characters'] = chars
        self.variables['num_players'] = len(ordered)

    def sort_characters(self, characters, frame):
        """Sorting algorithm that ranks based on quadrant."""
        ordered = list()
        center_y, center_x, _ = frame.shape
        center_x /= 2
        center_y /= 2
        for char in characters:
            # Quadrant 2 (Player 1)
            if characters[char][0][1] < (center_x - 10) and characters[char][0][0] < (center_y - 10):
                ordered.append((char, 1, characters[char][1]))
            elif characters[char][0][1] > (center_x + 10) and characters[char][0][0] < (center_y - 10):
                ordered.append((char, 2, characters[char][1]))
            elif characters[char][0][1] < (center_x - 10) and characters[char][0][0] > (center_y + 10):
                ordered.append((char, 3, characters[char][1]))
            else:
                ordered.append((char, 4, characters[char][1]))
        ordered = sorted(ordered, key=lambda x: x[1])
        return ordered

class Map(Detector):
    """Determines which map is being played."""
    def __init__(self, masks_dir, freq, threshold, default_shape, buf_len=None):
        """
        Instantiates necessary variables and passes control off to superclass constructor.
        Overrides superclass method.
        """
        self.waiting_black = False
        self.map = ''
        super(Map, self).__init__(masks_dir, freq, threshold, default_shape, buf_len)

    def detect(self, frame, cur_count, player):
        """Determines whether and how to process current frame. Overrides superclass method."""
        if self.variables['is_black']:
            if self.waiting_black:
                self.variables['map'] = self.map.split('.')[0]
                self.waiting_black = False
                if DEBUG_LEVEL > 0:
                    print 'Locked in map as %s' % (self.map.split('.')[0])
            else:
                return
        if (cur_count % self.freq) == 0:
            self.process(frame, cur_count, player)

    def process(self, frame, cur_count, player):
        """ Compares pre-loaded masks to current frame. Overrides superclass method."""
        best_val = 1
        best_mask = None

        if frame != None:
            # Convert to grayscale and threshold
            binary = cv.cvtColor(frame , cv.COLOR_BGR2GRAY)
            _, binary = cv.threshold(binary, 120, 255, cv.THRESH_BINARY)
            binary = cv.GaussianBlur(binary, (3, 3), 1)
            # Scale and grayscale the masks
            if len(self.default_shape) != 1:
                binary_roi = self.constrain_roi(binary)
                for mask, shape in zip(self.masks, self.default_shape):
                    if frame.shape != shape:
                        scaled_mask = (cv.cvtColor(utility.scaleImage(frame, mask[0], shape),
                            cv.COLOR_BGR2GRAY), mask[1])
                    else:
                        scaled_mask = (cv.cvtColor(mask[0], cv.COLOR_BGR2GRAY), mask[1])
                    distances = cv.matchTemplate(binary_roi, scaled_mask[0], cv.TM_SQDIFF_NORMED)
                    minval, _, minloc, _ = cv.minMaxLoc(distances)
                    if minval <= self.threshold and minval < best_val:
                        best_val = minval
                        best_mask = scaled_mask
                    if DEBUG_LEVEL > 2:
                        cv.imshow('Map Thresh', binary_roi)
                        cv.waitKey(1)
                if best_mask is not None:
                    self.handle(frame, player, best_mask, cur_count, minloc)
                    if DEBUG_LEVEL > 0:
                        print "[%s]: Found %s :-) ------> %s" % (self.name(), best_mask[1], best_val)
            else:
                binary_roi = self.constrain_roi(binary)
                for mask in self.masks:
                    if frame.shape != self.default_shape[0]:
                            scaled_mask = (cv.cvtColor(utility.scaleImage(frame,mask[0], self.default_shape[0]),
                                cv.COLOR_BGR2GRAY), mask[1])
                    else:
                        scaled_mask = (cv.cvtColor(mask[0], cv.COLOR_BGR2GRAY), mask[1])
                    distances = cv.matchTemplate(binary_roi, scaled_mask[0], cv.TM_SQDIFF_NORMED)
                    minval, _, minloc, _ = cv.minMaxLoc(distances)
                    if minval <= self.threshold and minval < best_val:
                        best_val = minval
                        best_mask = scaled_mask
                    if DEBUG_LEVEL > 2:
                        cv.imshow('Map Thresh', binary_roi)
                        cv.waitKey(1)
                if best_mask is not None:
                    self.handle(frame, player, best_mask, cur_count, minloc)
                    if DEBUG_LEVEL > 0:
                        print "[%s]: Found %s :-) ------> %s" % (self.name(), best_mask[1], best_val)

    def handle(self, frame, player, mask, cur_count, location):
        """Set variables. Overrides superclass method."""
        self.map = mask[1]
        self.waiting_black = True
        if DEBUG_LEVEL > 0:
            print '[%s]: Map is: %s' % (self.name(), mask[1].split('.')[0])

    def constrain_roi(self, frame):
        """"Constrains frame w.r.t. Maps. Overrides superclass method."""
        h, w = frame.shape
        frame = frame[np.ceil(h * 0.4):, np.ceil(w * 0.45):np.ceil(w * 0.92)]
        return frame

class StartRace(Detector):
    """Handles the beginning of a race in phase_0"""
    def handle(self, frame, player, mask, cur_count, location):
        """Store variables and toggle detectors. Overrides superclass method."""
        self.variables['is_started'] = True
        self.deactivate()
        self.activate('EndRace')
        self.deactivate('Characters')
        self.deactivate('Map')
        # Lock in player boxes (should be sorted alreadY)
        self.variables['player_regions'] = self.variables['player_regions'][0:self.variables['num_players']]
        # Populate dictionary with start time
        self.variables['start_time'] = np.floor(cur_count / self.variables['frame_rate']) - 2
        if DEBUG_LEVEL > 0:
            print '[%s]: Race started at %d seconds' % (self.name(), self.variables['start_time'])
            cv.waitKey(1)
    def constrain_roi(self, frame):
        """Constrains frame w.r.t. StartRace/BeginRace. Overrides superclass method."""
        h, w, _ = frame.shape
        frame = frame[0:np.ceil(h * 0.5), np.ceil(w * 0.4):np.ceil(w * 0.75)]
        return frame

class EndRace(Detector):
    """Handles the end of a race (phase_0)"""
    def __init__(self):
        """Class constructor. Overrides superclass method."""
        pass

    def detect(self, frame, cur_count, player):
        """Determines whether and how to process current frame. Overrides superclass method."""
        if self.variables['is_started']:
            if self.variables['is_black']:
                # Either rage-quit or clean race finish (we'll handle rage quits later)
                self.handle(cur_count)
        else:
            self.deactivate()

    def handle(self, cur_count):
        """Store variables and toggle detectors. Overrides superclass method."""
        self.variables['is_started'] = False
        # Populate dictionary with race duration
        self.variables['duration'] = np.ceil((cur_count / self.variables['frame_rate']) - self.variables['start_time'])
        self.variables['events'].append([(self.variables['start_time'], self.variables['duration'])])
        # On end race, deactivate EndRaceDetector, activate StartRaceDetector, BoxExtractor, and CharDetector
        self.deactivate()
        self.activate('StartRace')
        self.activate('Characters')
        self.activate('Map')
        self.activate('BoxExtractor')
        # Store as event for splitting
        self.create_event(start_time=self.variables['start_time'],
                          duration=self.variables['duration'])
        if DEBUG_LEVEL > 0:
            print "[%s] End of race detected at t=%2.2f seconds" % (self.name(), self.variables['duration'])
<|endoftext|>


<commit_before>#########################################################################
#     Wifiscanner.py - A simple python script which records and logs wifi probe requests.
#     Author - D4rKP01s0n
#     Requirements - Scapy and Datetime
#     Inspiration - Tim Tomes (LaNMaSteR53)'s WUDS https://bitbucket.org/LaNMaSteR53/wuds/
#     Reminder - Change mon0 (around line 65) to your monitor-mode enabled wifi interface
#########################################################################

from datetime import datetime
from scapy.all import sniff, Dot11
#import numpy
import logging
import time
#Devices which are known to be constantly probing
IGNORE_LIST = set(['00:00:00:00:00:00', '01:01:01:01:01:01'])
#SEEN_DEVICES = set() #Devices which have had their probes recieved, removed due to memory use
d = {'00:00:00:00:00:00':'Example MAC Address'} #Dictionary of all named devices

#knownfile = open('knowndevices.txt', 'a')
#knownfile.write(str(d))

#This exports known dictionary to a text file, but should only be used if the existing file is removed and the dictionary has been updated, as otherwise it will append the list to the existing file

#class colors: # These allow for color-coded output
#    HEADER = '\033[95m'    #    An example of using this would be as follows
#    OKBLUE = '\033[94m'    #    print bcolors.WARNING + "Warning: No active frommets remain. Continue?" + bcolors.ENDC
#    OKGREEN = '\033[92m'   #    Credit: Joeld of StackOverflow: http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
#    WARNING = '\033[93m'
#    FAIL = '\033[91m'
#    ENDC = '\033[0m'       #    End every colored line with this, or else everything following will be the same color
#    BOLD = '\033[1m'
#    UNDERLINE = '\033[4m'

def handle_packet(pkt):
  if not pkt.haslayer(Dot11):
		return
	if pkt.type == 0 and pkt.subtype == 4: #subtype used to be 8 (APs) but is now 4 (Probe Requests)
		#logging.debug('Probe Recorded with MAC ' + curmac)
		curmac = pkt.addr2
		curmac = curmac.upper() #Assign variable to packet mac and make it uppercase
		#SEEN_DEVICES.add(curmac) #Add to set of known devices (sets ignore duplicates so it is not a problem)
		if curmac not in IGNORE_LIST: #If not registered as ignored
			if curmac in d:
				logging.info('\033[95m' + 'Probe Recorded from ' + '\033[93m' + d[curmac] + '\033[95m' + ' with MAC ' + curmac + '\033[0m') #Log to file wifiscanner.log with purple color
                                print('\033[95m' + 'Probe MAC Address: ' + pkt.addr2 + ' from device ' + '\033[93m' + d[curmac] + '\033[0m')
                                        #'with SSID: {pkt.info}'.format(pkt=pkt)) #Print to command line with purple color
			else:
				logging.info('\033[92m' + 'Probe Recorded from MAC ' + pkt.addr2 + '\033[0m') #Log to file wifiscanner.log with green color
				print('\033[95m' + 'Device MAC: {pkt.addr2} '
					'with SSID: {pkt.info}'.format(pkt=pkt) + '\033[0m') #Print to command line with green color
			#print SEEN_DEVICES #Just for debug, prints all known devices
			#dump()

def main():

	logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p',filename='wifiscanner.log',level=logging.DEBUG) #setup logging to file
	logging.info('\n' + '\033[93m' + 'Wifi Scanner Initialized' + '\033[0m' + '\n') #announce that it has started to log file with yellow color
	print('\n' + '\033[93m' + 'Wifi Scanner Initialized' + '\033[0m' + '\n') #announce that it has started to command line with yellow color		(/n is newline)

	import argparse
	parser = argparse.ArgumentParser()
	parser.add_argument('--interface', '-i', default='mon0', # Change mon0 to your monitor-mode enabled wifi interface
				help='monitor mode enabled interface')
	args = parser.parse_args()
	sniff(iface=args.interface, prn=handle_packet, store=0, count=0) #start sniffin, store=0 makes sniff not save captured packets and count=0 makes sniff continously scan packets
if __name__ == '__main__':
	main()
<commit_msg>Added wificard variable<commit_after>#########################################################################
#     Wifiscanner.py - A simple python script which records and logs wifi probe requests.
#     Author - D4rKP01s0n
#     Requirements - Scapy and Datetime
#########################################################################

from datetime import datetime
from scapy.all import sniff, Dot11
#import numpy
import logging
import time
#Devices which are known to be constantly probing
IGNORE_LIST = set(['00:00:00:00:00:00', '01:01:01:01:01:01'])
#SEEN_DEVICES = set() #Devices which have had their probes recieved, removed due to memory use
d = {'00:00:00:00:00:00':'Example MAC Address'} #Dictionary of all named devices

wificard = 'mon0' #    Change this to your monitor-mode enabled wifi interface

#knownfile = open('knowndevices.txt', 'a')
#knownfile.write(str(d))

#This exports known dictionary to a text file, but should only be used if the existing file is removed and the dictionary has been updated, as otherwise it will append the list to the existing file

#class colors: # These allow for color-coded output
#    HEADER = '\033[95m'    #    An example of using this would be as follows
#    OKBLUE = '\033[94m'    #    print bcolors.WARNING + "Warning: No active frommets remain. Continue?" + bcolors.ENDC
#    OKGREEN = '\033[92m'   #    Credit: Joeld of StackOverflow: http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
#    WARNING = '\033[93m'
#    FAIL = '\033[91m'
#    ENDC = '\033[0m'       #    End every colored line with this, or else everything following will be the same color
#    BOLD = '\033[1m'
#    UNDERLINE = '\033[4m'

def handle_packet(pkt):
  if not pkt.haslayer(Dot11):
		return
	if pkt.type == 0 and pkt.subtype == 4: #subtype used to be 8 (APs) but is now 4 (Probe Requests)
		#logging.debug('Probe Recorded with MAC ' + curmac)
		curmac = pkt.addr2
		curmac = curmac.upper() #Assign variable to packet mac and make it uppercase
		#SEEN_DEVICES.add(curmac) #Add to set of known devices (sets ignore duplicates so it is not a problem)
		if curmac not in IGNORE_LIST: #If not registered as ignored
			if curmac in d:
				logging.info('\033[95m' + 'Probe Recorded from ' + '\033[93m' + d[curmac] + '\033[95m' + ' with MAC ' + curmac + '\033[0m') #Log to file wifiscanner.log with purple color
                                print('\033[95m' + 'Probe MAC Address: ' + pkt.addr2 + ' from device ' + '\033[93m' + d[curmac] + '\033[0m')
                                        #'with SSID: {pkt.info}'.format(pkt=pkt)) #Print to command line with purple color
			else:
				logging.info('\033[92m' + 'Probe Recorded from MAC ' + pkt.addr2 + '\033[0m') #Log to file wifiscanner.log with green color
				print('\033[95m' + 'Device MAC: {pkt.addr2} '
					'with SSID: {pkt.info}'.format(pkt=pkt) + '\033[0m') #Print to command line with green color
			#print SEEN_DEVICES #Just for debug, prints all known devices
			#dump()

def main():

	logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p',filename='wifiscanner.log',level=logging.DEBUG) #setup logging to file
	logging.info('\n' + '\033[93m' + 'Wifi Scanner Initialized' + '\033[0m' + '\n') #announce that it has started to log file with yellow color
	print('\n' + '\033[93m' + 'Wifi Scanner Initialized' + '\033[0m' + '\n') #announce that it has started to command line with yellow color		(/n is newline)

	import argparse
	parser = argparse.ArgumentParser()
	parser.add_argument('--interface', '-i', default=wificard, # Change mon0 to your monitor-mode enabled wifi interface
				help='monitor mode enabled interface')
	args = parser.parse_args()
	sniff(iface=args.interface, prn=handle_packet, store=0, count=0) #start sniffin, store=0 makes sniff not save captured packets and count=0 makes sniff continously scan packets
if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>from distutils.core import setup

setup(
  scripts=[
    'byteyears.py',
    'checkpyc.py',
    'copytime.py',
    'crlf.py',
    'dutree.py',
    'ftpmirror.py',
    'h2py.py',
    'lfcr.py',
    '../../Lib/tabnanny.py',
    'untabify.py',
    ],
  )
<commit_msg>add logmerge.py (pydoc is already installed)<commit_after>from distutils.core import setup

setup(
  scripts=[
    'byteyears.py',
    'checkpyc.py',
    'copytime.py',
    'crlf.py',
    'dutree.py',
    'ftpmirror.py',
    'h2py.py',
    'lfcr.py',
    'logmerge.py',
    '../../Lib/tabnanny.py',
    'untabify.py',
    ],
  )
<|endoftext|>


<commit_before>"""
Extension that omits the creation of file `skeleton.py`.

Since the isolated builds with PEP517/PEP518 are not completely backward compatible,
this extension provides a escape hatch for people that want to maintain the legacy
behaviour.

This extension might be excluded once support for isolated builds stabilises and the
community moves more to use it more exclusively.
"""

from typing import List

from .. import structure
from ..actions import Action, ActionParams, ScaffoldOpts, Structure
from . import Extension

class NoPyProject(Extension):
    """Do not include a pyproject.toml file in the project root, and thus avoid isolated
    builds as defined in PEP517/518.
    """

    name = "no_pyproject"

    def activate(self, actions: List[Action]) -> List[Action]:
        actions = self.register(actions, ensure_option, before="get_default_options")
        return self.register(actions, remove_files, after="define_structure")

def ensure_option(struct: Structure, opts: ScaffoldOpts) -> ActionParams:
    """Make option available in non-CLI calls (used by other parts of PyScaffold)"""
    return struct, {**opts, "pyproject": False, "isolated_build": False}

def remove_files(struct: Structure, opts: ScaffoldOpts) -> ActionParams:
    struct = structure.reject(struct, "pyproject.toml")
    return struct, opts
<commit_msg>Fix wrong name in no_pyproject docs<commit_after>"""
Extension that omits the creation of file `pyproject.toml`.

Since the isolated builds with PEP517/PEP518 are not completely backward compatible,
this extension provides a escape hatch for people that want to maintain the legacy
behaviour.

This extension might be excluded once support for isolated builds stabilises and the
community moves more to use it more exclusively.
"""

from typing import List

from .. import structure
from ..actions import Action, ActionParams, ScaffoldOpts, Structure
from . import Extension

class NoPyProject(Extension):
    """Do not include a pyproject.toml file in the project root, and thus avoid isolated
    builds as defined in PEP517/518.
    """

    name = "no_pyproject"

    def activate(self, actions: List[Action]) -> List[Action]:
        actions = self.register(actions, ensure_option, before="get_default_options")
        return self.register(actions, remove_files, after="define_structure")

def ensure_option(struct: Structure, opts: ScaffoldOpts) -> ActionParams:
    """Make option available in non-CLI calls (used by other parts of PyScaffold)"""
    return struct, {**opts, "pyproject": False, "isolated_build": False}

def remove_files(struct: Structure, opts: ScaffoldOpts) -> ActionParams:
    struct = structure.reject(struct, "pyproject.toml")
    return struct, opts
<|endoftext|>


<commit_before>from django.core.exceptions import ObjectDoesNotExist

from notification.backends.base import NotificationBackend

import facebook

import logging
log = logging.getLogger(__name__)

class FacebookBackend(NotificationBackend):
    def graph_api(self, user):
        """Return an instance of facebook.GraphAPI authorized with the `user`'s
        OAUth token.
        """
        return user.get_profile().facebook_graph_api

    def facebook_user_id(self, user):
        """Return the Facebook OpenGraph ID for the user or None if they do not
        have one in the system."""
        try:
            user_id = unicode(user.get_profile().facebook_id)
        except ObjectDoesNotExist:
            user_id = None
        return user_id

    def facebook_token(self, user):
        """Return the Facebook OpenGraph OAuth token for the user or None if
        they do not have one in the system."""
        try:
            token = user.get_profile().facebook_auth.token
        except ObjectDoesNotExist:
            token = None
        return token

    def should_send(self, sender, recipient, notice_type, *args, **kwargs):
        """Return true if the sender has an OAuth token and the recipient has at
        least a Facebook OpenGraph ID.
        """
        send = super(FacebookBackend, self).should_send(sender, recipient, notice_type)
        return (send and self.facebook_token(sender)
                and self.facebook_user_id(recipient))

class FacebookWallPostBackend(FacebookBackend):
    slug = u"facebook_wall_post"
    display_name = u"Facebook Wall Post"

    def send(self, sender, recipient, notice_type, context, *args, **kwargs):
        if not self.should_send(sender, recipient, notice_type):
            return False

        message = self.render_message(notice_type.label,
                'notification/facebook/wall_post.txt', 'notice.html', context)
        try:
            graph = self.graph_api(sender)
            graph.put_wall_post(message,
                    profile_id=self.facebook_user_id(recipient))
        except facebook.GraphAPIError:
            log.exception("Received an error when making a wall post from "
                    "%s to %s" % (sender, recipient))
            return False
        else:
            return True
<commit_msg>Check for a user before checking for their FB info.<commit_after>from django.core.exceptions import ObjectDoesNotExist

from notification.backends.base import NotificationBackend

import facebook

import logging
log = logging.getLogger(__name__)

class FacebookBackend(NotificationBackend):
    def graph_api(self, user):
        """Return an instance of facebook.GraphAPI authorized with the `user`'s
        OAUth token.
        """
        return user.get_profile().facebook_graph_api

    def facebook_user_id(self, user):
        """Return the Facebook OpenGraph ID for the user or None if they do not
        have one in the system."""
        if not user:
            return
        try:
            user_id = unicode(user.get_profile().facebook_id)
        except ObjectDoesNotExist:
            user_id = None
        return user_id

    def facebook_token(self, user):
        """Return the Facebook OpenGraph OAuth token for the user or None if
        they do not have one in the system."""
        if not user:
            return
        try:
            token = user.get_profile().facebook_auth.token
        except ObjectDoesNotExist:
            token = None
        return token

    def should_send(self, sender, recipient, notice_type, *args, **kwargs):
        """Return true if the sender has an OAuth token and the recipient has at
        least a Facebook OpenGraph ID.
        """
        send = super(FacebookBackend, self).should_send(sender, recipient,
                notice_type)
        return (send and self.facebook_token(sender)
                and self.facebook_user_id(recipient))

class FacebookWallPostBackend(FacebookBackend):
    slug = u"facebook_wall_post"
    display_name = u"Facebook Wall Post"

    def send(self, sender, recipient, notice_type, context, *args, **kwargs):
        if not self.should_send(sender, recipient, notice_type):
            return False

        message = self.render_message(notice_type.label,
                'notification/facebook/wall_post.txt', 'notice.html', context)
        try:
            graph = self.graph_api(sender)
            graph.put_wall_post(message,
                    profile_id=self.facebook_user_id(recipient))
        except facebook.GraphAPIError:
            log.exception("Received an error when making a wall post from "
                    "%s to %s" % (sender, recipient))
            return False
        else:
            return True
<|endoftext|>


<commit_before>import json
import socket

import requests

from flask import request, jsonify

from dadd.worker import app
from dadd.worker.proc import ChildProcess

@app.route('/run/', methods=['POST'])
def run_process():
    proc = ChildProcess(request.json)
    proc.run()
    return jsonify(proc.info())

@app.route('/register/', methods=['POST'])
def register_with_master():
    register(app)
    return jsonify({'message': 'ok'})

def register(host, port):
    sess = requests.Session()

    if 'USERNAME' in app.config and 'PASSWORD' in app.config:
        sess.auth = (app.config['USERNAME'], app.config['PASSWORD'])
    sess.headers = {'content-type': 'application/json'}

    try:
        url = app.config['MASTER_URL'] + '/api/hosts/'
        resp = sess.post(url, data=json.dumps({
            'host': socket.getfqdn(), 'port': port
        }))
        if not resp.ok:
            app.logger.warning('Error registering with master: %s' %
                               app.config['MASTER_URL'])
    except Exception as e:
        app.logger.warning('Connection Error: %s' % e)
<commit_msg>Allow setting the HOSTNAME reported to the master via an env var; Stub and endpoint for tailing the log of a process assuming you know the path.<commit_after>import os
import json
import socket

import requests

from flask import request, jsonify, Response, abort

from dadd.worker import app
from dadd.worker.proc import ChildProcess

@app.route('/run/', methods=['POST'])
def run_process():
    proc = ChildProcess(request.json)
    proc.run()
    return jsonify(proc.info())

@app.route('/register/', methods=['POST'])
def register_with_master():
    register(app)
    return jsonify({'message': 'ok'})

def register(host, port):
    sess = requests.Session()

    if 'USERNAME' in app.config and 'PASSWORD' in app.config:
        sess.auth = (app.config['USERNAME'], app.config['PASSWORD'])
    sess.headers = {'content-type': 'application/json'}

    try:
        url = app.config['MASTER_URL'] + '/api/hosts/'
        resp = sess.post(url, data=json.dumps({
            'host': app.config.get('HOSTNAME', socket.getfqdn()),
            'port': port
        }))
        if not resp.ok:
            app.logger.warning('Error registering with master: %s' %
                               app.config['MASTER_URL'])
    except Exception as e:
        app.logger.warning('Connection Error: %s' % e)

@app.route('/logs/<path>', methods=['GET'])
def tail_log(path):
    if os.path.exists(path) and path.startswith('/tmp/'):
        return Response(open(path), content_type='text/plain')
    abort(404)
<|endoftext|>


<commit_before>import h5py
import os
import csv
import math
import itertools
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.image as mimg

#### Load Training Dataset (TDS) (Label Vector and Feature Matrix)
def load_tds(file_name, list_name):
	'''
	INPUT:
		input_directory of .h5 training data
		file_name of .h5 training data
		list_name of label vector contained within file_name
	RETURNS:
		tds = [label_vector, training_feature_matrix]
	'''

	## Load the training data
	with h5py.File(file_name, 'r') as training_file:
		label_vector = training_file[list_name][:]
		segments = training_file['segment_list'][:]
		training_feature_matrix = training_file['feature_matrix'][:]

	## Convert inputs to python lists
	label_vector = label_vector.tolist()
	training_feature_matrix = training_feature_matrix.tolist()
	# Remove feature lists that dont have an associated label
	training_feature_matrix = training_feature_matrix[:len(label_vector)]
	# print "__"
	# print len(label_vector)
	## Remove the segments labeled "unknown" (0)
	while 0 in label_vector:
		i = label_vector.index(0)
		label_vector.pop(i)
		training_feature_matrix.pop(i)

	## Remove the segments labeled "mixed" (0)
	while 6 in label_vector:
		i = label_vector.index(6)
		label_vector.pop(i)
		training_feature_matrix.pop(i)

	## Remove the segments labeled "Shadow" (0)
	while 5 in label_vector:
		i = label_vector.index(5)
		label_vector.pop(i)
		training_feature_matrix.pop(i)

	# Combine the label vector and training feature matrix into one variable.
	tds = [label_vector,training_feature_matrix]

	return tds

#### Save classification results
def save_results(csv_name, path, image_name, pixel_counts):
	'''
	INPUT:
		path: location where the output will write
		image_name: name of the image that was classified
		pixel_clounts: number of pixels in each classification category

	Saves a csv with the input information. Appends to existing csv if one already exists

	NOTES:
		Only works with 5 classification categories:
			[white ice, gray ice, melt ponds, open water, shadow]
	'''

	csv_name = os.path.splitext(csv_name)[0] + ".csv"

	num_pixels = 0
	for i in range(len(pixel_counts)):
		num_pixels +=  pixel_counts[i]
	percentages = []
	for i in range(len(pixel_counts)):
		percentages.append(float(pixel_counts[i]/num_pixels))

	try:
		output_csv = os.path.join(path, csv_name)
		if not os.path.isfile(output_csv):
			with open(output_csv, "wb") as csvfile:
				writer = csv.writer(csvfile)
				writer.writerow(["Source", "White Ice", "Gray Ice", "Melt Ponds", "Open Water", "Shadow",
					"Prcnt White Ice", "Prcnt Gray Ice", "Prcnt Melt Ponds", "Prcnt Open Water", "Prcnt Shadow"])
				writer.writerow([image_name, pixel_counts[0], pixel_counts[1], pixel_counts[2], pixel_counts[3], pixel_counts[4],
					percentages[0], percentages[1], percentages[2], percentages[3], percentages[4]])

		else:
			with open(output_csv, "ab+") as csvfile:
				writer = csv.writer(csvfile)
				writer.writerow([image_name, pixel_counts[0], pixel_counts[1], pixel_counts[2], pixel_counts[3], pixel_counts[4],
					percentages[0], percentages[1], percentages[2], percentages[3], percentages[4]])
	except:
		print "error saving csv"
		print pixel_counts

#### Recombine classified image splits
def stitch(image_files, save_path=None):
	'''
	INPUT:
		image_files: list of the image splits for recombination
	RETURN:
		full_classification: full image stitched back together

	NOTES:
		Currently only implemented to recombine classified images, but the
		method could work with any image data.
		There are two levels of recombination. Recompiling the subimages (see
		method below) and recompiling the splits (this method)
	'''

	# Check to see if we have a square number of images
	#	This method relies on floating point precision, but
	#	will be accurate within the scope of this method
	root = math.sqrt(len(image_files))
	if int(root) != root:
		print "Incomplete set of images!"
		return None

	original_list = []
	classified_list = []

	image_files.sort()

	## Read the classified data and the original image data
	for image in image_files:
		with h5py.File(image,'r') as inputfile:
			original_image = inputfile['original'][:]
			original_list.append(original_image)
			classified_image = inputfile['classified'][:]
			classified_list.append(classified_image)

	# Find the right dimensions for stitching the images back together
	box_side = int(math.sqrt(len(classified_list)))
	# Get the number of bands based on the input list
	try:
		num_bands = np.shape(original_list)[3]
	except IndexError:
		num_bands = 1
	# Stitch the original back together
	full_original = compile_subimages(original_list,box_side,box_side,num_bands)
	# Stitch the classified image back together
	full_classification = compile_subimages(classified_list,box_side,box_side)

	if os.path.isdir(save_path):
		output_name = os.path.join(save_path, os.path.split(image_files[0])[1][:-18])
		save_color(full_classification, output_name + "_classified_image.png")
		fout = h5py.File(output_name + "_classified.h5",'w')
		fout.create_dataset('classified',data=full_classification,compression='gzip',compression_opts=9)
		fout.create_dataset('original',data=full_original,compression='gzip',compression_opts=9)
		fout.close()
	else:
		save_color(full_classification, image_files[0][:-18] + "_classified_image.png")
		fout = h5py.File(image_files[0][:-18] + "_classified.h5",'w')
		fout.create_dataset('classified',data=full_classification,compression='gzip',compression_opts=9)
		fout.create_dataset('original',data=full_original,compression='gzip',compression_opts=9)
		fout.close()

	return full_classification

#### Compiles the subimages of a split
def compile_subimages(subimage_list, num_x_subimages, num_y_subimages, bands=1):
	'''
	INPUT:
		subimage_list: the list of subimages, in left to right top to bottom order
		num_x_subimages: number of subimages in the x dimension
		num_y_subimages: number of subimages in the y dimension
		bands: number of spectral bands of the input image
	RETURNS:
		compiled_image: single [x,y,b] image
	'''
	x_size = np.shape(subimage_list[0])[1]
	y_size = np.shape(subimage_list[0])[0]

	if bands != 1:
		compiled_image = np.zeros([num_y_subimages*y_size,num_x_subimages*x_size,bands],dtype='uint8')
		counter = 0
		for y in range(num_y_subimages):
			for x in range(num_x_subimages):
				compiled_image[y*y_size:(y+1)*y_size, x*x_size:(x+1)*x_size, :] = subimage_list[counter]
				counter += 1
	else:
		compiled_image = np.zeros([num_y_subimages*y_size,num_x_subimages*x_size],dtype='uint8')
		counter = 0
		for y in range(num_y_subimages):
			for x in range(num_x_subimages):
				compiled_image[y*y_size:(y+1)*y_size, x*x_size:(x+1)*x_size] = subimage_list[counter]
				counter += 1

	return compiled_image

#### Saves an image with custom colormap
def save_color(image, save_name, custom_colormap=False):
	''''
	INPUTS:
		image: The image you want to save
		save_name: full name and filepath where you want the image to go
		custom_colormap: matplotlib colormap if you want to use your own
			defaults to nwright's 5 class colormap

	Saves a .png of the input image with desired colormap
	'''

	if custom_colormap is False:
		# Colors for the output image
		empty_color = [.1,.1,.1]		#Almost black
		snow_color = [.9,.9,.9]			#Almost white
		pond_color = [.31,.431,.647]	#Blue
		gray_color = [.5,.5,.5]			#Gray
		water_color = [0.,0.,0.]		#Black
		shadow_color = [1.0, .545, .0]  #Orange
		cloud_color = [.27, .15, .50]		#Purple

		custom_colormap = [empty_color,snow_color,gray_color,pond_color,water_color,shadow_color,cloud_color]
		custom_colormap = colors.ListedColormap(custom_colormap)

		#Making sure there is atleast one of every pixel so the colors map properly (only changes
		# display image, not saved data)
		image[0][0] = 0
		image[1][0] = 1
		image[2][0] = 2
		image[3][0] = 3
		image[4][0] = 4
		image[5][0] = 5
		image[6][0] = 6

	mimg.imsave(save_name, image, format='png', cmap=custom_colormap)

#### Count the number of pixels in each classification category of given image
def count_features(classified_image):

	sum_snow = float(len(classified_image[classified_image==1.0]))
	sum_gray_ice = float(len(classified_image[classified_image==2.0]))
	sum_melt_ponds = float(len(classified_image[classified_image==3.0]))
	sum_open_water = float(len(classified_image[classified_image==4.0]))
	sum_shadow = float(len(classified_image[classified_image==5.0]))

	# num_pixels =  sum_snow + sum_gray_ice + sum_melt_ponds + sum_open_water

	return sum_snow, sum_gray_ice, sum_melt_ponds, sum_open_water, sum_shadow

# Code from http://chriskiehl.com/article/parallelism-in-one-line/
# Returns a list of .h5 files in the given folder.
def get_image_paths(folder,keyword='.h5'):
	return (os.path.join(folder, f)
		for f in os.listdir(folder)
		if keyword in f)

# Remove hidden folders and files from the given list of strings (mac)
def remove_hidden(folder):
	i = 0
	while i < len(folder):
		if folder[i][0] == '.':
			folder.pop(i)
		else:
			i+=1

# Plots a confusion matrix. Adapted from
# http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py
#
def plot_confusion_matrix(cm,categories,ylabel,xlabel,
							normalize=False,
							title='',
							cmap=plt.cm.Blues):
	"""
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
	font = {'family' : 'Times New Roman',
		'weight' : 'bold',
		'size'   : 12}

	matplotlib.rc('font', **font)

	plt.imshow(cm, interpolation='nearest', cmap=cmap)
	plt.title(title)
	tick_marks = np.arange(len(categories))
	plt.xticks(tick_marks, categories, rotation=45)
	plt.yticks(tick_marks, categories)

	if normalize:
		cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
		print("Normalized confusion matrix")
	else:
	    print('Confusion matrix, without normalization')

	print(cm)

	thresh = cm.max() / 4.
	for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
		plt.text(j, i, cm[i, j],
				horizontalalignment="center",
				color="white" if cm[i, j] > thresh else "black")

	plt.tight_layout()
	plt.ylabel(ylabel)
	plt.xlabel(xlabel)
	plt.show()
<commit_msg>Removed saving of original image data along with classified results<commit_after>import h5py
import os
import csv
import math
import itertools
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.image as mimg

#### Load Training Dataset (TDS) (Label Vector and Feature Matrix)
def load_tds(file_name, list_name):
	'''
	INPUT:
		input_directory of .h5 training data
		file_name of .h5 training data
		list_name of label vector contained within file_name
	RETURNS:
		tds = [label_vector, training_feature_matrix]
	'''

	## Load the training data
	with h5py.File(file_name, 'r') as training_file:
		label_vector = training_file[list_name][:]
		segments = training_file['segment_list'][:]
		training_feature_matrix = training_file['feature_matrix'][:]

	## Convert inputs to python lists
	label_vector = label_vector.tolist()
	training_feature_matrix = training_feature_matrix.tolist()
	# Remove feature lists that dont have an associated label
	training_feature_matrix = training_feature_matrix[:len(label_vector)]
	# print "__"
	# print len(label_vector)
	## Remove the segments labeled "unknown" (0)
	while 0 in label_vector:
		i = label_vector.index(0)
		label_vector.pop(i)
		training_feature_matrix.pop(i)

	## Remove the segments labeled "mixed" (0)
	while 6 in label_vector:
		i = label_vector.index(6)
		label_vector.pop(i)
		training_feature_matrix.pop(i)

	## Remove the segments labeled "Shadow" (0)
	while 5 in label_vector:
		i = label_vector.index(5)
		label_vector.pop(i)
		training_feature_matrix.pop(i)

	# Combine the label vector and training feature matrix into one variable.
	tds = [label_vector,training_feature_matrix]

	return tds

#### Save classification results
def save_results(csv_name, path, image_name, pixel_counts):
	'''
	INPUT:
		path: location where the output will write
		image_name: name of the image that was classified
		pixel_clounts: number of pixels in each classification category

	Saves a csv with the input information. Appends to existing csv if one already exists

	NOTES:
		Only works with 5 classification categories:
			[white ice, gray ice, melt ponds, open water, shadow]
	'''

	csv_name = os.path.splitext(csv_name)[0] + ".csv"

	num_pixels = 0
	for i in range(len(pixel_counts)):
		num_pixels +=  pixel_counts[i]
	percentages = []
	for i in range(len(pixel_counts)):
		percentages.append(float(pixel_counts[i]/num_pixels))

	try:
		output_csv = os.path.join(path, csv_name)
		if not os.path.isfile(output_csv):
			with open(output_csv, "wb") as csvfile:
				writer = csv.writer(csvfile)
				writer.writerow(["Source", "White Ice", "Gray Ice", "Melt Ponds", "Open Water", "Shadow",
					"Prcnt White Ice", "Prcnt Gray Ice", "Prcnt Melt Ponds", "Prcnt Open Water", "Prcnt Shadow"])
				writer.writerow([image_name, pixel_counts[0], pixel_counts[1], pixel_counts[2], pixel_counts[3], pixel_counts[4],
					percentages[0], percentages[1], percentages[2], percentages[3], percentages[4]])

		else:
			with open(output_csv, "ab+") as csvfile:
				writer = csv.writer(csvfile)
				writer.writerow([image_name, pixel_counts[0], pixel_counts[1], pixel_counts[2], pixel_counts[3], pixel_counts[4],
					percentages[0], percentages[1], percentages[2], percentages[3], percentages[4]])
	except:
		print "error saving csv"
		print pixel_counts

#### Recombine classified image splits
def stitch(image_files, save_path=None):
	'''
	INPUT:
		image_files: list of the image splits for recombination
	RETURN:
		full_classification: full image stitched back together

	NOTES:
		Currently only implemented to recombine classified images, but the
		method could work with any image data.
		There are two levels of recombination. Recompiling the subimages (see
		method below) and recompiling the splits (this method)
	'''

	# Check to see if we have a square number of images
	#	This method relies on floating point precision, but
	#	will be accurate within the scope of this method
	root = math.sqrt(len(image_files))
	if int(root) != root:
		print "Incomplete set of images!"
		return None

	original_list = []
	classified_list = []

	image_files.sort()

	## Read the classified data and the original image data
	for image in image_files:
		with h5py.File(image,'r') as inputfile:
			classified_image = inputfile['classified'][:]
			classified_list.append(classified_image)

	# Find the right dimensions for stitching the images back together
	box_side = int(math.sqrt(len(classified_list)))
	# Get the number of bands based on the input list
	try:
		num_bands = np.shape(original_list)[3]
	except IndexError:
		num_bands = 1
	# Stitch the original back together
	full_original = compile_subimages(original_list,box_side,box_side,num_bands)
	# Stitch the classified image back together
	full_classification = compile_subimages(classified_list,box_side,box_side)

	if os.path.isdir(save_path):
		output_name = os.path.join(save_path, os.path.split(image_files[0])[1][:-18])
		fout = h5py.File(output_name + "_classified.h5",'w')
		fout.create_dataset('classified',data=full_classification,compression='gzip',compression_opts=9)
		fout.close()
	else:
		save_color(full_classification, image_files[0][:-18] + "_classified_image.png")
		fout = h5py.File(image_files[0][:-18] + "_classified.h5",'w')
		fout.create_dataset('classified',data=full_classification,compression='gzip',compression_opts=9)
		fout.close()

	return full_classification

#### Compiles the subimages of a split
def compile_subimages(subimage_list, num_x_subimages, num_y_subimages, bands=1):
	'''
	INPUT:
		subimage_list: the list of subimages, in left to right top to bottom order
		num_x_subimages: number of subimages in the x dimension
		num_y_subimages: number of subimages in the y dimension
		bands: number of spectral bands of the input image
	RETURNS:
		compiled_image: single [x,y,b] image
	'''
	x_size = np.shape(subimage_list[0])[1]
	y_size = np.shape(subimage_list[0])[0]

	if bands != 1:
		compiled_image = np.zeros([num_y_subimages*y_size,num_x_subimages*x_size,bands],dtype='uint8')
		counter = 0
		for y in range(num_y_subimages):
			for x in range(num_x_subimages):
				compiled_image[y*y_size:(y+1)*y_size, x*x_size:(x+1)*x_size, :] = subimage_list[counter]
				counter += 1
	else:
		compiled_image = np.zeros([num_y_subimages*y_size,num_x_subimages*x_size],dtype='uint8')
		counter = 0
		for y in range(num_y_subimages):
			for x in range(num_x_subimages):
				compiled_image[y*y_size:(y+1)*y_size, x*x_size:(x+1)*x_size] = subimage_list[counter]
				counter += 1

	return compiled_image

#### Saves an image with custom colormap
def save_color(image, save_name, custom_colormap=False):
	''''
	INPUTS:
		image: The image you want to save
		save_name: full name and filepath where you want the image to go
		custom_colormap: matplotlib colormap if you want to use your own
			defaults to nwright's 5 class colormap

	Saves a .png of the input image with desired colormap
	'''

	if custom_colormap is False:
		# Colors for the output image
		empty_color = [.1,.1,.1]		#Almost black
		snow_color = [.9,.9,.9]			#Almost white
		pond_color = [.31,.431,.647]	#Blue
		gray_color = [.5,.5,.5]			#Gray
		water_color = [0.,0.,0.]		#Black
		shadow_color = [1.0, .545, .0]  #Orange
		cloud_color = [.27, .15, .50]		#Purple

		custom_colormap = [empty_color,snow_color,gray_color,pond_color,water_color,shadow_color,cloud_color]
		custom_colormap = colors.ListedColormap(custom_colormap)

		#Making sure there is atleast one of every pixel so the colors map properly (only changes
		# display image, not saved data)
		image[0][0] = 0
		image[1][0] = 1
		image[2][0] = 2
		image[3][0] = 3
		image[4][0] = 4
		image[5][0] = 5
		image[6][0] = 6

	mimg.imsave(save_name, image, format='png', cmap=custom_colormap)

#### Count the number of pixels in each classification category of given image
def count_features(classified_image):

	sum_snow = float(len(classified_image[classified_image==1.0]))
	sum_gray_ice = float(len(classified_image[classified_image==2.0]))
	sum_melt_ponds = float(len(classified_image[classified_image==3.0]))
	sum_open_water = float(len(classified_image[classified_image==4.0]))
	sum_shadow = float(len(classified_image[classified_image==5.0]))

	# num_pixels =  sum_snow + sum_gray_ice + sum_melt_ponds + sum_open_water

	return sum_snow, sum_gray_ice, sum_melt_ponds, sum_open_water, sum_shadow

# Code from http://chriskiehl.com/article/parallelism-in-one-line/
# Returns a list of .h5 files in the given folder.
def get_image_paths(folder,keyword='.h5'):
	return (os.path.join(folder, f)
		for f in os.listdir(folder)
		if keyword in f)

# Remove hidden folders and files from the given list of strings (mac)
def remove_hidden(folder):
	i = 0
	while i < len(folder):
		if folder[i][0] == '.':
			folder.pop(i)
		else:
			i+=1

# Plots a confusion matrix. Adapted from
# http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py
#
def plot_confusion_matrix(cm,categories,ylabel,xlabel,
							normalize=False,
							title='',
							cmap=plt.cm.Blues):
	"""
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
	font = {'family' : 'Times New Roman',
		'weight' : 'bold',
		'size'   : 12}

	matplotlib.rc('font', **font)

	plt.imshow(cm, interpolation='nearest', cmap=cmap)
	plt.title(title)
	tick_marks = np.arange(len(categories))
	plt.xticks(tick_marks, categories, rotation=45)
	plt.yticks(tick_marks, categories)

	if normalize:
		cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
		print("Normalized confusion matrix")
	else:
	    print('Confusion matrix, without normalization')

	print(cm)

	thresh = cm.max() / 4.
	for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
		plt.text(j, i, cm[i, j],
				horizontalalignment="center",
				color="white" if cm[i, j] > thresh else "black")

	plt.tight_layout()
	plt.ylabel(ylabel)
	plt.xlabel(xlabel)
	plt.show()
<|endoftext|>


<commit_before>def memory_usage():
    """return memory usage of python process in MB

    from http://fa.bianp.net/blog/2013/different-ways-to-get-memory-consumption-or-lessons-learned-from-memory_profiler/
    psutil is quicker

    >>> isinstance(memory_usage(),float)
    True

    """
    try:
        import psutil, os
    except ImportError:
        return _memory_usage_ps()

    process = psutil.Process(os.getpid())
    mem = process.memory_info()[0] / float(2 ** 20)
    return mem

def _memory_usage_ps():
    """return memory usage of python process in MB

    >>> isinstance(_memory_usage_ps(),float)
    True

    """
    import subprocess, os
    out = subprocess.Popen(['ps', 'v', '-p', str(os.getpid())],
    stdout=subprocess.PIPE).communicate()[0].split(b'\n')
    vsz_index = out[0].split().index(b'RSS')
    mem = float(out[1].split()[vsz_index]) / 1024
    return mem

def load_memit():
    """load memory usage ipython magic,
    require memory_profiler package to be installed

    to get usage: %memit?

    Author: Vlad Niculae <vlad@vene.ro>
    Makes use of memory_profiler from Fabian Pedregosa
    available at https://github.com/fabianp/memory_profiler
    """
    from IPython.core.magic import Magics, line_magic, magics_class
    from memory_profiler import memory_usage as _mu

    try:
        ip = get_ipython()
    except NameError as err:
        raise Exception('not in ipython/jupyter kernel:\n {}'.format(err))

    @magics_class
    class MemMagics(Magics):
        @line_magic
        def memit(self, line='', setup='pass'):
            """Measure memory usage of a Python statement

            Usage, in line mode:
              %memit [-ir<R>t<T>] statement

            Options:
            -r<R>: repeat the loop iteration <R> times and take the best result.
            Default: 3

            -i: run the code in the current environment, without forking a new
            process. This is required on some MacOS versions of Accelerate if your
            line contains a call to `np.dot`.

            -t<T>: timeout after <T> seconds. Unused if `-i` is active.
            Default: None

            Examples
            --------
            ::

              In [1]: import numpy as np

              In [2]: %memit np.zeros(1e7)
              maximum of 3: 76.402344 MB per loop

              In [3]: %memit np.ones(1e6)
              maximum of 3: 7.820312 MB per loop

              In [4]: %memit -r 10 np.empty(1e8)
              maximum of 10: 0.101562 MB per loop

              In [5]: memit -t 3 while True: pass;
              Subprocess timed out.
              Subprocess timed out.
              Subprocess timed out.
              ERROR: all subprocesses exited unsuccessfully. Try again with the
              `-i` option.
              maximum of 3: -inf MB per loop

            """
            opts, stmt = self.parse_options(line, 'r:t:i', posix=False,
                                            strict=False)
            repeat = int(getattr(opts, 'r', 3))
            if repeat < 1:
                repeat == 1
            timeout = int(getattr(opts, 't', 0))
            if timeout <= 0:
                timeout = None
            run_in_place = hasattr(opts, 'i')

            # Don't depend on multiprocessing:
            try:
                import multiprocessing as pr
                from multiprocessing.queues import SimpleQueue
                q = SimpleQueue()
            except ImportError:
                class ListWithPut(list):
                    "Just a list where the `append` method is aliased to `put`."
                    def put(self, x):
                        self.append(x)
                q = ListWithPut()
                print ('WARNING: cannot import module `multiprocessing`. Forcing '
                       'the `-i` option.')
                run_in_place = True

            ns = self.shell.user_ns

            def _get_usage(q, stmt, setup='pass', ns={}):
                try:
                    exec(setup) in ns
                    _mu0 = _mu()[0]
                    exec(stmt) in ns
                    _mu1 = _mu()[0]
                    q.put(_mu1 - _mu0)
                except Exception as e:
                    q.put(float('-inf'))
                    raise e

            if run_in_place:
                for _ in xrange(repeat):
                    _get_usage(q, stmt, ns=ns)
            else:
                # run in consecutive subprocesses
                at_least_one_worked = False
                for _ in xrange(repeat):
                    p = pr.Process(target=_get_usage, args=(q, stmt, 'pass', ns))
                    p.start()
                    p.join(timeout=timeout)
                    if p.exitcode == 0:
                        at_least_one_worked = True
                    else:
                        p.terminate()
                        if p.exitcode == None:
                            print 'Subprocess timed out.'
                        else:
                            print 'Subprocess exited with code %d.' % p.exitcode
                        q.put(float('-inf'))

                if not at_least_one_worked:
                    print ('ERROR: all subprocesses exited unsuccessfully. Try '
                           'again with the `-i` option.')

            usages = [q.get() for _ in xrange(repeat)]
            usage = max(usages)
            print u"maximum of %d: %f MB per loop" % (repeat, usage)

    ip.register_magics(MemMagics)

<commit_msg>bug fix<commit_after>def memory_usage():
    """return memory usage of python process in MB

    from http://fa.bianp.net/blog/2013/different-ways-to-get-memory-consumption-or-lessons-learned-from-memory_profiler/
    psutil is quicker

    >>> isinstance(memory_usage(),float)
    True

    """
    try:
        import psutil, os
    except ImportError:
        return _memory_usage_ps()

    process = psutil.Process(os.getpid())
    mem = process.memory_info()[0] / float(2 ** 20)
    return mem

def _memory_usage_ps():
    """return memory usage of python process in MB

    >>> isinstance(_memory_usage_ps(),float)
    True

    """
    import subprocess, os
    out = subprocess.Popen(['ps', 'v', '-p', str(os.getpid())],
    stdout=subprocess.PIPE).communicate()[0].split(b'\n')
    vsz_index = out[0].split().index(b'RSS')
    mem = float(out[1].split()[vsz_index]) / 1024
    return mem

def load_memit():
    """load memory usage ipython magic,
    require memory_profiler package to be installed

    to get usage: %memit?

    Author: Vlad Niculae <vlad@vene.ro>
    Makes use of memory_profiler from Fabian Pedregosa
    available at https://github.com/fabianp/memory_profiler
    """
    from IPython.core.magic import Magics, line_magic, magics_class
    from memory_profiler import memory_usage as _mu

    try:
        ip = get_ipython()
    except NameError as err:
        raise Exception('not in ipython/jupyter kernel:\n {}'.format(err))

    @magics_class
    class MemMagics(Magics):
        @line_magic
        def memit(self, line='', setup='pass'):
            """Measure memory usage of a Python statement

            Usage, in line mode:
              %memit [-ir<R>t<T>] statement

            Options:
            -r<R>: repeat the loop iteration <R> times and take the best result.
            Default: 3

            -i: run the code in the current environment, without forking a new
            process. This is required on some MacOS versions of Accelerate if your
            line contains a call to `np.dot`.

            -t<T>: timeout after <T> seconds. Unused if `-i` is active.
            Default: None

            Examples
            --------
            ::

              In [1]: import numpy as np

              In [2]: %memit np.zeros(1e7)
              maximum of 3: 76.402344 MB per loop

              In [3]: %memit np.ones(1e6)
              maximum of 3: 7.820312 MB per loop

              In [4]: %memit -r 10 np.empty(1e8)
              maximum of 10: 0.101562 MB per loop

              In [5]: memit -t 3 while True: pass;
              Subprocess timed out.
              Subprocess timed out.
              Subprocess timed out.
              ERROR: all subprocesses exited unsuccessfully. Try again with the
              `-i` option.
              maximum of 3: -inf MB per loop

            """
            opts, stmt = self.parse_options(line, 'r:t:i', posix=False,
                                            strict=False)
            repeat = int(getattr(opts, 'r', 3))
            if repeat < 1:
                repeat == 1
            timeout = int(getattr(opts, 't', 0))
            if timeout <= 0:
                timeout = None
            run_in_place = hasattr(opts, 'i')

            # Don't depend on multiprocessing:
            try:
                import multiprocessing as pr
                from multiprocessing.queues import SimpleQueue
                q = SimpleQueue()
            except ImportError:
                class ListWithPut(list):
                    "Just a list where the `append` method is aliased to `put`."
                    def put(self, x):
                        self.append(x)
                q = ListWithPut()
                print ('WARNING: cannot import module `multiprocessing`. Forcing '
                       'the `-i` option.')
                run_in_place = True

            ns = self.shell.user_ns

            def _get_usage(q, stmt, setup='pass', ns={}):
                try:
                    exec(setup) in ns
                    _mu0 = _mu()[0]
                    exec(stmt) in ns
                    _mu1 = _mu()[0]
                    q.put(_mu1 - _mu0)
                except Exception as e:
                    q.put(float('-inf'))
                    raise e

            if run_in_place:
                for _ in xrange(repeat):
                    _get_usage(q, stmt, ns=ns)
            else:
                # run in consecutive subprocesses
                at_least_one_worked = False
                for _ in xrange(repeat):
                    p = pr.Process(target=_get_usage, args=(q, stmt, 'pass', ns))
                    p.start()
                    p.join(timeout=timeout)
                    if p.exitcode == 0:
                        at_least_one_worked = True
                    else:
                        p.terminate()
                        if p.exitcode == None:
                            print('Subprocess timed out.')
                        else:
                            print('Subprocess exited with code %d.' % p.exitcode)
                        q.put(float('-inf'))

                if not at_least_one_worked:
                    print('ERROR: all subprocesses exited unsuccessfully. Try '
                           'again with the `-i` option.')

            usages = [q.get() for _ in xrange(repeat)]
            usage = max(usages)
            print("maximum of %d: %f MB per loop" % (repeat, usage))

    ip.register_magics(MemMagics)

<|endoftext|>


<commit_before>from django.contrib import admin

from .models import DeviceToken

admin.site.register(DeviceToken)
<commit_msg>Add a list filter in the admin to filter by platform.<commit_after>from django.contrib import admin

from .models import DeviceToken

class DeviceTokenAdmin(admin.ModelAdmin):
    list_filter = ["platform"]

admin.site.register(DeviceToken, DeviceTokenAdmin)
<|endoftext|>