<commit_before>#!/usr/bin/env python
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import rospy
import sys
import json
import time
from statistics.srv import SessionQuery

class GeolocationDataException(Exception):
    pass

class FileWriter:
    """
    - periodically collects session events from /statistics/session service
    - wraps session events with metadata and renders json files to filesystem
    """

    def __init__(self):
        self.node = self._init_node()
        self.path = rospy.get_param('~json_output_path', '/tmp')
        self.interval = rospy.get_param('~aggregation_interval', 300)
        self.country = rospy.get_param('~country', None)
        self.store_id = rospy.get_param('~store_id', None)
        self.retailer = rospy.get_param('~retailer', None)
        if self.country is None or\
           self.store_id is None or\
           self.retailer is None:
               raise GeolocationDataException("You need to specify proper store metadata")
        self._wait_for_service()
        pass

    def _wait_for_service(self):
        rospy.loginfo("Waiting for the /statistics/session service to become available")
        rospy.wait_for_service('statistics/session')
        rospy.loginfo("Statistics aggretator service has become available")
        pass

    def _call_service(self):
        service_call = rospy.ServiceProxy('statistics/session', SessionQuery)
        response = service_call(erase=True)
        rospy.logdebug("Received response from service: %s" % response)
        return response

    def _sleep_between_requests(self):
        rospy.logdebug("Sleeping for %s seconds before writing stats" % self.interval)
        rospy.sleep(self.interval)
        pass

    def _init_node(self):
        rospy.init_node('statistics')
        pass

    def _jsonize_string(self, string):
        sessions = string.replace("'{", "{").replace("}'", "}")
        rospy.logdebug("Deserialized sessions => %s" % sessions)
        return json.loads(sessions)

    def _finalize_report_data(self, report_contents):
        """
        - fill report_time and end_ts with current time
        - fill start_ts with current_time minus self.interval
        """
        unix_now = int(time.time())
        report_contents['report_time'] = unix_now
        report_contents['start_ts'] = unix_now - self.interval
        report_contents['end_ts'] = unix_now
        return report_contents

    def _compose_and_write_json(self, sessions):
        """
        - jsonize the sessions string received from json aggregator
        - convert sessions list to json
        - finalize report and write the file
        """
        sessions = self._jsonize_string(sessions)
        report_contents = {"report_time": 0,
                           "start_ts": 0,
                           "end_ts": 0,
                           "metadata": {"country": self.country,
                                        "store_id": self.store_id,
                                        "retailer": self.retailer
                                        },
                           "sessions": []
                           }
        for session in sessions:
            report_contents['sessions'].append(session)

        report_contents = self._finalize_report_data(report_contents)
        with open(self.path + "/" +  str(int(time.time())) + ".json", 'w') as json_file:
            rospy.loginfo("Writing file %s" % json_file.name)
            json_file.write(json.dumps(report_contents, sort_keys=True, indent=4, separators=(',', ': ')))
            del report_contents

    def _spin(self):
        try:
            response = self._call_service()
        except rospy.service.ServiceException, e:
            rospy.logfatal("Could not connect to /statistics/session service because: %s" % e)
        self._compose_and_write_json(response.sessions)
        pass

    def run(self):
        rospy.logdebug("Starting session statistics service")
        while not rospy.is_shutdown():
            self._sleep_between_requests()
            self._spin()
            pass
        pass

if __name__ == '__main__':
    try:
        FileWriter().run()
    except rospy.ROSInterruptException:
        pass
    except rospy.service.ServiceException:
        pass
<commit_msg>Proper fields in proper places<commit_after>#!/usr/bin/env python
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import rospy
import sys
import json
import time
from statistics.srv import SessionQuery

class GeolocationDataException(Exception):
    pass

class FileWriter:
    """
    - periodically collects session events from /statistics/session service
    - wraps session events with metadata and renders json files to filesystem
    """

    def __init__(self):
        self.node = self._init_node()
        self.path = rospy.get_param('~json_output_path', '/tmp')
        self.interval = rospy.get_param('~aggregation_interval', 300)
        self.country = rospy.get_param('~country', None)
        self.store_id = rospy.get_param('~store_id', None)
        self.retailer = rospy.get_param('~retailer', None)
        if self.country is None or\
           self.store_id is None or\
           self.retailer is None:
               raise GeolocationDataException("You need to specify proper store metadata")
        self._wait_for_service()
        pass

    def _wait_for_service(self):
        rospy.loginfo("Waiting for the /statistics/session service to become available")
        rospy.wait_for_service('statistics/session')
        rospy.loginfo("Statistics aggretator service has become available")
        pass

    def _call_service(self):
        service_call = rospy.ServiceProxy('statistics/session', SessionQuery)
        response = service_call(erase=True)
        rospy.logdebug("Received response from service: %s" % response)
        return response

    def _sleep_between_requests(self):
        rospy.logdebug("Sleeping for %s seconds before writing stats" % self.interval)
        rospy.sleep(self.interval)
        pass

    def _init_node(self):
        rospy.init_node('statistics')
        pass

    def _jsonize_string(self, string):
        sessions = string.replace("'{", "{").replace("}'", "}")
        rospy.logdebug("Deserialized sessions => %s" % sessions)
        return json.loads(sessions)

    def _finalize_report_data(self, report_contents):
        """
        - fill report_time and end_ts with current time
        - fill start_ts with current_time minus self.interval
        """
        unix_now = int(time.time())
        report_contents['report_time'] = unix_now
        report_contents['start_ts'] = unix_now - self.interval
        report_contents['end_ts'] = unix_now
        return report_contents

    def _compose_and_write_json(self, sessions):
        """
        - jsonize the sessions string received from json aggregator
        - convert sessions list to json
        - finalize report and write the file
        """
        sessions = self._jsonize_string(sessions)
        report_contents = {"report_time": 0,
                           "start_ts": 0,
                           "status": "on",
                           "experience_type": "portal",
                           "end_ts": 0,
                           "metadata": {"country": self.country,
                                        "store_id": self.store_id,
                                        "retailer": self.retailer
                                        },
                           "sessions": []
                           }
        for session in sessions:
            report_contents['sessions'].append(session)

        report_contents = self._finalize_report_data(report_contents)
        with open(self.path + "/" +  str(int(time.time())) + ".json", 'w') as json_file:
            rospy.loginfo("Writing file %s" % json_file.name)
            json_file.write(json.dumps(report_contents, sort_keys=True, indent=4, separators=(',', ': ')))
            del report_contents

    def _spin(self):
        try:
            response = self._call_service()
        except rospy.service.ServiceException, e:
            rospy.logfatal("Could not connect to /statistics/session service because: %s" % e)
        self._compose_and_write_json(response.sessions)
        pass

    def run(self):
        rospy.logdebug("Starting session statistics service")
        while not rospy.is_shutdown():
            self._sleep_between_requests()
            self._spin()
            pass
        pass

if __name__ == '__main__':
    try:
        FileWriter().run()
    except rospy.ROSInterruptException:
        pass
    except rospy.service.ServiceException:
        pass
<|endoftext|>


<commit_before>#partly based on Thibaut Munzer's script

import time
from importlib import import_module
from ..job import Job

job_queue_class={
	'local':'local.LocalJobQueue',
	'local_multiprocess':'local.MultiProcessJobQueue',
	'avakas':'avakas.AvakasJobQueue',
	'plafrim':'plafrim.PlafrimJobQueue'
}

def JobQueue(jq_type='local', **jq_cfg2):
	tempstr = jq_type
	if tempstr in job_queue_class.keys():
		tempstr = job_queue_class[tempstr]
	templist = tempstr.split('.')
	temppath = '.'.join(templist[:-1])
	tempclass = templist[-1]
	_tempmod = import_module('.'+temppath,package=__name__)
	return getattr(_tempmod,tempclass)(**jq_cfg2)

class BaseJobQueue(object):
	def __init__(self, erase=True):
		self.job_list = []
		self.erase = erase

	def add_job(self, job):
		job.status = 'pending'
		if not job in self.job_list:
			self.job_list.append(job)
		else:
			print 'Job already in queue!'

	def update_queue(self):
		for j in self.job_list:
			if j.status == 'pending':
				j.save()
				self.submit_job(j)
			elif j.status == 'running':
				if not self.check_job_running(j):
					self.retrieve_job(j)
					if j.status == 'pending':
						j.status = 'missubmitted'
			if j.status == 'unfinished':
				j.fix()
			elif j.status == 'done':
				os.chdir(j.get_path())
				j.get_data()
				os.chdir(j.get_back_path())
				j.unpack_data()
				j.data = None
				self.job_list.remove(j)
				if (not self.erase) and (not j.erase):
					j.clean()
			elif j.status == 'missubmitted':
				print('Missubmitted job: '+'_'.join([j.descr,j.uuid]))

	def auto_finish_queue(self,t=60):
		if not job_list == []:
			self.update_queue()
			time.sleep(t)
			self.auto_finish_queue(t=t)

	def check_virtualenvs(self):
		envs = []
		for j in self.job_list:
			env = str(j.virtual_env)
			if env not in envs:
				self.update_virtualenv(env)
				envs.append(env)

	def submit_job(self, job):
		pass

	def check_job_running(self, job):
		pass

	def set_virtualenv(self, virtual_env, requirements):
		pass

	def update_virtualenv(self, virtual_env, requirements):
		pass

	def retrieve_job(self, job):
		pass

<commit_msg>tests avakas<commit_after>#partly based on Thibaut Munzer's script

import os
import time
from importlib import import_module
from ..job import Job

job_queue_class={
	'local':'local.LocalJobQueue',
	'local_multiprocess':'local.MultiProcessJobQueue',
	'avakas':'avakas.AvakasJobQueue',
	'plafrim':'plafrim.PlafrimJobQueue'
}

def JobQueue(jq_type='local', **jq_cfg2):
	tempstr = jq_type
	if tempstr in job_queue_class.keys():
		tempstr = job_queue_class[tempstr]
	templist = tempstr.split('.')
	temppath = '.'.join(templist[:-1])
	tempclass = templist[-1]
	_tempmod = import_module('.'+temppath,package=__name__)
	return getattr(_tempmod,tempclass)(**jq_cfg2)

class BaseJobQueue(object):
	def __init__(self, erase=True):
		self.job_list = []
		self.erase = erase

	def add_job(self, job):
		job.status = 'pending'
		if not job in self.job_list:
			self.job_list.append(job)
		else:
			print 'Job already in queue!'

	def update_queue(self):
		for j in self.job_list:
			if j.status == 'pending':
				j.save()
				self.submit_job(j)
			elif j.status == 'running':
				if not self.check_job_running(j):
					self.retrieve_job(j)
					if j.status == 'pending':
						j.status = 'missubmitted'
			if j.status == 'unfinished':
				j.fix()
			elif j.status == 'done':
				os.chdir(j.get_path())
				j.get_data()
				os.chdir(j.get_back_path())
				j.unpack_data()
				j.data = None
				self.job_list.remove(j)
				if (not self.erase) and (not j.erase):
					j.clean()
			elif j.status == 'missubmitted':
				print('Missubmitted job: '+'_'.join([j.descr,j.uuid]))

	def auto_finish_queue(self,t=60):
		if not job_list == []:
			self.update_queue()
			time.sleep(t)
			self.auto_finish_queue(t=t)

	def check_virtualenvs(self):
		envs = []
		for j in self.job_list:
			env = str(j.virtual_env)
			if env not in envs:
				self.update_virtualenv(env)
				envs.append(env)

	def submit_job(self, job):
		pass

	def check_job_running(self, job):
		pass

	def set_virtualenv(self, virtual_env, requirements):
		pass

	def update_virtualenv(self, virtual_env, requirements):
		pass

	def retrieve_job(self, job):
		pass

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# This file is part of Export Layers.
#
# Copyright (C) 2013-2015 khalim19 <khalim19@gmail.com>
#
# Export Layers is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Export Layers is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Export Layers.  If not, see <http://www.gnu.org/licenses/>.
#

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

str = unicode

import inspect
import os
import shutil

import unittest

import gimp

pdb = gimp.pdb

from ..pygimplib import pgpdb
from ..pygimplib import pgsettinggroup

from .. import exportlayers
from .. import settings_plugin

#===============================================================================

_current_module_dir = os.path.dirname(inspect.getfile(inspect.currentframe()))
RESOURCES_DIR = os.path.join(os.path.dirname(os.path.dirname(_current_module_dir)), "resources")
TEST_IMAGES_DIR = os.path.join(RESOURCES_DIR, "Test images")

EXPECTED_RESULTS_DIR = os.path.join(TEST_IMAGES_DIR, "Expected results")
OUTPUT_DIR = os.path.join(TEST_IMAGES_DIR, "Temp output")

#===============================================================================

class TestExportLayersCompareLayerContents(unittest.TestCase):

  @classmethod
  def setUpClass(cls):
    pdb.gimp_context_push()

    cls.test_image_filename = os.path.join(TEST_IMAGES_DIR, "test_export_layers_contents.xcf")
    cls.test_image = pdb.gimp_file_load(cls.test_image_filename, os.path.basename(cls.test_image_filename))

    cls.output_directory = OUTPUT_DIR

    if os.path.exists(cls.output_directory) and os.listdir(cls.output_directory):
      raise ValueError("Directory for temporary results '{0}' must be empty".format(cls.output_directory))

    cls.default_expected_layers_dir = EXPECTED_RESULTS_DIR
    # key: directory containing expected results
    # value: gimp.Image instance
    cls.expected_images = {}

  @classmethod
  def tearDownClass(cls):
    pdb.gimp_image_delete(cls.test_image)
    for image in cls.expected_images.values():
      pdb.gimp_image_delete(image)

    pdb.gimp_context_pop()

  def setUp(self):
    self.image_with_results = None

  def tearDown(self):
    if self.image_with_results is not None:
      pdb.gimp_image_delete(self.image_with_results)
    if os.path.exists(self.output_directory):
      shutil.rmtree(self.output_directory)

  def test_default_settings(self):
    self.compare()

  def test_ignore_layer_modes(self):
    self.compare({
                   'ignore_layer_modes': True
                 },
                 [('overlay', 'overlay_ignore-layer-modes')])

  def test_autocrop(self):
    self.compare({
                   'autocrop': True
                 },
                 [('left-frame-with-extra-borders', 'left-frame-with-extra-borders_autocrop'),
                  ('main-background', 'main-background_autocrop')])

  def test_use_image_size(self):
    self.compare({
                   'use_image_size': True
                 },
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'use_image_size'))

  def test_use_image_size_autocrop(self):
    self.compare({
                   'use_image_size': True,
                   'autocrop': True
                 },
                 [('left-frame-with-extra-borders', 'left-frame-with-extra-borders_autocrop'),
                  ('main-background', 'main-background_autocrop')],
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'use_image_size'))

  def test_merge_layer_groups(self):
    self.compare({
                   'merge_layer_groups': True
                 },
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'merge_layer_groups'))

  def test_background(self):
    self.compare({
                   'square_bracketed_mode': 1   # 1 = "Treat as background layers"
                 },
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'background'))

  def test_background_autocrop(self):
    self.compare({
                   'square_bracketed_mode': 1,   # 1 = "Treat as background layers"
                   'autocrop': True
                 },
                 [('bottom-frame-semi-transparent', 'bottom-frame-semi-transparent_background_autocrop'),
                  ('left-frame-with-extra-borders', 'left-frame-with-extra-borders_autocrop')])

  def test_background_autocrop_use_image_size(self):
    self.compare({
                   'square_bracketed_mode': 1,   # 1 = "Treat as background layers"
                   'autocrop': True,
                   'use_image_size': True
                 },
                 [('left-frame-with-extra-borders', 'left-frame-with-extra-borders_autocrop_use-image-size')],
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'background'))

  def test_background_autocrop_crop_to_background(self):
    self.compare({
                   'square_bracketed_mode': 1,   # 1 = "Treat as background layers"
                   'autocrop': True,
                   'crop_to_background': True
                 },
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'background',
                                                   'crop_to_background'))

  def test_background_autocrop_crop_to_background_use_image_size(self):
    self.compare({
                   'square_bracketed_mode': 1,   # 1 = "Treat as background layers"
                   'autocrop': True,
                   'crop_to_background': True,
                   'use_image_size': True
                 },
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'background',
                                                   'crop_to_background', 'use_image_size'))

  def compare(self, different_settings=None, different_results_and_expected_layers=None,
              expected_results_dir=None):
    settings = settings_plugin.create_settings()
    settings['special']['image'].set_value(self.test_image)
    settings['main']['output_directory'].set_value(self.output_directory)

    if different_settings is not None:
      for setting_name, setting_value in different_settings.items():
        settings['main'][setting_name].set_value(setting_value)

    if expected_results_dir is None:
      expected_results_dir = self.default_expected_layers_dir

    if expected_results_dir not in self.expected_images:
      self.expected_images[expected_results_dir], expected_layers = (
        self._load_layers_from_dir(expected_results_dir))
    else:
      expected_layers = {layer.name: layer for layer in self.expected_images[expected_results_dir].layers}

    param_values = pgsettinggroup.PdbParamCreator.list_param_values([settings])
    pdb.plug_in_export_layers(*param_values)

    self.image_with_results, layers = self._load_layers_from_dir(self.output_directory)

    if different_results_and_expected_layers is not None:
      for layer_name, expected_layer_name in different_results_and_expected_layers:
        expected_layers[layer_name] = expected_layers[expected_layer_name]

    for layer in layers.values():
      self._compare_layers(layer, expected_layers[layer.name])

  def _compare_layers(self, layer, expected_layer):
    self.assertEqual(pgpdb.compare_layers([layer, expected_layer]), True,
      msg=("Layers are not identical:\n"
           "processed layer: {0}\n"
           "expected layer: {1}"
           .format(layer.name, expected_layer.name)))

  @classmethod
  def _load_layers(cls, layers_filenames):
    """
    Load layers from specified filenames into a new image. Return the image and
    a dict with (layer name: gimp.Layer instance) pairs.
    """

    image = pgpdb.load_layers(layers_filenames, image=None, strip_file_extension=True)
    return image, {layer.name: layer for layer in image.layers}

  @classmethod
  def _load_layers_from_dir(cls, layers_dir):
    return cls._load_layers(cls._list_layers_files(layers_dir))

  @classmethod
  def _list_layers_files(cls, layers_dir):
    layers_files = []

    for filename in os.listdir(layers_dir):
      path = os.path.join(layers_dir, filename)
      if os.path.isfile(path):
        layers_files.append(path)

    return layers_files

#===============================================================================

def test_file_formats(layer_exporter, export_settings):
  """
  Test all file formats at once.
  """

  file_extensions = [
    "data", "xcf", "pix", "matte", "mask", "alpha", "als", "fli", "flc",
    "xcf.bz2", "xcfbz2", "c", "h", "xhtml", "dds", "dcm", "dicom", "eps",
    "fit", "fits", "gif", "gbr", "gih", "pat", "xcf.gz", "xcfgz",
    "html", "htm", "jpg", "jpeg", "jpe", "cel", "ico", "mng", "ora", "pbm",
    "pgm", "psd", "png", "pnm", "pdf", "ps", "ppm", "sgi", "rgb", "rgba",
    "bw", "icon", "im1", "im8", "im24", "im32", "rs", "ras", "tga", "tif",
    "tiff", "bmp", "xbm", "bitmap", "xpm", "xwd", "pcx", "pcc"
  ]

  orig_output_directory = export_settings['output_directory'].value

  for file_extension in file_extensions:
    export_settings['file_extension'].set_value(file_extension)
    export_settings['output_directory'].set_value(os.path.join(orig_output_directory, file_extension))
    try:
      layer_exporter.export_layers()
    except exportlayers.ExportLayersError:
      # Do not stop execution when one file format causes an error.
      continue
<commit_msg>test_exportlayers: Preserve settings from before execution of tests<commit_after># -*- coding: utf-8 -*-
#
# This file is part of Export Layers.
#
# Copyright (C) 2013-2015 khalim19 <khalim19@gmail.com>
#
# Export Layers is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Export Layers is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Export Layers.  If not, see <http://www.gnu.org/licenses/>.
#

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

str = unicode

import inspect
import os
import shutil

import unittest

import gimp

pdb = gimp.pdb

from ..pygimplib import pgpdb
from ..pygimplib import pgsettinggroup
from ..pygimplib import pgsettingpersistor

from .. import constants
from .. import exportlayers
from .. import settings_plugin

#===============================================================================

_current_module_dir = os.path.dirname(inspect.getfile(inspect.currentframe()))
RESOURCES_DIR = os.path.join(os.path.dirname(os.path.dirname(_current_module_dir)), "resources")
TEST_IMAGES_DIR = os.path.join(RESOURCES_DIR, "Test images")

EXPECTED_RESULTS_DIR = os.path.join(TEST_IMAGES_DIR, "Expected results")
OUTPUT_DIR = os.path.join(TEST_IMAGES_DIR, "Temp output")

#===============================================================================

class TestExportLayersCompareLayerContents(unittest.TestCase):

  @classmethod
  def setUpClass(cls):
    pdb.gimp_context_push()

    cls.test_image_filename = os.path.join(TEST_IMAGES_DIR, "test_export_layers_contents.xcf")
    cls.test_image = pdb.gimp_file_load(cls.test_image_filename, os.path.basename(cls.test_image_filename))

    cls.output_directory = OUTPUT_DIR

    if os.path.exists(cls.output_directory) and os.listdir(cls.output_directory):
      raise ValueError("Directory for temporary results '{0}' must be empty".format(cls.output_directory))

    cls.default_expected_layers_dir = EXPECTED_RESULTS_DIR
    # key: directory containing expected results
    # value: gimp.Image instance
    cls.expected_images = {}

    cls._backup_plugin_settings_from_before_execution()

  @classmethod
  def tearDownClass(cls):
    pdb.gimp_image_delete(cls.test_image)
    for image in cls.expected_images.values():
      pdb.gimp_image_delete(image)

    pdb.gimp_context_pop()

    cls._restore_plugin_settings_from_before_execution()

  @classmethod
  def _backup_plugin_settings_from_before_execution(cls):
    cls.orig_settings = settings_plugin.create_settings()
    cls.session_setting_source = pgsettingpersistor.SessionPersistentSettingSource(
      constants.SESSION_SOURCE_NAME)
    cls.persistent_setting_source = pgsettingpersistor.PersistentSettingSource(
      constants.PERSISTENT_SOURCE_NAME)
    pgsettingpersistor.SettingPersistor.load(
      [cls.orig_settings], [cls.session_setting_source, cls.persistent_setting_source])

  @classmethod
  def _restore_plugin_settings_from_before_execution(cls):
    pgsettingpersistor.SettingPersistor.save([cls.orig_settings], [cls.session_setting_source])

  def setUp(self):
    self.image_with_results = None

  def tearDown(self):
    if self.image_with_results is not None:
      pdb.gimp_image_delete(self.image_with_results)
    if os.path.exists(self.output_directory):
      shutil.rmtree(self.output_directory)

  def test_default_settings(self):
    self.compare()

  def test_ignore_layer_modes(self):
    self.compare({
                   'ignore_layer_modes': True
                 },
                 [('overlay', 'overlay_ignore-layer-modes')])

  def test_autocrop(self):
    self.compare({
                   'autocrop': True
                 },
                 [('left-frame-with-extra-borders', 'left-frame-with-extra-borders_autocrop'),
                  ('main-background', 'main-background_autocrop')])

  def test_use_image_size(self):
    self.compare({
                   'use_image_size': True
                 },
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'use_image_size'))

  def test_use_image_size_autocrop(self):
    self.compare({
                   'use_image_size': True,
                   'autocrop': True
                 },
                 [('left-frame-with-extra-borders', 'left-frame-with-extra-borders_autocrop'),
                  ('main-background', 'main-background_autocrop')],
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'use_image_size'))

  def test_merge_layer_groups(self):
    self.compare({
                   'merge_layer_groups': True
                 },
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'merge_layer_groups'))

  def test_background(self):
    self.compare({
                   'square_bracketed_mode': 1   # 1 = "Treat as background layers"
                 },
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'background'))

  def test_background_autocrop(self):
    self.compare({
                   'square_bracketed_mode': 1,   # 1 = "Treat as background layers"
                   'autocrop': True
                 },
                 [('bottom-frame-semi-transparent', 'bottom-frame-semi-transparent_background_autocrop'),
                  ('left-frame-with-extra-borders', 'left-frame-with-extra-borders_autocrop')])

  def test_background_autocrop_use_image_size(self):
    self.compare({
                   'square_bracketed_mode': 1,   # 1 = "Treat as background layers"
                   'autocrop': True,
                   'use_image_size': True
                 },
                 [('left-frame-with-extra-borders', 'left-frame-with-extra-borders_autocrop_use-image-size')],
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'background'))

  def test_background_autocrop_crop_to_background(self):
    self.compare({
                   'square_bracketed_mode': 1,   # 1 = "Treat as background layers"
                   'autocrop': True,
                   'crop_to_background': True
                 },
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'background',
                                                   'crop_to_background'))

  def test_background_autocrop_crop_to_background_use_image_size(self):
    self.compare({
                   'square_bracketed_mode': 1,   # 1 = "Treat as background layers"
                   'autocrop': True,
                   'crop_to_background': True,
                   'use_image_size': True
                 },
                 expected_results_dir=os.path.join(self.default_expected_layers_dir, 'background',
                                                   'crop_to_background', 'use_image_size'))

  def compare(self, different_settings=None, different_results_and_expected_layers=None,
              expected_results_dir=None):
    settings = settings_plugin.create_settings()
    settings['special']['image'].set_value(self.test_image)
    settings['main']['output_directory'].set_value(self.output_directory)

    if different_settings is not None:
      for setting_name, setting_value in different_settings.items():
        settings['main'][setting_name].set_value(setting_value)

    if expected_results_dir is None:
      expected_results_dir = self.default_expected_layers_dir

    if expected_results_dir not in self.expected_images:
      self.expected_images[expected_results_dir], expected_layers = (
        self._load_layers_from_dir(expected_results_dir))
    else:
      expected_layers = {layer.name: layer for layer in self.expected_images[expected_results_dir].layers}

    param_values = pgsettinggroup.PdbParamCreator.list_param_values([settings])
    pdb.plug_in_export_layers(*param_values)

    self.image_with_results, layers = self._load_layers_from_dir(self.output_directory)

    if different_results_and_expected_layers is not None:
      for layer_name, expected_layer_name in different_results_and_expected_layers:
        expected_layers[layer_name] = expected_layers[expected_layer_name]

    for layer in layers.values():
      self._compare_layers(layer, expected_layers[layer.name])

  def _compare_layers(self, layer, expected_layer):
    self.assertEqual(pgpdb.compare_layers([layer, expected_layer]), True,
      msg=("Layers are not identical:\n"
           "processed layer: {0}\n"
           "expected layer: {1}"
           .format(layer.name, expected_layer.name)))

  @classmethod
  def _load_layers(cls, layers_filenames):
    """
    Load layers from specified filenames into a new image. Return the image and
    a dict with (layer name: gimp.Layer instance) pairs.
    """

    image = pgpdb.load_layers(layers_filenames, image=None, strip_file_extension=True)
    return image, {layer.name: layer for layer in image.layers}

  @classmethod
  def _load_layers_from_dir(cls, layers_dir):
    return cls._load_layers(cls._list_layers_files(layers_dir))

  @classmethod
  def _list_layers_files(cls, layers_dir):
    layers_files = []

    for filename in os.listdir(layers_dir):
      path = os.path.join(layers_dir, filename)
      if os.path.isfile(path):
        layers_files.append(path)

    return layers_files

#===============================================================================

def test_file_formats(layer_exporter, export_settings):
  """
  Test all file formats at once.
  """

  file_extensions = [
    "data", "xcf", "pix", "matte", "mask", "alpha", "als", "fli", "flc",
    "xcf.bz2", "xcfbz2", "c", "h", "xhtml", "dds", "dcm", "dicom", "eps",
    "fit", "fits", "gif", "gbr", "gih", "pat", "xcf.gz", "xcfgz",
    "html", "htm", "jpg", "jpeg", "jpe", "cel", "ico", "mng", "ora", "pbm",
    "pgm", "psd", "png", "pnm", "pdf", "ps", "ppm", "sgi", "rgb", "rgba",
    "bw", "icon", "im1", "im8", "im24", "im32", "rs", "ras", "tga", "tif",
    "tiff", "bmp", "xbm", "bitmap", "xpm", "xwd", "pcx", "pcc"
  ]

  orig_output_directory = export_settings['output_directory'].value

  for file_extension in file_extensions:
    export_settings['file_extension'].set_value(file_extension)
    export_settings['output_directory'].set_value(os.path.join(orig_output_directory, file_extension))
    try:
      layer_exporter.export_layers()
    except exportlayers.ExportLayersError:
      # Do not stop execution when one file format causes an error.
      continue
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Builder of instances."""

import sys
import os
import argparse
import subprocess
# from process_dc_env import pythonGetEnv, dcEnvCheckArgs
from base import Base
from python import Python
from logging import Logging
from supervisor import Supervisor
# ==============================================================================
__version__ = "0.1"

__copyright__ = "Copyright 2017, devops.center"
__credits__ = ["Bob Lozano", "Gregg Jensen"]
__license__ = ' \
   # Copyright 2014-2017 devops.center llc                                    \
   #                                                                          \
   # Licensed under the Apache License, Version 2.0 (the "License");          \
   # you may not use this file except in compliance with the License.         \
   # You may obtain a copy of the License at                                  \
   #                                                                          \
   #   http://www.apache.org/licenses/LICENSE-2.0                             \
   #                                                                          \
   # Unless required by applicable law or agreed to in writing, software      \
   # distributed under the License is distributed on an "AS IS" BASIS,        \
   # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. \
   # See the License for the specific language governing permissions and      \
   # limitations under the License.                                           \
   # '
__status__ = "Development"
# ==============================================================================

class InstanceBuilder:
    """Builder of instances."""

    def __init__(self, argsDict, testMode):
        """Constructor for the InstanceBuilder."""
        self.test = testMode
        self.argList = argsDict
        self.elementsToInclude = argsDict["ELEMENTS_TO_INCLUDE"]
        self.elementDependency = [
            "base",
            "python",
            "supervisor",
            "deployenv",
            "logging",
            "mount",
            "postgres",
            "nginx",
            "pgpool",
            "redis",
            "web",
            "worker"
        ]

    def buildIt(self):
        """Execute the build based upon the elements."""
        # first setup some standardized directories
        self.setupStandardDirectories()

        for item in self.elementDependency:
            print("item: {}".format(item))
            for element in self.elementsToInclude:
                print("Element: {}".format(element))
                if element == item:
                    elementClassName = element[:1].upper() + element[1:]
                    aClassName = globals()[elementClassName]
                    theElement = aClassName(self.argList)
                    # theElement.run()
                    break

    def setupStandardDirectories(self):
        """Create the standard set of directories."""
        # if this script is run with sudo then we can get the uid
        # and gid for actual user who ran this with sudo
        # uid = int(os.environ.get('SUDO_UID'))
        # gid = int(os.environ.get('SUDO_GID'))
        # and then to use it after you make a directory
        # os.chown(mediaDBRestoreDir, uid, gid)

        # first make the /data directory
        dataDir = "/data"
        if not os.path.exists(dataDir):
            cmdToRun = ("sudo mkdir " + dataDir
                        + " ; sudo chmod 755 " + dataDir)
            subprocess.call(cmdToRun, shell=True)

        # ----------------------------------------------------------------------
        # If this will have an attached scratch volume, then prepare and mount
        # it, then make sure that code deploys go onto the scratch volume as
        # well.
        # ----------------------------------------------------------------------
        deployDir = "/data/deploy"
        if "SCRATCHVOLUME" in self.elementsToInclude:
            # hold the original directory
            originalDir = os.getcwd()
            # and change the dcStack postgres directory so that we can run the
            # mount script
            destDir = os.path.expanduser(self.stackDir + "/db/postgres")
            os.chdir(destDir)
            # TODO callexternal process i-mount.sh "/media/data"
#             sudo . / i - mount.sh "/media/data"

            # now we need to make the deploy directory and then set up
            # a symbolic link for legacy reasons
            mediaDeployDir = "/media/data/deploy"
            deployDir = "/data/deploy"
            if not os.path.exists(mediaDeployDir):
                cmdToRun = ("sudo mkdir -p " + mediaDeployDir
                            + " ; sudo chmod 755 " + mediaDeployDir)
                subprocess.call(cmdToRun, shell=True)
                cmdToRun = ("sudo ln -s " + mediaDeployDir + " "
                            + deployDir)
                subprocess.call(cmdToRun, shell=True)
                os.symlink(mediaDeployDir, deployDir)

                # and now go back to the original directory to proceed
                # with processing
                os.chdir(originalDir)
        else:
            if not os.path.exists(deployDir):
                cmdToRun = ("sudo mkdir -p " + deployDir
                            + " ; sudo chmod 755 " + deployDir)
                subprocess.call(cmdToRun, shell=True)

        # Create standard temp directory, then set up a symlink
        # to a previous standard, for compatibility reasons
        # Also create a standard directory for db restores.
        # sudo mkdir - p / media / data / tmp
        mediaTmpDir = "/media/data/tmp"
        scratchDir = "/data/scratch"
        if not os.path.exists(mediaTmpDir):
            cmdToRun = ("sudo mkdir -p " + mediaTmpDir
                        + " ; sudo chmod 777 " + mediaTmpDir)
            subprocess.call(cmdToRun, shell=True)
            cmdToRun = ("sudo ln -s " + mediaTmpDir + " "
                        + scratchDir)
            subprocess.call(cmdToRun, shell=True)

        # and now make the db_restore directory
        mediaDBRestoreDir = "/media/data/db_restore"
        if not os.path.exists(mediaDBRestoreDir):
            cmdToRun = ("sudo mkdir -p " + mediaDBRestoreDir
                        + " ; sudo chmod 777 " + mediaDBRestoreDir)
            subprocess.call(cmdToRun, shell=True)

def checkArgs():
    """Check the command line arguments."""
    parser = argparse.ArgumentParser(
        description=('comment'))
    parser.add_argument('-i', '--elementsToInclude', help='The list of '
                        'elements names that will correspond to the '
                        'modules to be added to the instance.',
                        required=True)
    parser.add_argument('-c', '--configFile', help='Config file that '
                        'holds variables that define paths on the local'
                        'system.',
                        required=False)
    parser.add_argument('-p', '--profile',
                        help='This is the AWS profile that needs to be used.',
                        required=True)
    parser.add_argument('-r', '--region',
                        help='This is the AWS profile that needs to be used.',
                        required=False)
    parser.add_argument('--suffix',
                        help='This is the defined suffix for this instance.',
                        required=False)
    parser.add_argument('--stack',
                        help='This is the name of the stack to use.',
                        required=False)
    parser.add_argument('--stackDir',
                        help='This is the path to where the stack is.',
                        required=False)
    parser.add_argument('-t', '--test', help='Will run the script but '
                        'will not actually execute the shell commands.'
                        'Think of this as a dry run or a run to be used'
                        ' with a testing suite',
                        action="store_true",
                        required=False)
    args, unknown = parser.parse_known_args()

    retArgs = {}

    if args.elementsToInclude:
        retElements = [x.strip() for x in args.elementsToInclude.split(',')]
        retArgs["ELEMENTS_TO_INCLUDE"] = retElements

    if args.configFile:
        retArgs["CONFIG_FILE"] = args.configFile

    if args.profile:
        retArgs["PROFILE"] = args.profile

    if args.region:
        retArgs["REGION"] = args.region

    if args.suffix:
        retArgs["SUFFIX"] = args.suffix

    if args.stack:
        retArgs["STACK"] = args.stack

    if args.stackDir:
        retArgs["STACK_DIR"] = args.stackDir

    retTest = ""
    if args.test:
        retTest = args.test

    return (retArgs, retTest)

def main(argv):
    """Main code goes here."""
    try:
        (cmdLineArgs, testMode) = checkArgs()
    except SystemExit:
        # dcEnvCheckArgs()
        print("Some arguments provided are not supported")
        sys.exit(1)

        # get the list of key/value pairs from the app-utils
        # envList = pythonGetEnv()

        # take the key/value arguments from the command line
        # and create one list of a merged set of key/value pairs
        # from the environment and from the command line
    argsList = cmdLineArgs.copy()
    # argsList.update(envList)

    # and then use that combined list to create an instance of
    # an InstanceBuilder to create the needed instance
    aBuilder = InstanceBuilder(argsList, testMode)

    aBuilder.buildIt()

if __name__ == "__main__":
    main(sys.argv[1:])

# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
<commit_msg>now that the correct order is going to be used put the run script back in and add one print out line to see where it will end up.<commit_after>#!/usr/bin/env python
"""Builder of instances."""

import sys
import os
import argparse
import subprocess
# from process_dc_env import pythonGetEnv, dcEnvCheckArgs
from base import Base
from python import Python
from logging import Logging
from supervisor import Supervisor
# ==============================================================================
__version__ = "0.1"

__copyright__ = "Copyright 2017, devops.center"
__credits__ = ["Bob Lozano", "Gregg Jensen"]
__license__ = ' \
   # Copyright 2014-2017 devops.center llc                                    \
   #                                                                          \
   # Licensed under the Apache License, Version 2.0 (the "License");          \
   # you may not use this file except in compliance with the License.         \
   # You may obtain a copy of the License at                                  \
   #                                                                          \
   #   http://www.apache.org/licenses/LICENSE-2.0                             \
   #                                                                          \
   # Unless required by applicable law or agreed to in writing, software      \
   # distributed under the License is distributed on an "AS IS" BASIS,        \
   # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. \
   # See the License for the specific language governing permissions and      \
   # limitations under the License.                                           \
   # '
__status__ = "Development"
# ==============================================================================

class InstanceBuilder:
    """Builder of instances."""

    def __init__(self, argsDict, testMode):
        """Constructor for the InstanceBuilder."""
        self.test = testMode
        self.argList = argsDict
        self.elementsToInclude = argsDict["ELEMENTS_TO_INCLUDE"]
        self.elementDependency = [
            "base",
            "python",
            "supervisor",
            "deployenv",
            "logging",
            "mount",
            "postgres",
            "nginx",
            "pgpool",
            "redis",
            "web",
            "worker"
        ]

    def buildIt(self):
        """Execute the build based upon the elements."""
        # first setup some standardized directories
        self.setupStandardDirectories()

        for item in self.elementDependency:
            for element in self.elementsToInclude:
                if element == item:
                    print("Building element: %s", element)
                    elementClassName = element[:1].upper() + element[1:]
                    aClassName = globals()[elementClassName]
                    theElement = aClassName(self.argList)
                    theElement.run()
                    break

    def setupStandardDirectories(self):
        """Create the standard set of directories."""
        # if this script is run with sudo then we can get the uid
        # and gid for actual user who ran this with sudo
        # uid = int(os.environ.get('SUDO_UID'))
        # gid = int(os.environ.get('SUDO_GID'))
        # and then to use it after you make a directory
        # os.chown(mediaDBRestoreDir, uid, gid)

        # first make the /data directory
        dataDir = "/data"
        if not os.path.exists(dataDir):
            cmdToRun = ("sudo mkdir " + dataDir
                        + " ; sudo chmod 755 " + dataDir)
            subprocess.call(cmdToRun, shell=True)

        # ----------------------------------------------------------------------
        # If this will have an attached scratch volume, then prepare and mount
        # it, then make sure that code deploys go onto the scratch volume as
        # well.
        # ----------------------------------------------------------------------
        deployDir = "/data/deploy"
        if "SCRATCHVOLUME" in self.elementsToInclude:
            # hold the original directory
            originalDir = os.getcwd()
            # and change the dcStack postgres directory so that we can run the
            # mount script
            destDir = os.path.expanduser(self.stackDir + "/db/postgres")
            os.chdir(destDir)
            # TODO callexternal process i-mount.sh "/media/data"
#             sudo . / i - mount.sh "/media/data"

            # now we need to make the deploy directory and then set up
            # a symbolic link for legacy reasons
            mediaDeployDir = "/media/data/deploy"
            deployDir = "/data/deploy"
            if not os.path.exists(mediaDeployDir):
                cmdToRun = ("sudo mkdir -p " + mediaDeployDir
                            + " ; sudo chmod 755 " + mediaDeployDir)
                subprocess.call(cmdToRun, shell=True)
                cmdToRun = ("sudo ln -s " + mediaDeployDir + " "
                            + deployDir)
                subprocess.call(cmdToRun, shell=True)
                os.symlink(mediaDeployDir, deployDir)

                # and now go back to the original directory to proceed
                # with processing
                os.chdir(originalDir)
        else:
            if not os.path.exists(deployDir):
                cmdToRun = ("sudo mkdir -p " + deployDir
                            + " ; sudo chmod 755 " + deployDir)
                subprocess.call(cmdToRun, shell=True)

        # Create standard temp directory, then set up a symlink
        # to a previous standard, for compatibility reasons
        # Also create a standard directory for db restores.
        # sudo mkdir - p / media / data / tmp
        mediaTmpDir = "/media/data/tmp"
        scratchDir = "/data/scratch"
        if not os.path.exists(mediaTmpDir):
            cmdToRun = ("sudo mkdir -p " + mediaTmpDir
                        + " ; sudo chmod 777 " + mediaTmpDir)
            subprocess.call(cmdToRun, shell=True)
            cmdToRun = ("sudo ln -s " + mediaTmpDir + " "
                        + scratchDir)
            subprocess.call(cmdToRun, shell=True)

        # and now make the db_restore directory
        mediaDBRestoreDir = "/media/data/db_restore"
        if not os.path.exists(mediaDBRestoreDir):
            cmdToRun = ("sudo mkdir -p " + mediaDBRestoreDir
                        + " ; sudo chmod 777 " + mediaDBRestoreDir)
            subprocess.call(cmdToRun, shell=True)

def checkArgs():
    """Check the command line arguments."""
    parser = argparse.ArgumentParser(
        description=('comment'))
    parser.add_argument('-i', '--elementsToInclude', help='The list of '
                        'elements names that will correspond to the '
                        'modules to be added to the instance.',
                        required=True)
    parser.add_argument('-c', '--configFile', help='Config file that '
                        'holds variables that define paths on the local'
                        'system.',
                        required=False)
    parser.add_argument('-p', '--profile',
                        help='This is the AWS profile that needs to be used.',
                        required=True)
    parser.add_argument('-r', '--region',
                        help='This is the AWS profile that needs to be used.',
                        required=False)
    parser.add_argument('--suffix',
                        help='This is the defined suffix for this instance.',
                        required=False)
    parser.add_argument('--stack',
                        help='This is the name of the stack to use.',
                        required=False)
    parser.add_argument('--stackDir',
                        help='This is the path to where the stack is.',
                        required=False)
    parser.add_argument('-t', '--test', help='Will run the script but '
                        'will not actually execute the shell commands.'
                        'Think of this as a dry run or a run to be used'
                        ' with a testing suite',
                        action="store_true",
                        required=False)
    args, unknown = parser.parse_known_args()

    retArgs = {}

    if args.elementsToInclude:
        retElements = [x.strip() for x in args.elementsToInclude.split(',')]
        retArgs["ELEMENTS_TO_INCLUDE"] = retElements

    if args.configFile:
        retArgs["CONFIG_FILE"] = args.configFile

    if args.profile:
        retArgs["PROFILE"] = args.profile

    if args.region:
        retArgs["REGION"] = args.region

    if args.suffix:
        retArgs["SUFFIX"] = args.suffix

    if args.stack:
        retArgs["STACK"] = args.stack

    if args.stackDir:
        retArgs["STACK_DIR"] = args.stackDir

    retTest = ""
    if args.test:
        retTest = args.test

    return (retArgs, retTest)

def main(argv):
    """Main code goes here."""
    try:
        (cmdLineArgs, testMode) = checkArgs()
    except SystemExit:
        # dcEnvCheckArgs()
        print("Some arguments provided are not supported")
        sys.exit(1)

        # get the list of key/value pairs from the app-utils
        # envList = pythonGetEnv()

        # take the key/value arguments from the command line
        # and create one list of a merged set of key/value pairs
        # from the environment and from the command line
    argsList = cmdLineArgs.copy()
    # argsList.update(envList)

    # and then use that combined list to create an instance of
    # an InstanceBuilder to create the needed instance
    aBuilder = InstanceBuilder(argsList, testMode)

    aBuilder.buildIt()

if __name__ == "__main__":
    main(sys.argv[1:])

# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before>from intelmq.lib.bot import Bot, sys
from intelmq.lib.message import Event
from intelmq.bots import utils

import csv
import StringIO
import lib

class DCUParserBot(Bot):
    """ Parses DCU-Collector output. """

    def process(self):
        report = self.receive_message()

        if report:
            report = report.strip()
            buffered = StringIO.StringIO(report)
            rows = csv.DictReader(buffered,
                                  delimiter="\t",
                                  fieldnames=lib.dcu_headers())

            for row in rows:
                event = Event()
                event.add("feed", "microsoft-dcu")
                add_dcu_fields(event, row)

                event = utils.generate_observation_time(event, "observation_time")
                event = utils.generate_reported_fields(event)

                self.send_message(event)
        self.acknowledge_message()

if __name__ == "__main__":
    bot = DCUParserBot(sys.argv[1])
    bot.start()
<commit_msg>Fix incoporating dcu library into parser<commit_after>from intelmq.lib.bot import Bot, sys
from intelmq.lib.message import Event
from intelmq.bots import utils

import csv
import StringIO
import lib

class DCUParserBot(Bot):
    """ Parses DCU-Collector output. """

    def process(self):
        report = self.receive_message()

        if report:
            report = report.strip()
            buffered = StringIO.StringIO(report)
            rows = csv.DictReader(buffered,
                                  delimiter="\t",
                                  fieldnames=lib.dcu_headers())

            for row in rows:
                event = Event(lib.convert_dcu_fields(row))
                event.add("feed", "microsoft-dcu")

                event = utils.generate_observation_time(event, "observation_time")
                event = utils.generate_reported_fields(event)

                self.send_message(event)
        self.acknowledge_message()

if __name__ == "__main__":
    bot = DCUParserBot(sys.argv[1])
    bot.start()
<|endoftext|>


<commit_before>import numpy as np
import pandas as pd
from sklearn import preprocessing

import matplotlib.pyplot as plt
import matplotlib

#
# TODO: Parameters to play around with
PLOT_TYPE_TEXT = False    # If you'd like to see indices
PLOT_VECTORS = True       # If you'd like to see your original features in P.C.-Space

matplotlib.style.use('ggplot') # Look Pretty
c = ['red', 'green', 'blue', 'orange', 'yellow', 'brown']

def drawVectors(transformed_features, components_, columns, plt):
  num_columns = len(columns)

  # This function will project your *original* feature (columns)
  # onto your principal component feature-space, so that you can
  # visualize how "important" each one was in the
  # multi-dimensional scaling

  # Scale the principal components by the max value in
  # the transformed set belonging to that component
  xvector = components_[0] * max(transformed_features[:,0])
  yvector = components_[1] * max(transformed_features[:,1])

  ## Visualize projections

  # Sort each column by its length. These are your *original*
  # columns, not the principal components.
  import math
  important_features = { columns[i] : math.sqrt(xvector[i]**2 + yvector[i]**2) for i in range(num_columns) }
  important_features = sorted(zip(important_features.values(), important_features.keys()), reverse=True)
  print "Projected Features by importance:\n", important_features

  ax = plt.axes()

  for i in range(num_columns):
    # Use an arrow to project each original feature as a
    # labeled vector on your principal component axes
    plt.arrow(0, 0, xvector[i], yvector[i], color='b', width=0.0005, head_width=0.02, alpha=0.75, zorder=600000)
    plt.text(xvector[i]*1.2, yvector[i]*1.2, list(columns)[i], color='b', alpha=0.75, zorder=600000)
  return ax

def doPCA(data, dimensions=2):
  from sklearn.decomposition import RandomizedPCA
  model = RandomizedPCA(n_components=dimensions)
  model.fit(data)
  return model

def doKMeans(data, clusters=0):
  #
  # TODO: Do the KMeans clustering here, passing in the # of clusters parameter
  # and fit it against your data. Then, return a tuple containing the cluster
  # centers and the labels
  #
  # .. your code here ..

  model = KMeans(n_clusters=clusters)
  model.fit(data)

  return model.cluster_centers_, model.labels_

#
# TODO: Load up the dataset. It has may or may not have nans in it. Make
# sure you catch them and destroy them, by setting them to '0'. This is valid
# for this dataset, since if the value is missing, you can assume no $ was spent
# on it.
#
# .. your code here ..
df=pd.read_csv('c:/Users/User/workspace/DAT210x/Module5/Datasets/Wholesale customers data.csv')
df.fillna(0)
#
# TODO: As instructed, get rid of the 'Channel' and 'Region' columns, since
# you'll be investigating as if this were a single location wholesaler, rather
# than a national / international one. Leaving these fields in here would cause
# KMeans to examine and give weight to them.
#
# .. your code here ..
df = df.drop(labels=['Channel', 'Region'], axis=1)

#
# TODO: Before unitizing / standardizing / normalizing your data in preparation for
# K-Means, it's a good idea to get a quick peek at it. You can do this using the
# .describe() method, or even by using the built-in pandas df.plot.hist()
#
# .. your code here ..
df.describe()
#fig = plt.figure()
#df.Milk.plot.hist(bins=50)
#df.Detergents_Paper.plot.hist(bins=50,title='Milk and Detergents_Paper')

#
# INFO: Having checked out your data, you may have noticed there's a pretty big gap
# between the top customers in each feature category and the rest. Some feature
# scaling algos won't get rid of outliers for you, so it's a good idea to handle that
# manually---particularly if your goal is NOT to determine the top customers. After
# all, you can do that with a simple Pandas .sort_values() and not a machine
# learning clustering algorithm. From a business perspective, you're probably more
# interested in clustering your +/- 2 standard deviation customers, rather than the
# creme dela creme, or bottom of the barrel'ers
#
# Remove top 5 and bottom 5 samples for each column:
drop = {}
for col in df.columns:
  # Bottom 5
  sort = df.sort_values(by=col, ascending=True)
  if len(sort) > 5: sort=sort[:5]
  for index in sort.index: drop[index] = True # Just store the index once

  # Top 5
  sort = df.sort_values(by=col, ascending=False)
  if len(sort) > 5: sort=sort[:5]
  for index in sort.index: drop[index] = True # Just store the index once

#
# INFO Drop rows by index. We do this all at once in case there is a
# collision. This way, we don't end up dropping more rows than we have
# to, if there is a single row that satisfies the drop for multiple columns.
# Since there are 6 rows, if we end up dropping < 5*6*2 = 60 rows, that means
# there indeed were collisions.
print "Dropping {0} Outliers...".format(len(drop))
df.drop(inplace=True, labels=drop.keys(), axis=0)
print df.describe()

#
# INFO: What are you interested in?
#
# Depending on what you're interested in, you might take a different approach
# to normalizing/standardizing your data.
#
# You should note that all columns left in the dataset are of the same unit.
# You might ask yourself, do I even need to normalize / standardize the data?
# The answer depends on what you're trying to accomplish. For instance, although
# all the units are the same (generic money unit), the price per item in your
# store isn't. There may be some cheap items and some expensive one. If your goal
# is to find out what items people buy tend to buy together but you didn't
# unitize properly before running kMeans, the contribution of the lesser priced
# item would be dwarfed by the more expensive item.
#
# For a great overview on a few of the normalization methods supported in SKLearn,
# please check out: https://stackoverflow.com/questions/30918781/right-function-for-normalizing-input-of-sklearn-svm
#
# Suffice to say, at the end of the day, you're going to have to know what question
# you want answered and what data you have available in order to select the best
# method for your purpose. Luckily, SKLearn's interfaces are easy to switch out
# so in the mean time, you can experiment with all of them and see how they alter
# your results.
#
#
# 5-sec summary before you dive deeper online:
#
# NORMALIZATION: Let's say your user spend a LOT. Normalization divides each item by
#                the average overall amount of spending. Stated differently, your
#                new feature is = the contribution of overall spending going into
#                that particular item: $spent on feature / $overall spent by sample
#
# MINMAX:        What % in the overall range of $spent by all users on THIS particular
#                feature is the current sample's feature at? When you're dealing with
#                all the same units, this will produce a near face-value amount. Be
#                careful though: if you have even a single outlier, it can cause all
#                your data to get squashed up in lower percentages.
#                Imagine your buyers usually spend $100 on wholesale milk, but today
#                only spent $20. This is the relationship you're trying to capture
#                with MinMax. NOTE: MinMax doesn't standardize (std. dev.); it only
#                normalizes / unitizes your feature, in the mathematical sense.
#                MinMax can be used as an alternative to zero mean, unit variance scaling.
#                [(sampleFeatureValue-min) / (max-min)] * (max-min) + min
#                Where min and max are for the overall feature values for all samples.

#
# TODO: Un-comment just ***ONE*** of lines at a time and see how alters your results
# Pay attention to the direction of the arrows, as well as their LENGTHS
#T = preprocessing.StandardScaler().fit_transform(df)
#T = preprocessing.MinMaxScaler().fit_transform(df)
#T = preprocessing.MaxAbsScaler().fit_transform(df)
#T = preprocessing.Normalizer().fit_transform(df)
T = df # No Change

#
# INFO: Sometimes people perform PCA before doing KMeans, so that KMeans only
# operates on the most meaningful features. In our case, there are so few features
# that doing PCA ahead of time isn't really necessary, and you can do KMeans in
# feature space. But keep in mind you have the option to transform your data to
# bring down its dimensionality. If you take that route, then your Clusters will
# already be in PCA-transformed feature space, and you won't have to project them
# again for visualization.

# Do KMeans
n_clusters = 3
from sklearn.cluster import KMeans
centroids, labels = doKMeans(T, n_clusters)

#
# TODO: Print out your centroids. They're currently in feature-space, which
# is good. Print them out before you transform them into PCA space for viewing
#
# .. your code here ..
print centroids

# Do PCA *after* to visualize the results. Project the centroids as well as
# the samples into the new 2D feature space for visualization purposes.
display_pca = doPCA(T)
T = display_pca.transform(T)
CC = display_pca.transform(centroids)

# Visualize all the samples. Give them the color of their cluster label
fig = plt.figure()
ax = fig.add_subplot(111)
if PLOT_TYPE_TEXT:
  # Plot the index of the sample, so you can further investigate it in your dset
  for i in range(len(T)): ax.text(T[i,0], T[i,1], df.index[i], color=c[labels[i]], alpha=0.75, zorder=600000)
  ax.set_xlim(min(T[:,0])*1.2, max(T[:,0])*1.2)
  ax.set_ylim(min(T[:,1])*1.2, max(T[:,1])*1.2)
else:
  # Plot a regular scatter plot
  sample_colors = [ c[labels[i]] for i in range(len(T)) ]
  ax.scatter(T[:, 0], T[:, 1], c=sample_colors, marker='o', alpha=0.2)

# Plot the Centroids as X's, and label them
ax.scatter(CC[:, 0], CC[:, 1], marker='x', s=169, linewidths=3, zorder=1000, c=c)
for i in range(len(centroids)): ax.text(CC[i, 0], CC[i, 1], str(i), zorder=500010, fontsize=18, color=c[i])

# Display feature vectors for investigation:
if PLOT_VECTORS: drawVectors(T, display_pca.components_, df.columns, plt)

# Add the cluster label back into the dataframe and display it:
df['label'] = pd.Series(labels, index=df.index)
print df

plt.show()
<commit_msg>assignment 4<commit_after>import numpy as np
import pandas as pd
from sklearn import preprocessing

import matplotlib.pyplot as plt
import matplotlib

# TODO: Parameters to play around with
PLOT_TYPE_TEXT = False    # If you'd like to see indices
PLOT_VECTORS = True       # If you'd like to see your original features in P.C.-Space

matplotlib.style.use('ggplot') # Look Pretty
c = ['red', 'green', 'blue', 'orange', 'yellow', 'brown']

def drawVectors(transformed_features, components_, columns, plt):
  num_columns = len(columns)

  # This function will project your *original* feature (columns)
  # onto your principal component feature-space, so that you can
  # visualize how "important" each one was in the
  # multi-dimensional scaling

  # Scale the principal components by the max value in
  # the transformed set belonging to that component
  xvector = components_[0] * max(transformed_features[:,0])
  yvector = components_[1] * max(transformed_features[:,1])

  ## Visualize projections

  # Sort each column by its length. These are your *original*
  # columns, not the principal components.
  import math
  important_features = { columns[i] : math.sqrt(xvector[i]**2 + yvector[i]**2) for i in range(num_columns) }
  important_features = sorted(zip(important_features.values(), important_features.keys()), reverse=True)
  print "Projected Features by importance:\n", important_features

  ax = plt.axes()

  for i in range(num_columns):
    # Use an arrow to project each original feature as a
    # labeled vector on your principal component axes
    plt.arrow(0, 0, xvector[i], yvector[i], color='b', width=0.0005, head_width=0.02, alpha=0.75, zorder=600000)
    plt.text(xvector[i]*1.2, yvector[i]*1.2, list(columns)[i], color='b', alpha=0.75, zorder=600000)
  return ax

def doPCA(data, dimensions=2):
  from sklearn.decomposition import RandomizedPCA
  model = RandomizedPCA(n_components=dimensions)
  model.fit(data)
  return model

def doKMeans(data, clusters=0):
  #
  # TODO: Do the KMeans clustering here, passing in the # of clusters parameter
  # and fit it against your data. Then, return a tuple containing the cluster
  # centers and the labels
  #
  # .. your code here ..

  model = KMeans(n_clusters=clusters)
  model.fit(data)

  return model.cluster_centers_, model.labels_

#
# TODO: Load up the dataset. It has may or may not have nans in it. Make
# sure you catch them and destroy them, by setting them to '0'. This is valid
# for this dataset, since if the value is missing, you can assume no $ was spent
# on it.
#
# .. your code here ..
df=pd.read_csv('c:/Users/User/workspace/DAT210x/Module5/Datasets/Wholesale customers data.csv')
df.fillna(0)
#
# TODO: As instructed, get rid of the 'Channel' and 'Region' columns, since
# you'll be investigating as if this were a single location wholesaler, rather
# than a national / international one. Leaving these fields in here would cause
# KMeans to examine and give weight to them.
#
# .. your code here ..
df = df.drop(labels=['Channel', 'Region'], axis=1)

#
# TODO: Before unitizing / standardizing / normalizing your data in preparation for
# K-Means, it's a good idea to get a quick peek at it. You can do this using the
# .describe() method, or even by using the built-in pandas df.plot.hist()
#
# .. your code here ..
df.describe()
#fig = plt.figure()
#df.Milk.plot.hist(bins=50)
#df.Detergents_Paper.plot.hist(bins=50,title='Milk and Detergents_Paper')

#
# INFO: Having checked out your data, you may have noticed there's a pretty big gap
# between the top customers in each feature category and the rest. Some feature
# scaling algos won't get rid of outliers for you, so it's a good idea to handle that
# manually---particularly if your goal is NOT to determine the top customers. After
# all, you can do that with a simple Pandas .sort_values() and not a machine
# learning clustering algorithm. From a business perspective, you're probably more
# interested in clustering your +/- 2 standard deviation customers, rather than the
# creme dela creme, or bottom of the barrel'ers
#
# Remove top 5 and bottom 5 samples for each column:
drop = {}
for col in df.columns:
  # Bottom 5
  sort = df.sort_values(by=col, ascending=True)
  if len(sort) > 5: sort=sort[:5]
  for index in sort.index: drop[index] = True # Just store the index once

  # Top 5
  sort = df.sort_values(by=col, ascending=False)
  if len(sort) > 5: sort=sort[:5]
  for index in sort.index: drop[index] = True # Just store the index once

#
# INFO Drop rows by index. We do this all at once in case there is a
# collision. This way, we don't end up dropping more rows than we have
# to, if there is a single row that satisfies the drop for multiple columns.
# Since there are 6 rows, if we end up dropping < 5*6*2 = 60 rows, that means
# there indeed were collisions.
print "Dropping {0} Outliers...".format(len(drop))
df.drop(inplace=True, labels=drop.keys(), axis=0)
print df.describe()

#
# INFO: What are you interested in?
#
# Depending on what you're interested in, you might take a different approach
# to normalizing/standardizing your data.
#
# You should note that all columns left in the dataset are of the same unit.
# You might ask yourself, do I even need to normalize / standardize the data?
# The answer depends on what you're trying to accomplish. For instance, although
# all the units are the same (generic money unit), the price per item in your
# store isn't. There may be some cheap items and some expensive one. If your goal
# is to find out what items people buy tend to buy together but you didn't
# unitize properly before running kMeans, the contribution of the lesser priced
# item would be dwarfed by the more expensive item.
#
# For a great overview on a few of the normalization methods supported in SKLearn,
# please check out: https://stackoverflow.com/questions/30918781/right-function-for-normalizing-input-of-sklearn-svm
#
# Suffice to say, at the end of the day, you're going to have to know what question
# you want answered and what data you have available in order to select the best
# method for your purpose. Luckily, SKLearn's interfaces are easy to switch out
# so in the mean time, you can experiment with all of them and see how they alter
# your results.
#
#
# 5-sec summary before you dive deeper online:
#
# NORMALIZATION: Let's say your user spend a LOT. Normalization divides each item by
#                the average overall amount of spending. Stated differently, your
#                new feature is = the contribution of overall spending going into
#                that particular item: $spent on feature / $overall spent by sample
#
# MINMAX:        What % in the overall range of $spent by all users on THIS particular
#                feature is the current sample's feature at? When you're dealing with
#                all the same units, this will produce a near face-value amount. Be
#                careful though: if you have even a single outlier, it can cause all
#                your data to get squashed up in lower percentages.
#                Imagine your buyers usually spend $100 on wholesale milk, but today
#                only spent $20. This is the relationship you're trying to capture
#                with MinMax. NOTE: MinMax doesn't standardize (std. dev.); it only
#                normalizes / unitizes your feature, in the mathematical sense.
#                MinMax can be used as an alternative to zero mean, unit variance scaling.
#                [(sampleFeatureValue-min) / (max-min)] * (max-min) + min
#                Where min and max are for the overall feature values for all samples.

#
# TODO: Un-comment just ***ONE*** of lines at a time and see how alters your results
# Pay attention to the direction of the arrows, as well as their LENGTHS
#T = preprocessing.StandardScaler().fit_transform(df)
#T = preprocessing.MinMaxScaler().fit_transform(df)
#T = preprocessing.MaxAbsScaler().fit_transform(df)
#T = preprocessing.Normalizer().fit_transform(df)
T = df # No Change

#
# INFO: Sometimes people perform PCA before doing KMeans, so that KMeans only
# operates on the most meaningful features. In our case, there are so few features
# that doing PCA ahead of time isn't really necessary, and you can do KMeans in
# feature space. But keep in mind you have the option to transform your data to
# bring down its dimensionality. If you take that route, then your Clusters will
# already be in PCA-transformed feature space, and you won't have to project them
# again for visualization.

# Do KMeans
n_clusters = 3
from sklearn.cluster import KMeans
centroids, labels = doKMeans(T, n_clusters)

#
# TODO: Print out your centroids. They're currently in feature-space, which
# is good. Print them out before you transform them into PCA space for viewing
#
# .. your code here ..
print centroids

# Do PCA *after* to visualize the results. Project the centroids as well as
# the samples into the new 2D feature space for visualization purposes.
display_pca = doPCA(T)
T = display_pca.transform(T)
CC = display_pca.transform(centroids)

# Visualize all the samples. Give them the color of their cluster label
fig = plt.figure()
ax = fig.add_subplot(111)
if PLOT_TYPE_TEXT:
  # Plot the index of the sample, so you can further investigate it in your dset
  for i in range(len(T)): ax.text(T[i,0], T[i,1], df.index[i], color=c[labels[i]], alpha=0.75, zorder=600000)
  ax.set_xlim(min(T[:,0])*1.2, max(T[:,0])*1.2)
  ax.set_ylim(min(T[:,1])*1.2, max(T[:,1])*1.2)
else:
  # Plot a regular scatter plot
  sample_colors = [ c[labels[i]] for i in range(len(T)) ]
  ax.scatter(T[:, 0], T[:, 1], c=sample_colors, marker='o', alpha=0.2)

# Plot the Centroids as X's, and label them
ax.scatter(CC[:, 0], CC[:, 1], marker='x', s=169, linewidths=3, zorder=1000, c=c)
for i in range(len(centroids)): ax.text(CC[i, 0], CC[i, 1], str(i), zorder=500010, fontsize=18, color=c[i])

# Display feature vectors for investigation:
if PLOT_VECTORS: drawVectors(T, display_pca.components_, df.columns, plt)

# Add the cluster label back into the dataframe and display it:
df['label'] = pd.Series(labels, index=df.index)
print df

plt.show()
<|endoftext|>