<commit_before>from collections import defaultdict
from django.core.management.base import BaseCommand, CommandError
from django.contrib.sites.models import Site
from eulfedora.server import Repository, RequestFailed
from pidservices.djangowrapper.shortcuts import DjangoPidmanRestClient
from progressbar import ProgressBar, Bar, Percentage, \
         ETA, Counter
import requests
import signal

from readux.books.models import Page

class Command(BaseCommand):
    '''Update Readux Page ARK default target from old-style page URLs
(Release 1) to the new format of page urls in Release 1.3.
'''
    help = __doc__

    v_normal = 1

    interrupted = False

    def handle(self, *pids, **options):
        # bind a handler for interrupt signal
        signal.signal(signal.SIGINT, self.interrupt_handler)

        verbosity = int(options.get('verbosity', self.v_normal))

        repo = Repository()
        try:
            pidman = DjangoPidmanRestClient()
        except Exception as err:
            # error if pid manager config options not in localsettings
            raise CommandError(err)

        old_page_target = '%s/books/pages/' % Site.objects.get_current().domain
        search_args = {'type':'ark', 'target': old_page_target, 'count': 100}
        results = pidman.search_pids(**search_args)
        total = results['results_count']
        if verbosity >= self.v_normal:
            print 'Found %d total page ARKs with targets to be updated' % total

        pbar = ProgressBar(widgets=[Percentage(),
            ' (', Counter(), ')',
            Bar(),
            ETA()],
            maxval=total).start()

        stats = defaultdict(int)
        self.processed = set()
        # NOTE: this could be a very large list in production
        # but otherwise, the list changes as we process updates
        # and we end up skipping results...
        arks_to_change = list(self.get_search_results(results, pidman, search_args))
        for ark in arks_to_change:
            self.processed.add(ark['pid'])
            # get fedora pid from target uri
            target_uri = ark['targets'][0]['target_uri']
            baseurl, pid = target_uri.rstrip('/').rsplit('/', 1)
            try:
                page = repo.get_object(pid, type=Page)
                # this should probably only happen in dev/qa
                if not page.exists:
                    if verbosity > self.v_normal:
                        self.stderr.write('Page %s does not exist' % pid)
                    stats['notfound'] += 1
                else:
                    # check if volume exists?
                    pidman.update_ark_target(ark['pid'], target_uri=page.absolute_url)
                    stats['updated'] += 1
            except RequestFailed as rf:
                print 'Error accessing %s: %s' % (pid, rf)
                stats['error'] += 1

            pbar.update(len(self.processed))
            if self.interrupted:
                break

        if not self.interrupted:
            pbar.finish()
        # summarize
        self.stderr.write('Updated %(updated)d, %(error)d error(s), %(notfound)d not found' \
            % stats)

    def get_search_results(self, search_results, pidman, search_args):
        # generator to page through pidman search results and
        # return the results
        for res in search_results['results']:
            if res['pid'] not in self.processed:
                yield res
        if 'next_page_link' in search_results and \
          search_results['next_page_link'] is not None:
            next_page = int(search_results['current_page_number']) + 1
            try:
                results = pidman.search_pids(page=next_page, **search_args)
                for res in self.get_search_results(results, pidman, search_args):
                    yield res
            except requests.exceptions.HTTPError:
                # 404 should mean we hit the end of the search results
                pass

    def interrupt_handler(self, signum, frame):
        '''Gracefully handle a SIGINT.  Stop and report what was done.'''
        if signum == signal.SIGINT:
            # restore default signal handler so a second SIGINT can be used to quit
            signal.signal(signal.SIGINT, signal.SIG_DFL)
            # set interrupt flag so main loop knows to quit
            self.interrupted = True

<commit_msg>Modify update page ark script to try to handle volume of arks in prod<commit_after>from collections import defaultdict
from django.core.management.base import BaseCommand, CommandError
from django.contrib.sites.models import Site
from eulfedora.server import Repository, RequestFailed
from pidservices.djangowrapper.shortcuts import DjangoPidmanRestClient
from progressbar import ProgressBar, Bar, Percentage, \
         ETA, Counter
import requests
import signal

from readux.books.models import Page

class Command(BaseCommand):
    '''Update Readux Page ARK default target from old-style page URLs
(Release 1) to the new format of page urls in Release 1.3.
'''
    help = __doc__

    v_normal = 1

    interrupted = False

    def handle(self, *pids, **options):
        # bind a handler for interrupt signal
        signal.signal(signal.SIGINT, self.interrupt_handler)

        verbosity = int(options.get('verbosity', self.v_normal))

        repo = Repository()
        try:
            pidman = DjangoPidmanRestClient()
        except Exception as err:
            # error if pid manager config options not in localsettings
            raise CommandError(err)

        old_page_target = '%s/books/pages/' % Site.objects.get_current().domain
        search_args = {'type':'ark', 'target': old_page_target, 'count': 10}
        # get a small result set to retrieve the total
        results = pidman.search_pids(**search_args)
        total = results['results_count']
        # then set a larger page size for actual processing
        search_args['count'] = 100
        if verbosity >= self.v_normal:
            print 'Found %d total page ARKs with targets to be updated' % total

        pbar = ProgressBar(widgets=[Percentage(),
            ' (', Counter(), ')',
            Bar(),
            ETA()],
            maxval=total).start()

        self.stats = defaultdict(int)
        self.processed = set()
        for ark in self.get_search_results(pidman, search_args):
            self.processed.add(ark['pid'])
            # get fedora pid from target uri
            target_uri = ark['targets'][0]['target_uri']
            baseurl, pid = target_uri.rstrip('/').rsplit('/', 1)
            try:
                page = repo.get_object(pid, type=Page)
                # this should probably only happen in dev/qa
                if not page.exists:
                    if verbosity > self.v_normal:
                        self.stderr.write('Page %s does not exist' % pid)
                    self.stats['notfound'] += 1
                else:
                    # check if volume exists?
                    pidman.update_ark_target(ark['pid'], target_uri=page.absolute_url)
                    self.stats['updated'] += 1
            except RequestFailed as rf:
                print 'Error accessing %s: %s' % (pid, rf)
                self.stats['error'] += 1

            pbar.update(len(self.processed))
            if self.interrupted:
                break

        if not self.interrupted:
            pbar.finish()
        # summarize
        self.stderr.write('Updated %(updated)d, %(error)d error(s), %(notfound)d not found' \
            % self.stats)

    def get_search_results(self, pidman, search_args, page=None):
        # generator to page through pidman search results and
        # return the results
        current_search_args = search_args.copy()
        if page is not None:
            current_search_args['page'] = page

        results = pidman.search_pids(**current_search_args)
        total = results.get('results_count', 0)

        if total and 'results' in results:
            for res in results['results']:
                if res['pid'] not in self.processed:
                    yield res

        if 'next_page_link' in results and \
          results['next_page_link'] is not None:
            next_page = int(results['current_page_number']) + 1
            try:
                results = pidman.search_pids(page=next_page, **search_args)
                for res in self.get_search_results(pidman, search_args, page=next_page):
                    yield res
            except requests.exceptions.HTTPError:
                # 404 should mean we hit the end of the search results
                pass

        # get count for the same search to check if we need to loop through again
        current_search_args = search_args.copy()
        current_search_args['count'] = 10
        results = pidman.search_pids(**search_args)
        total = results.get('results_count', 0)
        # if there are more search results than we have error/not found,
        # then there are still pids to update
        if total and total > (self.stats['error'] + self.stats['notfound']) :
            for res in self.get_search_results(pidman, search_args):
                yield res

    def get_search_results_OLD(self, search_results, pidman, search_args):
        # generator to page through pidman search results and
        # return the results
        if 'results' in search_results:
            for res in search_results['results']:
                if res['pid'] not in self.processed:
                    yield res

        if 'next_page_link' in search_results and \
          search_results['next_page_link'] is not None:
            next_page = int(search_results['current_page_number']) + 1
            try:
                results = pidman.search_pids(page=next_page, **search_args)
                for res in self.get_search_results(results, pidman, search_args):
                    yield res
            except requests.exceptions.HTTPError:
                # 404 should mean we hit the end of the search results
                pass

        # because the list changes as we iterate through it, run the
        # same search again

    def interrupt_handler(self, signum, frame):
        '''Gracefully handle a SIGINT.  Stop and report what was done.'''
        if signum == signal.SIGINT:
            # restore default signal handler so a second SIGINT can be used to quit
            signal.signal(signal.SIGINT, signal.SIG_DFL)
            # set interrupt flag so main loop knows to quit
            self.interrupted = True

<|endoftext|>


<commit_before>from statistics_handler import StatisticsHandler
from node_statistics_handler import NodeStatisticsHandler
from host_status import HostStatus
from node_manager import NodeManager
from threading import Thread
import arni_msgs
from arni_msgs.msg import HostStatistics
from arni_msgs.srv import NodeReaction
import psutil
import xmlrpclib
import socket
import rosnode
import rospy
import sensors
import threading

class HostStatisticsHandler( StatisticsHandler):

    """
    Represents a host , limited to one instance per host.
    Collects statistics about the current state of the host and s
    ends them using the publisher-subscriber mechanism.
    """

    def __init__(self, hostid):

        super(HostStatisticsHandler, self).__init__(hostid)

        self.__update_intervall = 0
        self.__publish_intervall = 0

        self.__init_params()
        self.__register_service()
        self.__lock = threading.Lock()
        self.__dict_lock = threading.Lock()

        self.pub = rospy.Publisher('/statistics_host', HostStatistics, queue_size = 500)
        #: Used to store information about the host's status.
        self._status = HostStatus(rospy.Time.now())

        #: Interface to restart and stop nodes
        #or executing other commands.
        self.__node_manager = NodeManager()

        #: Dictionary holding all nodes currently running on the host.
        self.__node_list = {}

        #Base-stats for I/O
        self.__bandwidth_base = {}
        self.__msg_freq_base = {}
        self.__disk_write_base = {}
        self.__disk_read_base = {}

        self.__set_bases()

    def __set_bases(self):

        psutil.cpu_percent()
        psutil.cpu_percent(percpu = True)

        for interface in psutil.net_io_counters(True):

            netint = psutil.net_io_counters(True)[interface]
            total_bytes = netint.bytes_recv + netint.bytes_sent
            total_packages = netint.packets_sent + netint.packets_recv

            self.__bandwidth_base[interface] = total_bytes
            self.__msg_freq_base[interface] = total_packages

        dev_names = ''
        for disk in psutil.disk_partitions():
            if all(['cdrom' not in disk.opts, 'sr' not in disk.device]):
                    dev_names += disk.device + ';'

        for key in psutil.disk_io_counters(True):
            if key in dev_names:
                disk = psutil.disk_io_counters(True)[key]
                self.__disk_read_base[key] = disk.read_bytes
                self.__disk_write_base[key] = disk.write_bytes

    def __register_service(self):
        """
        Register all services
        """
        rospy.Service(
            "/execute_node_reaction/%s" % self._id,
            NodeReaction, self.execute_reaction)

    def measure_status(self, event):
        """
        Collects information about the host's current status using psutils.
        Triggered periodically.
        """

        self.__lock.acquire()
        #update node list
        self.__dict_lock.acquire()
        self.update_nodes()
        self.__dict_lock.realease()

        #CPU
        self._status.add_cpu_usage(psutil.cpu_percent())
        self._status.add_cpu_usage_core(psutil.cpu_percent(percpu = True))
        #RAM
        self._status.add_ram_usage(psutil.virtual_memory().percent)
        #temp
        sensor_list = []
        self.get_sensors(sensor_list)

        #Bandwidth and message frequency
        self.__measure_network_usage()
        #Disk usage
        self.__measure_disk_usage()
        self.__lock.release()

    def __measure_network_usage(self):
        """
        measure current network_io_counters
        """
        network_interfaces = psutil.net_io_counters(True)

        for key in network_interfaces:
            total_bytes = (network_interfaces[key].bytes_sent + network_interfaces[key].bytes_recv) - self.__bandwidth_base[key]
            total_packages = (network_interfaces[key].packets_sent + network_interfaces[key].packets_recv) - self.__msg_freq_base[key]

            bandwidth = total_bytes / self.__update_intervall
            msg_frequency = total_packages / self.__update_intervall
            self._status.add_bandwidth(key, bandwidth)
            self._status.add_msg_frequency(key, msg_frequency)

            #update base stats for next iteration
            self.__bandwidth_base[key] += total_bytes
            self.__msg_freq_base[key] += total_packages

    def __measure_disk_usage(self):
        """
        measure current disk usage
        """
        # Free Space on disks
        disks = psutil.disk_partitions()
        dev_name = ''
        for x in disks:
            if 'cdrom' not in x.opts and 'sr' not in x.device:
                free_space = psutil.disk_usage(x.mountpoint).free / 2**20
                self._status.add_drive_space(x.device , free_space)
                dev_name += x.device + ';'

        #Drive I/O
        drive_io = psutil.disk_io_counters(True)
        for key in drive_io:
            if key in dev_name:
                readb = drive_io[key].read_bytes - self.__disk_read_base[key]
                writeb = drive_io[key].write_bytes - self.__disk_write_base[key]

                read_rate = readb / self.__update_intervall
                write_rate = writeb / self.__update_intervall
                self._status.add_drive_read(key, read_rate)
                self._status.add_drive_write(key, write_rate)
                ##update base stats for next iteration
                self.__disk_read_base[key] += readb
                self.__disk_write_base[key] += writeb

    def publish_status(self, event):
        """
        Publishes the current status to a topic using ROS's publisher-subscriber mechanism.
        Triggered periodically.
        """
        self.__lock.acquire()
        self._status.time_end = rospy.Time.now()
        stats = self.__calc_statistics()
        self.__dict_lock.acquire()
        self.__publish_nodes()
        self.__dict_lock.release()
        self.pub.publish(stats)
        self._status.reset()
        self._status.time_start = rospy.Time.now()
        self.__lock.release()

    def __publish_nodes(self):
        """
        publishes current status of all nodes.
        """
        for node in self.__node_list:
            Thread(target = self.__node_list[node].publish_status).start()

    def __init_params(self):
        """
        Initializes params on the parameter server,
        """

        rospy.set_param('/update_intervall', 1)
        rospy.set_param('/publish_intervall', 10)

        self.__update_intervall =  rospy.get_param('/update_intervall')
        self.__publish_intervall =  rospy.get_param('/publish_intervall')

        #set Topic statistics to same time_window as update_intervall
        rospy.set_param('/enable_statistics', True )
        rospy.set_param('/statistics_window_min_elements',
                        self.__update_intervall )
        rospy.set_param('/statistics_window_max_elements',
                        self.__publish_intervall)

    def shut_down_hook(self):

        for key in self.__node_list:
            self.__node_list[key].pub.unregister()
        self.pub.unregister()

    def __calc_statistics(self):
        """
        Calculates statistics like mean, standard deviation and max from the status.
        Returns an instance of HostStatistics which can be published.

        :returns: HostStatistics
        """
        stats_dict = self._status.calc_stats()

        hs = HostStatistics()

        hs.host = self._id
        hs.window_start = self._status.time_start
        hs.window_stop = self._status.time_end

        hs.cpu_usage_mean = stats_dict['cpu_usage_mean']
        hs.cpu_usage_stddev = stats_dict['cpu_usage_stddev']
        hs.cpu_usage_max = stats_dict['cpu_usage_max']

        hs.cpu_temp_mean = stats_dict['cpu_temp_mean']
        hs.cpu_temp_stddev = stats_dict['cpu_temp_stddev']
        hs.cpu_temp_max = stats_dict['cpu_temp_max']

        hs.cpu_usage_core_mean = stats_dict['cpu_usage_core_mean']
        hs.cpu_usage_core_stddev = stats_dict['cpu_usage_core_stddev']
        hs.cpu_usage_core_max = stats_dict['cpu_usage_core_max']

        hs.cpu_temp_core_mean = stats_dict['cpu_temp_core_mean']
        hs.cpu_temp_core_stddev = stats_dict['cpu_temp_core_stddev']
        hs.cpu_temp_core_max = stats_dict['cpu_temp_core_max']

        hs.ram_usage_mean = stats_dict['ram_usage_mean']
        hs.ram_usage_stddev = stats_dict['ram_usage_stddev']
        hs.ram_usage_max = stats_dict['ram_usage_max']

        hs.interface_name = stats_dict['interface_name']
        hs.message_frequency_mean =  map(int, stats_dict['message_frequency_mean'])
        hs.message_frequency_stddev = map(int, stats_dict['message_frequency_stddev'])
        hs.message_frequency_max = map(int, stats_dict['message_frequency_max'])

        hs.bandwidth_mean =  map(int, stats_dict['bandwidth_mean'])
        hs.bandwidth_stddev = map(int, stats_dict['bandwidth_stddev'])
        hs.bandwidth_max = map(int, stats_dict['bandwidth_max'])

        hs.drive_name = stats_dict['drive_name']
        hs.drive_free_space = stats_dict['drive_free_space']
        hs.drive_write = stats_dict['drive_write_mean']
        #hs.drive_write_stddev = stats_dict['drive_write_stddev']
        #hs.drive_write_max = stats_dict['drive_write_max']
        hs.drive_read = stats_dict['drive_read_mean']
        #hs.drive_read_stddev = stats_dict['drive_read_stddev']
        #hs.drive_read_max = stats_dict['drive_read_max']

        return hs

    def execute_reaction(self, reaction):
        """
        Parses through the reaction and
        calls the appropriate method from the NodeManager. Uses ROS Services.
        Returns a message about operation's success.

        :param reaction: Reaction to be executed and node affected.
        :type reaction: NodeReaction.
        :returns: String
        """

        msg = ''
        if reaction.node not in self.__node_list:
            return 'spefified Node is not running on this Host'

        if reaction.action == 'restart':
            node = self.__node_list[reaction.node]
            msg = self.__node_manager.restart_node(node)
            self.remove_node(reaction.node)
        elif reaction.action == 'stop':
            msg = self.__node_manager.stop_node(reaction.node)
            self.remove_node(reaction.node)
        elif reaction.action == 'command':
            msg = self.__node_manager.execute_command(reaction.command)
        else:
            msg = 'Failed to execute reaction, %s is no valid argument'%reaction.action
        return msg

    def remove_node(self, node):
        """
        Removes the Node with the given id from the host.

        :param node_id: id of the node to be removed.
        :type node_id: String
        """
        if node in self.__node_list:
            del self.__node_list[node]

    def get_node_info(self, event):
        """
        Get the list of all Nodes running on the host.
        Update the __node_list
        """
        nodes = {}
        for node_name in rosnode.get_node_names():
            node_api = rosnode.get_api_uri(rospy.get_master(), node_name)
            if self._id.replace('_','.') in node_api[2]:

        for node in nodes:
            if node not in self.__node_list:
                node_api = rosnode.get_api_uri(rospy.get_master(), node_name)
                try:
                    pid = self.node_server_proxy(node_api)
                    if not pid:
                        continue
                    node_process = psutil.Process(pid)
                    new_node = NodeStatisticsHandler(self._id, node_name, node_process)
                    self.__dict_lock.acquire()
                    self.__node_list[node] = new_node
                    self.__dict_lock.realease()
                except:
                    self.__dict_lock.realease()
                    continue

        self.__dict_lock.acquire()
        to_remove = []
        for node_name in self.__node_list:
            if node_name not in nodes:
                to_remove.append(node_name)
        for node_name in to_remove:
            self.remove_node(node_name)
        self.__dict_lock.realease()

    def node_server_proxy(self, node_api):
        socket.setdefaulttimeout(3)
        try:
            code,msg,pid = xmlrpclib.ServerProxy(node_api[2]).getPid('/NODEINFO')
            return pid
        except socket.Error:
            return False

    def update_nodes(self):
        """
        update the status of each node in its own threading
        """

        for node in self.__node_list:
            Thread(target = self.__node_list[node].measure_status).start()

    def get_sensors(self, sensor_list):

        sensors.init()
        try:
            for chip in sensors.iter_detected_chips():
                for feature in chip:
                    if feature.name.startswith(b'temp'):
                        sensor_list.append(feature)
        except sensors.SensorsError:
            pass

    @property
    def update_intervall(self):
        return self.__update_intervall

    @property
    def publish_intervall(self):
        return self.__publish_intervall

<commit_msg>small fix<commit_after>from statistics_handler import StatisticsHandler
from node_statistics_handler import NodeStatisticsHandler
from host_status import HostStatus
from node_manager import NodeManager
from threading import Thread
import arni_msgs
from arni_msgs.msg import HostStatistics
from arni_msgs.srv import NodeReaction
import psutil
import xmlrpclib
import socket
import rosnode
import rospy
import sensors
import threading

class HostStatisticsHandler( StatisticsHandler):

    """
    Represents a host , limited to one instance per host.
    Collects statistics about the current state of the host and s
    ends them using the publisher-subscriber mechanism.
    """

    def __init__(self, hostid):

        super(HostStatisticsHandler, self).__init__(hostid)

        self.__update_intervall = 0
        self.__publish_intervall = 0

        self.__init_params()
        self.__register_service()
        self.__lock = threading.Lock()
        self.__dict_lock = threading.Lock()

        self.pub = rospy.Publisher('/statistics_host', HostStatistics, queue_size = 500)
        #: Used to store information about the host's status.
        self._status = HostStatus(rospy.Time.now())

        #: Interface to restart and stop nodes
        #or executing other commands.
        self.__node_manager = NodeManager()

        #: Dictionary holding all nodes currently running on the host.
        self.__node_list = {}

        #Base-stats for I/O
        self.__bandwidth_base = {}
        self.__msg_freq_base = {}
        self.__disk_write_base = {}
        self.__disk_read_base = {}

        self.__set_bases()

    def __set_bases(self):

        psutil.cpu_percent()
        psutil.cpu_percent(percpu = True)

        for interface in psutil.net_io_counters(True):

            netint = psutil.net_io_counters(True)[interface]
            total_bytes = netint.bytes_recv + netint.bytes_sent
            total_packages = netint.packets_sent + netint.packets_recv

            self.__bandwidth_base[interface] = total_bytes
            self.__msg_freq_base[interface] = total_packages

        dev_names = ''
        for disk in psutil.disk_partitions():
            if all(['cdrom' not in disk.opts, 'sr' not in disk.device]):
                    dev_names += disk.device + ';'

        for key in psutil.disk_io_counters(True):
            if key in dev_names:
                disk = psutil.disk_io_counters(True)[key]
                self.__disk_read_base[key] = disk.read_bytes
                self.__disk_write_base[key] = disk.write_bytes

    def __register_service(self):
        """
        Register all services
        """
        rospy.Service(
            "/execute_node_reaction/%s" % self._id,
            NodeReaction, self.execute_reaction)

    def measure_status(self, event):
        """
        Collects information about the host's current status using psutils.
        Triggered periodically.
        """

        self.__lock.acquire()
        #update node list
        self.__dict_lock.acquire()
        self.update_nodes()
        self.__dict_lock.realease()

        #CPU
        self._status.add_cpu_usage(psutil.cpu_percent())
        self._status.add_cpu_usage_core(psutil.cpu_percent(percpu = True))
        #RAM
        self._status.add_ram_usage(psutil.virtual_memory().percent)
        #temp
        sensor_list = []
        self.get_sensors(sensor_list)

        #Bandwidth and message frequency
        self.__measure_network_usage()
        #Disk usage
        self.__measure_disk_usage()
        self.__lock.release()

    def __measure_network_usage(self):
        """
        measure current network_io_counters
        """
        network_interfaces = psutil.net_io_counters(True)

        for key in network_interfaces:
            total_bytes = (network_interfaces[key].bytes_sent + network_interfaces[key].bytes_recv) - self.__bandwidth_base[key]
            total_packages = (network_interfaces[key].packets_sent + network_interfaces[key].packets_recv) - self.__msg_freq_base[key]

            bandwidth = total_bytes / self.__update_intervall
            msg_frequency = total_packages / self.__update_intervall
            self._status.add_bandwidth(key, bandwidth)
            self._status.add_msg_frequency(key, msg_frequency)

            #update base stats for next iteration
            self.__bandwidth_base[key] += total_bytes
            self.__msg_freq_base[key] += total_packages

    def __measure_disk_usage(self):
        """
        measure current disk usage
        """
        # Free Space on disks
        disks = psutil.disk_partitions()
        dev_name = ''
        for x in disks:
            if 'cdrom' not in x.opts and 'sr' not in x.device:
                free_space = psutil.disk_usage(x.mountpoint).free / 2**20
                self._status.add_drive_space(x.device , free_space)
                dev_name += x.device + ';'

        #Drive I/O
        drive_io = psutil.disk_io_counters(True)
        for key in drive_io:
            if key in dev_name:
                readb = drive_io[key].read_bytes - self.__disk_read_base[key]
                writeb = drive_io[key].write_bytes - self.__disk_write_base[key]

                read_rate = readb / self.__update_intervall
                write_rate = writeb / self.__update_intervall
                self._status.add_drive_read(key, read_rate)
                self._status.add_drive_write(key, write_rate)
                ##update base stats for next iteration
                self.__disk_read_base[key] += readb
                self.__disk_write_base[key] += writeb

    def publish_status(self, event):
        """
        Publishes the current status to a topic using ROS's publisher-subscriber mechanism.
        Triggered periodically.
        """
        self.__lock.acquire()
        self._status.time_end = rospy.Time.now()
        stats = self.__calc_statistics()
        self.__dict_lock.acquire()
        self.__publish_nodes()
        self.__dict_lock.release()
        self.pub.publish(stats)
        self._status.reset()
        self._status.time_start = rospy.Time.now()
        self.__lock.release()

    def __publish_nodes(self):
        """
        publishes current status of all nodes.
        """
        for node in self.__node_list:
            Thread(target = self.__node_list[node].publish_status).start()

    def __init_params(self):
        """
        Initializes params on the parameter server,
        """

        rospy.set_param('/update_intervall', 1)
        rospy.set_param('/publish_intervall', 10)

        self.__update_intervall =  rospy.get_param('/update_intervall')
        self.__publish_intervall =  rospy.get_param('/publish_intervall')

        #set Topic statistics to same time_window as update_intervall
        rospy.set_param('/enable_statistics', True )
        rospy.set_param('/statistics_window_min_elements',
                        self.__update_intervall )
        rospy.set_param('/statistics_window_max_elements',
                        self.__publish_intervall)

    def shut_down_hook(self):

        for key in self.__node_list:
            self.__node_list[key].pub.unregister()
        self.pub.unregister()

    def __calc_statistics(self):
        """
        Calculates statistics like mean, standard deviation and max from the status.
        Returns an instance of HostStatistics which can be published.

        :returns: HostStatistics
        """
        stats_dict = self._status.calc_stats()

        hs = HostStatistics()

        hs.host = self._id
        hs.window_start = self._status.time_start
        hs.window_stop = self._status.time_end

        hs.cpu_usage_mean = stats_dict['cpu_usage_mean']
        hs.cpu_usage_stddev = stats_dict['cpu_usage_stddev']
        hs.cpu_usage_max = stats_dict['cpu_usage_max']

        hs.cpu_temp_mean = stats_dict['cpu_temp_mean']
        hs.cpu_temp_stddev = stats_dict['cpu_temp_stddev']
        hs.cpu_temp_max = stats_dict['cpu_temp_max']

        hs.cpu_usage_core_mean = stats_dict['cpu_usage_core_mean']
        hs.cpu_usage_core_stddev = stats_dict['cpu_usage_core_stddev']
        hs.cpu_usage_core_max = stats_dict['cpu_usage_core_max']

        hs.cpu_temp_core_mean = stats_dict['cpu_temp_core_mean']
        hs.cpu_temp_core_stddev = stats_dict['cpu_temp_core_stddev']
        hs.cpu_temp_core_max = stats_dict['cpu_temp_core_max']

        hs.ram_usage_mean = stats_dict['ram_usage_mean']
        hs.ram_usage_stddev = stats_dict['ram_usage_stddev']
        hs.ram_usage_max = stats_dict['ram_usage_max']

        hs.interface_name = stats_dict['interface_name']
        hs.message_frequency_mean =  map(int, stats_dict['message_frequency_mean'])
        hs.message_frequency_stddev = map(int, stats_dict['message_frequency_stddev'])
        hs.message_frequency_max = map(int, stats_dict['message_frequency_max'])

        hs.bandwidth_mean =  map(int, stats_dict['bandwidth_mean'])
        hs.bandwidth_stddev = map(int, stats_dict['bandwidth_stddev'])
        hs.bandwidth_max = map(int, stats_dict['bandwidth_max'])

        hs.drive_name = stats_dict['drive_name']
        hs.drive_free_space = stats_dict['drive_free_space']
        hs.drive_write = stats_dict['drive_write_mean']
        #hs.drive_write_stddev = stats_dict['drive_write_stddev']
        #hs.drive_write_max = stats_dict['drive_write_max']
        hs.drive_read = stats_dict['drive_read_mean']
        #hs.drive_read_stddev = stats_dict['drive_read_stddev']
        #hs.drive_read_max = stats_dict['drive_read_max']

        return hs

    def execute_reaction(self, reaction):
        """
        Parses through the reaction and
        calls the appropriate method from the NodeManager. Uses ROS Services.
        Returns a message about operation's success.

        :param reaction: Reaction to be executed and node affected.
        :type reaction: NodeReaction.
        :returns: String
        """

        msg = ''
        if reaction.node not in self.__node_list:
            return 'spefified Node is not running on this Host'

        if reaction.action == 'restart':
            node = self.__node_list[reaction.node]
            msg = self.__node_manager.restart_node(node)
            self.remove_node(reaction.node)
        elif reaction.action == 'stop':
            msg = self.__node_manager.stop_node(reaction.node)
            self.remove_node(reaction.node)
        elif reaction.action == 'command':
            msg = self.__node_manager.execute_command(reaction.command)
        else:
            msg = 'Failed to execute reaction, %s is no valid argument'%reaction.action
        return msg

    def remove_node(self, node):
        """
        Removes the Node with the given id from the host.

        :param node_id: id of the node to be removed.
        :type node_id: String
        """
        if node in self.__node_list:
            del self.__node_list[node]

    def get_node_info(self, event):
        """
        Get the list of all Nodes running on the host.
        Update the __node_list
        """
        nodes = []
        for node_name in rosnode.get_node_names():
            node_api = rosnode.get_api_uri(rospy.get_master(), node_name)
            if self._id.replace('_','.') in node_api[2]:
                nodes.append(node_name)

        for node in nodes:
            if node not in self.__node_list:
                node_api = rosnode.get_api_uri(rospy.get_master(), node)
                try:
                    pid = self.node_server_proxy(node_api)
                    if not pid:
                        continue
                    node_process = psutil.Process(pid)
                    new_node = NodeStatisticsHandler(self._id, node, node_process)
                    self.__dict_lock.acquire()
                    self.__node_list[node] = new_node
                    self.__dict_lock.realease()
                except:
                    self.__dict_lock.realease()
                    continue

        self.__dict_lock.acquire()
        to_remove = []
        for node_name in self.__node_list:
            if node_name not in nodes:
                to_remove.append(node_name)
        for node_name in to_remove:
            self.remove_node(node_name)
        self.__dict_lock.realease()

    def node_server_proxy(self, node_api):
        socket.setdefaulttimeout(3)
        try:
            code,msg,pid = xmlrpclib.ServerProxy(node_api[2]).getPid('/NODEINFO')
            return pid
        except socket.Error:
            return False

    def update_nodes(self):
        """
        update the status of each node in its own threading
        """

        for node in self.__node_list:
            Thread(target = self.__node_list[node].measure_status).start()

    def get_sensors(self, sensor_list):

        sensors.init()
        try:
            for chip in sensors.iter_detected_chips():
                for feature in chip:
                    if feature.name.startswith(b'temp'):
                        sensor_list.append(feature)
        except sensors.SensorsError:
            pass

    @property
    def update_intervall(self):
        return self.__update_intervall

    @property
    def publish_intervall(self):
        return self.__publish_intervall

<|endoftext|>


<commit_before>import xbmc
import xbmcgui
import requests
from contextlib import closing, nested
from xbmctorrent import plugin
from xbmctorrent.scrapers import scraper, ungenerate
from xbmctorrent.common import url_for_magnet, UA
from xbmctorrent.ga import tracked
from xbmctorrent import tmdb
from xbmcswift2.listitem import ListItem
from itertools import izip, chain
from multiprocessing.pool import ThreadPool

BASE_URL = "https://yify-torrents.com"
YOUTUBE_ACTION = "plugin://plugin.video.youtube/?path=/root/video&action=play_video&videoid=%s"
MOVIES_PER_PAGE = 20

GENRES = [
    "Action",
    "Adventure",
    "Animation",
    "Biography",
    "Comedy",
    "Crime",
    "Documentary",
    "Drama"
    "Family",
    "Fantasy",
    "Film-Noir",
    "Game-Show",
    "History",
    "Horror",
    "Music",
    "Musical",
    "Mystery",
    "News",
    "Reality-TV",
    "Romance",
    "Sci-Fi",
    "Sport",
    "Talk-Show",
    "Thriller",
    "War",
    "Western",
]

def get(yify_id):
    return requests.get("%s/api/movie.json" % BASE_URL, params={"id": yify_id}).json()

def get_list(yify_ids, cb=None):
    f = get
    if cb:
        def _get_cb(*args, **kwargs):
            ret = get(*args, **kwargs)
            cb(ret)
            return ret
        f = _get_cb
    return pool().map(f, yify_ids)

@scraper("YIFY Torrents - Movies", "http://fbcdn-sphotos-h-a.akamaihd.net/hphotos-ak-frc3/204323_207963335901313_5804989_o.jpg")
@plugin.route("/yify")
@tracked
def yify_index():
    return [
        {"label": "Search", "path": plugin.url_for("yify_search")},
        {"label": "Browse by Genre", "path": plugin.url_for("yify_genres")},

        {"label": "Most Popular", "path": plugin.url_for("yify_show", sort="seeds", order="desc", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 720p", "path": plugin.url_for("yify_show", sort="seeds", order="desc", quality="720p", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 1080p", "path": plugin.url_for("yify_show", sort="seeds", order="desc", quality="1080p", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 3D", "path": plugin.url_for("yify_show", sort="seeds", order="desc", quality="3D", set=1, limit=MOVIES_PER_PAGE)},

        {"label": "Best Rated", "path": plugin.url_for("yify_show", sort="rating", order="desc", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 720p", "path": plugin.url_for("yify_show", sort="rating", order="desc", quality="720p", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 1080p", "path": plugin.url_for("yify_show", sort="rating", order="desc", quality="1080p", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 3D", "path": plugin.url_for("yify_show", sort="rating", order="desc", quality="3D", set=1, limit=MOVIES_PER_PAGE)},

        {"label": "Most Recent", "path": plugin.url_for("yify_show", sort="date", order="desc", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 720p", "path": plugin.url_for("yify_show", sort="date", order="desc", quality="720p", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 1080p", "path": plugin.url_for("yify_show", sort="date", order="desc", quality="1080p", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 3D", "path": plugin.url_for("yify_show", sort="date", order="desc", quality="3D", set=1, limit=MOVIES_PER_PAGE)},
    ]

@plugin.route("/yify/show")
def yify_show():
    plugin.set_content("movies")
    # xbmc.executebuiltin("Container.SetViewMode(lowlist)")
    args = dict((k, v[0]) for k, v in plugin.request.args.items())

    current_page = int(args["set"])
    limit = int(args["limit"])

    with closing(xbmcgui.DialogProgress()) as dialog:
        dialog.create(plugin.name)

        plugin.log.info(plugin.request.args)
        r = requests.get("%s/api/list.json" % BASE_URL, params=args)
        search_result = r.json()
        movies = search_result["MovieList"]

        state = {"done": 0}
        def on_movie(data):
            state["done"] += 1
            dialog.update(
                percent=int(state["done"] * 100.0 / (len(movies) * 2)),
                line1="Fetching movie metadata...",
                line2=data.get("title") or data.get("MovieTitleClean") or "",
            )

        with nested(closing(ThreadPool(2)), closing(ThreadPool(2))) as (pool_yify, pool_tmdb):
            yify_list = [pool_yify.apply_async(get, [movie["MovieID"]], callback=on_movie) for movie in movies]
            tmdb_list = [pool_tmdb.apply_async(tmdb.get, [movie["ImdbCode"]], callback=on_movie) for movie in movies]
            while not all(job.ready() for job in chain(yify_list, tmdb_list)):
                if dialog.iscanceled():
                    dialog.close()
                    return
                xbmc.sleep(50)

    yify_list = map(lambda job: job.get(), yify_list)
    tmdb_list = map(lambda job: job.get(), tmdb_list)

    for movie, tmdb_meta in izip(yify_list, tmdb_list):
        if tmdb_meta:
            item = tmdb.get_list_item(tmdb_meta)
            if not args.get("quality") and movie["Quality"] != "720p":
                item["label"] = "%s (%s)" % (item["label"], movie["Quality"])
            item.update({
                "path": url_for_magnet(movie["TorrentMagnetUrl"]),
                "is_playable": True,
            })
            item.setdefault("info", {}).update({
                "count": movie["MovieID"],
                "genre": "%s (%s S:%s P:%s)" % (item["info"]["genre"], movie["Size"], movie["TorrentSeeds"], movie["TorrentPeers"]),
                "trailer": YOUTUBE_ACTION % movie["YoutubeTrailerID"],
                "plot_outline": movie["ShortDescription"],
                "video_codec": "h264",
                "mpaa": movie["AgeRating"],
            })
            width, height = map(int, movie["Resolution"].split("*"))
            item.setdefault("stream_info", {}).update({
                "video": {
                    "codec": "h264",
                    "width": width,
                    "height": height,
                    "aspect": float(width) / float(height),
                },
                "audio": {
                    "language": movie["Language"],
                },
            })
            yield item

    if current_page < (int(search_result["MovieCount"]) / limit):
        next_args = args.copy()
        next_args["set"] = int(next_args["set"]) + 1
        yield {
            "label": ">> Next page",
            "path": plugin.url_for("yify_show", **next_args),
        }

@plugin.route("/yify/genres")
def yify_genres():
    for genre in GENRES:
        yield {
            "label": genre,
            "path": plugin.url_for("yify_show", genre=genre, sort="alphabet", order="asc", set=1, limit=MOVIES_PER_PAGE),
        }

@plugin.route("/yify/search")
@tracked
def yify_search():
    query = plugin.request.args.get("query")
    if query:
        query = query[0]
    else:
        kb = xbmc.Keyboard("", "XBMCtorrent - YIFY - Search")
        kb.doModal()
        query = kb.isConfirmed() and kb.getText() or None

    if query:
        plugin.redirect(plugin.url_for("yify_show", keywords=query, set=1, limit=MOVIES_PER_PAGE))
<commit_msg>Analytics for YIFY, albeit very unclear<commit_after>import xbmc
import xbmcgui
import requests
from contextlib import closing, nested
from xbmctorrent import plugin
from xbmctorrent.scrapers import scraper, ungenerate
from xbmctorrent.common import url_for_magnet, UA
from xbmctorrent.ga import tracked
from xbmctorrent import tmdb
from xbmcswift2.listitem import ListItem
from itertools import izip, chain
from multiprocessing.pool import ThreadPool

BASE_URL = "https://yify-torrents.com"
YOUTUBE_ACTION = "plugin://plugin.video.youtube/?path=/root/video&action=play_video&videoid=%s"
MOVIES_PER_PAGE = 20

GENRES = [
    "Action",
    "Adventure",
    "Animation",
    "Biography",
    "Comedy",
    "Crime",
    "Documentary",
    "Drama"
    "Family",
    "Fantasy",
    "Film-Noir",
    "Game-Show",
    "History",
    "Horror",
    "Music",
    "Musical",
    "Mystery",
    "News",
    "Reality-TV",
    "Romance",
    "Sci-Fi",
    "Sport",
    "Talk-Show",
    "Thriller",
    "War",
    "Western",
]

def get(yify_id):
    return requests.get("%s/api/movie.json" % BASE_URL, params={"id": yify_id}).json()

def get_list(yify_ids, cb=None):
    f = get
    if cb:
        def _get_cb(*args, **kwargs):
            ret = get(*args, **kwargs)
            cb(ret)
            return ret
        f = _get_cb
    return pool().map(f, yify_ids)

@scraper("YIFY Torrents - Movies", "http://fbcdn-sphotos-h-a.akamaihd.net/hphotos-ak-frc3/204323_207963335901313_5804989_o.jpg")
@plugin.route("/yify")
@tracked
def yify_index():
    return [
        {"label": "Search", "path": plugin.url_for("yify_search")},
        {"label": "Browse by Genre", "path": plugin.url_for("yify_genres")},

        {"label": "Most Popular", "path": plugin.url_for("yify_show", sort="seeds", order="desc", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 720p", "path": plugin.url_for("yify_show", sort="seeds", order="desc", quality="720p", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 1080p", "path": plugin.url_for("yify_show", sort="seeds", order="desc", quality="1080p", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 3D", "path": plugin.url_for("yify_show", sort="seeds", order="desc", quality="3D", set=1, limit=MOVIES_PER_PAGE)},

        {"label": "Best Rated", "path": plugin.url_for("yify_show", sort="rating", order="desc", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 720p", "path": plugin.url_for("yify_show", sort="rating", order="desc", quality="720p", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 1080p", "path": plugin.url_for("yify_show", sort="rating", order="desc", quality="1080p", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 3D", "path": plugin.url_for("yify_show", sort="rating", order="desc", quality="3D", set=1, limit=MOVIES_PER_PAGE)},

        {"label": "Most Recent", "path": plugin.url_for("yify_show", sort="date", order="desc", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 720p", "path": plugin.url_for("yify_show", sort="date", order="desc", quality="720p", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 1080p", "path": plugin.url_for("yify_show", sort="date", order="desc", quality="1080p", set=1, limit=MOVIES_PER_PAGE)},
        {"label": "- in 3D", "path": plugin.url_for("yify_show", sort="date", order="desc", quality="3D", set=1, limit=MOVIES_PER_PAGE)},
    ]

@plugin.route("/yify/show")
@tracked
def yify_show():
    plugin.set_content("movies")
    # xbmc.executebuiltin("Container.SetViewMode(lowlist)")
    args = dict((k, v[0]) for k, v in plugin.request.args.items())

    current_page = int(args["set"])
    limit = int(args["limit"])

    with closing(xbmcgui.DialogProgress()) as dialog:
        dialog.create(plugin.name)

        plugin.log.info(plugin.request.args)
        r = requests.get("%s/api/list.json" % BASE_URL, params=args)
        search_result = r.json()
        movies = search_result["MovieList"]

        state = {"done": 0}
        def on_movie(data):
            state["done"] += 1
            dialog.update(
                percent=int(state["done"] * 100.0 / (len(movies) * 2)),
                line1="Fetching movie metadata...",
                line2=data.get("title") or data.get("MovieTitleClean") or "",
            )

        with nested(closing(ThreadPool(2)), closing(ThreadPool(2))) as (pool_yify, pool_tmdb):
            yify_list = [pool_yify.apply_async(get, [movie["MovieID"]], callback=on_movie) for movie in movies]
            tmdb_list = [pool_tmdb.apply_async(tmdb.get, [movie["ImdbCode"]], callback=on_movie) for movie in movies]
            while not all(job.ready() for job in chain(yify_list, tmdb_list)):
                if dialog.iscanceled():
                    dialog.close()
                    return
                xbmc.sleep(50)

    yify_list = map(lambda job: job.get(), yify_list)
    tmdb_list = map(lambda job: job.get(), tmdb_list)

    for movie, tmdb_meta in izip(yify_list, tmdb_list):
        if tmdb_meta:
            item = tmdb.get_list_item(tmdb_meta)
            if not args.get("quality") and movie["Quality"] != "720p":
                item["label"] = "%s (%s)" % (item["label"], movie["Quality"])
            item.update({
                "path": url_for_magnet(movie["TorrentMagnetUrl"]),
                "is_playable": True,
            })
            item.setdefault("info", {}).update({
                "count": movie["MovieID"],
                "genre": "%s (%s S:%s P:%s)" % (item["info"]["genre"], movie["Size"], movie["TorrentSeeds"], movie["TorrentPeers"]),
                "trailer": YOUTUBE_ACTION % movie["YoutubeTrailerID"],
                "plot_outline": movie["ShortDescription"],
                "video_codec": "h264",
                "mpaa": movie["AgeRating"],
            })
            width, height = map(int, movie["Resolution"].split("*"))
            item.setdefault("stream_info", {}).update({
                "video": {
                    "codec": "h264",
                    "width": width,
                    "height": height,
                    "aspect": float(width) / float(height),
                },
                "audio": {
                    "language": movie["Language"],
                },
            })
            yield item

    if current_page < (int(search_result["MovieCount"]) / limit):
        next_args = args.copy()
        next_args["set"] = int(next_args["set"]) + 1
        yield {
            "label": ">> Next page",
            "path": plugin.url_for("yify_show", **next_args),
        }

@plugin.route("/yify/genres")
@tracked
def yify_genres():
    for genre in GENRES:
        yield {
            "label": genre,
            "path": plugin.url_for("yify_show", genre=genre, sort="alphabet", order="asc", set=1, limit=MOVIES_PER_PAGE),
        }

@plugin.route("/yify/search")
@tracked
def yify_search():
    query = plugin.request.args.get("query")
    if query:
        query = query[0]
    else:
        kb = xbmc.Keyboard("", "XBMCtorrent - YIFY - Search")
        kb.doModal()
        query = kb.isConfirmed() and kb.getText() or None

    if query:
        plugin.redirect(plugin.url_for("yify_show", keywords=query, set=1, limit=MOVIES_PER_PAGE))
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""Program for the bulk renaming of files in an ordered manner."""

import os
from FileName import *
import RenamerHelp
import splitstring
import osnav

class RenamerFile(FileName):

    def __init__(self, string, renamer):
        super().__init__(string)
        self.tempName = ""
        self._oldName  = string
        self.renamer = renamer

    def rename(self):
        """Renames the file from the current name (string) to the new one (file)."""
        self.newName = self.file + self.extention
        os.rename(self.string, self.newName)
        #print(self.string + "-->" + self.newName)

    def renamePrint(self):
        """Functions like rename but prints the change from original filename to new."""
        self.newName = self.file + self.extention
        try:
            os.rename(self.string, self.newName)
            print(self._oldName + " --> " + self.newName)
        except FileExistsError:
            raise

    def setFileName(self, name, num):
        self.file = splitstring.stitch(name, self.file, num)

class Renamer(object):
    """Takes a directory and renames the files in that directory in an ordered manner."""

    def __init__(self):
        self.restart()

    def restart(self):
        self.mainList     = []
        self.subList      = []
        self.fill         = 0
        self.startNum     = 1
        self.toName       = ""
        self.string       = ""
        self.term1, term2 = 0, 0

    def printList(self):
        """Prints the names of the current files in the directoy."""
        i = 1
        for item in self.mainList:
            print(str(i).zfill(self.fill) + ": " + item.getName())
            i += 1
        print()

    def printSubList(self):
        """Prints the list of items selected by the select function."""
        for item in self.subList:
            print(item.getName())
        print()

    def inString(self, string):
        """Stores the raw input from the command line as a string."""
        self.string = string

    def swap(self, cmd):
        """Swaps two items in the list."""
        if len(self.mainList) > 0:
            term1, term2 = eval(cmd[1])-1, eval(cmd[2])-1
            self.mainList[term1], self.mainList[term2] = self.mainList[term2], self.mainList[term1]
            self.printList()
        else:
            print("Must have a sorted list before using swap")

    def _indexCheck(self, container, index):
        if index <= 0 or index > len(container):
            return False
        else:
            return True

    def select(self, cmd):
        """Creates a sublist between the two selected elements (1 indexed all inclusive)."""
        if len(self.mainList) > 0:
            self.subList = []
            if len(cmd) == 3:
                self.term1, self.term2 = eval(cmd[1]), eval(cmd[2])
            elif len(cmd) == 2:
                self.term1, self.term2 = eval(cmd[1]), eval(cmd[1])
            else:
                print("Invalid number of terms for select")
                return None
            # Make sure term1 is always the lower term.
            if self.term1 > self.term2:
                self.term1, self.term2 = self.term2, self.term1
            # Check that the numbers are valid list items.
            if self._indexCheck(self.mainList, self.term1) and self._indexCheck(self.mainList, self.term2):
                self.term1 -= 1
                self.subList = self.mainList[self.term1 : self.term2]
                self.printSubList()
            else:
                print("Must select items in list number\n")
        else:
            print("Must have a sorted list before using select\n")

    def insert(self, cmd):
        """Inserts the sublist into the selected place
the sublist cannot be inserted into itself

If the insert position is an element element of the list than the first item of the sublist
the first item of the sublist shall be at the insert position

If the insert position is an element greater than the last item of the sublist
the last item of the sublist will be at the insert position."""
        if len(self.mainList) > 0:
            if len(cmd) == 2:
                if len(self.subList) > 0:
                    iIn = eval(cmd[1])
                    if iIn > self.term1 and iIn <= self.term2:
                        print("Cannot insert the selected list into itself")
                    else:
                        # Inserting to a number lower than selected.
                        if iIn <= self.term1:
                            self.mainList[self.term1 : self.term2] = []
                            for j in range(len(self.subList)):
                                self.mainList.insert(iIn-1+j, self.subList[j])
                        # Inserting to a humber higher than selected.
                        elif iIn > self.term2:
                            self.mainList[self.term1 : self.term2] = []
                            for j in range(len(self.subList)):
                                self.mainList.insert(iIn-len(self.subList)+j, self.subList[j])
                        # Common to both case hence the else then an if and not an elif
                        self.printList()
                        self.subList = []
                # if len(self.subList) > 0:
                else:
                    print("Nothing to insert\n")
            # if len(cmd) == 2:
            elif len(cmd) == 3: #Insersts first term into second one so user does not need to use sel
                if eval(cmd[1]) != eval(cmd[2]):
                    self.select(["sel", cmd[1], cmd[1]])
                    self.insert(["ins", cmd[2]])
                else:
                    print("Cannot insert an item into itself\n")
        # if len(self.mainList) > 0:
        else:
            print("Must have a sorted list before using select\n")

    def set(self, cmd):
        """Input commands that determine how renamer will function."""
        if cmd[1] == "name":
           self.setName(cmd)
        if cmd[1] == "start":
            self.setStart(cmd)
        if cmd[1] == "zfill":
            self.setZFill(cmd)
        if cmd[1] == "mode":
            self.setMode(cmd)

    def setName(self, cmd):
        """Allows the user to set the name that the files will be renamed to."""
        if len(self.mainList) > 0:
            self.toName = self.string[self._lenCmdStr(cmd,2):]
            print()
            print("New name set to: " + self.toName)
            print("Example: " + splitstring.stitch(self.toName, self.mainList[0].file, self._zFillNum()))
            print()
        else:
            print("Must have a sorted list before setting name\n")

    def setStart(self, cmd):
        """Sets the starting number from which the files will be renamed."""
        if len(self.mainList) > 0:
            self.startNum = eval(cmd[2])
            self._checkZFill()
            print()
            print("New starting number is " + str(self.startNum))
            print("zfill is " + str(self.fill))
            print("Example: " + splitstring.stitch(self.toName, self.mainList[0].file, self._zFillNum()))
        else:
            print("Must have a sorted list before setting the start num\n")

    def setZFill(self, cmd):
        """Sets a custom zfill amount greater than the automatic zfill."""
        if len(self.mainList) > 0:
            self._checkZFill()
            if eval(cmd[2]) >= self.fill:
                self.fill = eval(cmd[2])
                print()
                print("New zfill set to: " + str(self.fill))
                print("Example: " + splitstring.stitch(self.toName, self.mainList[0].file, self._zFillNum()))
                print()
            else:
                print("Cannot set zfill to be less than the automatic zfill\n")

    def _checkZFill(self):
        """Check the zfill so that it is correct when a new start number has been added."""
        fill    = self.fill
        newFill = fill

        if len(str(self.startNum)) > fill:
            newFill = len(str(self.startNum))

        if (len(self.mainList) + self.startNum) > 10**(newFill):
            newFill += 1

        if newFill > self.fill:
            self.fill = newFill

    def _lenCmdStr(self, cmd, ind):
        """Returns the length sum of the length of substring elements to the index"""
        # This is used in conjunction with the input string to remove the commands from the input

        result = 0
        if len(cmd) >= ind:
            for i in range(ind):
                result += len(cmd[i]) + 1
        return result

    def _zFillNum(self):
        """Shorthand function to keep code clean"""
        return str(self.startNum).zfill(self.fill)

    def setMode(self, cmd):
        #Legacy from when prefix was a command
        pass

    def show(self, cmd):
        """Shows aspects of the program depend on the command entered after show."""
        if len(self.mainList) > 0:
            if len(cmd) > 1:
                if cmd[1] == "ext":
                    i = 1
                    for item in self.mainList:
                        print(str(i).zfill(self.fill) + ": " + item.getFileExt())
                        i += 1
                elif cmd[1] == "file":
                    i = 1
                    for item in self.mainList:
                        print(str(i).zfill(self.fill) + ": " + item.getFileName())
                        i +=1
                elif cmd[1] == "list" or cmd[1] == "lst":
                    self.printList()
                elif cmd[1] == "sublist":
                    self.printSubList()
            else:
                print("please select a subcommand for show,\nSubcommands are ext or file.")
        else:
            print("Must have a sorted list befor using show.")

    def sort(self):
        """Used to set the current directory to that in which the files shall be renamed."""
        self.mainList = []
        dirList = os.listdir(os.getcwd())
        for i in dirList:
            if os.path.isfile(i):
                fileName = RenamerFile(i, self)
                self.mainList.append(fileName)

        self.fill = len(str(len(self.mainList)))
        i = 1
        for item in self.mainList:
            print(str(i).zfill(self.fill) + ": " + item.getName())
            i += 1

    def remove(self, cmd):
        """If a single number is passed then that number is removed from the list
        If two numbers are passed than the range of those numbers is removed from the list"""
        if len(cmd) == 2:
            self.mainList.pop(eval(cmd[1])-1)
            self.printList()
        elif len(cmd) == 3:
            if eval(cmd[1]) < eval(cmd[2]):
                list1 = self.mainList[0:eval(cmd[1])-1]
                list2 = self.mainList[eval(cmd[2]):len(self.mainList)]
                self.mainList = list1 + list2
                self.printList()
            else:
                print("Number range for removal is incorrect err#5")

    def rename(self):
        """Renames the files in the list to their new name."""
        #Need to update to deal with naming conflicts.
        if len(self.mainList) > 0:
            i = self.startNum
            for item in self.mainList:
                item.setFileName(self.toName, str(i).zfill(self.fill))
                item.renamePrint()
                # except FileExistsError:
                #     print("Catching exception here")
                i += 1
                #self.printList()
            self.restart()
        else:
            print('No files have been selected be to renamed. Use sort before rename.')

    def help(self, cmd):
        """Shows help depending on what the user enters."""
        RenamerHelp.help(cmd)

    def dir(self):
        """An attempted to replicate the command line dir function for navigation."""
        osnav.dir()

    def cd(self, cmd):
        """An attempted to replicate the command line cd function for navigation."""
        osnav.cd(self.string)

    def processInput(self, cmd):
        """Takes a list of the command input split by spaces and calls functions according to the input."""
        if cmd[0]   == "swt" or cmd[0] == "swp" or cmd[0] == "swap":
            self.swap(cmd)
        elif cmd[0] == "sel" or cmd[0] == "select":
            self.select(cmd)
        elif cmd[0] == "ins" or cmd[0] == "insert":
            self.insert(cmd)
        elif cmd[0] == "lst" or cmd[0] == "list":
            self.printList()
        elif cmd[0] == "rst" or cmd[0] == "reset" or cmd[0] == "restart":
            self.restart()
        elif cmd[0] == "set":
            self.set(cmd)
        elif cmd[0] == "dir" or cmd[0] == "ls":
            self.dir()
        elif cmd[0] == "cd":
            self.cd(cmd)
        elif cmd[0] == "srt" or cmd[0] == "sort":
            self.sort()
        elif cmd[0] == "rm" or cmd[0] == "remove":
            self.remove(cmd)
        elif cmd[0] == "shw" or cmd[0] == "show":
            self.show(cmd)
        elif cmd[0] == "help":
            self.help(cmd)
        elif cmd[0] == "rename":
            self.rename()

def main():
    renamer = Renamer()
    print("""Renamer

Copyright (c) 2014 Julian Pritchard

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.""")
    print()
    print("Type help functions for a list of commands")
    while True:
            var = input("Enter a Command: ")
            if var == "exit":
                    break
            renamer.inString( var)
            var = var.split(" ")
            renamer.processInput( var)

if __name__ == '__main__':
    main()
<commit_msg>Added error checking to the swap function.<commit_after>#!/usr/bin/env python3
"""Program for the bulk renaming of files in an ordered manner."""

import os
from FileName import *
import RenamerHelp
import splitstring
import osnav

class RenamerFile(FileName):

    def __init__(self, string, renamer):
        super().__init__(string)
        self.tempName = ""
        self._oldName  = string
        self.renamer = renamer

    def rename(self):
        """Renames the file from the current name (string) to the new one (file)."""
        self.newName = self.file + self.extention
        os.rename(self.string, self.newName)
        #print(self.string + "-->" + self.newName)

    def renamePrint(self):
        """Functions like rename but prints the change from original filename to new."""
        self.newName = self.file + self.extention
        try:
            os.rename(self.string, self.newName)
            print(self._oldName + " --> " + self.newName)
        except FileExistsError:
            raise

    def setFileName(self, name, num):
        self.file = splitstring.stitch(name, self.file, num)

class Renamer(object):
    """Takes a directory and renames the files in that directory in an ordered manner."""

    def __init__(self):
        self.restart()

    def restart(self):
        self.mainList     = []
        self.subList      = []
        self.fill         = 0
        self.startNum     = 1
        self.toName       = ""
        self.string       = ""
        self.term1, term2 = 0, 0

    def printList(self):
        """Prints the names of the current files in the directoy."""
        i = 1
        for item in self.mainList:
            print(str(i).zfill(self.fill) + ": " + item.getName())
            i += 1
        print()

    def printSubList(self):
        """Prints the list of items selected by the select function."""
        for item in self.subList:
            print(item.getName())
        print()

    def inString(self, string):
        """Stores the raw input from the command line as a string."""
        self.string = string

    def swap(self, cmd):
        """Swaps two items in the list."""
        if len(self.mainList) > 0:
            term1, term2 = eval(cmd[1])-1, eval(cmd[2])-1
            if self._indexCheck(term1) && self._indexCheck(term2):
                self.mainList[term1], self.mainList[term2] = self.mainList[term2], self.mainList[term1]
                self.printList()
            else:
                print("Must swap items in the list.\n")
        else:
            print("Must have a sorted list before using swap.\n")

    def _indexCheck(self, container, index):
        if index <= 0 or index > len(container):
            return False
        else:
            return True

    def select(self, cmd):
        """Creates a sublist between the two selected elements (1 indexed all inclusive)."""
        if len(self.mainList) > 0:
            self.subList = []
            # Selects a range of items.
            if len(cmd) == 3:
                self.term1, self.term2 = eval(cmd[1]), eval(cmd[2])
            # Selects a single items.
            elif len(cmd) == 2:
                self.term1, self.term2 = eval(cmd[1]), eval(cmd[1])
            # Tell the user they did something wrong, and end the function.
            else:
                print("Invalid number of terms for select.\n")
                return None
            # Make sure term1 is always the lower term.
            if self.term1 > self.term2:
                self.term1, self.term2 = self.term2, self.term1
            # Check that the numbers are valid list items.
            if self._indexCheck(self.mainList, self.term1) and self._indexCheck(self.mainList, self.term2):
                # Select the number(s) and put them into a container.
                self.term1 -= 1
                self.subList = self.mainList[self.term1 : self.term2]
                # Print the selection.
                self.printSubList()
            else:
                print("Must select items in list number\n")
        else:
            print("Must have a sorted list before using select\n")

    def insert(self, cmd):
        """Inserts the sublist into the selected place
the sublist cannot be inserted into itself

If the insert position is an element element of the list than the first item of the sublist
the first item of the sublist shall be at the insert position

If the insert position is an element greater than the last item of the sublist
the last item of the sublist will be at the insert position."""
        if len(self.mainList) > 0:
            if len(cmd) == 2:
                if len(self.subList) > 0:
                    iIn = eval(cmd[1])
                    if iIn > self.term1 and iIn <= self.term2:
                        print("Cannot insert the selected list into itself")
                    else:
                        # Inserting to a number lower than selected.
                        if iIn <= self.term1:
                            self.mainList[self.term1 : self.term2] = []
                            for j in range(len(self.subList)):
                                self.mainList.insert(iIn-1+j, self.subList[j])
                        # Inserting to a humber higher than selected.
                        elif iIn > self.term2:
                            self.mainList[self.term1 : self.term2] = []
                            for j in range(len(self.subList)):
                                self.mainList.insert(iIn-len(self.subList)+j, self.subList[j])
                        # Common to both case hence the else then an if and not an elif
                        self.printList()
                        self.subList = []
                # if len(self.subList) > 0:
                else:
                    print("Nothing to insert\n")
            # if len(cmd) == 2:
            elif len(cmd) == 3: #Insersts first term into second one so user does not need to use sel
                if eval(cmd[1]) != eval(cmd[2]):
                    self.select(["sel", cmd[1], cmd[1]])
                    self.insert(["ins", cmd[2]])
                else:
                    print("Cannot insert an item into itself\n")
        # if len(self.mainList) > 0:
        else:
            print("Must have a sorted list before using select\n")

    def set(self, cmd):
        """Input commands that determine how renamer will function."""
        if cmd[1] == "name":
           self.setName(cmd)
        if cmd[1] == "start":
            self.setStart(cmd)
        if cmd[1] == "zfill":
            self.setZFill(cmd)
        if cmd[1] == "mode":
            self.setMode(cmd)

    def setName(self, cmd):
        """Allows the user to set the name that the files will be renamed to."""
        if len(self.mainList) > 0:
            self.toName = self.string[self._lenCmdStr(cmd,2):]
            print()
            print("New name set to: " + self.toName)
            print("Example: " + splitstring.stitch(self.toName, self.mainList[0].file, self._zFillNum()))
            print()
        else:
            print("Must have a sorted list before setting name\n")

    def setStart(self, cmd):
        """Sets the starting number from which the files will be renamed."""
        if len(self.mainList) > 0:
            self.startNum = eval(cmd[2])
            self._checkZFill()
            print()
            print("New starting number is " + str(self.startNum))
            print("zfill is " + str(self.fill))
            print("Example: " + splitstring.stitch(self.toName, self.mainList[0].file, self._zFillNum()))
        else:
            print("Must have a sorted list before setting the start num\n")

    def setZFill(self, cmd):
        """Sets a custom zfill amount greater than the automatic zfill."""
        if len(self.mainList) > 0:
            self._checkZFill()
            if eval(cmd[2]) >= self.fill:
                self.fill = eval(cmd[2])
                print()
                print("New zfill set to: " + str(self.fill))
                print("Example: " + splitstring.stitch(self.toName, self.mainList[0].file, self._zFillNum()))
                print()
            else:
                print("Cannot set zfill to be less than the automatic zfill\n")

    def _checkZFill(self):
        """Check the zfill so that it is correct when a new start number has been added."""
        fill    = self.fill
        newFill = fill

        if len(str(self.startNum)) > fill:
            newFill = len(str(self.startNum))

        if (len(self.mainList) + self.startNum) > 10**(newFill):
            newFill += 1

        if newFill > self.fill:
            self.fill = newFill

    def _lenCmdStr(self, cmd, ind):
        """Returns the length sum of the length of substring elements to the index"""
        # This is used in conjunction with the input string to remove the commands from the input

        result = 0
        if len(cmd) >= ind:
            for i in range(ind):
                result += len(cmd[i]) + 1
        return result

    def _zFillNum(self):
        """Shorthand function to keep code clean"""
        return str(self.startNum).zfill(self.fill)

    def setMode(self, cmd):
        #Legacy from when prefix was a command
        pass

    def show(self, cmd):
        """Shows aspects of the program depend on the command entered after show."""
        if len(self.mainList) > 0:
            if len(cmd) > 1:
                if cmd[1] == "ext":
                    i = 1
                    for item in self.mainList:
                        print(str(i).zfill(self.fill) + ": " + item.getFileExt())
                        i += 1
                elif cmd[1] == "file":
                    i = 1
                    for item in self.mainList:
                        print(str(i).zfill(self.fill) + ": " + item.getFileName())
                        i +=1
                elif cmd[1] == "list" or cmd[1] == "lst":
                    self.printList()
                elif cmd[1] == "sublist":
                    self.printSubList()
            else:
                print("please select a subcommand for show,\nSubcommands are ext or file.")
        else:
            print("Must have a sorted list befor using show.")

    def sort(self):
        """Used to set the current directory to that in which the files shall be renamed."""
        self.mainList = []
        dirList = os.listdir(os.getcwd())
        for i in dirList:
            if os.path.isfile(i):
                fileName = RenamerFile(i, self)
                self.mainList.append(fileName)

        self.fill = len(str(len(self.mainList)))
        i = 1
        for item in self.mainList:
            print(str(i).zfill(self.fill) + ": " + item.getName())
            i += 1

    def remove(self, cmd):
        """If a single number is passed then that number is removed from the list
        If two numbers are passed than the range of those numbers is removed from the list"""
        if len(cmd) == 2:
            self.mainList.pop(eval(cmd[1])-1)
            self.printList()
        elif len(cmd) == 3:
            if eval(cmd[1]) < eval(cmd[2]):
                list1 = self.mainList[0:eval(cmd[1])-1]
                list2 = self.mainList[eval(cmd[2]):len(self.mainList)]
                self.mainList = list1 + list2
                self.printList()
            else:
                print("Number range for removal is incorrect err#5")

    def rename(self):
        """Renames the files in the list to their new name."""
        #Need to update to deal with naming conflicts.
        if len(self.mainList) > 0:
            i = self.startNum
            for item in self.mainList:
                item.setFileName(self.toName, str(i).zfill(self.fill))
                item.renamePrint()
                # except FileExistsError:
                #     print("Catching exception here")
                i += 1
                #self.printList()
            self.restart()
        else:
            print('No files have been selected be to renamed. Use sort before rename.')

    def help(self, cmd):
        """Shows help depending on what the user enters."""
        RenamerHelp.help(cmd)

    def dir(self):
        """An attempted to replicate the command line dir function for navigation."""
        osnav.dir()

    def cd(self, cmd):
        """An attempted to replicate the command line cd function for navigation."""
        osnav.cd(self.string)

    def processInput(self, cmd):
        """Takes a list of the command input split by spaces and calls functions according to the input."""
        if cmd[0]   == "swt" or cmd[0] == "swp" or cmd[0] == "swap":
            self.swap(cmd)
        elif cmd[0] == "sel" or cmd[0] == "select":
            self.select(cmd)
        elif cmd[0] == "ins" or cmd[0] == "insert":
            self.insert(cmd)
        elif cmd[0] == "lst" or cmd[0] == "list":
            self.printList()
        elif cmd[0] == "rst" or cmd[0] == "reset" or cmd[0] == "restart":
            self.restart()
        elif cmd[0] == "set":
            self.set(cmd)
        elif cmd[0] == "dir" or cmd[0] == "ls":
            self.dir()
        elif cmd[0] == "cd":
            self.cd(cmd)
        elif cmd[0] == "srt" or cmd[0] == "sort":
            self.sort()
        elif cmd[0] == "rm" or cmd[0] == "remove":
            self.remove(cmd)
        elif cmd[0] == "shw" or cmd[0] == "show":
            self.show(cmd)
        elif cmd[0] == "help":
            self.help(cmd)
        elif cmd[0] == "rename":
            self.rename()

def main():
    renamer = Renamer()
    print("""Renamer

Copyright (c) 2014 Julian Pritchard

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.""")
    print()
    print("Type help functions for a list of commands")
    while True:
            var = input("Enter a Command: ")
            if var == "exit":
                    break
            renamer.inString( var)
            var = var.split(" ")
            renamer.processInput( var)

if __name__ == '__main__':
    main()
<|endoftext|>