<commit_before># -*- coding: utf-8 -*-

"""
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License,
    or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    See the GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>.

    @author: RaNaN
"""

import sys
import re
from os import stat
from os.path import join, exists
from time import time

from module.ConfigParser import IGNORE
from module.network.RequestFactory import getURL
from module.plugins.Hook import threaded, Expose, Hook

class UpdateManager(Hook):
    __name__ = "UpdateManager"
    __version__ = "0.12"
    __description__ = """checks for updates"""
    __config__ = [("activated", "bool", "Activated", "True"),
        ("interval", "int", "Check interval in minutes", "480"),
        ("debug", "bool", "Check for plugin changes when in debug mode", False)]
    __author_name__ = ("RaNaN")
    __author_mail__ = ("ranan@pyload.org")

    URL = "http://get.pyload.org/check2/%s/"
    MIN_TIME = 3 * 60 * 60 # 3h minimum check interval

    @property
    def debug(self):
        return self.core.debug and self.getConfig("debug")

    def setup(self):
        if self.debug:
            self.logDebug("Monitoring file changes")
            self.interval = 4
            self.last_check = 0 #timestamp of updatecheck
            self.old_periodical = self.periodical
            self.periodical = self.checkChanges
            self.mtimes = {}  #recordes times
        else:
            self.interval = max(self.getConfig("interval") * 60, self.MIN_TIME)

        self.updated = False
        self.reloaded = True
        self.version = "None"

        self.info = {"pyload": False, "plugins": False}

    @threaded
    def periodical(self):

        updates = self.checkForUpdate()
        if updates:
            self.checkPlugins(updates)

        if self.updated and not self.reloaded:
            self.info["plugins"] = True
            self.log.info(_("*** Plugins have been updated, please restart pyLoad ***"))
        elif self.updated and self.reloaded:
            self.log.info(_("Plugins updated and reloaded"))
            self.updated = False
        elif self.version == "None":
            self.log.info(_("No plugin updates available"))

    @Expose
    def recheckForUpdates(self):
        """recheck if updates are available"""
        self.periodical()

    def checkForUpdate(self):
        """checks if an update is available, return result"""

        try:
            if self.version == "None": # No updated known
                version_check = getURL(self.URL % self.core.api.getServerVersion()).splitlines()
                self.version = version_check[0]

                # Still no updates, plugins will be checked
                if self.version == "None":
                    self.log.info(_("No Updates for pyLoad"))
                    return version_check[1:]

            self.info["pyload"] = True
            self.log.info(_("***  New pyLoad Version %s available  ***") % self.version)
            self.log.info(_("***  Get it here: http://pyload.org/download  ***"))

        except:
            self.log.warning(_("Not able to connect server for updates"))

        return None # Nothing will be done

    def checkPlugins(self, updates):
        """ checks for plugins updates"""

        # plugins were already updated
        if self.info["plugins"]: return

        reloads = []

        vre = re.compile(r'__version__.*=.*("|\')([0-9.]+)')
        url = updates[0]
        schema = updates[1].split("|")
        updates = updates[2:]

        for plugin in updates:
            info = dict(zip(schema, plugin.split("|")))
            filename = info["name"]
            prefix = info["type"]
            version = info["version"]

            if filename.endswith(".pyc"):
                name = filename[:filename.find("_")]
            else:
                name = filename.replace(".py", "")

            #TODO: obsolete in 0.5.0
            if prefix.endswith("s"):
                type = prefix[:-1]
            else:
                type = prefix

            plugins = getattr(self.core.pluginManager, "%sPlugins" % type)

            if name in plugins:
                if float(plugins[name]["v"]) >= float(version):
                    continue

            if name in IGNORE or (type, name) in IGNORE:
                continue

            self.log.info(_("New version of %(type)s|%(name)s : %(version).2f") % {
                "type": type,
                "name": name,
                "version": float(version)
            })

            try:
                content = getURL(url % info)
            except Exception, e:
                self.logWarning(_("Error when updating %s") % filename, str(e))
                continue

            m = vre.search(content)
            if not m or m.group(2) != version:
                self.logWarning(_("Error when updating %s") % name, _("Version mismatch"))
                continue

            f = open(join("userplugins", prefix, filename), "wb")
            f.write(content)
            f.close()
            self.updated = True

            reloads.append((prefix, name))

        self.reloaded = self.core.pluginManager.reloadPlugins(reloads)

    def checkChanges(self):

        if self.last_check + max(self.getConfig("interval") * 60, self.MIN_TIME) < time():
            self.old_periodical()
            self.last_check = time()

        modules = filter(
            lambda m: m and (m.__name__.startswith("module.plugins.") or m.__name__.startswith("userplugins.")) and m.__name__.count(".") >= 2,
            sys.modules.itervalues())

        reloads = []

        for m in modules:
            root, type, name = m.__name__.rsplit(".", 2)
            id = (type, name)
            if type in self.core.pluginManager.plugins:
                f = m.__file__.replace(".pyc", ".py")
                if not exists(f): continue

                mtime = stat(f).st_mtime

                if id not in self.mtimes:
                    self.mtimes[id] = mtime
                elif self.mtimes[id] < mtime:
                    reloads.append(id)
                    self.mtimes[id] = mtime

        self.core.pluginManager.reloadPlugins(reloads)
<commit_msg>forget to bump version<commit_after># -*- coding: utf-8 -*-

"""
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License,
    or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    See the GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>.

    @author: RaNaN
"""

import sys
import re
from os import stat
from os.path import join, exists
from time import time

from module.ConfigParser import IGNORE
from module.network.RequestFactory import getURL
from module.plugins.Hook import threaded, Expose, Hook

class UpdateManager(Hook):
    __name__ = "UpdateManager"
    __version__ = "0.13"
    __description__ = """checks for updates"""
    __config__ = [("activated", "bool", "Activated", "True"),
        ("interval", "int", "Check interval in minutes", "480"),
        ("debug", "bool", "Check for plugin changes when in debug mode", False)]
    __author_name__ = ("RaNaN")
    __author_mail__ = ("ranan@pyload.org")

    URL = "http://get.pyload.org/check2/%s/"
    MIN_TIME = 3 * 60 * 60 # 3h minimum check interval

    @property
    def debug(self):
        return self.core.debug and self.getConfig("debug")

    def setup(self):
        if self.debug:
            self.logDebug("Monitoring file changes")
            self.interval = 4
            self.last_check = 0 #timestamp of updatecheck
            self.old_periodical = self.periodical
            self.periodical = self.checkChanges
            self.mtimes = {}  #recordes times
        else:
            self.interval = max(self.getConfig("interval") * 60, self.MIN_TIME)

        self.updated = False
        self.reloaded = True
        self.version = "None"

        self.info = {"pyload": False, "plugins": False}

    @threaded
    def periodical(self):

        updates = self.checkForUpdate()
        if updates:
            self.checkPlugins(updates)

        if self.updated and not self.reloaded:
            self.info["plugins"] = True
            self.log.info(_("*** Plugins have been updated, please restart pyLoad ***"))
        elif self.updated and self.reloaded:
            self.log.info(_("Plugins updated and reloaded"))
            self.updated = False
        elif self.version == "None":
            self.log.info(_("No plugin updates available"))

    @Expose
    def recheckForUpdates(self):
        """recheck if updates are available"""
        self.periodical()

    def checkForUpdate(self):
        """checks if an update is available, return result"""

        try:
            if self.version == "None": # No updated known
                version_check = getURL(self.URL % self.core.api.getServerVersion()).splitlines()
                self.version = version_check[0]

                # Still no updates, plugins will be checked
                if self.version == "None":
                    self.log.info(_("No Updates for pyLoad"))
                    return version_check[1:]

            self.info["pyload"] = True
            self.log.info(_("***  New pyLoad Version %s available  ***") % self.version)
            self.log.info(_("***  Get it here: http://pyload.org/download  ***"))

        except:
            self.log.warning(_("Not able to connect server for updates"))

        return None # Nothing will be done

    def checkPlugins(self, updates):
        """ checks for plugins updates"""

        # plugins were already updated
        if self.info["plugins"]: return

        reloads = []

        vre = re.compile(r'__version__.*=.*("|\')([0-9.]+)')
        url = updates[0]
        schema = updates[1].split("|")
        updates = updates[2:]

        for plugin in updates:
            info = dict(zip(schema, plugin.split("|")))
            filename = info["name"]
            prefix = info["type"]
            version = info["version"]

            if filename.endswith(".pyc"):
                name = filename[:filename.find("_")]
            else:
                name = filename.replace(".py", "")

            #TODO: obsolete in 0.5.0
            if prefix.endswith("s"):
                type = prefix[:-1]
            else:
                type = prefix

            plugins = getattr(self.core.pluginManager, "%sPlugins" % type)

            if name in plugins:
                if float(plugins[name]["v"]) >= float(version):
                    continue

            if name in IGNORE or (type, name) in IGNORE:
                continue

            self.log.info(_("New version of %(type)s|%(name)s : %(version).2f") % {
                "type": type,
                "name": name,
                "version": float(version)
            })

            try:
                content = getURL(url % info)
            except Exception, e:
                self.logWarning(_("Error when updating %s") % filename, str(e))
                continue

            m = vre.search(content)
            if not m or m.group(2) != version:
                self.logWarning(_("Error when updating %s") % name, _("Version mismatch"))
                continue

            f = open(join("userplugins", prefix, filename), "wb")
            f.write(content)
            f.close()
            self.updated = True

            reloads.append((prefix, name))

        self.reloaded = self.core.pluginManager.reloadPlugins(reloads)

    def checkChanges(self):

        if self.last_check + max(self.getConfig("interval") * 60, self.MIN_TIME) < time():
            self.old_periodical()
            self.last_check = time()

        modules = filter(
            lambda m: m and (m.__name__.startswith("module.plugins.") or m.__name__.startswith("userplugins.")) and m.__name__.count(".") >= 2,
            sys.modules.itervalues())

        reloads = []

        for m in modules:
            root, type, name = m.__name__.rsplit(".", 2)
            id = (type, name)
            if type in self.core.pluginManager.plugins:
                f = m.__file__.replace(".pyc", ".py")
                if not exists(f): continue

                mtime = stat(f).st_mtime

                if id not in self.mtimes:
                    self.mtimes[id] = mtime
                elif self.mtimes[id] < mtime:
                    reloads.append(id)
                    self.mtimes[id] = mtime

        self.core.pluginManager.reloadPlugins(reloads)
<|endoftext|>


<commit_before>import pysnmp.entity.rfc3413.oneliner.cmdgen as cmdgen
from pysnmp.smi import builder, view
from pysnmp.smi.error import SmiError

class Snmpy(object):
    '''
    Simple pysnmp wrapper.
    '''
    ERROR_MSG = 'SNMP%s of %s on %s failed.'
    PREVIOUS_EXPORT = 'already exported'
    VERSION_2_2C = 1

    def __init__(self, host, public, private, port=161, timeout=1, retries=2):
        '''
        Create the snmpy object.
        :param host: hostname or IP address of the device to communicate with
        :type host: str
        :param public: public community string
        :type public: str
        :param private: private community string
        :type private: str
        :param port: port number to connect on (default 161)
        :type port: int
        :param timeout: timeout, in seconds (default 1s)
        :type timeout: int
        :param retries: number of retries (default 2)
        :type retries: int
        :returns:
        '''
        self._host = host
        self._port = port
        self._timeout = timeout
        self._retries = retries
        self._transport = cmdgen.UdpTransportTarget((self._host, self._port),
                                                    timeout=self._timeout,
                                                    retries=self._retries)
        self._public = cmdgen.CommunityData(public, self.VERSION_2_2C)
        self._private = cmdgen.CommunityData(private, self.VERSION_2_2C)

        self._mibBuilder = builder.MibBuilder()
        self._mibViewController = view.MibViewController(self._mibBuilder)

        # Preload some commonly used modules.
        self.load_mibs('SNMPv2-MIB', 'IF-MIB', 'IP-MIB',
                       'HOST-RESOURCES-MIB', 'FIBRE-CHANNEL-FE-MIB')

    def add_mib_path(self, *path):
        '''
        Add an additional directory to the MIB search path.
        :param path: path to additional MIBs
        '''
        mibPath = self._mibBuilder.getMibPath() + path
        self._mibBuilder.setMibPath(*mibPath)

    def load_mibs(self, *modules):
        '''
        Load one or more additional MIBs.
        :param modules: modules to load
        '''
        for module in modules:
            try:
                self._mibBuilder.loadModules(module)
            except SmiError, e:
                if self.PREVIOUS_EXPORT in str(e):
                    continue
                raise

    def get(self, oid):
        '''
        Get a specific value from an OID in the SNMP tree.
        :param oid: OID to get
        :returns: value from the specified OID
        '''
        noid = self.__node_id(oid)
        (errorIndication, errorStatus, errorIndex, varBinds) = \
            cmdgen.CommandGenerator().getCmd(
                self._public,
                self._transport,
                noid)
        if errorIndication:
            raise RuntimeError(self.ERROR_MSG % ('get', oid, self._host))

        return varBinds[0][1]

    def set(self, oid, value):
        '''
        Set a specific value to an OID in the SNMP tree.
        :param oid: OID to set
        :param value: value to set
        '''
        initial_value = self.get(oid)
        set_value = type(initial_value)(str(value))
        noid = self.__node_id(oid)
        (errorIndication, errorStatus, errorIndex, varBinds) = \
            cmdgen.CommandGenerator().setCmd(
                self._private,
                self._public,
                (noid, set_value)
            )
        if errorIndication:
            raise RuntimeError(self.ERROR_MSG % ('set', oid, self._host))

        return varBinds[0][1]

    def __node_id(self, oid):
        '''
        Translate a named OID to the dotted-decimal format.
        :param oid: OID to translate
        '''
        ids = oid.split('.')
        symbols = ids[0].split('::')
        ids = tuple([int(x) for x in ids[1:]])
        mibnode, = self._mibBuilder.importSymbols(*symbols)
        oid = mibnode.getName() + ids

        return oid
<commit_msg>Setting `mpModel` explicitly. Without it, authentication does not work correctly using `pysnmp==4.3.0`.<commit_after>import pysnmp.entity.rfc3413.oneliner.cmdgen as cmdgen
from pysnmp.smi import builder, view
from pysnmp.smi.error import SmiError

class Snmpy(object):
    '''
    Simple pysnmp wrapper.
    '''
    ERROR_MSG = 'SNMP%s of %s on %s failed.'
    PREVIOUS_EXPORT = 'already exported'
    VERSION_2_2C = 1

    def __init__(self, host, public, private, port=161, timeout=1, retries=2):
        '''
        Create the snmpy object.
        :param host: hostname or IP address of the device to communicate with
        :type host: str
        :param public: public community string
        :type public: str
        :param private: private community string
        :type private: str
        :param port: port number to connect on (default 161)
        :type port: int
        :param timeout: timeout, in seconds (default 1s)
        :type timeout: int
        :param retries: number of retries (default 2)
        :type retries: int
        :returns:
        '''
        self._host = host
        self._port = port
        self._timeout = timeout
        self._retries = retries
        self._transport = cmdgen.UdpTransportTarget((self._host, self._port),
                                                    timeout=self._timeout,
                                                    retries=self._retries)
        self._public = cmdgen.CommunityData(public, mpModel=self.VERSION_2_2C)
        self._private = cmdgen.CommunityData(private, mpModel=self.VERSION_2_2C)

        self._mibBuilder = builder.MibBuilder()
        self._mibViewController = view.MibViewController(self._mibBuilder)

        # Preload some commonly used modules.
        self.load_mibs('SNMPv2-MIB', 'IF-MIB', 'IP-MIB',
                       'HOST-RESOURCES-MIB', 'FIBRE-CHANNEL-FE-MIB')

    def add_mib_path(self, *path):
        '''
        Add an additional directory to the MIB search path.
        :param path: path to additional MIBs
        '''
        mibPath = self._mibBuilder.getMibPath() + path
        self._mibBuilder.setMibPath(*mibPath)

    def load_mibs(self, *modules):
        '''
        Load one or more additional MIBs.
        :param modules: modules to load
        '''
        for module in modules:
            try:
                self._mibBuilder.loadModules(module)
            except SmiError, e:
                if self.PREVIOUS_EXPORT in str(e):
                    continue
                raise

    def get(self, oid):
        '''
        Get a specific value from an OID in the SNMP tree.
        :param oid: OID to get
        :returns: value from the specified OID
        '''
        noid = self.__node_id(oid)
        (errorIndication, errorStatus, errorIndex, varBinds) = \
            cmdgen.CommandGenerator().getCmd(
                self._public,
                self._transport,
                noid)
        if errorIndication:
            raise RuntimeError(self.ERROR_MSG % ('get', oid, self._host))

        return varBinds[0][1]

    def set(self, oid, value):
        '''
        Set a specific value to an OID in the SNMP tree.
        :param oid: OID to set
        :param value: value to set
        '''
        initial_value = self.get(oid)
        set_value = type(initial_value)(str(value))
        noid = self.__node_id(oid)
        (errorIndication, errorStatus, errorIndex, varBinds) = \
            cmdgen.CommandGenerator().setCmd(
                self._private,
                self._public,
                (noid, set_value)
            )
        if errorIndication:
            raise RuntimeError(self.ERROR_MSG % ('set', oid, self._host))

        return varBinds[0][1]

    def __node_id(self, oid):
        '''
        Translate a named OID to the dotted-decimal format.
        :param oid: OID to translate
        '''
        ids = oid.split('.')
        symbols = ids[0].split('::')
        ids = tuple([int(x) for x in ids[1:]])
        mibnode, = self._mibBuilder.importSymbols(*symbols)
        oid = mibnode.getName() + ids

        return oid
<|endoftext|>


<commit_before>import binascii
from hextobase64 import convertHex

# predefined strings for testing the function
str1 = b"1c0111001f010100061a024b53535009181c"
str2 = b"686974207468652062756c6c277320657965"

# function to XOR two strings
def xorStr(user1, user2):
    if(len(user1)==len(user2)):
        # Make an iterator that aggregates elements from each of the raw strings.
        zipp = zip(convertHex(user1),convertHex(user2))
        # XOR of elements
        theList = ((chr((a^b)) for a,b in zipp))
        # making of the string
        theString = ("".join(theList))
        # turning string to bytes
        finalBytes = theString.encode()
        # turning bytes to hex
        finalHex = binascii.b2a_hex(finalBytes)
        #skadooooooosh
        print (finalHex)

    else:
        print("Enter two strings with same length")

if __name__ == "__main__":
    xorStr(str1, str2)
<commit_msg>Update 2.py<commit_after>import binascii
from hextobase64 import convertHex

str1 = b"1c0111001f010100061a024b53535009181c"
str2 = b"686974207468652062756c6c277320657965"
zipp = zip(convertHex(user1),convertHex(user2))
theList = ((chr((a^b)) for a,b in zipp))
theString = ("".join(theList))
finalBytes = theString.encode()
finalHex = binascii.b2a_hex(finalBytes)
print (finalHex)
<|endoftext|>


<commit_before>from apps.rss_feeds.models import FeedUpdateHistory
# from apps.rss_feeds.models import FeedXML
from django.core.cache import cache
from django.conf import settings
from apps.reader.models import UserSubscription
from apps.rss_feeds.importer import PageImporter
from utils import feedparser
from django.db import IntegrityError
from utils.story_functions import pre_process_story
from utils import log as logging
import sys
import time
import datetime
import traceback
import multiprocessing
import urllib2
import xml.sax
import socket
import mongoengine

# Refresh feed code adapted from Feedjack.
# http://feedjack.googlecode.com

VERSION = '0.9'
URL = 'http://www.newsblur.com/'
USER_AGENT = 'NewsBlur Fetcher %s - %s' % (VERSION, URL)
SLOWFEED_WARNING = 10
ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)
FEED_OK, FEED_SAME, FEED_ERRPARSE, FEED_ERRHTTP, FEED_ERREXC = range(5)

def mtime(ttime):
    """ datetime auxiliar function.
    """
    return datetime.datetime.fromtimestamp(time.mktime(ttime))

import threading
class TimeoutError(Exception): pass
def timelimit(timeout):
    """borrowed from web.py"""
    def _1(function):
        def _2(*args, **kw):
            class Dispatch(threading.Thread):
                def __init__(self):
                    threading.Thread.__init__(self)
                    self.result = None
                    self.error = None

                    self.setDaemon(True)
                    self.start()

                def run(self):
                    try:
                        self.result = function(*args, **kw)
                    except:
                        self.error = sys.exc_info()

            c = Dispatch()
            c.join(timeout)
            if c.isAlive():
                raise TimeoutError, 'took too long'
            if c.error:
                raise c.error[0], c.error[1]
            return c.result
        return _2
    return _1

class FetchFeed:
    def __init__(self, feed, options):
        self.feed = feed
        self.options = options
        self.fpf = None

    @timelimit(20)
    def fetch(self):
        """ Downloads and parses a feed.
        """
        socket.setdefaulttimeout(30)
        identity = self.get_identity()
        log_msg = u'%2s ---> [%-30s] Fetching feed (%d)' % (identity,
                                                 unicode(self.feed)[:30],
                                                 self.feed.id)
        logging.debug(log_msg)

        # Check if feed still needs to be updated
        # feed = Feed.objects.get(pk=self.feed.pk)
        # if feed.next_scheduled_update > datetime.datetime.now() and not self.options.get('force'):
        #     log_msg = u'        ---> Already fetched %s (%d)' % (self.feed.feed_title,
        #                                                          self.feed.id)
        #     logging.debug(log_msg)
        #     feed.save_feed_history(303, "Already fetched")
        #     return FEED_SAME, None
        # else:
        self.feed.set_next_scheduled_update()

        etag=self.feed.etag
        modified = self.feed.last_modified.utctimetuple()[:7] if self.feed.last_modified else None

        if self.options.get('force'):
            modified = None
            etag = None

        self.fpf = feedparser.parse(self.feed.feed_address,
                                    agent=USER_AGENT,
                                    etag=etag,
                                    modified=modified)

        return FEED_OK, self.fpf

    def get_identity(self):
        identity = "X"

        current_process = multiprocessing.current_process()
        if current_process._identity:
            identity = current_process._identity[0]

        return identity

class ProcessFeed:
    def __init__(self, feed, fpf, db, options):
        self.feed = feed
        self.options = options
        self.fpf = fpf
        self.lock = multiprocessing.Lock()
        self.db = db
        self.entry_trans = {
            ENTRY_NEW:'new',
            ENTRY_UPDATED:'updated',
            ENTRY_SAME:'same',
            ENTRY_ERR:'error'}
        self.entry_keys = sorted(self.entry_trans.keys())

    def process(self):
        """ Downloads and parses a feed.
        """

        ret_values = {
            ENTRY_NEW:0,
            ENTRY_UPDATED:0,
            ENTRY_SAME:0,
            ENTRY_ERR:0}

        # logging.debug(u' ---> [%d] Processing %s' % (self.feed.id, self.feed.feed_title))

        if hasattr(self.fpf, 'status'):
            if self.options['verbose']:
                logging.debug(u'   ---> [%-30s] Fetched feed, HTTP status %d: %s (bozo: %s)' % (unicode(self.feed)[:30],
                                                     self.fpf.status,
                                                     self.feed.feed_address,
                                                     self.fpf.bozo))
                if self.fpf.bozo and self.fpf.status != 304:
                    logging.debug(u'   ---> [%-30s] BOZO exception: %s' % (
                                  unicode(self.feed)[:30],
                                  self.fpf.bozo_exception,))
            if self.fpf.status == 304:
                self.feed.save()
                self.feed.save_feed_history(304, "Not modified")
                return FEED_SAME, ret_values

            if self.fpf.status >= 400:
                self.feed.save()
                self.feed.save_feed_history(self.fpf.status, "HTTP Error")
                return FEED_ERRHTTP, ret_values

        if self.fpf.bozo and isinstance(self.fpf.bozo_exception, feedparser.NonXMLContentType):
            if not self.fpf.entries:
                logging.debug("   ---> [%-30s] Feed is Non-XML. Checking address..." % unicode(self.feed)[:30])
                fixed_feed = self.feed.check_feed_address_for_feed_link()
                if not fixed_feed:
                    self.feed.save_feed_history(502, 'Non-xml feed', self.fpf.bozo_exception)
                return FEED_ERRPARSE, ret_values
        elif self.fpf.bozo and isinstance(self.fpf.bozo_exception, xml.sax._exceptions.SAXException):
            logging.debug("   ---> [%-30s] Feed is Bad XML (SAX). Checking address..." % unicode(self.feed)[:30])
            if not self.fpf.entries:
                fixed_feed = self.feed.check_feed_address_for_feed_link()
                if not fixed_feed:
                    self.feed.save_feed_history(503, 'SAX Exception', self.fpf.bozo_exception)
                return FEED_ERRPARSE, ret_values

        # the feed has changed (or it is the first time we parse it)
        # saving the etag and last_modified fields
        self.feed.etag = self.fpf.get('etag')[:255]
        # some times this is None (it never should) *sigh*
        if self.feed.etag is None:
            self.feed.etag = ''

        try:
            self.feed.last_modified = mtime(self.fpf.modified)
        except:
            pass

        self.feed.feed_title = self.fpf.feed.get('title', self.feed.feed_title)
        self.feed.feed_tagline = self.fpf.feed.get('tagline', self.feed.feed_tagline)
        self.feed.feed_link = self.fpf.feed.get('link', self.feed.feed_link)
        self.feed.last_update = datetime.datetime.now()

        guids = []
        for entry in self.fpf.entries:
            if entry.get('id', ''):
                guids.append(entry.get('id', ''))
            elif entry.title:
                guids.append(entry.title)
            elif entry.link:
                guids.append(entry.link)

        self.lock.acquire()
        try:
            self.feed.save()
        finally:
            self.lock.release()

        # Compare new stories to existing stories, adding and updating
        start_date = datetime.datetime.now()
        end_date = datetime.datetime.now()
        story_guids = []
        for entry in self.fpf.entries:
            story = pre_process_story(entry)
            if story.get('published') < start_date:
                start_date = story.get('published')
            if story.get('published') > end_date:
                end_date = story.get('published')
            story_guids.append(story.get('guid') or story.get('link'))
        existing_stories = self.db.stories.find({
            'story_feed_id': self.feed.pk,
            '$or': [
                {
                    'story_date': {'$gte': start_date},
                    'story_date': {'$lte': end_date}
                },
                {
                    'story_guid': {'$in': story_guids}
                }
            ]
        })
        # MStory.objects(
        #     (Q(story_date__gte=start_date) & Q(story_date__lte=end_date))
        #     | (Q(story_guid__in=story_guids)),
        #     story_feed=self.feed
        # ).order_by('-story_date')
        ret_values = self.feed.add_update_stories(self.fpf.entries, existing_stories, self.db)

        logging.debug(u'   ---> [%-30s] Parsed Feed: %s' % (
                      unicode(self.feed)[:30],
                      u' '.join(u'%s=%d' % (self.entry_trans[key],
                              ret_values[key]) for key in self.entry_keys),))
        self.feed.update_all_statistics(lock=self.lock)
        self.feed.save_feed_history(200, "OK")

        return FEED_OK, ret_values

class Dispatcher:
    def __init__(self, options, num_threads):
        self.options = options
        self.entry_stats = {
            ENTRY_NEW:0,
            ENTRY_UPDATED:0,
            ENTRY_SAME:0,
            ENTRY_ERR:0}
        self.feed_stats = {
            FEED_OK:0,
            FEED_SAME:0,
            FEED_ERRPARSE:0,
            FEED_ERRHTTP:0,
            FEED_ERREXC:0}
        self.feed_trans = {
            FEED_OK:'ok',
            FEED_SAME:'unchanged',
            FEED_ERRPARSE:'cant_parse',
            FEED_ERRHTTP:'http_error',
            FEED_ERREXC:'exception'}
        self.feed_keys = sorted(self.feed_trans.keys())
        self.num_threads = num_threads
        self.time_start = datetime.datetime.now()
        self.workers = []

    def process_feed_wrapper(self, feed_queue):
        """ wrapper for ProcessFeed
        """
        if not self.options['single_threaded']:
            # Close the DB so the connection can be re-opened on a per-process basis
            from django.db import connection
            connection.close()
        delta = None

        MONGO_DB = settings.MONGO_DB
        db = mongoengine.connection.connect(db=MONGO_DB['NAME'], host=MONGO_DB['HOST'], port=MONGO_DB['PORT'])

        current_process = multiprocessing.current_process()

        identity = "X"
        if current_process._identity:
            identity = current_process._identity[0]
        for feed in feed_queue:
            ret_entries = {
                ENTRY_NEW: 0,
                ENTRY_UPDATED: 0,
                ENTRY_SAME: 0,
                ENTRY_ERR: 0
            }
            start_time = datetime.datetime.now()

            ### Uncomment to test feed fetcher
            # from random import randint
            # if randint(0,10) < 10:
            #     continue

            try:
                ffeed = FetchFeed(feed, self.options)
                ret_feed, fetched_feed = ffeed.fetch()

                if ((fetched_feed and ret_feed == FEED_OK) or self.options['force']):
                    pfeed = ProcessFeed(feed, fetched_feed, db, self.options)
                    ret_feed, ret_entries = pfeed.process()

                    if ret_entries.get(ENTRY_NEW) or self.options['force']:
                        user_subs = UserSubscription.objects.filter(feed=feed)
                        logging.debug(u'   ---> [%-30s] Computing scores for all feed subscribers: %s subscribers' % (unicode(feed)[:30], user_subs.count()))
                        for sub in user_subs:
                            cache.delete('usersub:%s' % sub.user_id)
                            silent = False if self.options['verbose'] >= 2 else True
                            sub.calculate_feed_scores(silent=silent)
                    cache.delete('feed_stories:%s-%s-%s' % (feed.id, 0, 25))
                    # if ret_entries.get(ENTRY_NEW) or ret_entries.get(ENTRY_UPDATED) or self.options['force']:
                    #     feed.get_stories(force=True)
            except KeyboardInterrupt:
                break
            except urllib2.HTTPError, e:
                feed.save_feed_history(e.code, e.msg, e.fp.read())
                fetched_feed = None
            except Exception, e:
                logging.debug('[%d] ! -------------------------' % (feed.id,))
                tb = traceback.format_exc()
                logging.debug(tb)
                logging.debug('[%d] ! -------------------------' % (feed.id,))
                ret_feed = FEED_ERREXC
                feed.save_feed_history(500, "Error", tb)
                fetched_feed = None

            if ((self.options['force']) or
                (fetched_feed and
                 feed.feed_link and
                 (ret_feed == FEED_OK or
                  (ret_feed == FEED_SAME and feed.stories_last_month > 10)))):

                logging.debug(u'   ---> [%-30s] Fetching page' % (unicode(feed)[:30]))
                page_importer = PageImporter(feed.feed_link, feed)
                page_importer.fetch_page()

            delta = datetime.datetime.now() - start_time

            feed.last_load_time = max(1, delta.seconds)
            feed.fetched_once = True
            try:
                feed.save()
            except IntegrityError:
                logging.debug("   ---> [%-30s] IntegrityError on feed: %s" % (unicode(feed)[:30], feed.feed_address,))

            done_msg = (u'%2s ---> [%-30s] Processed in %s [%s]' % (
                identity, feed.feed_title[:30], unicode(delta),
                self.feed_trans[ret_feed],))
            logging.debug(done_msg)

            self.feed_stats[ret_feed] += 1
            for key, val in ret_entries.items():
                self.entry_stats[key] += val

        time_taken = datetime.datetime.now() - self.time_start
        history = FeedUpdateHistory(
            number_of_feeds=len(feed_queue),
            seconds_taken=time_taken.seconds
        )
        history.save()
        if not self.options['single_threaded']:
            logging.debug("---> DONE WITH PROCESS: %s" % current_process.name)
            sys.exit()

    def add_jobs(self, feeds_queue, feeds_count=1):
        """ adds a feed processing job to the pool
        """
        self.feeds_queue = feeds_queue
        self.feeds_count = feeds_count

    def run_jobs(self):
        if self.options['single_threaded']:
            self.process_feed_wrapper(self.feeds_queue[0])
        else:
            for i in range(self.num_threads):
                feed_queue = self.feeds_queue[i]
                self.workers.append(multiprocessing.Process(target=self.process_feed_wrapper,
                                                            args=(feed_queue,)))
                # worker.setName("Thread #%s" % (i+1))
                # worker.setDaemon(True)
            for i in range(self.num_threads):
                self.workers[i].start()

    def poll(self):
        """ polls the active threads
        """
        if not self.options['single_threaded']:
            for i in range(self.num_threads):
                self.workers[i].join()
            done = (u'* DONE in %s\n* Feeds: %s\n* Entries: %s' % (
                    unicode(datetime.datetime.now() - self.time_start),
                    u' '.join(u'%s=%d' % (self.feed_trans[key],
                              self.feed_stats[key])
                              for key in self.feed_keys),
                    u' '.join(u'%s=%d' % (self.entry_trans[key],
                              self.entry_stats[key])
                              for key in self.entry_keys)
                    ))
            logging.debug(done)
            return

                <commit_msg>Whoops, sometimes there is no etag.<commit_after>from apps.rss_feeds.models import FeedUpdateHistory
# from apps.rss_feeds.models import FeedXML
from django.core.cache import cache
from django.conf import settings
from apps.reader.models import UserSubscription
from apps.rss_feeds.importer import PageImporter
from utils import feedparser
from django.db import IntegrityError
from utils.story_functions import pre_process_story
from utils import log as logging
import sys
import time
import datetime
import traceback
import multiprocessing
import urllib2
import xml.sax
import socket
import mongoengine

# Refresh feed code adapted from Feedjack.
# http://feedjack.googlecode.com

VERSION = '0.9'
URL = 'http://www.newsblur.com/'
USER_AGENT = 'NewsBlur Fetcher %s - %s' % (VERSION, URL)
SLOWFEED_WARNING = 10
ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)
FEED_OK, FEED_SAME, FEED_ERRPARSE, FEED_ERRHTTP, FEED_ERREXC = range(5)

def mtime(ttime):
    """ datetime auxiliar function.
    """
    return datetime.datetime.fromtimestamp(time.mktime(ttime))

import threading
class TimeoutError(Exception): pass
def timelimit(timeout):
    """borrowed from web.py"""
    def _1(function):
        def _2(*args, **kw):
            class Dispatch(threading.Thread):
                def __init__(self):
                    threading.Thread.__init__(self)
                    self.result = None
                    self.error = None

                    self.setDaemon(True)
                    self.start()

                def run(self):
                    try:
                        self.result = function(*args, **kw)
                    except:
                        self.error = sys.exc_info()

            c = Dispatch()
            c.join(timeout)
            if c.isAlive():
                raise TimeoutError, 'took too long'
            if c.error:
                raise c.error[0], c.error[1]
            return c.result
        return _2
    return _1

class FetchFeed:
    def __init__(self, feed, options):
        self.feed = feed
        self.options = options
        self.fpf = None

    @timelimit(20)
    def fetch(self):
        """ Downloads and parses a feed.
        """
        socket.setdefaulttimeout(30)
        identity = self.get_identity()
        log_msg = u'%2s ---> [%-30s] Fetching feed (%d)' % (identity,
                                                 unicode(self.feed)[:30],
                                                 self.feed.id)
        logging.debug(log_msg)

        # Check if feed still needs to be updated
        # feed = Feed.objects.get(pk=self.feed.pk)
        # if feed.next_scheduled_update > datetime.datetime.now() and not self.options.get('force'):
        #     log_msg = u'        ---> Already fetched %s (%d)' % (self.feed.feed_title,
        #                                                          self.feed.id)
        #     logging.debug(log_msg)
        #     feed.save_feed_history(303, "Already fetched")
        #     return FEED_SAME, None
        # else:
        self.feed.set_next_scheduled_update()

        etag=self.feed.etag
        modified = self.feed.last_modified.utctimetuple()[:7] if self.feed.last_modified else None

        if self.options.get('force'):
            modified = None
            etag = None

        self.fpf = feedparser.parse(self.feed.feed_address,
                                    agent=USER_AGENT,
                                    etag=etag,
                                    modified=modified)

        return FEED_OK, self.fpf

    def get_identity(self):
        identity = "X"

        current_process = multiprocessing.current_process()
        if current_process._identity:
            identity = current_process._identity[0]

        return identity

class ProcessFeed:
    def __init__(self, feed, fpf, db, options):
        self.feed = feed
        self.options = options
        self.fpf = fpf
        self.lock = multiprocessing.Lock()
        self.db = db
        self.entry_trans = {
            ENTRY_NEW:'new',
            ENTRY_UPDATED:'updated',
            ENTRY_SAME:'same',
            ENTRY_ERR:'error'}
        self.entry_keys = sorted(self.entry_trans.keys())

    def process(self):
        """ Downloads and parses a feed.
        """

        ret_values = {
            ENTRY_NEW:0,
            ENTRY_UPDATED:0,
            ENTRY_SAME:0,
            ENTRY_ERR:0}

        # logging.debug(u' ---> [%d] Processing %s' % (self.feed.id, self.feed.feed_title))

        if hasattr(self.fpf, 'status'):
            if self.options['verbose']:
                logging.debug(u'   ---> [%-30s] Fetched feed, HTTP status %d: %s (bozo: %s)' % (unicode(self.feed)[:30],
                                                     self.fpf.status,
                                                     self.feed.feed_address,
                                                     self.fpf.bozo))
                if self.fpf.bozo and self.fpf.status != 304:
                    logging.debug(u'   ---> [%-30s] BOZO exception: %s' % (
                                  unicode(self.feed)[:30],
                                  self.fpf.bozo_exception,))
            if self.fpf.status == 304:
                self.feed.save()
                self.feed.save_feed_history(304, "Not modified")
                return FEED_SAME, ret_values

            if self.fpf.status >= 400:
                self.feed.save()
                self.feed.save_feed_history(self.fpf.status, "HTTP Error")
                return FEED_ERRHTTP, ret_values

        if self.fpf.bozo and isinstance(self.fpf.bozo_exception, feedparser.NonXMLContentType):
            if not self.fpf.entries:
                logging.debug("   ---> [%-30s] Feed is Non-XML. Checking address..." % unicode(self.feed)[:30])
                fixed_feed = self.feed.check_feed_address_for_feed_link()
                if not fixed_feed:
                    self.feed.save_feed_history(502, 'Non-xml feed', self.fpf.bozo_exception)
                return FEED_ERRPARSE, ret_values
        elif self.fpf.bozo and isinstance(self.fpf.bozo_exception, xml.sax._exceptions.SAXException):
            logging.debug("   ---> [%-30s] Feed is Bad XML (SAX). Checking address..." % unicode(self.feed)[:30])
            if not self.fpf.entries:
                fixed_feed = self.feed.check_feed_address_for_feed_link()
                if not fixed_feed:
                    self.feed.save_feed_history(503, 'SAX Exception', self.fpf.bozo_exception)
                return FEED_ERRPARSE, ret_values

        # the feed has changed (or it is the first time we parse it)
        # saving the etag and last_modified fields
        self.feed.etag = self.fpf.get('etag')
        if self.feed.etag:
            self.feed.etag = self.feed.etag[:255]
        # some times this is None (it never should) *sigh*
        if self.feed.etag is None:
            self.feed.etag = ''

        try:
            self.feed.last_modified = mtime(self.fpf.modified)
        except:
            pass

        self.feed.feed_title = self.fpf.feed.get('title', self.feed.feed_title)
        self.feed.feed_tagline = self.fpf.feed.get('tagline', self.feed.feed_tagline)
        self.feed.feed_link = self.fpf.feed.get('link', self.feed.feed_link)
        self.feed.last_update = datetime.datetime.now()

        guids = []
        for entry in self.fpf.entries:
            if entry.get('id', ''):
                guids.append(entry.get('id', ''))
            elif entry.title:
                guids.append(entry.title)
            elif entry.link:
                guids.append(entry.link)

        self.lock.acquire()
        try:
            self.feed.save()
        finally:
            self.lock.release()

        # Compare new stories to existing stories, adding and updating
        start_date = datetime.datetime.now()
        end_date = datetime.datetime.now()
        story_guids = []
        for entry in self.fpf.entries:
            story = pre_process_story(entry)
            if story.get('published') < start_date:
                start_date = story.get('published')
            if story.get('published') > end_date:
                end_date = story.get('published')
            story_guids.append(story.get('guid') or story.get('link'))
        existing_stories = self.db.stories.find({
            'story_feed_id': self.feed.pk,
            '$or': [
                {
                    'story_date': {'$gte': start_date},
                    'story_date': {'$lte': end_date}
                },
                {
                    'story_guid': {'$in': story_guids}
                }
            ]
        })
        # MStory.objects(
        #     (Q(story_date__gte=start_date) & Q(story_date__lte=end_date))
        #     | (Q(story_guid__in=story_guids)),
        #     story_feed=self.feed
        # ).order_by('-story_date')
        ret_values = self.feed.add_update_stories(self.fpf.entries, existing_stories, self.db)

        logging.debug(u'   ---> [%-30s] Parsed Feed: %s' % (
                      unicode(self.feed)[:30],
                      u' '.join(u'%s=%d' % (self.entry_trans[key],
                              ret_values[key]) for key in self.entry_keys),))
        self.feed.update_all_statistics(lock=self.lock)
        self.feed.save_feed_history(200, "OK")

        return FEED_OK, ret_values

class Dispatcher:
    def __init__(self, options, num_threads):
        self.options = options
        self.entry_stats = {
            ENTRY_NEW:0,
            ENTRY_UPDATED:0,
            ENTRY_SAME:0,
            ENTRY_ERR:0}
        self.feed_stats = {
            FEED_OK:0,
            FEED_SAME:0,
            FEED_ERRPARSE:0,
            FEED_ERRHTTP:0,
            FEED_ERREXC:0}
        self.feed_trans = {
            FEED_OK:'ok',
            FEED_SAME:'unchanged',
            FEED_ERRPARSE:'cant_parse',
            FEED_ERRHTTP:'http_error',
            FEED_ERREXC:'exception'}
        self.feed_keys = sorted(self.feed_trans.keys())
        self.num_threads = num_threads
        self.time_start = datetime.datetime.now()
        self.workers = []

    def process_feed_wrapper(self, feed_queue):
        """ wrapper for ProcessFeed
        """
        if not self.options['single_threaded']:
            # Close the DB so the connection can be re-opened on a per-process basis
            from django.db import connection
            connection.close()
        delta = None

        MONGO_DB = settings.MONGO_DB
        db = mongoengine.connection.connect(db=MONGO_DB['NAME'], host=MONGO_DB['HOST'], port=MONGO_DB['PORT'])

        current_process = multiprocessing.current_process()

        identity = "X"
        if current_process._identity:
            identity = current_process._identity[0]
        for feed in feed_queue:
            ret_entries = {
                ENTRY_NEW: 0,
                ENTRY_UPDATED: 0,
                ENTRY_SAME: 0,
                ENTRY_ERR: 0
            }
            start_time = datetime.datetime.now()

            ### Uncomment to test feed fetcher
            # from random import randint
            # if randint(0,10) < 10:
            #     continue

            try:
                ffeed = FetchFeed(feed, self.options)
                ret_feed, fetched_feed = ffeed.fetch()

                if ((fetched_feed and ret_feed == FEED_OK) or self.options['force']):
                    pfeed = ProcessFeed(feed, fetched_feed, db, self.options)
                    ret_feed, ret_entries = pfeed.process()

                    if ret_entries.get(ENTRY_NEW) or self.options['force']:
                        user_subs = UserSubscription.objects.filter(feed=feed)
                        logging.debug(u'   ---> [%-30s] Computing scores for all feed subscribers: %s subscribers' % (unicode(feed)[:30], user_subs.count()))
                        for sub in user_subs:
                            cache.delete('usersub:%s' % sub.user_id)
                            silent = False if self.options['verbose'] >= 2 else True
                            sub.calculate_feed_scores(silent=silent)
                    cache.delete('feed_stories:%s-%s-%s' % (feed.id, 0, 25))
                    # if ret_entries.get(ENTRY_NEW) or ret_entries.get(ENTRY_UPDATED) or self.options['force']:
                    #     feed.get_stories(force=True)
            except KeyboardInterrupt:
                break
            except urllib2.HTTPError, e:
                feed.save_feed_history(e.code, e.msg, e.fp.read())
                fetched_feed = None
            except Exception, e:
                logging.debug('[%d] ! -------------------------' % (feed.id,))
                tb = traceback.format_exc()
                logging.debug(tb)
                logging.debug('[%d] ! -------------------------' % (feed.id,))
                ret_feed = FEED_ERREXC
                feed.save_feed_history(500, "Error", tb)
                fetched_feed = None

            if ((self.options['force']) or
                (fetched_feed and
                 feed.feed_link and
                 (ret_feed == FEED_OK or
                  (ret_feed == FEED_SAME and feed.stories_last_month > 10)))):

                logging.debug(u'   ---> [%-30s] Fetching page' % (unicode(feed)[:30]))
                page_importer = PageImporter(feed.feed_link, feed)
                page_importer.fetch_page()

            delta = datetime.datetime.now() - start_time

            feed.last_load_time = max(1, delta.seconds)
            feed.fetched_once = True
            try:
                feed.save()
            except IntegrityError:
                logging.debug("   ---> [%-30s] IntegrityError on feed: %s" % (unicode(feed)[:30], feed.feed_address,))

            done_msg = (u'%2s ---> [%-30s] Processed in %s [%s]' % (
                identity, feed.feed_title[:30], unicode(delta),
                self.feed_trans[ret_feed],))
            logging.debug(done_msg)

            self.feed_stats[ret_feed] += 1
            for key, val in ret_entries.items():
                self.entry_stats[key] += val

        time_taken = datetime.datetime.now() - self.time_start
        history = FeedUpdateHistory(
            number_of_feeds=len(feed_queue),
            seconds_taken=time_taken.seconds
        )
        history.save()
        if not self.options['single_threaded']:
            logging.debug("---> DONE WITH PROCESS: %s" % current_process.name)
            sys.exit()

    def add_jobs(self, feeds_queue, feeds_count=1):
        """ adds a feed processing job to the pool
        """
        self.feeds_queue = feeds_queue
        self.feeds_count = feeds_count

    def run_jobs(self):
        if self.options['single_threaded']:
            self.process_feed_wrapper(self.feeds_queue[0])
        else:
            for i in range(self.num_threads):
                feed_queue = self.feeds_queue[i]
                self.workers.append(multiprocessing.Process(target=self.process_feed_wrapper,
                                                            args=(feed_queue,)))
                # worker.setName("Thread #%s" % (i+1))
                # worker.setDaemon(True)
            for i in range(self.num_threads):
                self.workers[i].start()

    def poll(self):
        """ polls the active threads
        """
        if not self.options['single_threaded']:
            for i in range(self.num_threads):
                self.workers[i].join()
            done = (u'* DONE in %s\n* Feeds: %s\n* Entries: %s' % (
                    unicode(datetime.datetime.now() - self.time_start),
                    u' '.join(u'%s=%d' % (self.feed_trans[key],
                              self.feed_stats[key])
                              for key in self.feed_keys),
                    u' '.join(u'%s=%d' % (self.entry_trans[key],
                              self.entry_stats[key])
                              for key in self.entry_keys)
                    ))
            logging.debug(done)
            return

                <|endoftext|>


<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

from UM.Controller import Controller
from UM.PluginRegistry import PluginRegistry
from UM.Mesh.MeshFileHandler import MeshFileHandler
from UM.Settings.MachineSettings import MachineSettings
from UM.Resources import Resources
from UM.Operations.OperationStack import OperationStack
from UM.Event import CallFunctionEvent
from UM.Signal import Signal, SignalEmitter
from UM.Logger import Logger
from UM.Preferences import Preferences
from UM.OutputDevice.OutputDeviceManager import OutputDeviceManager
from UM.Settings.MachineManager import MachineManager
from UM.i18n import i18nCatalog

import threading
import argparse
import os
import urllib.parse
import sys

##  Central object responsible for running the main event loop and creating other central objects.
#
#   The Application object is a central object for accessing other important objects. It is also
#   responsible for starting the main event loop. It is passed on to plugins so it can be easily
#   used to access objects required for those plugins.
class Application(SignalEmitter):
    ##  Init method
    #
    #   \param name \type{string} The name of the application.
    #   \param version \type{string} Version, formatted as major.minor.rev
    def __init__(self, name, version,  **kwargs):
        if(Application._instance != None):
            raise ValueError("Duplicate singleton creation")

        # If the constructor is called and there is no instance, set the instance to self.
        # This is done because we can't make constructor private
        Application._instance = self

        self._application_name = name
        self._version = version

        os.putenv("UBUNTU_MENUPROXY","0") #For Ubuntu Unity this makes Qt use its own menu bar rather than pass it on to Unity.

        Signal._app = self
        Resources.ApplicationIdentifier = name
        i18nCatalog.setApplication(self)

        Resources.addSearchPath(os.path.dirname(sys.executable))
        Resources.addSearchPath(os.path.join(Application.getInstallPrefix(), "share", "uranium"))
        Resources.addSearchPath(os.path.join(Application.getInstallPrefix(), "Resources", "uranium"))
        Resources.addSearchPath(os.path.join(Application.getInstallPrefix(), "Resources", self.getApplicationName()))
        if not hasattr(sys, "frozen"):
            Resources.addSearchPath(os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."))

        self._main_thread = threading.current_thread()

        super().__init__(**kwargs) # Call super to make multiple inheritence work.

        self._renderer = None

        PluginRegistry.addType("backend", self.setBackend)
        PluginRegistry.addType("logger", Logger.addLogger)
        PluginRegistry.addType("extension", self.addExtension)

        preferences = Preferences.getInstance()
        preferences.addPreference("general/language", "en")
        try:
            preferences.readFromFile(Resources.getPath(Resources.Preferences, self._application_name + ".cfg"))
        except FileNotFoundError:
            pass

        self._controller = Controller(self)
        self._mesh_file_handler = MeshFileHandler()
        self._extensions = []
        self._backend = None
        self._output_device_manager = OutputDeviceManager()
        self._machine_manager = MachineManager(self._application_name)

        self._required_plugins = []

        self._operation_stack = OperationStack()

        self._plugin_registry = PluginRegistry.getInstance()

        self._plugin_registry.addPluginLocation(os.path.join(Application.getInstallPrefix(), "lib", "uranium"))
        self._plugin_registry.addPluginLocation(os.path.join(os.path.dirname(sys.executable), "plugins"))
        self._plugin_registry.addPluginLocation(os.path.join(Application.getInstallPrefix(), "Resources", "uranium", "plugins"))
        self._plugin_registry.addPluginLocation(os.path.join(Application.getInstallPrefix(), "Resources", self.getApplicationName(), "plugins"))
        # Locally installed plugins
        self._plugin_registry.addPluginLocation(os.path.join(Resources.getStoragePath(Resources.Resources), "plugins"))
        if not hasattr(sys, "frozen"):
            self._plugin_registry.addPluginLocation(os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "plugins"))

        self._plugin_registry.setApplication(self)

        self._parsed_command_line = None
        self.parseCommandLine()

        self._visible_messages = []
        self._message_lock = threading.Lock()
        self.showMessageSignal.connect(self.showMessage)
        self.hideMessageSignal.connect(self.hideMessage)

    ##  Emitted when the application window was closed and we need to shut down the application
    applicationShuttingDown = Signal()

    showMessageSignal = Signal()

    hideMessageSignal = Signal()

    def hideMessage(self, message):
        raise NotImplementedError

    def showMessage(self, message):
        raise NotImplementedError

    ##  Get the version of the application
    #   \returns version \type{string}
    def getVersion(self):
        return self._version

    ##  Add a message to the visible message list so it will be displayed.
    #   This should only be called by message object itself.
    #   To show a message, simply create it and call its .show() function.
    #   \param message \type{Message} message object
    #   \sa Message::show()
    #def showMessage(self, message):
    #    with self._message_lock:
    #        if message not in self._visible_messages:
    #            self._visible_messages.append(message)
    #            self.visibleMessageAdded.emit(message)

    visibleMessageAdded = Signal()

    ##  Remove a message from the visible message list so it will no longer be displayed.
    #   This should only be called by message object itself.
    #   in principle, this should only be called by the message itself (hide)
    #   \param message \type{Message} message object
    #   \sa Message::hide()
    #def hideMessage(self, message):
    #    with self._message_lock:
    #        if message in self._visible_messages:
    #            self._visible_messages.remove(message)
    #            self.visibleMessageRemoved.emit(message)

    ##  Hide message by ID (as provided by built-in id function)
    #   \param message_id \type{long}
    def hideMessageById(self, message_id):
        found_message = None
        with self._message_lock:
            for message in self._visible_messages:
                if id(message) == message_id:
                    found_message = message
        if found_message is not None:
            self.hideMessageSignal.emit(found_message)

    visibleMessageRemoved = Signal()

    ##  Get list of all visible messages
    #   \returns visible_messages \type{list}
    def getVisibleMessages(self):
        with self._message_lock:
            return self._visible_messages

    ##  Function that needs to be overriden by child classes with a list of plugin it needs.
    def _loadPlugins(self):
        pass

    def getCommandLineOption(self, name, default = None):
        if not self._parsed_command_line:
            self.parseCommandLine()

        return self._parsed_command_line.get(name, default)

    ##  Get name of the application.
    #   \returns application_name \type{string}
    def getApplicationName(self):
        return self._application_name

    ##  Set name of the application.
    #   \param application_name \type{string}
    def setApplicationName(self, application_name):
        self._application_name = application_name

    def getApplicationLanguage(self):
        override_lang = os.getenv("URANIUM_LANGUAGE")
        if override_lang:
            return override_lang

        preflang = Preferences.getInstance().getValue("general/language")
        if preflang:
            return preflang

        env_lang = os.getenv("LANGUAGE")
        if env_lang:
            return env_lang

        return "en"

    ##  Application has a list of plugins that it *must* have. If it does not have these, it cannot function.
    #   These plugins can not be disabled in any way.
    #   \returns required_plugins \type{list}
    def getRequiredPlugins(self):
        return self._required_plugins

    ##  Set the plugins that the application *must* have in order to function.
    #   \param plugin_names \type{list} List of strings with the names of the required plugins
    def setRequiredPlugins(self, plugin_names):
        self._required_plugins = plugin_names

    ##  Set the backend of the application (the program that does the heavy lifting).
    #   \param backend \type{Backend}
    def setBackend(self, backend):
        self._backend = backend

    def getMachineManager(self):
        return self._machine_manager

    ##  Get the backend of the application (the program that does the heavy lifting).
    #   \returns Backend \type{Backend}
    def getBackend(self):
        return self._backend

    ##  Get the PluginRegistry of this application.
    #   \returns PluginRegistry \type{PluginRegistry}
    def getPluginRegistry(self):
        return self._plugin_registry

    ##  Get the Controller of this application.
    #   \returns Controller \type{Controller}
    def getController(self):
        return self._controller

    ##  Get the MeshFileHandler of this application.
    #   \returns MeshFileHandler \type{MeshFileHandler}
    def getMeshFileHandler(self):
        return self._mesh_file_handler

    def getOperationStack(self):
        return self._operation_stack

    def getOutputDeviceManager(self):
        return self._output_device_manager

    ##  Run the main eventloop.
    #   This method should be reimplemented by subclasses to start the main event loop.
    #   \exception NotImplementedError
    def run(self):
        raise NotImplementedError("Run must be implemented by application")

    ##  Return an application-specific Renderer object.
    #   \exception NotImplementedError
    def getRenderer(self):
        raise NotImplementedError("getRenderer must be implemented by subclasses.")

    ##  Post a function event onto the event loop.
    #
    #   This takes a CallFunctionEvent object and puts it into the actual event loop.
    #   \exception NotImplementedError
    def functionEvent(self, event):
        raise NotImplementedError("functionEvent must be implemented by subclasses.")

    ##  Call a function the next time the event loop runs.
    #
    #   \param function The function to call.
    #   \param args The positional arguments to pass to the function.
    #   \param kwargs The keyword arguments to pass to the function.
    def callLater(self, function, *args, **kwargs):
        event = CallFunctionEvent(function, args, kwargs)
        self.functionEvent(event)

    ##  Get the application"s main thread.
    def getMainThread(self):
        return self._main_thread

    ##  Return the singleton instance of the application object
    @classmethod
    def getInstance(cls):
        # Note: Explicit use of class name to prevent issues with inheritance.
        if Application._instance is None:
            Application._instance = cls()

        return Application._instance

    def parseCommandLine(self):
        parser = argparse.ArgumentParser(prog = self.getApplicationName())
        parser.add_argument("--version", action="version", version="%(prog)s {0}".format(self.getVersion()))
        parser.add_argument("--external-backend",
                            dest="external-backend",
                            action="store_true", default=False,
                            help="Use an externally started backend instead of starting it automatically.")

        self.addCommandLineOptions(parser)

        self._parsed_command_line = vars(parser.parse_args())

    ##  Can be overridden to add additional command line options to the parser.
    #
    #   \param parser \type{argparse.ArgumentParser} The parser that will parse the command line.
    def addCommandLineOptions(self, parser):
        pass

    def addExtension(self, extension):
        self._extensions.append(extension)

    def getExtensions(self):
        return self._extensions

    @staticmethod
    def getInstallPrefix():
        if "python" in os.path.basename(sys.executable):
            return os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), ".."))
        else:
            return os.path.abspath(os.path.join(os.path.dirname(sys.executable), ".."))

    _instance = None
<commit_msg>Add command line arguments to log<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

from UM.Controller import Controller
from UM.PluginRegistry import PluginRegistry
from UM.Mesh.MeshFileHandler import MeshFileHandler
from UM.Settings.MachineSettings import MachineSettings
from UM.Resources import Resources
from UM.Operations.OperationStack import OperationStack
from UM.Event import CallFunctionEvent
from UM.Signal import Signal, SignalEmitter
from UM.Logger import Logger
from UM.Preferences import Preferences
from UM.OutputDevice.OutputDeviceManager import OutputDeviceManager
from UM.Settings.MachineManager import MachineManager
from UM.i18n import i18nCatalog

import threading
import argparse
import os
import urllib.parse
import sys

##  Central object responsible for running the main event loop and creating other central objects.
#
#   The Application object is a central object for accessing other important objects. It is also
#   responsible for starting the main event loop. It is passed on to plugins so it can be easily
#   used to access objects required for those plugins.
class Application(SignalEmitter):
    ##  Init method
    #
    #   \param name \type{string} The name of the application.
    #   \param version \type{string} Version, formatted as major.minor.rev
    def __init__(self, name, version,  **kwargs):
        if(Application._instance != None):
            raise ValueError("Duplicate singleton creation")

        # If the constructor is called and there is no instance, set the instance to self.
        # This is done because we can't make constructor private
        Application._instance = self

        self._application_name = name
        self._version = version

        os.putenv("UBUNTU_MENUPROXY","0") #For Ubuntu Unity this makes Qt use its own menu bar rather than pass it on to Unity.

        Signal._app = self
        Resources.ApplicationIdentifier = name
        i18nCatalog.setApplication(self)

        Resources.addSearchPath(os.path.dirname(sys.executable))
        Resources.addSearchPath(os.path.join(Application.getInstallPrefix(), "share", "uranium"))
        Resources.addSearchPath(os.path.join(Application.getInstallPrefix(), "Resources", "uranium"))
        Resources.addSearchPath(os.path.join(Application.getInstallPrefix(), "Resources", self.getApplicationName()))
        if not hasattr(sys, "frozen"):
            Resources.addSearchPath(os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."))

        self._main_thread = threading.current_thread()

        super().__init__(**kwargs) # Call super to make multiple inheritence work.

        self._renderer = None

        PluginRegistry.addType("backend", self.setBackend)
        PluginRegistry.addType("logger", Logger.addLogger)
        PluginRegistry.addType("extension", self.addExtension)

        preferences = Preferences.getInstance()
        preferences.addPreference("general/language", "en")
        try:
            preferences.readFromFile(Resources.getPath(Resources.Preferences, self._application_name + ".cfg"))
        except FileNotFoundError:
            pass

        self._controller = Controller(self)
        self._mesh_file_handler = MeshFileHandler()
        self._extensions = []
        self._backend = None
        self._output_device_manager = OutputDeviceManager()
        self._machine_manager = MachineManager(self._application_name)

        self._required_plugins = []

        self._operation_stack = OperationStack()

        self._plugin_registry = PluginRegistry.getInstance()

        self._plugin_registry.addPluginLocation(os.path.join(Application.getInstallPrefix(), "lib", "uranium"))
        self._plugin_registry.addPluginLocation(os.path.join(os.path.dirname(sys.executable), "plugins"))
        self._plugin_registry.addPluginLocation(os.path.join(Application.getInstallPrefix(), "Resources", "uranium", "plugins"))
        self._plugin_registry.addPluginLocation(os.path.join(Application.getInstallPrefix(), "Resources", self.getApplicationName(), "plugins"))
        # Locally installed plugins
        self._plugin_registry.addPluginLocation(os.path.join(Resources.getStoragePath(Resources.Resources), "plugins"))
        if not hasattr(sys, "frozen"):
            self._plugin_registry.addPluginLocation(os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "plugins"))

        self._plugin_registry.setApplication(self)

        self._parsed_command_line = None
        self.parseCommandLine()

        self._visible_messages = []
        self._message_lock = threading.Lock()
        self.showMessageSignal.connect(self.showMessage)
        self.hideMessageSignal.connect(self.hideMessage)

    ##  Emitted when the application window was closed and we need to shut down the application
    applicationShuttingDown = Signal()

    showMessageSignal = Signal()

    hideMessageSignal = Signal()

    def hideMessage(self, message):
        raise NotImplementedError

    def showMessage(self, message):
        raise NotImplementedError

    ##  Get the version of the application
    #   \returns version \type{string}
    def getVersion(self):
        return self._version

    ##  Add a message to the visible message list so it will be displayed.
    #   This should only be called by message object itself.
    #   To show a message, simply create it and call its .show() function.
    #   \param message \type{Message} message object
    #   \sa Message::show()
    #def showMessage(self, message):
    #    with self._message_lock:
    #        if message not in self._visible_messages:
    #            self._visible_messages.append(message)
    #            self.visibleMessageAdded.emit(message)

    visibleMessageAdded = Signal()

    ##  Remove a message from the visible message list so it will no longer be displayed.
    #   This should only be called by message object itself.
    #   in principle, this should only be called by the message itself (hide)
    #   \param message \type{Message} message object
    #   \sa Message::hide()
    #def hideMessage(self, message):
    #    with self._message_lock:
    #        if message in self._visible_messages:
    #            self._visible_messages.remove(message)
    #            self.visibleMessageRemoved.emit(message)

    ##  Hide message by ID (as provided by built-in id function)
    #   \param message_id \type{long}
    def hideMessageById(self, message_id):
        found_message = None
        with self._message_lock:
            for message in self._visible_messages:
                if id(message) == message_id:
                    found_message = message
        if found_message is not None:
            self.hideMessageSignal.emit(found_message)

    visibleMessageRemoved = Signal()

    ##  Get list of all visible messages
    #   \returns visible_messages \type{list}
    def getVisibleMessages(self):
        with self._message_lock:
            return self._visible_messages

    ##  Function that needs to be overriden by child classes with a list of plugin it needs.
    def _loadPlugins(self):
        pass

    def getCommandLineOption(self, name, default = None):
        if not self._parsed_command_line:
            self.parseCommandLine()
            Logger.log("d", "Command line options: %s", str(self._parsed_command_line))

        return self._parsed_command_line.get(name, default)

    ##  Get name of the application.
    #   \returns application_name \type{string}
    def getApplicationName(self):
        return self._application_name

    ##  Set name of the application.
    #   \param application_name \type{string}
    def setApplicationName(self, application_name):
        self._application_name = application_name

    def getApplicationLanguage(self):
        override_lang = os.getenv("URANIUM_LANGUAGE")
        if override_lang:
            return override_lang

        preflang = Preferences.getInstance().getValue("general/language")
        if preflang:
            return preflang

        env_lang = os.getenv("LANGUAGE")
        if env_lang:
            return env_lang

        return "en"

    ##  Application has a list of plugins that it *must* have. If it does not have these, it cannot function.
    #   These plugins can not be disabled in any way.
    #   \returns required_plugins \type{list}
    def getRequiredPlugins(self):
        return self._required_plugins

    ##  Set the plugins that the application *must* have in order to function.
    #   \param plugin_names \type{list} List of strings with the names of the required plugins
    def setRequiredPlugins(self, plugin_names):
        self._required_plugins = plugin_names

    ##  Set the backend of the application (the program that does the heavy lifting).
    #   \param backend \type{Backend}
    def setBackend(self, backend):
        self._backend = backend

    def getMachineManager(self):
        return self._machine_manager

    ##  Get the backend of the application (the program that does the heavy lifting).
    #   \returns Backend \type{Backend}
    def getBackend(self):
        return self._backend

    ##  Get the PluginRegistry of this application.
    #   \returns PluginRegistry \type{PluginRegistry}
    def getPluginRegistry(self):
        return self._plugin_registry

    ##  Get the Controller of this application.
    #   \returns Controller \type{Controller}
    def getController(self):
        return self._controller

    ##  Get the MeshFileHandler of this application.
    #   \returns MeshFileHandler \type{MeshFileHandler}
    def getMeshFileHandler(self):
        return self._mesh_file_handler

    def getOperationStack(self):
        return self._operation_stack

    def getOutputDeviceManager(self):
        return self._output_device_manager

    ##  Run the main eventloop.
    #   This method should be reimplemented by subclasses to start the main event loop.
    #   \exception NotImplementedError
    def run(self):
        raise NotImplementedError("Run must be implemented by application")

    ##  Return an application-specific Renderer object.
    #   \exception NotImplementedError
    def getRenderer(self):
        raise NotImplementedError("getRenderer must be implemented by subclasses.")

    ##  Post a function event onto the event loop.
    #
    #   This takes a CallFunctionEvent object and puts it into the actual event loop.
    #   \exception NotImplementedError
    def functionEvent(self, event):
        raise NotImplementedError("functionEvent must be implemented by subclasses.")

    ##  Call a function the next time the event loop runs.
    #
    #   \param function The function to call.
    #   \param args The positional arguments to pass to the function.
    #   \param kwargs The keyword arguments to pass to the function.
    def callLater(self, function, *args, **kwargs):
        event = CallFunctionEvent(function, args, kwargs)
        self.functionEvent(event)

    ##  Get the application"s main thread.
    def getMainThread(self):
        return self._main_thread

    ##  Return the singleton instance of the application object
    @classmethod
    def getInstance(cls):
        # Note: Explicit use of class name to prevent issues with inheritance.
        if Application._instance is None:
            Application._instance = cls()

        return Application._instance

    def parseCommandLine(self):
        parser = argparse.ArgumentParser(prog = self.getApplicationName())
        parser.add_argument("--version", action="version", version="%(prog)s {0}".format(self.getVersion()))
        parser.add_argument("--external-backend",
                            dest="external-backend",
                            action="store_true", default=False,
                            help="Use an externally started backend instead of starting it automatically.")

        self.addCommandLineOptions(parser)

        self._parsed_command_line = vars(parser.parse_args())

    ##  Can be overridden to add additional command line options to the parser.
    #
    #   \param parser \type{argparse.ArgumentParser} The parser that will parse the command line.
    def addCommandLineOptions(self, parser):
        pass

    def addExtension(self, extension):
        self._extensions.append(extension)

    def getExtensions(self):
        return self._extensions

    @staticmethod
    def getInstallPrefix():
        if "python" in os.path.basename(sys.executable):
            return os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), ".."))
        else:
            return os.path.abspath(os.path.join(os.path.dirname(sys.executable), ".."))

    _instance = None
<|endoftext|>