<commit_before># -*- coding: utf-8 -*-

"""WakaTime plugin for Eric6 and Pymakr."""

from __future__ import unicode_literals

import os
import platform
import re
import shutil
import ssl
import sys
import time
import threading
import traceback
from datetime import datetime
from zipfile import ZipFile
try:
    import ConfigParser as configparser
except ImportError:
    import configparser
try:
    from urllib2 import urlretrieve
except ImportError:
    from urllib.request import urlretrieve

is_py2 = (sys.version_info[0] == 2)
if is_py2:
    import codecs
    open = codecs.open

from PyQt5.QtGui import QIcon
from PyQt5.QtCore import QObject
try:
    from PyQt5.QtGui import QAction, QInputDialog, QLineEdit
except ImportError:
    from PyQt5.QtWidgets import QAction, QInputDialog, QLineEdit

from E5Gui.E5Application import e5App
from UI.Info import Program, Version

# Start-Of-Header
name = "WakaTime Plugin"
author = "WakaTime <support@wakatime.com>"
autoactivate = True
deactivateable = False
version = "1.0.2"
pluginType = "viewmanager"
pluginTypename = "WakaTime"
className = "WakaTimePlugin"
packageName = None
shortDescription = "Automatic time tracking and metrics about your IDE usage."
longDescription = "Automatic time tracking and metrics about your IDE usage."
pyqtApi = 2
python2Compatible = True
# End-Of-Header

error = ""

if platform.system() == 'Windows':
    RESOURCES_FOLDER = os.path.join(os.getenv('APPDATA'), 'WakaTime')
else:
    RESOURCES_FOLDER = os.path.join(os.path.expanduser('~'), '.wakatime')
if not os.path.exists(RESOURCES_FOLDER):
    os.makedirs(RESOURCES_FOLDER)

# add wakatime package to path
sys.path.insert(0, os.path.join(RESOURCES_FOLDER, 'wakatime-master'))

HEARTBEAT_FREQUENCY = 2
CLI_LOCATION = os.path.join(RESOURCES_FOLDER, 'wakatime-master', 'wakatime', 'cli.py')
PLUGIN = '{prog}/{progVer} {prog}-wakatime/{pluginVer}'.format(
    prog=Program,
    progVer=Version,
    pluginVer=version,
)
IS_DEBUG_ENABLED = False

# Log Levels
DEBUG = 'DEBUG'
INFO = 'INFO'
WARNING = 'WARNING'
ERROR = 'ERROR'

def getConfigData():
    return {}

def prepareUninstall():
    """
    Module function to prepare for an uninstallation.
    """
    pass

class WakaTimePlugin(QObject):
    """
    Class implementing the WakaTime plugin.
    """
    def __init__(self, ui):
        """
        Constructor

        @param ui reference to the user interface object (UI.UserInterface)
        """
        super(WakaTimePlugin, self).__init__(ui)
        self._ui = ui

        try:
            e5App().registerPluginObject(pluginTypename, self)
        except KeyError:
            pass    # ignore duplicate registration

    def getProjectHelper(self):
        """
        Public method to get a reference to the project helper object.

        @return reference to the project helper object
        """
        return self.__projectHelperObject

    def initToolbar(self, ui, toolbarManager):
        """
        Public slot to initialize the VCS toolbar.

        @param ui reference to the main window (UserInterface)
        @param toolbarManager reference to a toolbar manager object
            (E5ToolBarManager)
        """
        if self.__projectHelperObject:
            self.__projectHelperObject.initToolbar(ui, toolbarManager)

    def activate(self):
        global IS_DEBUG_ENABLED
        IS_DEBUG_ENABLED = self._getDebug()

        log(DEBUG, 'Initializing WakaTime v{ver}...'.format(ver=version))

        self._setupCli()

        apiKey = self._getApiKey()
        if not apiKey.strip():
            self._promptForApiKey()

        self._setupEventListeners()

        self._addMenu()

        log(DEBUG, 'Finished initializing WakaTime plugin.')
        return None, True

    def deactivate(self):
        """
        Public method to deactivate this plugin.
        """
        self.__object = None

        self.action.setVisible(False)
        self.action.setEnabled(False)

    def _setupCli(self):
        if self._isCliLatest():
            return

        thread = DownloadCLI()
        thread.start()

    def _setupEventListeners(self):
        e5App().getObject('ViewManager').cursorChanged.connect(self._cursorChanged)
        e5App().getObject('ViewManager').editorSaved.connect(self._editorSaved)
        #e5App().getObject('Project').projectOpened.connect(self._projectOpened)

    def _addMenu(self):
        """Adds the WakaTime menu item under the File menu."""
        self.action = QAction(QIcon(), 'WakaTime', self)
        self.action.triggered.connect(self._promptForApiKey)
        fileMenu = e5App().getObject('UserInterface').getMenu('file')
        fileMenu.addAction(self.action)

    def _editorSaved(self, fileName):
        self._handleActivity(fileName, isWrite=True)

    def _cursorChanged(self, editor):
        fileName = editor.getFileName()
        self._handleActivity(fileName)

    def _projectOpened(self):
        log(DEBUG, '_projectOpened')

    def _handleActivity(self, fileName, isWrite=False):
        fileName = os.path.abspath(os.path.realpath(fileName))
        if not hasattr(self, '_lastFile'):
            self._lastFile = None
        if fileName != self._lastFile or self._enoughTimePassed(isWrite):
            if not self._shouldExclude(fileName):
                self._sendHeartbeat(fileName, isWrite)

    def _enoughTimePassed(self, isWrite):
        now = time.time()
        if not hasattr(self, '_lastTime'):
            self._lastTime = 0
        if now - self._lastTime > HEARTBEAT_FREQUENCY * 60:
            return True
        if isWrite and now - self._lastTime > 2:
            return True
        return False

    def _shouldExclude(self, fileName):
        return os.path.basename(fileName) == '0.1'

    def _sendHeartbeat(self, fileName, isWrite):
        try:
            from wakatime.main import execute
        except ImportError:
            log(DEBUG, 'Skipping heartbeat because wakatime-cli not found.')
            return
        except:
            log(ERROR, traceback.format_exc())
            return

        self._lastFile = fileName
        self._lastTime = time.time()

        args = ['--entity', fileName, '--plugin', PLUGIN]
        if isWrite:
            args.append('--write')

        log(DEBUG, 'Sending heartbeat:')
        log(DEBUG, ' '.join(args))

        execute(args)

    def _promptForApiKey(self, *args):
        """Prompt the user to enter their api key."""

        ui = e5App().getObject('UserInterface')
        title = 'WakaTime'
        label = '               Enter your WakaTime API Key:               \n' + \
                '               (https://wakatime.com/settings)            '
        default = self._getApiKey()
        text, ok = QInputDialog.getText(ui, title, label, QLineEdit.Normal, default)
        if ok:
            configs = self._parseConfigFile()
            configs.set('settings', 'api_key', text)
            configFile = self._configFile()
            with open(configFile, 'w', encoding='utf-8') as fh:
                configs.write(fh)

    def _createConfigFile(self):
        """Creates the .wakatime.cfg INI file in $HOME directory, if it does
        not already exist.
        """
        configFile = self._configFile()
        try:
            with open(configFile) as fh:
                pass
        except IOError:
            try:
                with open(configFile, 'w') as fh:
                    fh.write("[settings]\n")
                    fh.write("debug = false\n")
                    fh.write("hidefilenames = false\n")
            except IOError:
                pass

    def _getApiKey(self):
        self._createConfigFile()
        key = ''
        try:
            configs = self._parseConfigFile()
            if configs is not None:
                if configs.has_option('settings', 'api_key'):
                    key = configs.get('settings', 'api_key')
        except:
            pass
        return key

    def _getDebug(self):
        self._createConfigFile()
        try:
            configs = self._parseConfigFile()
            if configs is not None:
                if configs.has_option('settings', 'debug'):
                    return configs.get('settings', 'debug') == 'true'
        except:
            pass
        return False

    def _configFile(self):
        home = os.environ.get('WAKATIME_HOME')
        if home:
            return os.path.join(os.path.expanduser(home), '.wakatime.cfg')

        return os.path.join(os.path.expanduser('~'), '.wakatime.cfg')

    def _parseConfigFile(self):
        """Returns a configparser.SafeConfigParser instance with configs
        read from the config file. Default location of the config file is
        at ~/.wakatime.cfg.
        """

        configFile = self._configFile()

        configs = configparser.SafeConfigParser()
        try:
            with open(configFile, 'r', encoding='utf-8') as fh:
                try:
                    configs.readfp(fh)
                    return configs
                except configparser.Error:
                    log(ERROR, traceback.format_exc())
                    return None
        except IOError:
            log(DEBUG, "Error: Could not read from config file {0}\n".format(configFile))
            return None

    def _isCliLatest(self):
        remoteVer = self._getLatestCliVersion()
        about = os.path.join(RESOURCES_FOLDER, 'wakatime-master', 'wakatime', '__about__.py')
        try:
            with open(about) as fh:
                localVer = self._extractVersion(fh)
            if remoteVer and localVer:
                return remoteVer.strip() == localVer.strip()
        except:
            log(DEBUG, traceback.format_exc())
        return False

    def _getLatestCliVersion(self):
        url = 'https://raw.githubusercontent.com/wakatime/wakatime/master/wakatime/__about__.py'
        try:
            output = os.path.join(RESOURCES_FOLDER, 'remoteVer')
            download(url, output)
            with open(output) as fh:
                return self._extractVersion(fh)
        except:
            return None

    def _extractVersion(self, fh):
        pattern = re.compile(r"^__version_info__ = \('([0-9]+)', '([0-9]+)', '([0-9]+)'\)")
        for line in fh.readlines():
            match = pattern.search(line)
            if match:
                return '{0}.{0}.{0}'.format(match.group(1), match.group(2), match.group(3))
        return None

    def prepareUninstall(self):
        """
        Public method to prepare for an uninstallation.
        """
        e5App().unregisterPluginObject(pluginTypename)

    def prepareUnload(self):
        """
        Public method to prepare for an unload.
        """
        if self.__projectHelperObject:
            self.__projectHelperObject.removeToolbar(
                self.__ui, e5App().getObject("ToolbarManager"))
        e5App().unregisterPluginObject(pluginTypename)

class DownloadCLI(threading.Thread):
    """Non-blocking thread for downloading latest wakatime-cli from GitHub.
    """

    def run(self):
        log(INFO, 'Downloading wakatime-cli...')

        try:
            shutil.rmtree(os.path.join(RESOURCES_FOLDER, 'wakatime-master'))
        except:
            pass

        try:
            url = 'https://github.com/wakatime/wakatime/archive/master.zip'
            zip_file = os.path.join(RESOURCES_FOLDER, 'wakatime.zip')
            download(url, zip_file)

            log(INFO, 'Extracting wakatime-cli...')
            with ZipFile(zip_file) as zf:
                zf.extractall(RESOURCES_FOLDER)
        except:
            log(DEBUG, traceback.format_exc())

        log(INFO, 'Finished extracting wakatime-cli.')

def download(url, filePath):
    try:
        urlretrieve(url, filePath)
    except IOError:
        ssl._create_default_https_context = ssl._create_unverified_context
        urlretrieve(url, filePath)

def log(lvl, msg, *args, **kwargs):
    if lvl == DEBUG and not IS_DEBUG_ENABLED:
        return
    if len(args) > 0:
        msg = msg.format(*args)
    elif len(kwargs) > 0:
        msg = msg.format(**kwargs)
    date = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
    try:
        print('[WakaTime] [{date}] [{lvl}] {msg}'.format(lvl=lvl, date=date, msg=msg))
    except UnicodeDecodeError:
        print('[WakaTime] [{date}] [{lvl}] {msg}'.format(lvl=lvl, date=date, msg=msg.decode('utf8', 'replace')))
<commit_msg>Use new GitHub repo name for wakatime-cli<commit_after># -*- coding: utf-8 -*-

"""WakaTime plugin for Eric6 and Pymakr."""

from __future__ import unicode_literals

import os
import platform
import re
import shutil
import ssl
import sys
import time
import threading
import traceback
from datetime import datetime
from zipfile import ZipFile
try:
    import ConfigParser as configparser
except ImportError:
    import configparser
try:
    from urllib2 import urlretrieve
except ImportError:
    from urllib.request import urlretrieve

is_py2 = (sys.version_info[0] == 2)
if is_py2:
    import codecs
    open = codecs.open

from PyQt5.QtGui import QIcon
from PyQt5.QtCore import QObject
try:
    from PyQt5.QtGui import QAction, QInputDialog, QLineEdit
except ImportError:
    from PyQt5.QtWidgets import QAction, QInputDialog, QLineEdit

from E5Gui.E5Application import e5App
from UI.Info import Program, Version

# Start-Of-Header
name = "WakaTime Plugin"
author = "WakaTime <support@wakatime.com>"
autoactivate = True
deactivateable = False
version = "1.0.2"
pluginType = "viewmanager"
pluginTypename = "WakaTime"
className = "WakaTimePlugin"
packageName = None
shortDescription = "Automatic time tracking and metrics about your IDE usage."
longDescription = "Automatic time tracking and metrics about your IDE usage."
pyqtApi = 2
python2Compatible = True
# End-Of-Header

error = ""

if platform.system() == 'Windows':
    RESOURCES_FOLDER = os.path.join(os.getenv('APPDATA'), 'WakaTime')
else:
    RESOURCES_FOLDER = os.path.join(os.path.expanduser('~'), '.wakatime')
if not os.path.exists(RESOURCES_FOLDER):
    os.makedirs(RESOURCES_FOLDER)

# add wakatime package to path
sys.path.insert(0, os.path.join(RESOURCES_FOLDER, 'legacy-python-cli-master'))

HEARTBEAT_FREQUENCY = 2
CLI_LOCATION = os.path.join(RESOURCES_FOLDER, 'legacy-python-cli-master', 'wakatime', 'cli.py')
PLUGIN = '{prog}/{progVer} {prog}-wakatime/{pluginVer}'.format(
    prog=Program,
    progVer=Version,
    pluginVer=version,
)
IS_DEBUG_ENABLED = False

# Log Levels
DEBUG = 'DEBUG'
INFO = 'INFO'
WARNING = 'WARNING'
ERROR = 'ERROR'

def getConfigData():
    return {}

def prepareUninstall():
    """
    Module function to prepare for an uninstallation.
    """
    pass

class WakaTimePlugin(QObject):
    """
    Class implementing the WakaTime plugin.
    """
    def __init__(self, ui):
        """
        Constructor

        @param ui reference to the user interface object (UI.UserInterface)
        """
        super(WakaTimePlugin, self).__init__(ui)
        self._ui = ui

        try:
            e5App().registerPluginObject(pluginTypename, self)
        except KeyError:
            pass    # ignore duplicate registration

    def getProjectHelper(self):
        """
        Public method to get a reference to the project helper object.

        @return reference to the project helper object
        """
        return self.__projectHelperObject

    def initToolbar(self, ui, toolbarManager):
        """
        Public slot to initialize the VCS toolbar.

        @param ui reference to the main window (UserInterface)
        @param toolbarManager reference to a toolbar manager object
            (E5ToolBarManager)
        """
        if self.__projectHelperObject:
            self.__projectHelperObject.initToolbar(ui, toolbarManager)

    def activate(self):
        global IS_DEBUG_ENABLED
        IS_DEBUG_ENABLED = self._getDebug()

        log(DEBUG, 'Initializing WakaTime v{ver}...'.format(ver=version))

        self._setupCli()

        apiKey = self._getApiKey()
        if not apiKey.strip():
            self._promptForApiKey()

        self._setupEventListeners()

        self._addMenu()

        log(DEBUG, 'Finished initializing WakaTime plugin.')
        return None, True

    def deactivate(self):
        """
        Public method to deactivate this plugin.
        """
        self.__object = None

        self.action.setVisible(False)
        self.action.setEnabled(False)

    def _setupCli(self):
        if self._isCliLatest():
            return

        thread = DownloadCLI()
        thread.start()

    def _setupEventListeners(self):
        e5App().getObject('ViewManager').cursorChanged.connect(self._cursorChanged)
        e5App().getObject('ViewManager').editorSaved.connect(self._editorSaved)
        #e5App().getObject('Project').projectOpened.connect(self._projectOpened)

    def _addMenu(self):
        """Adds the WakaTime menu item under the File menu."""
        self.action = QAction(QIcon(), 'WakaTime', self)
        self.action.triggered.connect(self._promptForApiKey)
        fileMenu = e5App().getObject('UserInterface').getMenu('file')
        fileMenu.addAction(self.action)

    def _editorSaved(self, fileName):
        self._handleActivity(fileName, isWrite=True)

    def _cursorChanged(self, editor):
        fileName = editor.getFileName()
        self._handleActivity(fileName)

    def _projectOpened(self):
        log(DEBUG, '_projectOpened')

    def _handleActivity(self, fileName, isWrite=False):
        fileName = os.path.abspath(os.path.realpath(fileName))
        if not hasattr(self, '_lastFile'):
            self._lastFile = None
        if fileName != self._lastFile or self._enoughTimePassed(isWrite):
            if not self._shouldExclude(fileName):
                self._sendHeartbeat(fileName, isWrite)

    def _enoughTimePassed(self, isWrite):
        now = time.time()
        if not hasattr(self, '_lastTime'):
            self._lastTime = 0
        if now - self._lastTime > HEARTBEAT_FREQUENCY * 60:
            return True
        if isWrite and now - self._lastTime > 2:
            return True
        return False

    def _shouldExclude(self, fileName):
        return os.path.basename(fileName) == '0.1'

    def _sendHeartbeat(self, fileName, isWrite):
        try:
            from wakatime.main import execute
        except ImportError:
            log(DEBUG, 'Skipping heartbeat because wakatime-cli not found.')
            return
        except:
            log(ERROR, traceback.format_exc())
            return

        self._lastFile = fileName
        self._lastTime = time.time()

        args = ['--entity', fileName, '--plugin', PLUGIN]
        if isWrite:
            args.append('--write')

        log(DEBUG, 'Sending heartbeat:')
        log(DEBUG, ' '.join(args))

        execute(args)

    def _promptForApiKey(self, *args):
        """Prompt the user to enter their api key."""

        ui = e5App().getObject('UserInterface')
        title = 'WakaTime'
        label = '               Enter your WakaTime API Key:               \n' + \
                '               (https://wakatime.com/settings)            '
        default = self._getApiKey()
        text, ok = QInputDialog.getText(ui, title, label, QLineEdit.Normal, default)
        if ok:
            configs = self._parseConfigFile()
            configs.set('settings', 'api_key', text)
            configFile = self._configFile()
            with open(configFile, 'w', encoding='utf-8') as fh:
                configs.write(fh)

    def _createConfigFile(self):
        """Creates the .wakatime.cfg INI file in $HOME directory, if it does
        not already exist.
        """
        configFile = self._configFile()
        try:
            with open(configFile) as fh:
                pass
        except IOError:
            try:
                with open(configFile, 'w') as fh:
                    fh.write("[settings]\n")
                    fh.write("debug = false\n")
                    fh.write("hidefilenames = false\n")
            except IOError:
                pass

    def _getApiKey(self):
        self._createConfigFile()
        key = ''
        try:
            configs = self._parseConfigFile()
            if configs is not None:
                if configs.has_option('settings', 'api_key'):
                    key = configs.get('settings', 'api_key')
        except:
            pass
        return key

    def _getDebug(self):
        self._createConfigFile()
        try:
            configs = self._parseConfigFile()
            if configs is not None:
                if configs.has_option('settings', 'debug'):
                    return configs.get('settings', 'debug') == 'true'
        except:
            pass
        return False

    def _configFile(self):
        home = os.environ.get('WAKATIME_HOME')
        if home:
            return os.path.join(os.path.expanduser(home), '.wakatime.cfg')

        return os.path.join(os.path.expanduser('~'), '.wakatime.cfg')

    def _parseConfigFile(self):
        """Returns a configparser.SafeConfigParser instance with configs
        read from the config file. Default location of the config file is
        at ~/.wakatime.cfg.
        """

        configFile = self._configFile()

        configs = configparser.SafeConfigParser()
        try:
            with open(configFile, 'r', encoding='utf-8') as fh:
                try:
                    configs.readfp(fh)
                    return configs
                except configparser.Error:
                    log(ERROR, traceback.format_exc())
                    return None
        except IOError:
            log(DEBUG, "Error: Could not read from config file {0}\n".format(configFile))
            return None

    def _isCliLatest(self):
        remoteVer = self._getLatestCliVersion()
        about = os.path.join(RESOURCES_FOLDER, 'legacy-python-cli-master', 'wakatime', '__about__.py')
        try:
            with open(about) as fh:
                localVer = self._extractVersion(fh)
            if remoteVer and localVer:
                return remoteVer.strip() == localVer.strip()
        except:
            log(DEBUG, traceback.format_exc())
        return False

    def _getLatestCliVersion(self):
        url = 'https://raw.githubusercontent.com/wakatime/wakatime/master/wakatime/__about__.py'
        try:
            output = os.path.join(RESOURCES_FOLDER, 'remoteVer')
            download(url, output)
            with open(output) as fh:
                return self._extractVersion(fh)
        except:
            return None

    def _extractVersion(self, fh):
        pattern = re.compile(r"^__version_info__ = \('([0-9]+)', '([0-9]+)', '([0-9]+)'\)")
        for line in fh.readlines():
            match = pattern.search(line)
            if match:
                return '{0}.{1}.{2}'.format(match.group(1), match.group(2), match.group(3))
        return None

    def prepareUninstall(self):
        """
        Public method to prepare for an uninstallation.
        """
        e5App().unregisterPluginObject(pluginTypename)

    def prepareUnload(self):
        """
        Public method to prepare for an unload.
        """
        if self.__projectHelperObject:
            self.__projectHelperObject.removeToolbar(
                self.__ui, e5App().getObject("ToolbarManager"))
        e5App().unregisterPluginObject(pluginTypename)

class DownloadCLI(threading.Thread):
    """Non-blocking thread for downloading latest wakatime-cli from GitHub.
    """

    def run(self):
        log(INFO, 'Downloading wakatime-cli...')

        try:
            shutil.rmtree(os.path.join(RESOURCES_FOLDER, 'legacy-python-cli-master'))
        except:
            pass

        try:
            url = 'https://github.com/wakatime/wakatime/archive/master.zip'
            zip_file = os.path.join(RESOURCES_FOLDER, 'legacy-python-cli-master.zip')
            download(url, zip_file)

            log(INFO, 'Extracting wakatime-cli...')
            with ZipFile(zip_file) as zf:
                zf.extractall(RESOURCES_FOLDER)
        except:
            log(DEBUG, traceback.format_exc())

        log(INFO, 'Finished extracting wakatime-cli.')

def download(url, filePath):
    try:
        urlretrieve(url, filePath)
    except IOError:
        ssl._create_default_https_context = ssl._create_unverified_context
        urlretrieve(url, filePath)

def log(lvl, msg, *args, **kwargs):
    if lvl == DEBUG and not IS_DEBUG_ENABLED:
        return
    if len(args) > 0:
        msg = msg.format(*args)
    elif len(kwargs) > 0:
        msg = msg.format(**kwargs)
    date = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
    try:
        print('[WakaTime] [{date}] [{lvl}] {msg}'.format(lvl=lvl, date=date, msg=msg))
    except UnicodeDecodeError:
        print('[WakaTime] [{date}] [{lvl}] {msg}'.format(lvl=lvl, date=date, msg=msg.decode('utf8', 'replace')))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import unittest

import tests.common
import core
from core.localisation import _
from core import Rpg
import models.settings
import json

class rpgTests(tests.common.common):
	pass
<commit_msg>test on the validity of the world added<commit_after># -*- coding: utf-8 -*-
import unittest

import tests.common
import core
from core.localisation import _
from core import Rpg
import models.settings
import json

class rpgTests(tests.common.common):
		def test_unknown_world(self):
			rpgEngine = Rpg.Rpg()
			try:
				rpgEngine.init("some/unexisting/world", "uselessLogin")
			except core.exception.exception as e:
				self.assertEquals(e.message, _('ERROR_UNKNOWN_SELECTED_WORLD'))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Started: February 7, 2016
# Finished: February 9, 2016
# GitHub Repository Name: MusicDownloader

# Modules:
import webbrowser # This module can control the browser
import json # Json encoder/decoder
from bs4 import BeautifulSoup # Module to sort through HTML
import lxml # Module to prepare html for BeautifulSoup
import urllib2 # Gets html
import sys # Allow more control over printing
import string # More ways to manipulate strings
import unidecode # Decodes weird characters
import youtube_dl # For downloading YouTube videos/audio
import eyed3 # For editing ID3 tags for mp3 file
import os # More control over Mac file system

# Prompt User for Keywords for Song
userSearch = raw_input("Search for song: ") # Reads input as a string
# userSearch = input("Search for song (use quotes): ") # Reads input as raw code
# print("Searching for " + userSearch)
userSearch = userSearch.strip() # Remove extraneous white space

# Search for song in iTunes Store
# Documentation: http://www.apple.com/itunes/affiliates/resources/documentation/itunes-store-web-service-search-api.html
baseURL = "https://itunes.apple.com/search?"
searchKeys = [
    ["term", userSearch],
    ["country", "US"],
    ["media", "music"],
    ["entity", "song"],
    ["limit", "50"],
    ["lang", "en_us"],
    ["explicit", "yes"]
]
finalURL = baseURL
for i in range(0, len(searchKeys)): # len() returns length of a variable
    # print "Term: %d" % (i)
    currentKey = searchKeys[i]
    criteria = str(currentKey[1]) #Make sure it's a string
    criteria = criteria.replace(" ", "%20") # %20 represents a space
    appendStr = currentKey[0] + "=" + criteria # Build url
    # print(appendStr)
    if i < (len(searchKeys) - 1):
        appendStr += "&"
    finalURL += appendStr

# print("Final URL: " + finalURL) # Debugging
# webbrowser.open(finalURL)

# Retrieve and Save iTunes JSON Data
response = urllib2.urlopen(finalURL) #Get HTML source code
html = response.read() #HTML source code
soup = BeautifulSoup(html, "lxml") # Using lxml parser
print("")
print("*********** Found iTunes data ***********")
print("")
# print(soup.prettify()) # Feedback

rawJSON = soup.find('p').text # Just the json text
rawJSON.strip() # Trim the white space

# Parse iTunes JSON Data
iTunesObj = json.loads(rawJSON) # Decode JSON
# print(iTunesObj)

numShow = 5

results = iTunesObj['results']
b = numShow
if len(results) < numShow:
    b = len(results)

for i in range(0, b):
    sys.stdout.write("(%i) Track Name: " % i)
    sys.stdout.flush() # No line break
    print results[i]['trackName'] # Adds a line break after
    print "    Artist: %s" % results[i]['artistName']
    print "    Album: %s" % results[i]['collectionName']
    print "    Genre: %s" % results[i]['primaryGenreName']
    print("")

print("Which song is the one you were looking for?")
iTunesSearchSelection = input("Type the respective index: ")
songData = results[iTunesSearchSelection]
print "" # Line break
print("Selected:")
print "%s by %s" % (songData['trackName'], songData['artistName'])
print "" # Line break

# *******************   Find song on YouTube   *******************

searchAudio = raw_input("Search for audio video? (y/n) ") # Ask if want to search for audio on YouTube
extra = ""
if searchAudio is "y": # If only want to search for audio videos
    extra = " Audio" # add on 'audio' to search

baseURL = "https://www.youtube.com/results?search_query="
YouTubeSearch = songData['trackName'] + " " + songData['artistName'] + extra
print "" # Line break

YouTubeSearch = unidecode.unidecode(YouTubeSearch) # Remove complex unicode characters
print "Searching for '%s' on YouTube" % YouTubeSearch
print "" # Line break
out = YouTubeSearch.translate(string.maketrans("",""), string.punctuation) # Remove punctuation
YouTubeSearch = YouTubeSearch.replace(" ", "+") # Remove spaces with '+'
finalURL = baseURL + YouTubeSearch # Final URL

response = urllib2.urlopen(finalURL) #Get HTML source code
html = response.read() #HTML source code
soup = BeautifulSoup(html, "lxml") # Using lxml parser

videoLinks = soup.findAll("a", { "class": "yt-uix-sessionlink yt-uix-tile-link yt-ui-ellipsis yt-ui-ellipsis-2       spf-link " })
videoUploaders = soup.findAll("a", { "class": "yt-uix-sessionlink g-hovercard      spf-link " })
videoTimes = soup.findAll("div", { "class": "yt-lockup-thumbnail" }) # In case there are playlists, find the div

videos = [];
# Stores all the results on the page except for the last 3 hits on the page
upper = len(videoTimes) - 3
numPlaylists = 0
for i in range(0, upper):
    time = videoTimes[i].findAll("span", { "class": "video-time" }) # Find within the larger div
    if not time: # If array is empty (ie. no time found for that video)
        numPlaylists += 1
        # print "Found a playlist"
    else: # If not a playlists
        # The video must be a playlist
        time = time[0] # First result

        link = "https://www.youtube.com" + videoLinks[i].get('href')

        # print videoLinks[i].contents[0]
        # Structure of array:
        # [name, link, uploader, length]
        videos.append(
            [
                videoLinks[i].contents[0],
                link,
                videoUploaders[i].contents[0],
                time.text
            ]
        )

# Only returns up to specified number
print "Found %s playlist(s)" % numPlaylists
for i in range(0, numShow):
    video = videos[i]
    sys.stdout.write("(%i) Video name: " % i)
    sys.stdout.flush() # No line break
    print video[0] # Adds a line break after
    print "    Link: %s" % video[1]
    print "    Uploader: %s" % video[2]
    print "    Length: %s" % video[3]
    print("")

milliseconds = songData['trackTimeMillis']
x = milliseconds / 1000
seconds = x % 60
x /= 60
minutes = x % 60

time = str(minutes) + ":" + str(seconds)

print "Which video is the one you were looking for?"
print "The iTunes version is: %s" % time
YouTubeSelection = input("Type the respective index: ")
print "" # Line break
data = videos[YouTubeSelection]

fileName = songData['artistName'] + " - " + songData['trackName'] # Declare file name
filePath = "~/Desktop/" # Declare file path

ydl_opts = { # Set options
    'format': 'bestaudio/best',
    # 'outtmpl': u'%(title)s-%(id)s.%(ext)s',
    'outtmpl': filePath + fileName + ".%(ext)s",
    'postprocessors': [{
        'key': 'FFmpegExtractAudio',
        'preferredcodec': 'mp3',
        'preferredquality': '192' # 128, 160, 192, 210, 256
    }],
}

with youtube_dl.YoutubeDL(ydl_opts) as ydl:
    ydl.download([data[1]]) # Download the song

# *******************   Find Image Artwork   *******************
# print("Finding Google Image for album artwork")
# Add code here:

# *******************   Update ID3 Tags   *******************

mp3Path = os.path.expanduser(filePath + fileName + ".mp3")
year = str(songData['releaseDate'])
year = int(year[:4])

audiofile = eyed3.load(mp3Path)
audiofile.tag.title = songData['trackName']
audiofile.tag.artist = songData['artistName']
audiofile.tag.album = songData['collectionName']
audiofile.tag.album_artist = songData['artistName'] # This needs to be changed - need to be able to find album artist, not song artist
audiofile.tag.track_num = (songData['trackNumber'], songData['trackCount'])
audiofile.tag.disc_num = (songData['discNumber'], songData['discCount'])
audiofile.tag.genre = songData['primaryGenreName']
audiofile.tag.release_date = year
audiofile.tag.orig_release_date = year
audiofile.tag.recording_date = year
audiofile.tag.encoding_date = year
audiofile.tag.taggin_date = year

# Append Image
# Reference: http://tuxpool.blogspot.com/2013/02/how-to-store-images-in-mp3-files-using.html
# imageData = open("test.jpg", "rb").read() # Stores image data
# audiofile.tag.images.set(3, imageData, "image/jpeg", "Description") # 3 for front cover, 4 for back, 0 for other

audiofile.tag.save()

print "" # Line break
print "Updated ID3 Tags"
# print "Song Year (Must Manually Add): %s" % year
print "" # Line break
print "**************   Complete   **************"
<commit_msg>Updated to work with new YouTube site structure<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Started: February 7, 2016
# Finished: February 9, 2016
# GitHub Repository Name: MusicDownloader

# Modules:
import webbrowser # This module can control the browser
import json # Json encoder/decoder
from bs4 import BeautifulSoup # Module to sort through HTML
import lxml # Module to prepare html for BeautifulSoup
import urllib2 # Gets html
import sys # Allow more control over printing
import string # More ways to manipulate strings
import unidecode # Decodes weird characters
import youtube_dl # For downloading YouTube videos/audio
import eyed3 # For editing ID3 tags for mp3 file
import os # More control over Mac file system

# Prompt User for Keywords for Song
userSearch = raw_input("Search for song: ") # Reads input as a string
# userSearch = input("Search for song (use quotes): ") # Reads input as raw code
# print("Searching for " + userSearch)
userSearch = userSearch.strip() # Remove extraneous white space

# Search for song in iTunes Store
# Documentation: http://www.apple.com/itunes/affiliates/resources/documentation/itunes-store-web-service-search-api.html
baseURL = "https://itunes.apple.com/search?"
searchKeys = [
    ["term", userSearch],
    ["country", "US"],
    ["media", "music"],
    ["entity", "song"],
    ["limit", "50"],
    ["lang", "en_us"],
    ["explicit", "yes"]
]
finalURL = baseURL
for i in range(0, len(searchKeys)): # len() returns length of a variable
    # print "Term: %d" % (i)
    currentKey = searchKeys[i]
    criteria = str(currentKey[1]) #Make sure it's a string
    criteria = criteria.replace(" ", "%20") # %20 represents a space
    appendStr = currentKey[0] + "=" + criteria # Build url
    # print(appendStr)
    if i < (len(searchKeys) - 1):
        appendStr += "&"
    finalURL += appendStr

# print("Final URL: " + finalURL) # Debugging
# webbrowser.open(finalURL)

# Retrieve and Save iTunes JSON Data
response = urllib2.urlopen(finalURL) #Get HTML source code
html = response.read() #HTML source code
soup = BeautifulSoup(html, "lxml") # Using lxml parser
print("")
print("*********** Found iTunes data ***********")
print("")
# print(soup.prettify()) # Feedback

rawJSON = soup.find('p').text # Just the json text
rawJSON.strip() # Trim the white space

# Parse iTunes JSON Data
iTunesObj = json.loads(rawJSON) # Decode JSON
# print(iTunesObj)

numShow = 5

results = iTunesObj['results']
b = numShow
if len(results) < numShow:
    b = len(results)

for i in range(0, b):
    sys.stdout.write("(%i) Track Name: " % i)
    sys.stdout.flush() # No line break
    print results[i]['trackName'] # Adds a line break after
    print "    Artist: %s" % results[i]['artistName']
    print "    Album: %s" % results[i]['collectionName']
    print "    Genre: %s" % results[i]['primaryGenreName']
    print("")

print("Which song is the one you were looking for?")
iTunesSearchSelection = input("Type the respective index: ")
songData = results[iTunesSearchSelection]
print "" # Line break
print("Selected:")
print "%s by %s" % (songData['trackName'], songData['artistName'])
print "" # Line break

# *******************   Find song on YouTube   *******************

searchAudio = raw_input("Search for audio video? (y/n) ") # Ask if want to search for audio on YouTube
extra = ""
if searchAudio is "y": # If only want to search for audio videos
    extra = " Audio" # add on 'audio' to search

baseURL = "https://www.youtube.com/results?search_query="
YouTubeSearch = songData['trackName'] + " " + songData['artistName'] + extra
print "" # Line break

YouTubeSearch = unidecode.unidecode(YouTubeSearch) # Remove complex unicode characters
print "Searching for '%s' on YouTube" % YouTubeSearch
print "" # Line break
out = YouTubeSearch.translate(string.maketrans("",""), string.punctuation) # Remove punctuation
YouTubeSearch = YouTubeSearch.replace(" ", "+") # Remove spaces with '+'
finalURL = baseURL + YouTubeSearch # Final URL

response = urllib2.urlopen(finalURL) #Get HTML source code
html = response.read() #HTML source code
soup = BeautifulSoup(html, "lxml") # Using lxml parser

videoLinks = soup.findAll("a", { "class": "yt-uix-sessionlink yt-uix-tile-link yt-ui-ellipsis yt-ui-ellipsis-2       spf-link " })
videoUploaders = soup.findAll("a", { "class": "g-hovercard yt-uix-sessionlink      spf-link " })
videoTimes = soup.findAll("div", { "class": "yt-lockup-thumbnail" }) # In case there are playlists, find the div
# print len(videoUploaders)

videos = [];
# Stores all the results on the page except for the last 3 hits on the page
upper = len(videoTimes) - 3
numPlaylists = 0
for i in range(0, upper):
    # print i
    time = videoTimes[i].findAll("span", { "class": "video-time" }) # Find within the larger div
    if not time: # If array is empty (ie. no time found for that video)
        numPlaylists += 1
        # print "Found a playlist"
    else: # If not a playlists
        # The video must be a playlist
        time = time[0] # First result

        link = "https://www.youtube.com" + videoLinks[i].get('href')
        # print link

        # print videoUploaders[i]

        # print videoLinks[i].contents[0]
        # Structure of array:
        # [name, link, uploader, length]
        videos.append(
            [
                videoLinks[i].contents[0],
                link,
                videoUploaders[i].contents[0],
                time.text
            ]
        )

# Only returns up to specified number
print "Found %s playlist(s)" % numPlaylists
for i in range(0, numShow):
    video = videos[i]
    sys.stdout.write("(%i) Video name: " % i)
    sys.stdout.flush() # No line break
    print video[0] # Adds a line break after
    print "    Link: %s" % video[1]
    print "    Uploader: %s" % video[2]
    print "    Length: %s" % video[3]
    print("")

milliseconds = songData['trackTimeMillis']
x = milliseconds / 1000
seconds = x % 60
x /= 60
minutes = x % 60

time = str(minutes) + ":" + str(seconds)

print "Which video is the one you were looking for?"
print "The iTunes version is: %s" % time
YouTubeSelection = input("Type the respective index: ")
print "" # Line break
data = videos[YouTubeSelection]

fileName = songData['artistName'] + " - " + songData['trackName'] # Declare file name
filePath = "~/Desktop/" # Declare file path

ydl_opts = { # Set options
    'format': 'bestaudio/best',
    # 'outtmpl': u'%(title)s-%(id)s.%(ext)s',
    'outtmpl': filePath + fileName + ".%(ext)s",
    'postprocessors': [{
        'key': 'FFmpegExtractAudio',
        'preferredcodec': 'mp3',
        'preferredquality': '192' # 128, 160, 192, 210, 256
    }],
}

with youtube_dl.YoutubeDL(ydl_opts) as ydl:
    ydl.download([data[1]]) # Download the song

# *******************   Find Image Artwork   *******************
# print("Finding Google Image for album artwork")
# Add code here:

# *******************   Update ID3 Tags   *******************

mp3Path = os.path.expanduser(filePath + fileName + ".mp3")
year = str(songData['releaseDate'])
year = int(year[:4])

audiofile = eyed3.load(mp3Path)
audiofile.tag.title = songData['trackName']
audiofile.tag.artist = songData['artistName']
audiofile.tag.album = songData['collectionName']
audiofile.tag.album_artist = songData['artistName'] # This needs to be changed - need to be able to find album artist, not song artist
audiofile.tag.track_num = (songData['trackNumber'], songData['trackCount'])
audiofile.tag.disc_num = (songData['discNumber'], songData['discCount'])
audiofile.tag.genre = songData['primaryGenreName']
audiofile.tag.release_date = year
audiofile.tag.orig_release_date = year
audiofile.tag.recording_date = year
audiofile.tag.encoding_date = year
audiofile.tag.taggin_date = year

# Append Image
# Reference: http://tuxpool.blogspot.com/2013/02/how-to-store-images-in-mp3-files-using.html
# imageData = open("test.jpg", "rb").read() # Stores image data
# audiofile.tag.images.set(3, imageData, "image/jpeg", "Description") # 3 for front cover, 4 for back, 0 for other

audiofile.tag.save()

print "" # Line break
print "Updated ID3 Tags"
# print "Song Year (Must Manually Add): %s" % year
print "" # Line break
print "**************   Complete   **************"
<|endoftext|>


<commit_before>import argparse
import logging
import logging.config
import requests
import ConfigParser
import os
from time import sleep
from urlparse import urljoin
from redis import Redis
from circus.client import CircusClient
from datetime import datetime
from pytz import timezone

import iso8601

logger = logging.getLogger(__name__)

def hook(watcher, arbiter, hook_name, **kwargs):
    logger.info("remove watcher {}".format(watcher.name))
    watcher = arbiter._watchers_names.pop(watcher.name)
    del arbiter.watchers[arbiter.watchers.index(watcher)]
    return True

class AuctionsDataBridge(object):
    """docstring for AuctionsDataBridge"""
    def __init__(self, config):
        super(AuctionsDataBridge, self).__init__()
        self.config = config
        self.tenders_url = urljoin(
            self.config_get('tenders_api_server'),
            '/api/{}/tenders'.format(
                self.config_get('tenders_api_version')
            )
        )
        self.tz = timezone('Europe/Kiev')
        self.couch_url = urljoin(self.config_get('couch_url'), self.config_get('auctions_db'))
        self.current_worker_port = int(self.config_get('starts_port'))
        self.mapings = Redis.from_url(self.config_get('redis_url'))
        self.circus_client = CircusClient(endpoint=self.config_get('circus_endpoint'))
        self.url = self.tenders_url

    def config_get(self, name):
        return self.config.get('main', name)

    def tender_url(self, tender_id):
        return urljoin(self.tenders_url, 'tenders/{}/auction'.format(tender_id))

    def get_teders_list(self):

        while True:
            params = {'offset': self.offset, 'opt_fields': 'status,auctionPeriod'}
            logger.debug('Start request to {}, params: {}'.format(
                self.url, params))
            response = requests.get(self.url, params=params)

            logger.debug('Request response: {}'.format(response.status_code))
            if response.ok:
                response_json = response.json()
                if len(response_json['data']) == 0:
                    logger.info("Change offset date to {}".format(response_json['next_page']['offset']))
                    self.offset = response_json['next_page']['offset']
                    break
                for item in response_json['data']:
                    if 'auctionPeriod' in item \
                            and 'startDate' in item['auctionPeriod'] \
                            and 'endDate' not in item['auctionPeriod'] \
                            and item['status'] == "auction":

                        date = iso8601.parse_date(item['auctionPeriod']['startDate'])
                        date = date.astimezone(self.tz)
                        if datetime.now(self.tz) > date:
                            continue

                        yield item
                logger.info("Change offset date to {}".format(response_json['next_page']['offset']))
                self.offset = response_json['next_page']['offset']

    def start_auction_worker(self, tender_item):
        self.mapings.set(tender_item['id'], "http://localhost:{}/".format(self.current_worker_port))
        logger.info(self.circus_client.call(
            {
                "command": "add",
                "properties": {
                    "cmd": self.config_get('auction_worker'),
                    "args": 'planning {} {} {}'.format(tender_item['id'],
                                                       self.current_worker_port,
                                                       self.couch_url),
                    "name": "auction_worker_{}".format(tender_item['id']),
                    "start": True,
                    "options": {
                        "shell": True,
                        "hooks": {
                            "after_stop": ("openprocurement.auction.databridge.hook",
                                           True)
                        },
                        "respawn": False,
                        "stdout_stream": {
                            "class": "StdoutStream"
                        },
                        "stderr_stream": {
                            "class": "StdoutStream"
                        }
                    }
                }
            }
        ))
        self.current_worker_port += 1

    def run(self):
        logger.info('Start Auctions Bridge')
        self.offset = ''
        while True:
            logger.info('Start data sync...')
            for tender_item in self.get_teders_list():
                logger.debug('Item {}'.format(tender_item))
                self.start_auction_worker(tender_item)
                sleep(3)
            logger.info('Wait...')
            sleep(100)

def main():
    parser = argparse.ArgumentParser(description='---- Auctions Bridge ----')
    parser.add_argument('config', type=str, help='Path to configuration file')
    params = parser.parse_args()
    if os.path.isfile(params.config):
        logging.config.fileConfig(params.config)
        config = ConfigParser.ConfigParser()
        config.read(params.config)
        AuctionsDataBridge(config).run()

##############################################################

if __name__ == "__main__":
    main()
<commit_msg>Improve planning of auction<commit_after>import argparse
import logging
import logging.config
import requests
import ConfigParser
import os
from time import sleep
from urlparse import urljoin
from redis import Redis
from circus.client import CircusClient
from datetime import datetime
from pytz import timezone
import subprocess

import iso8601

logger = logging.getLogger(__name__)

def hook(watcher, arbiter, hook_name, **kwargs):
    logger.info("remove watcher {}".format(watcher.name))
    watcher = arbiter._watchers_names.pop(watcher.name)
    del arbiter.watchers[arbiter.watchers.index(watcher)]
    return True

class AuctionsDataBridge(object):
    """docstring for AuctionsDataBridge"""
    def __init__(self, config):
        super(AuctionsDataBridge, self).__init__()
        self.config = config
        self.tenders_url = urljoin(
            self.config_get('tenders_api_server'),
            '/api/{}/tenders'.format(
                self.config_get('tenders_api_version')
            )
        )
        self.tz = timezone('Europe/Kiev')
        self.couch_url = urljoin(self.config_get('couch_url'), self.config_get('auctions_db'))
        self.current_worker_port = int(self.config_get('starts_port'))
        self.mapings = Redis.from_url(self.config_get('redis_url'))
        self.circus_client = CircusClient(endpoint=self.config_get('circus_endpoint'))
        self.url = self.tenders_url

    def config_get(self, name):
        return self.config.get('main', name)

    def tender_url(self, tender_id):
        return urljoin(self.tenders_url, 'tenders/{}/auction'.format(tender_id))

    def get_teders_list(self):

        while True:
            params = {'offset': self.offset, 'opt_fields': 'status,auctionPeriod'}
            logger.debug('Start request to {}, params: {}'.format(
                self.url, params))
            response = requests.get(self.url, params=params)

            logger.debug('Request response: {}'.format(response.status_code))
            if response.ok:
                response_json = response.json()
                if len(response_json['data']) == 0:
                    logger.info("Change offset date to {}".format(response_json['next_page']['offset']))
                    self.offset = response_json['next_page']['offset']
                    break
                for item in response_json['data']:
                    if 'auctionPeriod' in item \
                            and 'startDate' in item['auctionPeriod'] \
                            and 'endDate' not in item['auctionPeriod'] \
                            and item['status'] == "auction":

                        date = iso8601.parse_date(item['auctionPeriod']['startDate'])
                        date = date.astimezone(self.tz)
                        if datetime.now(self.tz) > date:
                            continue

                        yield item
                logger.info("Change offset date to {}".format(response_json['next_page']['offset']))
                self.offset = response_json['next_page']['offset']

    def start_auction_worker(self, tender_item):
        self.mapings.set(tender_item['id'], "http://localhost:{}/".format(self.current_worker_port))
        logger.info(
            subprocess.check_output(
                [self.config_get('auction_worker'),
                 'planning', str(tender_item['id']),
                 str(self.current_worker_port),
                 str(self.couch_url)
                 ]
            )
        )
        self.current_worker_port += 1

    def run(self):
        logger.info('Start Auctions Bridge')
        self.offset = ''
        while True:
            logger.info('Start data sync...')
            for tender_item in self.get_teders_list():
                logger.debug('Item {}'.format(tender_item))
                self.start_auction_worker(tender_item)
                sleep(3)
            logger.info('Wait...')
            sleep(100)

def main():
    parser = argparse.ArgumentParser(description='---- Auctions Bridge ----')
    parser.add_argument('config', type=str, help='Path to configuration file')
    params = parser.parse_args()
    if os.path.isfile(params.config):
        logging.config.fileConfig(params.config)
        config = ConfigParser.ConfigParser()
        config.read(params.config)
        AuctionsDataBridge(config).run()

##############################################################

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>__version__ = '0.28.0'
<commit_msg>update dev version after 0.28.0 tag [skip ci]<commit_after>__version__ = '0.28.0.dev2909'
<|endoftext|>


<commit_before>from __future__ import absolute_import
from sfepy.base.base import assert_
from sfepy.base.testing import TestCommon

##
# 28.08.2007, c
class Test( TestCommon ):

    ##
    # 28.08.2007, c
    def from_conf( conf, options ):
        return Test( conf = conf, options = options )
    from_conf = staticmethod( from_conf )

    ##
    # 28.08.2007, c
    def test_struct_add( self ):
        from sfepy.base.base import Struct
        from copy import deepcopy

        a = Struct( f1 = 0,
                    f2 = [1, 2, 3],
                    f3 = Struct( ff = 'abc' ),
                    f4 = 3.14 )
        a0 = deepcopy( a )
        b = Struct( f1 = 5,
                    f2 = [1],
                    f3 = Struct( ff = '', gg = 123 ),
                    f5 = 'new one' )
        c = a + b

        assert_( c.f1 == 0 )
        assert_( c.f2 == [1, 2, 3] )
        assert_( c.f3.ff == 'abc' )
        assert_( c.f3.gg == 123 )
        assert_( c.f4 == 3.14 )
        assert_( c.f5 == 'new one' )

        assert_( a.f1 == a0.f1 )
        assert_( a.f2 == a0.f2 )
        assert_( a.f3.ff == a0.f3.ff )
        assert_( a.f4 == a0.f4 )

        return True

    ##
    # 28.08.2007, c
    def test_struct_i_add( self ):
        from sfepy.base.base import Struct

        a = Struct( f1 = 0,
                    f2 = [1, 2, 3],
                    f3 = Struct( ff = 'abc' ) )
        b = Struct( f1 = 5,
                    f2 = [1],
                    f3 = Struct( ff = '', gg = 123 ),
                    f4 = 'new one' )
        a += b

        assert_( a.f1 == 0 )
        assert_( a.f2 == [1, 2, 3] )
        assert_( a.f3.ff == 'abc' )
        assert_( a.f3.gg == 123 )
        assert_( a.f4 == 'new one' )

        return True

    def test_container_add(self):
        from sfepy.base.base import Struct, Container

        a = Struct(name='a')
        b = Struct(name='b')

        c1 = Container()

        c1 = c1 + c1
        assert_(c1.names == [])

        c1 += Container([a, b])
        assert_(c1.names == ['a', 'b'])

        c2 = c1 + c1
        assert_(c2.names == 2 * ['a', 'b'])

        c2 += c2
        assert_(c2.names == 4 * ['a', 'b'])

        return True

    def test_verbose_output(self):
        try:
            from StringIO import StringIO  # Python 2
        except ImportError:
            from io import StringIO        # Python 3

        from sfepy.base.base import Output, goptions

        fd = StringIO()

        output = Output('test', filename=fd)

        output('test1')
        goptions['verbose'] = False
        output('test2')
        goptions['verbose'] = 1
        output('test3')

        _ok1 = bool(goptions['verbose'])
        _ok2 = fd.getvalue() == 'test test1\ntest test3\n'

        fd.close()

        ok = _ok1 and _ok2

        return ok

    def test_resolve_deps(self):
        from sfepy.base.resolve_deps import resolve

        deps = {
            'a' : ['a', 'b'],
            'b' : ['a', 'b'],
            'c' : ['b', 'c', 'd', 'e'],
            'd' : ['c', 'e'],
            'e' : ['d', 'e'],
            'f' : ['e', 'f', 'g'],
            'g' : ['g'],
        }
        order = resolve(deps)
        ok1 = order == [['g'], ['a', 'b'], ['c', 'd', 'e'], ['f']]

        deps = {
            'a' : ['b'],
            'b' : ['c'],
            'c' : ['a'],
        }
        order = resolve(deps)
        ok2 = order == [['a', 'b', 'c']]

        return ok1 and ok2

    def test_parse_conf(self):
        from sfepy.base.parse_conf import list_dict
        from sfepy.base.conf import dict_from_string
        ld = list_dict()

        def parse(x):
            out = ld.parseString(x, True)[0]
            return out

        def parse_dict(x):
            out = dict_from_string(x)
            return out

        assert_(parse("1,2") == ([1,2],{}))
        assert_(parse("1,[2]") == ([1,[2]],{}))
        assert_(parse("1,[2,4],c=3") == ([1,[2,4]],{'c':3}))
        assert_(parse("1,(2),c:3,uu=7") == ([1,(2,)],{'c':3,'uu':7}))
        assert_(parse("'long string ([\"',(2,5),c:3") ==
                     (['long string (["',(2,5)],{'c':3}))

        assert_(parse_dict('') == {})
        assert_(parse_dict('a:[]') == {'a':[]})
        assert_(parse_dict('a:{}') == {'a':{}})
        assert_(parse_dict('1:2,a:{},3:4') == {1:2,'a':{},3:4})

        assert_(parse('') == ([],{}))
        assert_(parse('[ ]') == ([[]],{}))
        assert_(parse('[]') == ([[]],{}))
        assert_(parse('[[]]') == ([[[]]],{}))
        assert_(parse('[[[]]]')==([[[[]]]],{}))
        assert_(parse('a,{},[],None,True,False,"False"') ==
                         (['a',{},[],None,True,False,"False"],{}))

        return True
<commit_msg>Code cleaning<commit_after>from __future__ import absolute_import
from sfepy.base.base import assert_
from sfepy.base.testing import TestCommon

##
# 28.08.2007, c
class Test( TestCommon ):

    ##
    # 28.08.2007, c
    def from_conf( conf, options ):
        return Test( conf = conf, options = options )
    from_conf = staticmethod( from_conf )

    ##
    # 28.08.2007, c
    def test_struct_add( self ):
        from sfepy.base.base import Struct
        from copy import deepcopy

        a = Struct( f1 = 0,
                    f2 = [1, 2, 3],
                    f3 = Struct( ff = 'abc' ),
                    f4 = 3.14 )
        a0 = deepcopy( a )
        b = Struct( f1 = 5,
                    f2 = [1],
                    f3 = Struct( ff = '', gg = 123 ),
                    f5 = 'new one' )
        c = a + b

        assert_( c.f1 == 0 )
        assert_( c.f2 == [1, 2, 3] )
        assert_( c.f3.ff == 'abc' )
        assert_( c.f3.gg == 123 )
        assert_( c.f4 == 3.14 )
        assert_( c.f5 == 'new one' )

        assert_( a.f1 == a0.f1 )
        assert_( a.f2 == a0.f2 )
        assert_( a.f3.ff == a0.f3.ff )
        assert_( a.f4 == a0.f4 )

        return True

    ##
    # 28.08.2007, c
    def test_struct_i_add( self ):
        from sfepy.base.base import Struct

        a = Struct( f1 = 0,
                    f2 = [1, 2, 3],
                    f3 = Struct( ff = 'abc' ) )
        b = Struct( f1 = 5,
                    f2 = [1],
                    f3 = Struct( ff = '', gg = 123 ),
                    f4 = 'new one' )
        a += b

        assert_( a.f1 == 0 )
        assert_( a.f2 == [1, 2, 3] )
        assert_( a.f3.ff == 'abc' )
        assert_( a.f3.gg == 123 )
        assert_( a.f4 == 'new one' )

        return True

    def test_container_add(self):
        from sfepy.base.base import Struct, Container

        a = Struct(name='a')
        b = Struct(name='b')

        c1 = Container()

        c1 = c1 + c1
        assert_(c1.names == [])

        c1 += Container([a, b])
        assert_(c1.names == ['a', 'b'])

        c2 = c1 + c1
        assert_(c2.names == 2 * ['a', 'b'])

        c2 += c2
        assert_(c2.names == 4 * ['a', 'b'])

        return True

    def test_verbose_output(self):
        try:
            from StringIO import StringIO  # Python 2
        except ImportError:
            from io import StringIO        # Python 3

        from sfepy.base.base import Output, goptions

        fd = StringIO()

        output = Output('test', filename=fd)

        output('test1')
        goptions['verbose'] = False
        output('test2')
        goptions['verbose'] = 1
        output('test3')

        _ok1 = bool(goptions['verbose'])
        _ok2 = fd.getvalue() == 'test test1\ntest test3\n'

        fd.close()

        ok = _ok1 and _ok2

        return ok

    def test_resolve_deps(self):
        from sfepy.base.resolve_deps import resolve

        deps = {
            'a' : ['a', 'b'],
            'b' : ['a', 'b'],
            'c' : ['b', 'c', 'd', 'e'],
            'd' : ['c', 'e'],
            'e' : ['d', 'e'],
            'f' : ['e', 'f', 'g'],
            'g' : ['g'],
        }
        order = resolve(deps)
        ok1 = order == [['g'], ['a', 'b'], ['c', 'd', 'e'], ['f']]

        deps = {
            'a' : ['b'],
            'b' : ['c'],
            'c' : ['a'],
        }
        order = resolve(deps)
        ok2 = order == [['a', 'b', 'c']]

        return ok1 and ok2

    def test_parse_conf(self):
        from sfepy.base.parse_conf import list_dict
        from sfepy.base.conf import dict_from_string as parse_dict
        ld = list_dict()

        def parse(x):
            out = ld.parseString(x, True)[0]
            return out

        assert_(parse("1,2") == ([1,2],{}))
        assert_(parse("1,[2]") == ([1,[2]],{}))
        assert_(parse("1,[2,4],c=3") == ([1,[2,4]],{'c':3}))
        assert_(parse("1,(2),c:3,uu=7") == ([1,(2,)],{'c':3,'uu':7}))
        assert_(parse("'long string ([\"',(2,5),c:3") ==
                     (['long string (["',(2,5)],{'c':3}))

        assert_(parse_dict('') == {})
        assert_(parse_dict('a:[]') == {'a':[]})
        assert_(parse_dict('a:{}') == {'a':{}})
        assert_(parse_dict('1:2,a:{},3:4') == {1:2,'a':{},3:4})

        assert_(parse('') == ([],{}))
        assert_(parse('[ ]') == ([[]],{}))
        assert_(parse('[]') == ([[]],{}))
        assert_(parse('[[]]') == ([[[]]],{}))
        assert_(parse('[[[]]]')==([[[[]]]],{}))
        assert_(parse('a,{},[],None,True,False,"False"') ==
                         (['a',{},[],None,True,False,"False"],{}))

        return True
<|endoftext|>


<commit_before>import pytest

from rocketchat_API.APIExceptions.RocketExceptions import RocketMissingParamException

def test_chat_post_notext_message(logged_rocket):
    chat_post_message = logged_rocket.chat_post_message(None, channel="GENERAL").json()
    assert chat_post_message.get("channel") == "GENERAL"
    assert chat_post_message.get("message").get("msg") == ""
    assert chat_post_message.get("success")

def test_chat_post_update_delete_message(logged_rocket):
    chat_post_message = logged_rocket.chat_post_message(
        "hello", channel="GENERAL"
    ).json()
    assert chat_post_message.get("channel") == "GENERAL"
    assert chat_post_message.get("message").get("msg") == "hello"
    assert chat_post_message.get("success")

    with pytest.raises(RocketMissingParamException):
        logged_rocket.chat_post_message(text="text")

    msg_id = chat_post_message.get("message").get("_id")
    chat_get_message = logged_rocket.chat_get_message(msg_id=msg_id).json()
    assert chat_get_message.get("message").get("_id") == msg_id

    chat_update = logged_rocket.chat_update(
        room_id=chat_post_message.get("channel"),
        msg_id=chat_post_message.get("message").get("_id"),
        text="hello again",
    ).json()

    assert chat_update.get("message").get("msg") == "hello again"
    assert chat_update.get("success")

    chat_delete = logged_rocket.chat_delete(
        room_id=chat_post_message.get("channel"),
        msg_id=chat_post_message.get("message").get("_id"),
    ).json()
    assert chat_delete.get("success")

def test_chat_post_notext_message(logged_rocket):
    message = {"rid": "GENERAL"}
    chat_send_message = logged_rocket.chat_send_message(message).json()
    assert chat_send_message.get("message").get("rid") == "GENERAL"
    assert chat_send_message.get("message").get("msg") == ""
    assert chat_send_message.get("success")

def test_chat_send_custom_id_message(logged_rocket):
    message = {"rid": "GENERAL", "msg": "Hallo Welt", "_id": "CUSTOM"}
    chat_send_message = logged_rocket.chat_send_message(message).json()
    assert chat_send_message.get("message").get("rid") == "GENERAL"
    assert chat_send_message.get("message").get("msg") == "Hallo Welt"
    assert chat_send_message.get("message").get("_id") == "CUSTOM"
    assert chat_send_message.get("success")

def test_chat_post_react(logged_rocket):
    message_id = (
        logged_rocket.chat_post_message("hello", channel="GENERAL")
        .json()
        .get("message")
        .get("_id")
    )
    chat_react = logged_rocket.chat_react(msg_id=message_id).json()
    assert chat_react.get("success")

def test_post_pin_unpin(logged_rocket):
    message_id = (
        logged_rocket.chat_post_message("hello", channel="GENERAL")
        .json()
        .get("message")
        .get("_id")
    )
    chat_pin_message = logged_rocket.chat_pin_message(message_id).json()
    assert chat_pin_message.get("success")
    assert chat_pin_message.get("message").get("t") == "message_pinned"

    chat_unpin_message = logged_rocket.chat_unpin_message(message_id).json()
    assert chat_unpin_message.get("success")

def test_post_star_unstar(logged_rocket):
    message_id = (
        logged_rocket.chat_post_message("hello", channel="GENERAL")
        .json()
        .get("message")
        .get("_id")
    )
    chat_star_message = logged_rocket.chat_star_message(message_id).json()
    assert chat_star_message.get("success")

    chat_unstar_message = logged_rocket.chat_unstar_message(message_id).json()
    assert chat_unstar_message.get("success")

def test_chat_search(logged_rocket):
    chat_search = logged_rocket.chat_search(
        room_id="GENERAL", search_text="hello"
    ).json()
    assert chat_search.get("success")

def test_chat_get_message_read_receipts(logged_rocket):
    message_id = (
        logged_rocket.chat_post_message("hello", channel="GENERAL")
        .json()
        .get("message")
        .get("_id")
    )
    chat_get_message_read_receipts = logged_rocket.chat_get_message_read_receipts(
        message_id=message_id
    ).json()
    assert chat_get_message_read_receipts.get("success")
    assert "receipts" in chat_get_message_read_receipts
<commit_msg>Updated Tests for empty message, custom message ID, updating and deleting<commit_after>import pytest

from rocketchat_API.APIExceptions.RocketExceptions import RocketMissingParamException

def test_chat_post_notext_message(logged_rocket):
    chat_post_message = logged_rocket.chat_post_message(None, channel="GENERAL").json()
    assert chat_post_message.get("channel") == "GENERAL"
    assert chat_post_message.get("message").get("msg") == ""
    assert chat_post_message.get("success")

def test_chat_post_update_delete_message(logged_rocket):
    chat_post_message = logged_rocket.chat_post_message(
        "hello", channel="GENERAL"
    ).json()
    assert chat_post_message.get("channel") == "GENERAL"
    assert chat_post_message.get("message").get("msg") == "hello"
    assert chat_post_message.get("success")

    with pytest.raises(RocketMissingParamException):
        logged_rocket.chat_post_message(text="text")

    msg_id = chat_post_message.get("message").get("_id")
    chat_get_message = logged_rocket.chat_get_message(msg_id=msg_id).json()
    assert chat_get_message.get("message").get("_id") == msg_id

    chat_update = logged_rocket.chat_update(
        room_id=chat_post_message.get("channel"),
        msg_id=chat_post_message.get("message").get("_id"),
        text="hello again",
    ).json()

    assert chat_update.get("message").get("msg") == "hello again"
    assert chat_update.get("success")

    chat_delete = logged_rocket.chat_delete(
        room_id=chat_post_message.get("channel"),
        msg_id=chat_post_message.get("message").get("_id"),
    ).json()
    assert chat_delete.get("success")

def test_chat_send_notext_message(logged_rocket):
    message = {"rid": "GENERAL"}
    chat_send_message = logged_rocket.chat_send_message(message).json()
    assert chat_send_message.get("message").get("rid") == "GENERAL"
    assert chat_send_message.get("message").get("msg") == ""
    assert chat_send_message.get("success")

def test_chat_send_custom_id_message(logged_rocket):
    message = {"rid": "GENERAL", "msg": "Hallo Welt", "_id": "1337"}
    chat_send_message = logged_rocket.chat_send_message(message).json()
    assert chat_send_message.get("success")
    assert chat_send_message.get("message").get("rid") == "GENERAL"
    assert chat_send_message.get("message").get("msg") == "Hallo Welt"
    assert chat_send_message.get("message").get("_id") == "1337"

def test_chat_send_update_delete_message(logged_rocket):
    chat_send_message = logged_rocket.chat_send_message(
        {"msg": "hello", "rid": "GENERAL"}
    ).json()
    assert chat_send_message.get("message").get("rid") == "GENERAL"
    assert chat_send_message.get("message").get("msg") == "hello"
    assert chat_send_message.get("success")

    with pytest.raises(RocketMissingParamException):
        logged_rocket.chat_send_message({"msg": "text"})

    msg_id = chat_send_message.get("message").get("_id")
    chat_get_message = logged_rocket.chat_get_message(msg_id=msg_id).json()
    assert chat_get_message.get("message").get("_id") == msg_id

    chat_update = logged_rocket.chat_update(
        room_id=chat_send_message.get("message").get("rid"),
        msg_id=chat_send_message.get("message").get("_id"),
        text="hello again",
    ).json()

    assert chat_update.get("message").get("msg") == "hello again"
    assert chat_update.get("success")

    chat_delete = logged_rocket.chat_delete(
        room_id=chat_send_message.get("message").get("rid"),
        msg_id=chat_send_message.get("message").get("_id"),
    ).json()
    assert chat_delete.get("success")

def test_chat_post_react(logged_rocket):
    message_id = (
        logged_rocket.chat_post_message("hello", channel="GENERAL")
        .json()
        .get("message")
        .get("_id")
    )
    chat_react = logged_rocket.chat_react(msg_id=message_id).json()
    assert chat_react.get("success")

def test_post_pin_unpin(logged_rocket):
    message_id = (
        logged_rocket.chat_post_message("hello", channel="GENERAL")
        .json()
        .get("message")
        .get("_id")
    )
    chat_pin_message = logged_rocket.chat_pin_message(message_id).json()
    assert chat_pin_message.get("success")
    assert chat_pin_message.get("message").get("t") == "message_pinned"

    chat_unpin_message = logged_rocket.chat_unpin_message(message_id).json()
    assert chat_unpin_message.get("success")

def test_post_star_unstar(logged_rocket):
    message_id = (
        logged_rocket.chat_post_message("hello", channel="GENERAL")
        .json()
        .get("message")
        .get("_id")
    )
    chat_star_message = logged_rocket.chat_star_message(message_id).json()
    assert chat_star_message.get("success")

    chat_unstar_message = logged_rocket.chat_unstar_message(message_id).json()
    assert chat_unstar_message.get("success")

def test_chat_search(logged_rocket):
    chat_search = logged_rocket.chat_search(
        room_id="GENERAL", search_text="hello"
    ).json()
    assert chat_search.get("success")

def test_chat_get_message_read_receipts(logged_rocket):
    message_id = (
        logged_rocket.chat_post_message("hello", channel="GENERAL")
        .json()
        .get("message")
        .get("_id")
    )
    chat_get_message_read_receipts = logged_rocket.chat_get_message_read_receipts(
        message_id=message_id
    ).json()
    assert chat_get_message_read_receipts.get("success")
    assert "receipts" in chat_get_message_read_receipts
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Release data for the IPython project.

$Id: Release.py 2681 2007-08-28 17:51:38Z vivainio $"""

#*****************************************************************************
#       Copyright (C) 2001-2006 Fernando Perez <fperez@colorado.edu>
#
#       Copyright (c) 2001 Janko Hauser <jhauser@zscout.de> and Nathaniel Gray
#       <n8gray@caltech.edu>
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#*****************************************************************************

# Name of the package for release purposes.  This is the name which labels
# the tarballs and RPMs made by distutils, so it's best to lowercase it.
name = 'ipython'

# For versions with substrings (like 0.6.16.svn), use an extra . to separate
# the new substring.  We have to avoid using either dashes or underscores,
# because bdist_rpm does not accept dashes (an RPM) convention, and
# bdist_deb does not accept underscores (a Debian convention).

revision = '2680M'

version = '0.8.2.svn.r' + revision.rstrip('M')

description = "An enhanced interactive Python shell."

long_description = \
"""
IPython provides a replacement for the interactive Python interpreter with
extra functionality.

Main features:

 * Comprehensive object introspection.

 * Input history, persistent across sessions.

 * Caching of output results during a session with automatically generated
   references.

 * Readline based name completion.

 * Extensible system of 'magic' commands for controlling the environment and
   performing many tasks related either to IPython or the operating system.

 * Configuration system with easy switching between different setups (simpler
   than changing $PYTHONSTARTUP environment variables every time).

 * Session logging and reloading.

 * Extensible syntax processing for special purpose situations.

 * Access to the system shell with user-extensible alias system.

 * Easily embeddable in other Python programs.

 * Integrated access to the pdb debugger and the Python profiler.

 The latest development version is always available at the IPython subversion
 repository_.

.. _repository: http://ipython.scipy.org/svn/ipython/ipython/trunk#egg=ipython-dev
 """

license = 'BSD'

authors = {'Fernando' : ('Fernando Perez','fperez@colorado.edu'),
           'Janko'    : ('Janko Hauser','jhauser@zscout.de'),
           'Nathan'   : ('Nathaniel Gray','n8gray@caltech.edu'),
           'Ville'    : ('Ville Vainio','vivainio@gmail.com')
           }

url = 'http://ipython.scipy.org'

download_url = 'http://ipython.scipy.org/dist'

platforms = ['Linux','Mac OSX','Windows XP/2000/NT','Windows 95/98/ME']

keywords = ['Interactive','Interpreter','Shell']
<commit_msg>update revnum<commit_after># -*- coding: utf-8 -*-
"""Release data for the IPython project.

$Id: Release.py 2711 2007-09-04 21:11:05Z vivainio $"""

#*****************************************************************************
#       Copyright (C) 2001-2006 Fernando Perez <fperez@colorado.edu>
#
#       Copyright (c) 2001 Janko Hauser <jhauser@zscout.de> and Nathaniel Gray
#       <n8gray@caltech.edu>
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#*****************************************************************************

# Name of the package for release purposes.  This is the name which labels
# the tarballs and RPMs made by distutils, so it's best to lowercase it.
name = 'ipython'

# For versions with substrings (like 0.6.16.svn), use an extra . to separate
# the new substring.  We have to avoid using either dashes or underscores,
# because bdist_rpm does not accept dashes (an RPM) convention, and
# bdist_deb does not accept underscores (a Debian convention).

revision = '2710'

version = '0.8.2.svn.r' + revision.rstrip('M')

description = "An enhanced interactive Python shell."

long_description = \
"""
IPython provides a replacement for the interactive Python interpreter with
extra functionality.

Main features:

 * Comprehensive object introspection.

 * Input history, persistent across sessions.

 * Caching of output results during a session with automatically generated
   references.

 * Readline based name completion.

 * Extensible system of 'magic' commands for controlling the environment and
   performing many tasks related either to IPython or the operating system.

 * Configuration system with easy switching between different setups (simpler
   than changing $PYTHONSTARTUP environment variables every time).

 * Session logging and reloading.

 * Extensible syntax processing for special purpose situations.

 * Access to the system shell with user-extensible alias system.

 * Easily embeddable in other Python programs.

 * Integrated access to the pdb debugger and the Python profiler.

 The latest development version is always available at the IPython subversion
 repository_.

.. _repository: http://ipython.scipy.org/svn/ipython/ipython/trunk#egg=ipython-dev
 """

license = 'BSD'

authors = {'Fernando' : ('Fernando Perez','fperez@colorado.edu'),
           'Janko'    : ('Janko Hauser','jhauser@zscout.de'),
           'Nathan'   : ('Nathaniel Gray','n8gray@caltech.edu'),
           'Ville'    : ('Ville Vainio','vivainio@gmail.com')
           }

url = 'http://ipython.scipy.org'

download_url = 'http://ipython.scipy.org/dist'

platforms = ['Linux','Mac OSX','Windows XP/2000/NT','Windows 95/98/ME']

keywords = ['Interactive','Interpreter','Shell']
<|endoftext|>


<commit_before>#  Copyright (c) 2020, Apple Inc. All rights reserved.
#
#  Use of this source code is governed by a BSD-3-clause license that can be
#  found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause

from coremltools.converters.mil.mil.types.symbolic import is_symbolic
from ._op_reqs import *
from ._utils import broadcast_shapes

@register_op(doc_str="")
class linear(Operation):
    """
    Perform  ``x * weight.T + bias`` where ``weight`` and ``bias`` are constant at
    compile time.

    Parameters
    ----------
    x: tensor<[*D,D_in], T> (required)
        * ``1 <= rank <= 3``.
        * ``0 <= rank(*D) <= 2``.
    weight: const tensor<[D_out,D_in], T> (required)
    bias: const tensor<[D_out],T> (optional)
        * Defaults to ``0``.

    Returns
    -------
    tensor<[*D,D_out], T>
        * same rank as the input ``x``.

    Attributes
    ----------
    T: fp32
    """
    input_spec = InputSpec(
        x=TensorInputType(),
        weight=TensorInputType(const=True),
        bias=TensorInputType(const=True, optional=True),
    )

    def __init__(self, **kwargs):
        super(linear, self).__init__(**kwargs)

    def type_inference(self):
        x_type = self.x.dtype
        x_shape = self.x.shape
        weight_shape = self.weight.shape
        assert len(weight_shape) == 2

        shape = list(x_shape)
        shape[-1] = weight_shape[0]
        return types.tensor(x_type, tuple(shape))

    @precondition(allow=VALUE)
    def value_inference(self):
        res = np.matmul(self.x.val, np.transpose(self.weight.val))
        if self.bias is not None:
            res += self.bias.val
        return res

@register_op(doc_str="")
class matmul(Operation):
    """
    Perform N-D batch matrix multiplication with Numpy-style broadcasting:

    * Rule 1: If both ``x, y`` are 1-D, return the scalar from the dot product.

    * Rule 2: If both ``x, y`` are 2D or higher, perform a broadcast on the batch
    dimensions (all dimensions except the last ``2``).

        * For example: ``x.shape == (10, 4, 3)``, ``y.shape == (5, 10, 3, 2)``,
        ``Matmul(x, y).shape == (5, 10, 4, 2)``.

        * Conventional matrix multiplication is a special case where both ``x, y`` are
        exactly 2D. For example: ``x.shape == (4, 3), y.shape == (3, 2)``,
        ``Matmul(x, y).shape == (4, 2)``.

    * If ``x`` is 1-D, and ``y`` is N-D where ``N >= 2``, ``x`` is first promoted to
    matrix ``xm``by prepending a ``1`` to its dimension, and the resulting ``xm`` is
    broadcast to ``y`` following Rule 2. Remove the inserted dimension afterwards.

        * For example: ``x.shape == (4), y.shape == (10, 4, 3)``,
        ``xm.shape == (1, 4)``. ``Matmul(xm, y).shape == (10, 1, 3)``. Removing the
        inserted dimension results in ``Matmul(x, y).shape == (10, 3)``.

        * Note that ``xm, Matmul(xm, y)`` are immaterial and only for illustration purposes.

    * If ``x`` is N-D where ``N >= 2``, and ``y`` is 1-D, ``y`` is first promoted to
    matrix ``ym`` by appending a ``1`` to its dimension, and the resulting ``ym`` is
    broadcast to ``x`` following Rule 2. Remove the inserted dimension afterwards.

        * For example: ``x.shape == (10, 3, 4), y.shape == (4,)``,
        ``ym.shape == (4, 1)``. ``Matmul(x, ym).shape == (10, 3, 1)``. Removing the
        inserted dimension results in ``Matmul(x, y).shape == (10, 3)``.

        * Note that ``ym, Matmul(x, ym)`` are immaterial and only for illustration purposes.

    Parameters
    ----------
    x: tensor<[*,K1], T> (required)
        * ``x`` must be 1D or higher.
    y: tensor<[*,K2], T> (required)
        * ``y`` must be 1D or higher.
    transpose_x: const bool (optional)
        * Defaults to ``False``.
        * ``True`` to transpose the last two dimensions of ``x`` before multiplication.
        It has no effect when
          ``x`` is 1D.
    transpose_y: const bool (optional)
        * Defaults to ``False``.
        * ``True`` to transpose the last two dimensions of ``y`` before multiplication.
        It has no effect when ``y`` is 1D.

    Returns
    -------
    tensor<*, T>
        * Scalar or tensor output.

    Attributes
    ----------
    T: fp32
    """
    input_spec = InputSpec(
        x=TensorInputType(),
        y=TensorInputType(),
        transpose_x=BoolInputType(const=True, default=False),
        transpose_y=BoolInputType(const=True, default=False),
    )

    def __init__(self, **kwargs):
        super(matmul, self).__init__(**kwargs)

    def type_inference(self):
        x_type = self.x.dtype
        x_shape = list(self.x.shape)
        y_shape = list(self.y.shape)

        if self.transpose_x.val:
            x_shape = list(x_shape)
            x_shape[-1], x_shape[-2] = x_shape[-2], x_shape[-1]
            x_shape = tuple(x_shape)
        if self.transpose_y.val:
            y_shape = list(y_shape)
            y_shape[-1], y_shape[-2] = y_shape[-2], y_shape[-1]
            y_shape = tuple(y_shape)
        if not (
            x_shape[-1] == y_shape[-2]
            or is_symbolic(x_shape[-1])
            or is_symbolic(y_shape[-2])
        ):
            msg = "Op {} (matmul): x {}, y {} are not broadcastable"
            raise ValueError(msg.format(self.name, self.x.shape, self.y.shape))

        ret_shape = list(broadcast_shapes(x_shape[:-2], y_shape[:-2]))
        ret_shape += [x_shape[-2], y_shape[-1]]
        return types.tensor(x_type, tuple(ret_shape))

    @precondition(allow=VALUE)
    def value_inference(self):
        x = self.x.val
        if self.transpose_x.val:
            x = np.transpose(x)
        y = self.y.val
        if self.transpose_y.val:
            y = np.transpose(y)
        return np.matmul(x, y)
<commit_msg>Docs: Update converter.mil.ops documentation for linear.py (#901)<commit_after>#  Copyright (c) 2020, Apple Inc. All rights reserved.
#
#  Use of this source code is governed by a BSD-3-clause license that can be
#  found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause

from coremltools.converters.mil.mil.types.symbolic import is_symbolic
from ._op_reqs import *
from ._utils import broadcast_shapes

@register_op(doc_str="")
class linear(Operation):
    """
    linear
    ======

    Perform  ``x * weight.T + bias`` where ``weight`` and ``bias`` are constant at
    compile time.

    Parameters
    ----------
    ``x: tensor<[*D,D_in], T>`` (required)
    * ``1 <= rank <= 3``.
    * ``0 <= rank(*D) <= 2``

    ``weight: const tensor<[D_out,D_in], T>`` (required)

    ``bias: const tensor<[D_out],T>`` (optional, defaults to ``0``)

    Returns
    -------
    ``tensor<[*D,D_out], T>``: Same rank as the input ``x``

    Attributes
    ----------
    ``T: fp32``
    """
    input_spec = InputSpec(
        x=TensorInputType(),
        weight=TensorInputType(const=True),
        bias=TensorInputType(const=True, optional=True),
    )

    def __init__(self, **kwargs):
        super(linear, self).__init__(**kwargs)

    def type_inference(self):
        x_type = self.x.dtype
        x_shape = self.x.shape
        weight_shape = self.weight.shape
        assert len(weight_shape) == 2

        shape = list(x_shape)
        shape[-1] = weight_shape[0]
        return types.tensor(x_type, tuple(shape))

    @precondition(allow=VALUE)
    def value_inference(self):
        res = np.matmul(self.x.val, np.transpose(self.weight.val))
        if self.bias is not None:
            res += self.bias.val
        return res

@register_op(doc_str="")
class matmul(Operation):
    """
    matmul
    ======

    Perform N-D batch matrix multiplication with Numpy-style broadcasting
    based on the following rules:

    Rules
    -----
    1. If both ``x, y`` are 1-D, return the scalar from the dot product.

    2. If both ``x, y`` are 2-D or higher, perform a broadcast on the batch dimensions
    (all dimensions except the last ``2``).

    For example:

    * ``x.shape == (10, 4, 3)``
    * ``y.shape == (5, 10, 3, 2)``
    * ``Matmul(x, y).shape == (5, 10, 4, 2)``

    Conventional matrix multiplication is a special case where both ``x, y`` are
    exactly 2-D. For example:

    * ``x.shape == (4, 3)``
    * ``y.shape == (3, 2)``
    * ``Matmul(x, y).shape == (4, 2)``

    If ``x`` is 1-D, and ``y`` is N-D where ``N >= 2``, ``x`` is first promoted to
    matrix ``xm``by prepending a ``1`` to its dimension, and the resulting ``xm`` is
    broadcast to ``y`` following Rule 2. Remove the inserted dimension afterwards.
    For example:

    * ``x.shape == (4)``
    * ``y.shape == (10, 4, 3)``
    * ``xm.shape == (1, 4)``
    * ``Matmul(xm, y).shape == (10, 1, 3)``
    * Removing the inserted dimension results in ``Matmul(x, y).shape == (10, 3)``.

    Note: ``xm`` and ``Matmul(xm, y)`` are for illustration only.

    If ``x`` is N-D where ``N >= 2``, and ``y`` is 1-D, ``y`` is first promoted to
    matrix ``ym`` by appending a ``1`` to its dimension, and the resulting ``ym`` is
    broadcast to ``x`` following Rule 2. Remove the inserted dimension afterwards.
    For example:

    * ``x.shape == (10, 3, 4)``
    * ``y.shape == (4,)``
    * ``ym.shape == (4, 1)``
    * ``Matmul(x, ym).shape == (10, 3, 1)``
    * Removing the inserted dimension results in ``Matmul(x, y).shape == (10, 3)``.

    Note: ``xm`` and ``Matmul(xm, y)`` are for illustration only.

    Parameters
    ----------
    ``x: tensor<[*,K1], T>`` (required): ``x`` must be 1-D or higher.

    ``y: tensor<[*,K2], T>`` (required): ``y`` must be 1-D or higher.

    ``transpose_x: const bool`` (optional, defaults to ``False``): Use ``True`` to
    transpose the last two dimensions of ``x`` before multiplication. It has no effect
    when ``x`` is 1-D.

    ``transpose_y: const bool`` (optional, defaults to ``False``): Use ```True`` to
    transpose the last two dimensions of ``y`` before multiplication. It has no effect
    when ``y`` is 1-D.

    Returns
    -------
    ``tensor<*, T>``: Scalar or tensor output.

    Attributes
    ----------
    ``T: fp32``
    """
    input_spec = InputSpec(
        x=TensorInputType(),
        y=TensorInputType(),
        transpose_x=BoolInputType(const=True, default=False),
        transpose_y=BoolInputType(const=True, default=False),
    )

    def __init__(self, **kwargs):
        super(matmul, self).__init__(**kwargs)

    def type_inference(self):
        x_type = self.x.dtype
        x_shape = list(self.x.shape)
        y_shape = list(self.y.shape)

        if self.transpose_x.val:
            x_shape = list(x_shape)
            x_shape[-1], x_shape[-2] = x_shape[-2], x_shape[-1]
            x_shape = tuple(x_shape)
        if self.transpose_y.val:
            y_shape = list(y_shape)
            y_shape[-1], y_shape[-2] = y_shape[-2], y_shape[-1]
            y_shape = tuple(y_shape)
        if not (
            x_shape[-1] == y_shape[-2]
            or is_symbolic(x_shape[-1])
            or is_symbolic(y_shape[-2])
        ):
            msg = "Op {} (matmul): x {}, y {} are not broadcastable"
            raise ValueError(msg.format(self.name, self.x.shape, self.y.shape))

        ret_shape = list(broadcast_shapes(x_shape[:-2], y_shape[:-2]))
        ret_shape += [x_shape[-2], y_shape[-1]]
        return types.tensor(x_type, tuple(ret_shape))

    @precondition(allow=VALUE)
    def value_inference(self):
        x = self.x.val
        if self.transpose_x.val:
            x = np.transpose(x)
        y = self.y.val
        if self.transpose_y.val:
            y = np.transpose(y)
        return np.matmul(x, y)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
#= Imports ====================================================================
import os.path

import requests
import dhtmlparser as d

from config import *

#= Variables ==================================================================
#= Functions & objects ========================================================
class Concept:
    def __init__(self, title, rel_link, session):
        self.title = title
        self.rel_link = rel_link
        self.link = ABCLINUXU_URL + rel_link

        self.meta = None
        self.session = session

    def _get(self, url, params=None, as_text=True):
        data = self.session.get(url, params=params)
        return data.text.encode("utf-8") if as_text else data.content

    def _init_metadata(self, data=None):
        if not data:
            data = self._get(self.link)

        if '
Správa zápisku
' not in data:
            raise ValueError(
                "Can't parse metadata! It looks like I am not logged in!"
            )

        data = data.split('
Správa zápisku
')[1]

        dom = d.parseString(data)
        meta_list = dom.find("div", {"class": "s_sekce"})[0]

        self.meta = {}
        for li in meta_list.find("li"):
            a = li.find("a")[0]
            self.meta[a.getContent().strip()] = a.params["href"]

    def get_full_text(self):
        data = self._get(self.link)

        if not self.meta:
            self._init_metadata(data)

        # data = data.split('
')[0]
        data = data.rsplit('', 1)[0]

        # find beginning of the concept text
        dom = d.parseString(data)
        meta_vypis = dom.find("p", {"class": "meta-vypis"})
        if not meta_vypis:
            raise ValueError("Can't find meta-vypis
!")
        data = data.split(str(meta_vypis[0]))[1]

        return data.strip()

    def edit(self, text, title=None):
        if not self.meta:
            self._init_metadata()

        data = self._get(ABCLINUXU_URL + self.meta["Uprav zápis"])
        data = data.text.encode("utf-8")
        # TODO: implement

    def remove(self):
        raise NotImplementedError("Not implemented yet.")

    def publish(self):
        raise NotImplementedError("Not implemented yet.")

    def add_picture(self, opened_file):
        """
        Add picture to the Concept.

        Args:
            opened_file (file): opened file object
        """
        # init meta
        if not self.meta:
            self._init_metadata()

        # get link to pic form
        data = self._get(ABCLINUXU_URL + self.meta["Přidej obrázek"])
        dom = d.parseString(data)

        # get information from pic form
        form = dom.find("form", {"enctype": "multipart/form-data"})[0]
        add_pic_url = form.params["action"]

        # send pic
        data = self.session.post(
            ABCLINUXU_URL + add_pic_url,
            data={
                "action": "addScreenshot2",
                "finish": "Nahrát"
            },
            files={"screenshot": opened_file}
        )
        data = data.text.encode("utf-8")
        check_error_div(data, '
')

    def list_pictures(self):
        raise NotImplementedError("Not implemented yet.")

    def __str__(self):
        return self.title
<commit_msg>Added some docstring, code cleaned a little.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
#= Imports ====================================================================
import requests
import dhtmlparser as d

from config import *

#= Variables ==================================================================
#= Functions & objects ========================================================
class Concept:
    """
    This class represents concept of the blog - it has all attributes of the
    blog, but it is invisible for the readers.
    """
    def __init__(self, title, rel_link, session):
        self.title = title
        self.rel_link = rel_link
        self.link = ABCLINUXU_URL + rel_link

        self.meta = None
        self.session = session

    def _get(self, url, params=None, as_text=True):
        data = self.session.get(url, params=params)
        return data.text.encode("utf-8") if as_text else data.content

    def _init_metadata(self, data=None):
        if not data:
            data = self._get(self.link)

        if '
Správa zápisku
' not in data:
            raise ValueError(
                "Can't parse metadata! It looks like I am not logged in!"
            )

        data = data.split('
Správa zápisku
')[1]

        dom = d.parseString(data)
        meta_list = dom.find("div", {"class": "s_sekce"})[0]

        self.meta = {}
        for li in meta_list.find("li"):
            a = li.find("a")[0]
            self.meta[a.getContent().strip()] = a.params["href"]

    def get_content(self):
        """
        Get content of this Concept.

        Returns:
            str: full HTML UTF-8 encoded text of the concept.
        """
        data = self._get(self.link)

        if not self.meta:
            self._init_metadata(data)

        # data = data.split('
')[0]
        data = data.rsplit('', 1)[0]

        # find beginning of the concept text
        dom = d.parseString(data)
        meta_vypis = dom.find("p", {"class": "meta-vypis"})
        if not meta_vypis:
            raise ValueError("Can't find meta-vypis
!")
        data = data.split(str(meta_vypis[0]))[1]

        return data.strip()

    def add_picture(self, opened_file):
        """
        Add picture to the Concept.

        Args:
            opened_file (file): opened file object
        """
        # init meta
        if not self.meta:
            self._init_metadata()

        # get link to pic form
        data = self._get(ABCLINUXU_URL + self.meta["Přidej obrázek"])
        dom = d.parseString(data)

        # get information from pic form
        form = dom.find("form", {"enctype": "multipart/form-data"})[0]
        add_pic_url = form.params["action"]

        # send pic
        data = self.session.post(
            ABCLINUXU_URL + add_pic_url,
            data={
                "action": "addScreenshot2",
                "finish": "Nahrát"
            },
            files={"screenshot": opened_file}
        )
        data = data.text.encode("utf-8")
        check_error_div(data, '
')

    def list_pictures(self):
        raise NotImplementedError("Not implemented yet.")

    def remove(self):
        raise NotImplementedError("Not implemented yet.")

    def publish(self):
        raise NotImplementedError("Not implemented yet.")

    def edit(self, text, title=None):
        raise NotImplementedError("Not implemented yet.")

        # if not self.meta:
        #     self._init_metadata()

        # data = self._get(ABCLINUXU_URL + self.meta["Uprav zápis"])
        # data = data.text.encode("utf-8")

    def __str__(self):
        return self.title
<|endoftext|>


<commit_before>#!/usr/bin/env python
import stat
import re
import subprocess
from subprocess import Popen

import lief
import pathlib

from utils import get_sample, has_recent_glibc, is_linux, is_x86_64

def test_freebl(tmp_path):
    tmp = pathlib.Path(tmp_path)

    libfreebl3_path   = get_sample('ELF/ELF64_x86-64_library_libfreebl3.so')

    output_ls         = tmp / "ls.new"
    output_libfreebl3 = tmp / "libfreebl3.so"

    libfreebl3 = lief.parse(libfreebl3_path)
    ls         = lief.parse("/usr/bin/ls")
    if ls is None:
        ls = lief.parse("/bin/ls")

    if lief.ELF.DYNAMIC_TAGS.FLAGS_1 in ls and ls[lief.ELF.DYNAMIC_TAGS.FLAGS_1].has(lief.ELF.DYNAMIC_FLAGS_1.PIE):
        ls[lief.ELF.DYNAMIC_TAGS.FLAGS_1].remove(lief.ELF.DYNAMIC_FLAGS_1.PIE)

    ls.add_library("libfreebl3.so")

    ls         += lief.ELF.DynamicEntryRunPath("$ORIGIN")
    libfreebl3 += lief.ELF.DynamicEntryRunPath("$ORIGIN")

    ls.write(output_ls.as_posix())
    libfreebl3.write(output_libfreebl3.as_posix())

    output_ls.chmod(output_ls.stat().st_mode | stat.S_IEXEC)
    output_libfreebl3.chmod(output_libfreebl3.stat().st_mode | stat.S_IEXEC)

    if is_linux() and is_x86_64() and has_recent_glibc():
        with Popen([output_ls, "--version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as proc:
            stdout = proc.stdout.read()
            print(stdout.decode("utf8"))
            assert re.search(r'ls \(GNU coreutils\) ', stdout.decode("utf8")) is not None
            proc.poll()
            assert proc.returncode == 0
<commit_msg>Fix test<commit_after>#!/usr/bin/env python
import stat
import re
import subprocess
from subprocess import Popen
import pytest

import lief
import pathlib

from utils import get_sample, has_recent_glibc, is_linux, is_x86_64

@pytest.mark.skipif(not (is_linux() and is_x86_64() and has_recent_glibc()), reason="incompatible env")
def test_freebl(tmp_path):
    tmp = pathlib.Path(tmp_path)

    libfreebl3_path   = get_sample('ELF/ELF64_x86-64_library_libfreebl3.so')

    output_ls         = tmp / "ls.new"
    output_libfreebl3 = tmp / "libfreebl3.so"

    libfreebl3 = lief.parse(libfreebl3_path)
    ls         = lief.parse("/usr/bin/ls")
    if ls is None:
        ls = lief.parse("/bin/ls")

    if lief.ELF.DYNAMIC_TAGS.FLAGS_1 in ls and ls[lief.ELF.DYNAMIC_TAGS.FLAGS_1].has(lief.ELF.DYNAMIC_FLAGS_1.PIE):
        ls[lief.ELF.DYNAMIC_TAGS.FLAGS_1].remove(lief.ELF.DYNAMIC_FLAGS_1.PIE)

    ls.add_library("libfreebl3.so")

    ls         += lief.ELF.DynamicEntryRunPath("$ORIGIN")
    libfreebl3 += lief.ELF.DynamicEntryRunPath("$ORIGIN")

    ls.write(output_ls.as_posix())
    libfreebl3.write(output_libfreebl3.as_posix())

    output_ls.chmod(output_ls.stat().st_mode | stat.S_IEXEC)
    output_libfreebl3.chmod(output_libfreebl3.stat().st_mode | stat.S_IEXEC)

    with Popen([output_ls, "--version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as proc:
        stdout = proc.stdout.read()
        print(stdout.decode("utf8"))
        assert re.search(r'ls \(GNU coreutils\) ', stdout.decode("utf8")) is not None
        proc.poll()
        assert proc.returncode == 0
<|endoftext|>