<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import ctypes
import stat
import os
import io

from . import api
from . import compat
from . import exceptions as exp

class Sftp(object):
    """
    Sftp wrapper.

    Exposes api for interacting with sftp subsystem: put or get files,
    open files with random read-write access, etc.

    :ivar ponter sftp: c sftp session pointer
    :ivar pointer session: c ssh session pointer

    :param pyssh.session.Session session: initialized and connected
        :py:class:`pyssh.session.Session` instance.
    """

    sftp = None
    session = None

    def __init__(self, session, buffer_size=2048):
        self.session_wrapper = session
        self.session = session.session

        self.buffer_size = buffer_size

        # TODO: handle exceptions
        self.sftp = api.library.sftp_new(self.session)
        api.library.sftp_init(self.sftp)

    def _get_file_metadata(self, file_ptr):
        attrs_ptr = api.library.sftp_fstat(file_ptr)
        if attrs_ptr is None:
            msg = api.library.ssh_get_error(self.session)
            raise exp.ConnectionError("Error raised by ssh: {0}".format(msg.decode("utf-8")))

        attrs = ctypes.cast(attrs_ptr, ctypes.POINTER(api.SftpAttributes))
        return attrs.contents

    def _open_remote_file(self, path):
        remote_file_ptr = api.library.sftp_open(self.sftp, path, os.O_RDONLY, stat.S_IRWXU)

        if remote_file_ptr is None:
            msg = api.library.ssh_get_error(self.session)
            raise exp.ConnectionError("Error raised by ssh: {0}".format(msg.decode("utf-8")))

        return remote_file_ptr

    def get(self, remote_path, local_path):
        """
        Get a remote file to local.

        :param str remote_path: remote file path
        :param str local_path:  local file path
        """
        remote_path = compat.to_bytes(remote_path)

        # Create new pointer to remote file
        remote_file_ptr = self._open_remote_file(remote_path)

        # Obtain metadata with remote file size
        remote_file_attrs = self._get_file_metadata(remote_file_ptr)

        # Create counters for reintent in case of error
        total_readed, total_size = 0, remote_file_attrs.size
        errors_counter = 0

        def read_pipeline(f):
            nonlocal total_readed, total_size, errors_counter

            while True:
                buffer = ctypes.create_string_buffer(self.buffer_size)
                readed = api.library.sftp_read(remote_file_ptr, ctypes.byref(buffer),
                                               self.buffer_size)
                if readed == 0:
                    if total_readed != total_size:
                        errors_counter += 1
                        return False
                    return True

                elif readed < 0:
                    raise exp.ConnectionError("Connection interrumped")

                else:
                    errors_counter = 0
                    total_readed += readed
                    f.write(buffer.value)

        try:
            with io.open(local_path, "wb") as f:
                while errors_counter < 3:
                    ok = read_pipeline(f)
                    if ok:
                        break

                    api.library.sftp_close(remote_file_ptr)
                    remote_file_ptr = self._open_remote_file(remote_path)

                if errors_counter >= 3:
                    raise exp.Connection("Connection errors repeated more than 3 times")

        except (exp.ConnectionError, RuntimeError):
            api.library.sftp_close(remote_file_ptr)
            raise

    def put(self, path, remote_path):
        """
        Puts the local file to remote host.

        :param str path: local file path
        :param str remote_path: remote file path
        """

        if not os.path.exists(path):
            raise RuntimeError("Path {0} does not exists".format(path))

        if isinstance(remote_path, compat.text_type):
            remote_path = compat.to_bytes(remote_path, "utf-8")

        access_type = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
        remote_file_ptr = api.library.sftp_open(self.sftp, remote_path, access_type, stat.S_IRWXU)

        if remote_file_ptr is None:
            msg = api.library.ssh_get_error(self.session)
            raise exp.ConnectionError("Error raised by ssh: {0}".format(msg.decode("utf-8")))

        with io.open(path, "rb") as f:
            while True:
                chuck = f.read(self.buffer_size)
                if not chuck:
                    break

                written = api.library.sftp_write(remote_file_ptr, chuck, len(chuck))
                if written != len(chuck):
                    raise RuntimeError("Can't write file")

        api.library.sftp_close(remote_file_ptr)

    def open(self, path, mode):
        """
        Open a remote file.

        :param str path: remote file path
        :param int mode: open file model
                         (see http://docs.python.org/3.3/library/os.html#open-flag-constants)

        :returns: SFTP File wrapper
        :rtype: pyssh.SftpFile
        """
        if isinstance(path, compat.text_type):
            path = compat.to_bytes(path, "utf-8")

        return SftpFile(path, mode, self)

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        api.library.sftp_free(self.sftp)

class SftpFile(object):
    """
    SFTP File wrapper
    """

    _closed = False

    def __init__(self, path, mode, sftp_wrapper):
        self.sftp_wrapper = sftp_wrapper
        self.sftp = sftp_wrapper.sftp

        self.file = api.library.sftp_open(self.sftp, path, mode, stat.S_IRWXU)

        if self.file is None:
            self._closed = True
            raise ext.SftpError("Can't open file {0}".format(path.decode("utf-8")))

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def write(self, data):
        """
        Write bytes to remote file.

        :param bytes data: bytes chunk of data
        :returns: number of bytes are written
        :rtype: int
        """
        written = api.library.sftp_write(self.file, data, len(data))
        if written != len(data):
            raise RuntimeError("Can't write file")

        return written

    def read(self, num=None, buffer_length=1024):
        """
        Read from remote file.

        :param int num: number of bytes to read, if num is None reads all.
        :returns: readed bytes chunk
        :rtype: bytes
        """
        if num is None:
            buffer_len = buffer_length
        else:
            buffer_len = num

        buffer = ctypes.create_string_buffer(buffer_len)
        readed = api.library.sftp_read(self.file, ctypes.byref(buffer),  buffer_len);

        if readed == 0:
            return b""

        if num is not None and num > 0:
            if buffer_len != readed:
                raise RuntimeError("Error on read")
            return buffer.value

        readed_data = [buffer.value]
        while True:
            buffer = ctypes.create_string_buffer(buffer_len)
            readed = api.library.sftp_read(self.file, ctypes.byref(buffer),  buffer_len);
            if readed == 0:
                break

            readed_data.append(buffer.value)
        return b"".join(readed_data)

    def seek(self, offset):
        """
        Change position on a remote file.

        :param int offset: file position
        :returns: boolean value if seek is success or not
        :rtype: bool
        """
        ret = api.library.sftp_seek64(self.file, offset);
        if ret != api.SSH_OK:
            return False

        return True

    def tell(self):
        """
        Query the current position on a file.

        :returns: a current position.
        :rtype: int
        """
        return api.library.sftp_tell64(self.file)

    def close(self):
        """
        Close a opened file.
        """
        if self._closed:
            raise ext.ResourceManagementError("SftpFile instance already closed.")

        self._closed = True
        api.library.sftp_close(self.file)
<commit_msg>Fixed wrong process bytes data on sftp.<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import ctypes
import stat
import os
import io

from . import api
from . import compat
from . import exceptions as exp

class Sftp(object):
    """
    Sftp wrapper.

    Exposes api for interacting with sftp subsystem: put or get files,
    open files with random read-write access, etc.

    :ivar ponter sftp: c sftp session pointer
    :ivar pointer session: c ssh session pointer

    :param pyssh.session.Session session: initialized and connected
        :py:class:`pyssh.session.Session` instance.
    """

    sftp = None
    session = None

    def __init__(self, session, buffer_size=1024*16):
        self.session_wrapper = session
        self.session = session.session

        self.buffer_size = buffer_size

        # TODO: handle exceptions
        self.sftp = api.library.sftp_new(self.session)
        api.library.sftp_init(self.sftp)

    def _get_file_metadata(self, file_ptr):
        attrs_ptr = api.library.sftp_fstat(file_ptr)
        if attrs_ptr is None:
            msg = api.library.ssh_get_error(self.session)
            raise exp.ConnectionError("Error raised by ssh: {0}".format(msg.decode("utf-8")))

        attrs = ctypes.cast(attrs_ptr, ctypes.POINTER(api.SftpAttributes))
        return attrs.contents

    def _open_remote_file(self, path):
        remote_file_ptr = api.library.sftp_open(self.sftp, path, os.O_RDONLY, stat.S_IRWXU)

        if remote_file_ptr is None:
            msg = api.library.ssh_get_error(self.session)
            raise exp.ConnectionError("Error raised by ssh: {0}".format(msg.decode("utf-8")))

        return remote_file_ptr

    def get(self, remote_path, local_path):
        """
        Get a remote file to local.

        :param str remote_path: remote file path
        :param str local_path:  local file path
        """
        remote_path = compat.to_bytes(remote_path)

        # Create new pointer to remote file
        remote_file_ptr = self._open_remote_file(remote_path)

        # Obtain metadata with remote file size
        remote_file_attrs = self._get_file_metadata(remote_file_ptr)

        # Create counters for reintent in case of error
        total_readed, total_size = 0, remote_file_attrs.size
        errors_counter = 0

        def read_pipeline(f):
            nonlocal total_readed, total_size, errors_counter

            while True:
                buffer = ctypes.create_string_buffer(self.buffer_size)
                readed = api.library.sftp_read(remote_file_ptr, ctypes.byref(buffer),
                                               self.buffer_size)
                if readed == 0:
                    if total_readed != total_size:
                        errors_counter += 1
                        return False
                    return True

                elif readed < 0:
                    raise exp.ConnectionError("Connection interrumped")

                else:
                    errors_counter = 0
                    total_readed += readed
                    f.write(buffer.raw[:readed])

        try:
            with io.open(local_path, "wb") as f:
                while errors_counter < 3:
                    ok = read_pipeline(f)
                    if ok:
                        break

                    api.library.sftp_close(remote_file_ptr)
                    remote_file_ptr = self._open_remote_file(remote_path)

                if errors_counter >= 3:
                    raise exp.Connection("Connection errors repeated more than 3 times")

        except (exp.ConnectionError, RuntimeError):
            api.library.sftp_close(remote_file_ptr)
            raise

    def put(self, path, remote_path):
        """
        Puts the local file to remote host.

        :param str path: local file path
        :param str remote_path: remote file path
        """

        if not os.path.exists(path):
            raise RuntimeError("Path {0} does not exists".format(path))

        if isinstance(remote_path, compat.text_type):
            remote_path = compat.to_bytes(remote_path, "utf-8")

        access_type = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
        remote_file_ptr = api.library.sftp_open(self.sftp, remote_path, access_type, stat.S_IRWXU)

        if remote_file_ptr is None:
            msg = api.library.ssh_get_error(self.session)
            raise exp.ConnectionError("Error raised by ssh: {0}".format(msg.decode("utf-8")))

        with io.open(path, "rb") as f:
            while True:
                chuck = f.read(self.buffer_size)
                if not chuck:
                    break

                written = api.library.sftp_write(remote_file_ptr, chuck, len(chuck))
                if written != len(chuck):
                    raise RuntimeError("Can't write file")

        api.library.sftp_close(remote_file_ptr)

    def open(self, path, mode):
        """
        Open a remote file.

        :param str path: remote file path
        :param int mode: open file model
                         (see http://docs.python.org/3.3/library/os.html#open-flag-constants)

        :returns: SFTP File wrapper
        :rtype: pyssh.SftpFile
        """
        if isinstance(path, compat.text_type):
            path = compat.to_bytes(path, "utf-8")

        return SftpFile(path, mode, self)

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        api.library.sftp_free(self.sftp)

class SftpFile(object):
    """
    SFTP File wrapper
    """

    _closed = False

    def __init__(self, path, mode, sftp_wrapper):
        self.sftp_wrapper = sftp_wrapper
        self.sftp = sftp_wrapper.sftp

        self.file = api.library.sftp_open(self.sftp, path, mode, stat.S_IRWXU)

        if self.file is None:
            self._closed = True
            raise ext.SftpError("Can't open file {0}".format(path.decode("utf-8")))

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def write(self, data):
        """
        Write bytes to remote file.

        :param bytes data: bytes chunk of data
        :returns: number of bytes are written
        :rtype: int
        """
        written = api.library.sftp_write(self.file, data, len(data))
        if written != len(data):
            raise RuntimeError("Can't write file")

        return written

    def read(self, num=None, buffer_length=1024):
        """
        Read from remote file.

        :param int num: number of bytes to read, if num is None reads all.
        :returns: readed bytes chunk
        :rtype: bytes
        """
        if num is None:
            buffer_len = buffer_length
        else:
            buffer_len = num

        buffer = ctypes.create_string_buffer(buffer_len)
        readed = api.library.sftp_read(self.file, ctypes.byref(buffer),  buffer_len);

        if readed == 0:
            return b""

        if num is not None and num > 0:
            if buffer_len != readed:
                raise RuntimeError("Error on read")
            return buffer.raw

        readed_data = [buffer.raw]
        while True:
            buffer = ctypes.create_string_buffer(buffer_len)

            readed = api.library.sftp_read(self.file, ctypes.byref(buffer),  buffer_len);
            if readed == 0:
                break

            readed_data.append(buffer.raw[:readed])
        return b"".join(readed_data)

    def seek(self, offset):
        """
        Change position on a remote file.

        :param int offset: file position
        :returns: boolean value if seek is success or not
        :rtype: bool
        """
        ret = api.library.sftp_seek64(self.file, offset);
        if ret != api.SSH_OK:
            return False

        return True

    def tell(self):
        """
        Query the current position on a file.

        :returns: a current position.
        :rtype: int
        """
        return api.library.sftp_tell64(self.file)

    def close(self):
        """
        Close a opened file.
        """
        if self._closed:
            raise ext.ResourceManagementError("SftpFile instance already closed.")

        self._closed = True
        api.library.sftp_close(self.file)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License,
    or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    See the GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>.

    @author: spoob
    @author: sebnapi
    @author: RaNaN
    @author: mkaay
    @version: v0.3
"""

CURRENT_VERSION = '0.3'

import ConfigParser
import gettext
from glob import glob
import logging
import logging.handlers
from os import mkdir
from os import sep
from os import chdir
from os.path import basename
from os.path import exists
from os.path import dirname
from os.path import abspath
import subprocess
from sys import argv
from sys import exit
from sys import path
from sys import stdout
import time
from time import sleep
import urllib2
from imp import find_module
from re import sub
try:
    find_module("Crypto")
except ImportError:
    print "Install pycrypto to use pyLoad"
    exit()
from module.file_list import File_List
from module.thread_list import Thread_List
from module.network.Request import Request
import thread

class Core(object):
    """ pyLoad Core """
    def __init__(self):
        if len(argv) > 1:
            if argv[1] == "-v":
                print "pyLoad", CURRENT_VERSION
                exit()

    def read_config(self):
        """ read config and sets preferences """
        self.configfile = ConfigParser.SafeConfigParser()
        self.configfile.read('config')
        for section in self.configfile.sections():
            self.config[section] = {}
            for option in self.configfile.options(section):
                self.config[section][option] = self.configfile.get(section, option)
                self.config[section][option] = False if self.config[section][option].lower() == 'false' else self.config[section][option]

    def set_option(self, section, option, value):
        self.config[option] = value
        self.configfile.set(section, option, str(value))
        self.configfile.write(open('config', "wb"))

    def read_option(self):
        return self.config

    def shutdown(self):
        "abort all downloads and exit"
        self.thread_list.pause = True

        for pyfile in self.thread_list.py_downloading:
            pyfile.plugin.req.abort = True

        while self.thread_list.py_downloading:
            sleep(1)
        self.logger.info("Going to shutdown pyLoad")
        exit()

    def toggle_pause(self):
        if self.thread_list.pause:
            self.thread_list.pause = False
            return False
        elif not self.thread_list.pause:
            self.thread_list.pause = True
            return True

    def start(self):
        """ starts the machine"""
        chdir(dirname(abspath(__file__)) + sep)

        self.config = {}
        self.plugin_folder = "module" + sep + "plugins"
        self.plugins_avaible = {}

        self.read_config()

        self.do_kill = False
        translation = gettext.translation("pyLoad", "locale", languages=[self.config['general']['language']])
        translation.install(unicode=True)

        self.check_install("pycurl", "pycurl for lower memory footprint while downloading")
        self.check_install("tesseract", "tesseract for captcha reading", False)
        self.check_install("gocr", "gocr for captcha reading", False)
        self.check_file(self.config['log']['log_folder'], _("folder for logs"))
        self.check_file(self.config['general']['download_folder'], _("folder for downloads"))
        self.check_file(self.config['general']['link_file'], _("file for links"), False)
        self.check_file(self.config['general']['failed_file'], _("file for failed links"), False)
        if self.config['ssl']['activated']:
            self.check_file(self.config['ssl']['cert'], _("ssl certificate"), False, False, True)
            self.check_file(self.config['ssl']['key'], _("ssl key"), False, False, True)

        if self.config['general']['debug_mode']:
            self.init_logger(logging.DEBUG) # logging level
        else:
            self.init_logger(logging.INFO) # logging level

        self.check_update()

        self.logger.info(_("Downloadtime: %s") % self.is_time_download()) # debug only

        path.append(self.plugin_folder)
        self.create_plugin_index()

        self.init_server()

        self.file_list = File_List(self)
        self.thread_list = Thread_List(self)

        self.read_url_list(self.config['general']['link_file'])

        while True:
            sleep(2)
            if self.do_kill:
                self.logger.info("pyLoad quits")
                exit()

    def init_server(self):
        try:
            server_addr = (self.config['remote']['listenaddr'], int(self.config['remote']['port']))
            usermap = { self.config['remote']['username']: self.config['remote']['password'] }
            if self.config['ssl']['activated']:
                Server = __import__("module.remote.SecureXMLRPCServer", globals(), locals(), "SecureXMLRPCServer", -1)
                self.server = Server.SecureXMLRPCServer(server_addr, self.config['ssl']['cert'], self.config['ssl']['key'], usermap)
                self.logger.info("Secure XMLRPC Server Started")
            else:
                Server = __import__("SimpleXMLRPCServer")
                self.server = Server.SimpleXMLRPCServer(server_addr)
                self.logger.info("Normal XMLRPC Server Started")

            self.server.register_introspection_functions()
            self.server.register_function(self.status_downloads)
            self.server.register_function(self.status_server)
            self.server.register_function(self.kill)
            self.server.register_function(self.del_links)
            self.server.register_function(self.del_packages)
            self.server.register_function(self.add_urls)
            self.server.register_function(self.get_queue)
            #self.server.register_function(self.move_urls_up)
            #self.server.register_function(self.move_urls_down)
            self.server.register_function(self.is_time_download)
            self.server.register_function(self.is_time_reconnect)
            self.server.register_function(self.get_conf_val)
            self.server.register_function(self.file_exists)
            self.server.register_function(self.get_server_version)
            thread.start_new_thread(self.server.serve_forever, ())
        except Exception, e:
            self.logger.error("Failed starting socket server, CLI and GUI will not be available: %s" % str(e))

    def init_logger(self, level):

        file_handler = logging.handlers.RotatingFileHandler(self.config['log']['log_folder'] + sep + 'log.txt', maxBytes=102400, backupCount=int(self.config['log']['log_count'])) #100 kib each
        console = logging.StreamHandler(stdout)

        frm = logging.Formatter("%(asctime)s: %(levelname)-8s  %(message)s", "%d.%m.%Y %H:%M:%S")
        file_handler.setFormatter(frm)
        console.setFormatter(frm)

        self.logger = logging.getLogger("log") # settable in config

        if self.config['log']['file_log']:
            self.logger.addHandler(file_handler)

        self.logger.addHandler(console) #if console logging
        self.logger.setLevel(level)

    def check_install(self, check_name, legend, python=True, essential=False):
        """check wether needed tools are installed"""
        try:
            if python:
                find_module(check_name)
            else:
                pipe = subprocess.PIPE
                subprocess.Popen(check_name, stdout=pipe, stderr=pipe)
        except:
            print "Install", legend
            if essential: exit()

    def check_file(self, check_name, legend, folder=True, empty=True, essential=False):
        """check wether needed files are exists"""
        if not exists(check_name):
            created = False
            if empty:
                try:
                    if folder:
                        mkdir(check_name)
                    else:
                        open(check_name, "w")
                    print _("%s created") % legend
                    created = True
                except:
                    print _("could not create %s: %s") % (legend, check_name)
            else:
                print _("could not find %s: %s") % (legend, check_name)
            if essential and not created:
                exit()

    def check_update(self):
        """checks newst version"""
        if not self.config['updates']['search_updates']:
            return False

        newst_version = Request().load("http://update.pyload.org/index.php?do=" + CURRENT_VERSION)
        if newst_version == "True":
            if not self.config['updates']['install_updates']:
                self.logger.info("New Version of pyLoad available")
            else:
                updater = __import__("pyLoadUpdater")
                updater.main()
        else:
            self.logger.info("No Updates for pyLoad")

    def create_plugin_index(self):
        for file_handler in glob(self.plugin_folder + sep + '*.py') + glob(self.plugin_folder + sep + 'DLC.pyc'):
            plugin_pattern = ""
            plugin_file = sub("(\.pyc|\.py)", "", basename(file_handler))
            if plugin_file == "DLC":
                plugin_pattern = "(?!http://).*\.dlc"
            else:
                for line in open(file_handler, "r").readlines():
                    if "props['pattern']" in line:
                        plugin_pattern = line.split("r\"")[1].split("\"")[0]
                        break
            if plugin_pattern != "":
                self.plugins_avaible[plugin_file] = plugin_pattern
                self.logger.debug(plugin_file + _(" added"))
        self.logger.info(_("created index of plugins"))

    def compare_time(self, start, end):
        if start == end: return True

        now  = time.localtime()[3:5]
        if start < now and end > now: return True
        elif start > end and (now > start or now < end): return True
        elif start < now and end < now and start > end: return True
        else: return False

    ####################################
    ########## XMLRPC Methods ##########
    ####################################

    def status_downloads(self):
        downloads = []
        for pyfile in self.thread_list.py_downloading:
            download = {}
            download['id'] = pyfile.id
            download['name'] = pyfile.status.filename
            download['speed'] = pyfile.status.get_speed()
            download['eta'] = pyfile.status.get_ETA()
            download['kbleft'] = pyfile.status.kB_left()
            download['size'] = pyfile.status.size()
            download['percent'] = pyfile.status.percent()
            download['status'] = pyfile.status.type
            download['wait_until'] = pyfile.status.waituntil
            download['plugin'] = pyfile.status.plugin
            downloads.append(download)
        return downloads

    def get_conf_val(self, cat, var):
        if var != "username" and var != "password":
            return self.config[cat][var]
        else:
            raise Exception("not allowed!")

    def status_server(self):
        status = {}
        status['pause'] = self.thread_list.pause
        status['queue'] = len(self.file_list.files)
        status['speed'] = 0

        for pyfile in self.thread_list.py_downloading:
            status['speed'] += pyfile.status.get_speed()

        return status

    def file_exists(self, path): #@XXX: security?!
        return exists(path)

    def get_server_version(self):
        return CURRENT_VERSION

    def add_urls(self, links):
        for link in links:
            self.file_list.collector.addLink(link)
        self.file_list.save()

    def del_links(self, ids):
        for id in ids:
            try:
                self.file_list.collector.removeFile(id)
            except:
                self.file_list.packages.removeFile(id)
        self.file_list.save()

    def del_packages(self, ids):
        for id in ids:
            self.file_list.packages.removePackage(id)
        self.file_list.save()

    def kill(self):
        self.do_kill = True
        return True

    def get_queue(self):
        data = []
        for q in self.file_list.data["queue"]:
            ds = {
                "id": q.data.id,
                "name": q.data.package_name,
                "folder": q.data.folder,
                "files": []
            }
            for f in q.links:
                ds["files"].append({
                    "name": f.status.name,
                    "status": f.status.type,
                    "url": f.url
                })
            data.append(ds)
        return data

    def get_collector_packages(self):
        data = []
        for q in self.file_list.data["packages"]:
            ds = {
                "id": q.data.id,
                "name": q.data.package_name,
                "folder": q.data.folder,
                "files": []
            }
            for f in q.links:
                ds["files"].append({
                    "name": f.status.name,
                    "status": f.status.type,
                    "url": f.url
                })
            data.append(ds)
        return data

    #def move_urls_up(self, ids):
    #    for id in ids:
    #        self.file_list.move(id)
    #    self.file_list.save()

    #def move_urls_down(self, ids):
    #    for id in ids:
    #        self.file_list.move(id, 1)
    #    self.file_list.save()

    def read_url_list(self, url_list):
        """read links from txt"""
        txt = open(self.config['general']['link_file'], 'r')
        new_links = 0
        links = txt.readlines()
        for link in links:
            if link != "\n":
                self.file_list.collector.addLink(link)
                new_links += 1

        txt.close()

        self.file_list.save()
        if new_links:
            self.logger.info("Parsed link from %s: %i" % (self.config['general']['link_file'], new_links))

        txt = open(self.config['general']['link_file'], 'w')
        txt.write("")
        txt.close()

    def is_time_download(self):
        start = self.config['downloadTime']['start'].split(":")
        end = self.config['downloadTime']['end'].split(":")
        return self.compare_time(start, end)

    def is_time_reconnect(self):
        start = self.config['reconnectTime']['start'].split(":")
        end = self.config['reconnectTime']['end'].split(":")
        return self.compare_time(start, end)

if __name__ == "__main__":
    pyload_core = Core()
    try:
        pyload_core.start()
    except KeyboardInterrupt:
        pyload_core.logger.info("killed pyLoad by Terminal")
        exit()

<commit_msg>Better Update Check<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License,
    or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    See the GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>.

    @author: spoob
    @author: sebnapi
    @author: RaNaN
    @author: mkaay
    @version: v0.3
"""

CURRENT_VERSION = '0.3'

import ConfigParser
import gettext
from glob import glob
import logging
import logging.handlers
from os import mkdir
from os import sep
from os import chdir
from os.path import basename
from os.path import exists
from os.path import dirname
from os.path import abspath
import subprocess
from sys import argv
from sys import exit
from sys import path
from sys import stdout
import time
from time import sleep
import urllib2
from imp import find_module
from re import sub
try:
    find_module("Crypto")
except ImportError:
    print "Install pycrypto to use pyLoad"
    exit()
from module.file_list import File_List
from module.thread_list import Thread_List
from module.network.Request import Request
import thread

class Core(object):
    """ pyLoad Core """
    def __init__(self):
        if len(argv) > 1:
            if argv[1] == "-v":
                print "pyLoad", CURRENT_VERSION
                exit()

    def read_config(self):
        """ read config and sets preferences """
        self.configfile = ConfigParser.SafeConfigParser()
        self.configfile.read('config')
        for section in self.configfile.sections():
            self.config[section] = {}
            for option in self.configfile.options(section):
                self.config[section][option] = self.configfile.get(section, option)
                self.config[section][option] = False if self.config[section][option].lower() == 'false' else self.config[section][option]

    def set_option(self, section, option, value):
        self.config[option] = value
        self.configfile.set(section, option, str(value))
        self.configfile.write(open('config', "wb"))

    def read_option(self):
        return self.config

    def shutdown(self):
        "abort all downloads and exit"
        self.thread_list.pause = True

        for pyfile in self.thread_list.py_downloading:
            pyfile.plugin.req.abort = True

        while self.thread_list.py_downloading:
            sleep(1)
        self.logger.info("Going to shutdown pyLoad")
        exit()

    def toggle_pause(self):
        if self.thread_list.pause:
            self.thread_list.pause = False
            return False
        elif not self.thread_list.pause:
            self.thread_list.pause = True
            return True

    def start(self):
        """ starts the machine"""
        chdir(dirname(abspath(__file__)) + sep)

        self.config = {}
        self.plugin_folder = "module" + sep + "plugins"
        self.plugins_avaible = {}

        self.read_config()

        self.do_kill = False
        translation = gettext.translation("pyLoad", "locale", languages=[self.config['general']['language']])
        translation.install(unicode=True)

        self.check_install("pycurl", "pycurl for lower memory footprint while downloading")
        self.check_install("tesseract", "tesseract for captcha reading", False)
        self.check_install("gocr", "gocr for captcha reading", False)
        self.check_file(self.config['log']['log_folder'], _("folder for logs"))
        self.check_file(self.config['general']['download_folder'], _("folder for downloads"))
        self.check_file(self.config['general']['link_file'], _("file for links"), False)
        self.check_file(self.config['general']['failed_file'], _("file for failed links"), False)
        if self.config['ssl']['activated']:
            self.check_file(self.config['ssl']['cert'], _("ssl certificate"), False, False, True)
            self.check_file(self.config['ssl']['key'], _("ssl key"), False, False, True)

        if self.config['general']['debug_mode']:
            self.init_logger(logging.DEBUG) # logging level
        else:
            self.init_logger(logging.INFO) # logging level

        self.check_update()

        self.logger.info(_("Downloadtime: %s") % self.is_time_download()) # debug only

        path.append(self.plugin_folder)
        self.create_plugin_index()

        self.init_server()

        self.file_list = File_List(self)
        self.thread_list = Thread_List(self)

        self.read_url_list(self.config['general']['link_file'])

        while True:
            sleep(2)
            if self.do_kill:
                self.logger.info("pyLoad quits")
                exit()

    def init_server(self):
        try:
            server_addr = (self.config['remote']['listenaddr'], int(self.config['remote']['port']))
            usermap = { self.config['remote']['username']: self.config['remote']['password'] }
            if self.config['ssl']['activated']:
                Server = __import__("module.remote.SecureXMLRPCServer", globals(), locals(), "SecureXMLRPCServer", -1)
                self.server = Server.SecureXMLRPCServer(server_addr, self.config['ssl']['cert'], self.config['ssl']['key'], usermap)
                self.logger.info("Secure XMLRPC Server Started")
            else:
                Server = __import__("SimpleXMLRPCServer")
                self.server = Server.SimpleXMLRPCServer(server_addr)
                self.logger.info("Normal XMLRPC Server Started")

            self.server.register_introspection_functions()
            self.server.register_function(self.status_downloads)
            self.server.register_function(self.status_server)
            self.server.register_function(self.kill)
            self.server.register_function(self.del_links)
            self.server.register_function(self.del_packages)
            self.server.register_function(self.add_urls)
            self.server.register_function(self.get_queue)
            #self.server.register_function(self.move_urls_up)
            #self.server.register_function(self.move_urls_down)
            self.server.register_function(self.is_time_download)
            self.server.register_function(self.is_time_reconnect)
            self.server.register_function(self.get_conf_val)
            self.server.register_function(self.file_exists)
            self.server.register_function(self.get_server_version)
            thread.start_new_thread(self.server.serve_forever, ())
        except Exception, e:
            self.logger.error("Failed starting socket server, CLI and GUI will not be available: %s" % str(e))

    def init_logger(self, level):

        file_handler = logging.handlers.RotatingFileHandler(self.config['log']['log_folder'] + sep + 'log.txt', maxBytes=102400, backupCount=int(self.config['log']['log_count'])) #100 kib each
        console = logging.StreamHandler(stdout)

        frm = logging.Formatter("%(asctime)s: %(levelname)-8s  %(message)s", "%d.%m.%Y %H:%M:%S")
        file_handler.setFormatter(frm)
        console.setFormatter(frm)

        self.logger = logging.getLogger("log") # settable in config

        if self.config['log']['file_log']:
            self.logger.addHandler(file_handler)

        self.logger.addHandler(console) #if console logging
        self.logger.setLevel(level)

    def check_install(self, check_name, legend, python=True, essential=False):
        """check wether needed tools are installed"""
        try:
            if python:
                find_module(check_name)
            else:
                pipe = subprocess.PIPE
                subprocess.Popen(check_name, stdout=pipe, stderr=pipe)
        except:
            print "Install", legend
            if essential: exit()

    def check_file(self, check_name, legend, folder=True, empty=True, essential=False):
        """check wether needed files are exists"""
        if not exists(check_name):
            created = False
            if empty:
                try:
                    if folder:
                        mkdir(check_name)
                    else:
                        open(check_name, "w")
                    print _("%s created") % legend
                    created = True
                except:
                    print _("could not create %s: %s") % (legend, check_name)
            else:
                print _("could not find %s: %s") % (legend, check_name)
            if essential and not created:
                exit()

    def check_update(self):
        """checks newst version"""
        if not self.config['updates']['search_updates']:
            return False

        newst_version = Request().load("http://update.pyload.org/s/" + CURRENT_VERSION)
        if newst_version == "True":
            if not self.config['updates']['install_updates']:
                self.logger.info("New Version of pyLoad available")
            else:
                updater = __import__("pyLoadUpdater")
                updater.main()
        else:
            self.logger.info("No Updates for pyLoad")

    def create_plugin_index(self):
        for file_handler in glob(self.plugin_folder + sep + '*.py') + glob(self.plugin_folder + sep + 'DLC.pyc'):
            plugin_pattern = ""
            plugin_file = sub("(\.pyc|\.py)", "", basename(file_handler))
            if plugin_file == "DLC":
                plugin_pattern = "(?!http://).*\.dlc"
            else:
                for line in open(file_handler, "r").readlines():
                    if "props['pattern']" in line:
                        plugin_pattern = line.split("r\"")[1].split("\"")[0]
                        break
            if plugin_pattern != "":
                self.plugins_avaible[plugin_file] = plugin_pattern
                self.logger.debug(plugin_file + _(" added"))
        self.logger.info(_("created index of plugins"))

    def compare_time(self, start, end):
        if start == end: return True

        now  = time.localtime()[3:5]
        if start < now and end > now: return True
        elif start > end and (now > start or now < end): return True
        elif start < now and end < now and start > end: return True
        else: return False

    ####################################
    ########## XMLRPC Methods ##########
    ####################################

    def status_downloads(self):
        downloads = []
        for pyfile in self.thread_list.py_downloading:
            download = {}
            download['id'] = pyfile.id
            download['name'] = pyfile.status.filename
            download['speed'] = pyfile.status.get_speed()
            download['eta'] = pyfile.status.get_ETA()
            download['kbleft'] = pyfile.status.kB_left()
            download['size'] = pyfile.status.size()
            download['percent'] = pyfile.status.percent()
            download['status'] = pyfile.status.type
            download['wait_until'] = pyfile.status.waituntil
            download['plugin'] = pyfile.status.plugin
            downloads.append(download)
        return downloads

    def get_conf_val(self, cat, var):
        if var != "username" and var != "password":
            return self.config[cat][var]
        else:
            raise Exception("not allowed!")

    def status_server(self):
        status = {}
        status['pause'] = self.thread_list.pause
        status['queue'] = len(self.file_list.files)
        status['speed'] = 0

        for pyfile in self.thread_list.py_downloading:
            status['speed'] += pyfile.status.get_speed()

        return status

    def file_exists(self, path): #@XXX: security?!
        return exists(path)

    def get_server_version(self):
        return CURRENT_VERSION

    def add_urls(self, links):
        for link in links:
            self.file_list.collector.addLink(link)
        self.file_list.save()

    def del_links(self, ids):
        for id in ids:
            try:
                self.file_list.collector.removeFile(id)
            except:
                self.file_list.packages.removeFile(id)
        self.file_list.save()

    def del_packages(self, ids):
        for id in ids:
            self.file_list.packages.removePackage(id)
        self.file_list.save()

    def kill(self):
        self.do_kill = True
        return True

    def get_queue(self):
        data = []
        for q in self.file_list.data["queue"]:
            ds = {
                "id": q.data.id,
                "name": q.data.package_name,
                "folder": q.data.folder,
                "files": []
            }
            for f in q.links:
                ds["files"].append({
                    "name": f.status.name,
                    "status": f.status.type,
                    "url": f.url
                })
            data.append(ds)
        return data

    def get_collector_packages(self):
        data = []
        for q in self.file_list.data["packages"]:
            ds = {
                "id": q.data.id,
                "name": q.data.package_name,
                "folder": q.data.folder,
                "files": []
            }
            for f in q.links:
                ds["files"].append({
                    "name": f.status.name,
                    "status": f.status.type,
                    "url": f.url
                })
            data.append(ds)
        return data

    #def move_urls_up(self, ids):
    #    for id in ids:
    #        self.file_list.move(id)
    #    self.file_list.save()

    #def move_urls_down(self, ids):
    #    for id in ids:
    #        self.file_list.move(id, 1)
    #    self.file_list.save()

    def read_url_list(self, url_list):
        """read links from txt"""
        txt = open(self.config['general']['link_file'], 'r')
        new_links = 0
        links = txt.readlines()
        for link in links:
            if link != "\n":
                self.file_list.collector.addLink(link)
                new_links += 1

        txt.close()

        self.file_list.save()
        if new_links:
            self.logger.info("Parsed link from %s: %i" % (self.config['general']['link_file'], new_links))

        txt = open(self.config['general']['link_file'], 'w')
        txt.write("")
        txt.close()

    def is_time_download(self):
        start = self.config['downloadTime']['start'].split(":")
        end = self.config['downloadTime']['end'].split(":")
        return self.compare_time(start, end)

    def is_time_reconnect(self):
        start = self.config['reconnectTime']['start'].split(":")
        end = self.config['reconnectTime']['end'].split(":")
        return self.compare_time(start, end)

if __name__ == "__main__":
    pyload_core = Core()
    try:
        pyload_core.start()
    except KeyboardInterrupt:
        pyload_core.logger.info("killed pyLoad by Terminal")
        exit()

<|endoftext|>


<commit_before>"""
This module describes the interfaces specific to the gaphor.diagram module.
These interfaces are:

 - IConnect
   Use to define adapters for connecting
 - Editor
   Text editor interface

"""

from functools import singledispatch
from gaphor.misc.generic.multidispatch import multidispatch

@singledispatch
def Editor(obj):
    pass

@multidispatch(object, object)
class Group:
    def __init__(self, parent, item):
        pass

@multidispatch(object, object)
class Group:
    def __init__(self, parent, item):
        pass

@multidispatch(object, object)
class IConnect:
    """
    This function is used by the HandleTool to allow connecting
    lines to element items. For each specific case (Element, Line) an
    adapter could be written.
    """

    def __init__(self, item, line_item):
        pass

    def allow(self, handle, port):
        return False
<commit_msg>Remove redundant Group declaration<commit_after>"""
This module describes the interfaces specific to the gaphor.diagram module.
These interfaces are:

 - IConnect
   Use to define adapters for connecting
 - Editor
   Text editor interface

"""

from functools import singledispatch
from gaphor.misc.generic.multidispatch import multidispatch

@singledispatch
def Editor(obj):
    pass

@multidispatch(object, object)
class Group:
    def __init__(self, parent, item):
        pass

@multidispatch(object, object)
class IConnect:
    """
    This function is used by the HandleTool to allow connecting
    lines to element items. For each specific case (Element, Line) an
    adapter could be written.
    """

    def __init__(self, item, line_item):
        pass

    def allow(self, handle, port):
        return False
<|endoftext|>


<commit_before>#########################################################################
#-*- coding:utf-8 -*-
# File Name: odd.py
#########################################################################
#!/bin/python

def is_odd(n):
    return n % 2 == 1

print list(filter(is_odd, [1,2,3,4,5,6,7,8,9]))
<commit_msg>change odd<commit_after>#########################################################################
#-*- coding:utf-8 -*-
# File Name: odd.py
#########################################################################
#!/bin/python

def is_odd(n):
    return n % 2 == 1

odd = lambda n : n %2 == 1

print list(filter(is_odd, [1,2,3,4,5,6,7,8,9]))
print list(filter(odd, [1,2,3,4,5,6,7,8,9]))
print odd(4)
<|endoftext|>


<commit_before>import numpy as np
from proteus import Domain, Context, Comm
from proteus.mprans import SpatialTools as st
import proteus.TwoPhaseFlow.TwoPhaseFlowProblem as TpFlow
from proteus import WaveTools as wt
from proteus.mprans import BoundaryConditions as bc
from proteus.Profiling import logEvent
from proteus import MeshTools as mt
from proteus.ctransportCoefficients import smoothedHeaviside
from proteus.ctransportCoefficients import smoothedHeaviside_integral

#   ____            _            _      ___        _   _
#  / ___|___  _ __ | |_ _____  _| |_   / _ \ _ __ | |_(_) ___  _ __  ___
# | |   / _ \| '_ \| __/ _ \ \/ / __| | | | | '_ \| __| |/ _ \| '_ \/ __|
# | |__| (_) | | | | ||  __/>  <| |_  | |_| | |_) | |_| | (_) | | | \__ \
#  \____\___/|_| |_|\__\___/_/\_\\__|  \___/| .__/ \__|_|\___/|_| |_|___/
#                                           |_|
# Context options
# used in command line directly with option -C
# e.g.: parun [...] -C "g=(0.,-9.81,0.) rho_0=998.2 genMesh=False"
#
# only change/add context options in the "other options section"
# other sections have variables used in _p and _n files

context_options = []
# physical constants
context_options += [
   ("rho_0", 998.2, "Water density"),
   ("nu_0", 1.004e-6, "Water kinematic viscosity m/sec^2"),
   ("rho_1", 1.205, "Air Densiy"),
   ("nu_1", 1.5e-5, "Air kinematic viscosity m/sec^2"),
   ("sigma_01", 0., "Surface tension"),
   ("g", np.array([0, -9.81, 0.]), "Gravitational acceleration vector"),
   ("U", np.array([0.1,0,0]), "inlet velocity"),
   ("outlet_level", 2.5, "outlet level")

]

#wave options
context_options +=[
    ("waveHeight", 0.25, "Wave Height"),
    ("period", 2., "Wave Period"),
    ("mwl", 2.5, "Water Level")

]

# run time options
context_options += [
    ("T", 0.1 ,"Simulation time in s"),
    ("dt_init", 0.0001 ,"Value of initial time step"),
    ("dt_fixed", None, "Value of maximum time step"),
    ("dt_output", 0.1, "number of saves per second"),
    ("cfl", 0.5 ,"Target CFL value")

]

#mesh refinement
context_options += [
    ("he", 1., "Set characteristic element size"),
    # numerical options
    ("genMesh", True, "True: generate new mesh every time. False: do not generate mesh if file 	    exists"),
    ("use_gmsh", False, "use_gmsh"),
    ("refinement", True, "ref"),
    ("refinement_freesurface", 0.05, "ref"),
    ("refinement_grading", 1.2, "ref"),
    ("movingDomain", False, "True/False")
]

# instantiate context options
opts=Context.Options(context_options)

#  ____                        _
# |  _ \  ___  _ __ ___   __ _(_)_ __
# | | | |/ _ \| '_ ` _ \ / _` | | '_ \
# | |_| | (_) | | | | | | (_| | | | | |
# |____/ \___/|_| |_| |_|\__,_|_|_| |_|
# Domain
# All geometrical options go here (but not mesh options)

domain = Domain.PiecewiseLinearComplexDomain()

# Wave
g = np.array([0., -9.81, 0.])
nu_0 = 1.004e-6
period=opts.period

wave = wt.MonochromaticWaves(period=period,
                             waveHeight=opts.waveHeight,
                             mwl=opts.mwl,
                             depth=opts.mwl,
                             g=g,
                             waveDir=(1,0,0),
                             waveType='Linear',
                             fast=True)

#ShapeSTL
SG=st.ShapeSTL(domain,'NWT.stl')

boundaryTags= SG.boundaryTags

SG.regions=np.array([[-13., 2.5, 0.], [-3., 2.5, 0.], [9., 2.5, 0.]]
SG.regionFlags=np.array([1, 2, 3])

dragAlpha = 5*(2*np.pi/period)/nu_0
smoothing = 3. * opts.he
nd = domain.nd
xTop = max(SG.vertices[:,1])

##Relaxation Zones
SG.setGenerationZones(flags=1,
                      epsFact_solid=2.,
                      center=np.array([-13., 2.5, 0.]),
                      orientation=np.array([1., 0., 0.]),
                      waves=wave,
                      dragAlpha=dragAlpha,
                      vert_axis=1,
                      smoothing=smoothing,
                      )

SG.setAbsorptionZones(flags=3,
                      epsFact_solid=4.,
                      center=np.array([9., 2.5, 0.]),
                      orientation=np.array([-1., 0., 0.]),
                      dragAlpha=dragAlpha,
                      vert_axis=1
                      )

SG.BC['Top_Gen'].setAtmosphere(orientation=np.array([0,+1,0]))
SG.BC['Bed_Gen'].setFreeSlip()
SG.BC['Wall_Gen'].setFreeSlip()
SG.BC['Top1'].setAtmosphere(orientation=np.array([0,+1,0]))
SG.BC['Bed1'].setFreeSlip()
SG.BC['Wall1'].setFreeSlip()
SG.BC['Top_Abs'].setAtmosphere(orientation=np.array([0,+1,0]))
SG.BC['Bed_Abs'].setFreeSlip()
SG.BC['Wall_Abs'].setFreeSlip()
SG.BC['Concrete1'].setFreeSlip()

SG.BC['Inlet1'].setNonMaterial()
SG.BC['Outlet1'].setNonMaterial()

SG.BC['Inlet_Gen'].setUnsteadyTwoPhaseVelocityInlet(wave=wave,
                                                    vert_axis=1,
                                                    smoothing=3.*opts.he,
                                                    orientation=np.array([-1.,0.,0.]),
                                                    vof_air=1.,
                                                    vof_water=0.)

SG.BC['Outlet_Abs'].setHydrostaticPressureOutletWithDepth(seaLevel= opts.mwl,
                                                          rhoUp=opts.rho_1,
                                                          rhoDown=opts.rho_0,
                                                          g=opts.g,
                                                          refLevel=xTop,
                                                          smoothing=smoothing,
                                                          orientation=np.array([1.,0.,0.]),
                                                          vert_axis=1)

#  ___       _ _   _       _    ____                _ _ _   _
# |_ _|_ __ (_) |_(_) __ _| |  / ___|___  _ __   __| (_) |_(_) ___  _ __  ___
#  | || '_ \| | __| |/ _` | | | |   / _ \| '_ \ / _` | | __| |/ _ \| '_ \/ __|
#  | || | | | | |_| | (_| | | | |__| (_) | | | | (_| | | |_| | (_) | | | \__ \
# |___|_| |_|_|\__|_|\__,_|_|  \____\___/|_| |_|\__,_|_|\__|_|\___/|_| |_|___/
#

vert_axis = np.where(abs(opts.g)>0)[0][0]

class At_Rest:
    def uOfXT(self, x, t):
        return 0.0
class PHI_IC:
    def uOfXT(self, x, t):
        return x[vert_axis]-opts.outlet_level
class VF_IC:
    def uOfXT(self, x, t):
        return smoothedHeaviside(smoothing,opts.outlet_level-x[vert_axis])
class P_IC:
    def uOfXT(self, x, t):
        p_top = 0.0
        phi_top = xTop
        phi = x[vert_axis] - opts.outlet_level
        return p_top - opts.g[vert_axis] * (opts.rho_0 * (phi_top - phi) +
                                       (opts.rho_1 - opts.rho_0) *
                                       (smoothedHeaviside_integral(smoothing, phi_top)-
                                        smoothedHeaviside_integral(smoothing, phi)))

initialConditions = {'pressure': P_IC(),
                     'vel_u': At_Rest(),
                     'vel_v': At_Rest(),
                     'vel_w':At_Rest(),
                     'vof': VF_IC(),
                     'ncls': PHI_IC(),
                     'rdls': PHI_IC()}
outputStepping = TpFlow.OutputStepping(final_time=opts.T,
                                       dt_init=opts.dt_init,
                                       dt_output=opts.dt_output,
                                       nDTout=None,
                                       dt_fixed=None)

myTpFlowProblem = TpFlow.TwoPhaseFlowProblem(ns_model=None,
                                             ls_model=None,
                                             nd=domain.nd,
                                             cfl=opts.cfl,
                                             outputStepping=outputStepping,
                                             structured=False,
                                             he=opts.he,
                                             nnx=None,
                                             nny=None,
                                             nnz=None,
                                             domain=domain,
                                             initialConditions=initialConditions,
                                             boundaryConditions=None)

params = myTpFlowProblem.Parameters
myTpFlowProblem.useSuperLu=False#True
params.physical.densityA = opts.rho_0  # water
params.physical.densityB = opts.rho_1  # air
params.physical.kinematicViscosityA = opts.nu_0  # water
params.physical.kinematicViscosityB = opts.nu_1  # air
params.physical.surf_tension_coeff = opts.sigma_01

m = params.Models

m.rdls.p.CoefficientsOptions.epsFact=0.75
m.rans2p.index = 0
m.vof.index = 1
m.ncls.index = 2
m.rdls.index = 3
m.mcorr.index = 4
m.rdls.n.maxLineSearches=0
m.rdls.n.maxNonlinearIts=50
#pdb.set_trace()

#Assemble domain
domain.MeshOptions.he = opts.he
mesh_fileprefix = 'mesh_he_'+str(int(opts.he*1000))
domain.MeshOptions.setOutputFiles(name=mesh_fileprefix)
st.assembleDomain(domain)
#pdb.set_trace()
myTpFlowProblem.Parameters.Models.rans2p.auxiliaryVariables += domain.auxiliaryVariables['twp']

<commit_msg>Delete STLMesh_Zones.py<commit_after><|endoftext|>


<commit_before># !usr/bin/env python
# catenary calculation, re-written in python - NO Elasticity!!!

import math
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import fsolve
#from inout import write_file

def cat(a):
	# defining catenary function
	#catenary eq (math): a*sinh(L/(2*a)+atanh(d/S))+a*sinh(L/(2*a)-atanh(d/S))-S=0
	return a*math.sinh(L/(2*a))+math.atanh(d/S)+a*math.sinh(L/(2*a))-math.atanh(d/S)-S

L=float(input("Horizontal Distance between supports [m]: "))
d=float(input ("Vertical Distance between supports [m]: "))
S=float(input("Length of cable [m] - must be greater than distance between supports:  "))
w=float(input("Unit weight of cable [kg/m]: "))
za=float(input("Elevation of higher support from reference plane [m]: "))

#checking if cable length is bigger than total distance between supports
distance=(L**2+d**2)**0.5
if S <= distance:
	print ("Length of cable must be greater than TOTAL distance between supports!")
	S=float(input("Length of cable [m]: "))
else:
	pass

# solving catenary function for 'a'

a=fsolve(cat, 1)

# hor. distance between lowest catenary point (P) to higher support point (La)
La=a*(L/(2*a)+math.atanh(d/S))
# hor. distance between lowest catenary point (P) to lower support point (Lb)
Lb=L-La
# vert. distance from higher support point to lowest point (P) in catenary (ha)
ha=a*math.cosh(La/a)-a
## calculating reaction forces and angles
# catenary lenght between support "A" (higher) and "P" - Sa
Sa=a*math.sinh(La/a)
# catenary lenght between support "B" )lower) and "P" - Sb
Sb=a*math.sinh(Lb/a)
# horizontal tension - constant through catenary: H
H=w*a
# vertical tension at "A"  (Va) and "B" (Vb)
Va=Sa*w
Vb=Sb*w
# tension at "A" (TA) and B (TB)
TA=(H**2+Va**2)**0.5
TB=(H**2+Vb**2)**0.5
# inclination angles from vertical at "A" (ThetA) and B (ThetB)
ThetA=math.atan(H/Va)
ThetB=math.atan(H/Vb)
ThetAd=ThetA*180/math.pi;
ThetBd=ThetB*180/math.pi;
# establishing A, B and P in coordinate system
# index "a" corresponding to point "A", "b" to "B"-point and "P" to lowest caten. point
zb=za-d
zp=za-ha
xa=La
xp=0
xb=-Lb

# writting results to file
fname='catenary_res.txt'
fn=open(fname, 'a')
print fn, "Horizontal Distance between supports in meters: ", round(L,3)
print fn, "Catenary length in meters: ", round(S,3)
print fn, "Vertical Distance Between supports in meters: ", round(d,3)
print fn, "Unit Weight of Catenary line in kg/m: ", round(w,3)
print fn, "Elevation of higher support (A) from reference plane in meters: ", round(za,3)
print fn, "\Catenary coef.: ", round(a,5)
print fn, "Horizontal tension in kg (constant along line: ", round(H,3)
print fn, "Vertical tension in A in kg: ", round(Va,3)
print fn, "Total tension in A in kg: ", round(TA,3)
print fn, "Total tension in B in kg: ", round(TB,3)
print fn, "Inclination angle from vertical at A in radians: ", round(ThetA,3)
print fn, "Inclination angle from vertical at B in radians: ", round(ThetB,3)
print fn, "Inclination angle from vertical at A in degrees: ", round(ThetAd,3)
print fn, "Inclination angle from vertical at B in degrees: ", round(ThetBd,3)
fn.close()

# graphing catenary curve - matplotlib & writting coordinates in file
xinc=L/100
y=[]
xc=[]
fncoords="catenary_coords.txt"
fn=open(fncoords, "a")

for x in np.arange (xb, xa+xinc, xinc):
	ycal=a*math.cosh(x/a)
	fn.write("\n")
	fn.write(str(round(x,3)))
	fn.write("\t")
	fn.write(str(round(ycal[0],3)))
	y.append(ycal)
	xc.append(x)
	fn.close()

	# plotting, finally
	plt.plot(xc,y)
	plt.xlabel("X-distance [m]")
	plt.ylabel("Y-distance [m]")
	plt.grid()
	plt.show()
<commit_msg>Update pycatenary.py   - Update pycatenary.py so that it will not attempt to make a plot.   - Update pycatenary.py so that it will not attempt to make a file of outputs, but instead only prints out.<commit_after># !usr/bin/env python
# catenary calculation, re-written in python - NO Elasticity!!!

import math
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import fsolve
#from inout import write_file

def cat(a):
	# defining catenary function
	#catenary eq (math): a*sinh(L/(2*a)+atanh(d/S))+a*sinh(L/(2*a)-atanh(d/S))-S=0
	return a*math.sinh(L/(2*a))+math.atanh(d/S)+a*math.sinh(L/(2*a))-math.atanh(d/S)-S

L=float(input("Horizontal Distance between supports [m]: "))
d=float(input ("Vertical Distance between supports [m]: "))
S=float(input("Length of cable [m] - must be greater than distance between supports:  "))
w=float(input("Unit weight of cable [kg/m]: "))
za=float(input("Elevation of higher support from reference plane [m]: "))

#checking if cable length is bigger than total distance between supports
distance=(L**2+d**2)**0.5
if S <= distance:
	print ("Length of cable must be greater than TOTAL distance between supports!")
	S=float(input("Length of cable [m]: "))
else:
	pass

# solving catenary function for 'a'

a=fsolve(cat, 1)

# hor. distance between lowest catenary point (P) to higher support point (La)
La=a*(L/(2*a)+math.atanh(d/S))
# hor. distance between lowest catenary point (P) to lower support point (Lb)
Lb=L-La
# vert. distance from higher support point to lowest point (P) in catenary (ha)
ha=a*math.cosh(La/a)-a
## calculating reaction forces and angles
# catenary lenght between support "A" (higher) and "P" - Sa
Sa=a*math.sinh(La/a)
# catenary lenght between support "B" )lower) and "P" - Sb
Sb=a*math.sinh(Lb/a)
# horizontal tension - constant through catenary: H
H=w*a
# vertical tension at "A"  (Va) and "B" (Vb)
Va=Sa*w
Vb=Sb*w
# tension at "A" (TA) and B (TB)
TA=(H**2+Va**2)**0.5
TB=(H**2+Vb**2)**0.5
# inclination angles from vertical at "A" (ThetA) and B (ThetB)
ThetA=math.atan(H/Va)
ThetB=math.atan(H/Vb)
ThetAd=ThetA*180/math.pi;
ThetBd=ThetB*180/math.pi;
# establishing A, B and P in coordinate system
# index "a" corresponding to point "A", "b" to "B"-point and "P" to lowest caten. point
zb=za-d
zp=za-ha
xa=La
xp=0
xb=-Lb

# writting results to file
#fname='catenary_res.txt'
#fn=open(fname, 'a')
print fn, "Horizontal Distance between supports in meters: ", round(L,3)
print fn, "Catenary length in meters: ", round(S,3)
print fn, "Vertical Distance Between supports in meters: ", round(d,3)
print fn, "Unit Weight of Catenary line in kg/m: ", round(w,3)
print fn, "Elevation of higher support (A) from reference plane in meters: ", round(za,3)
print fn, "\Catenary coef.: ", round(a,5)
print fn, "Horizontal tension in kg (constant along line: ", round(H,3)
print fn, "Vertical tension in A in kg: ", round(Va,3)
print fn, "Total tension in A in kg: ", round(TA,3)
print fn, "Total tension in B in kg: ", round(TB,3)
print fn, "Inclination angle from vertical at A in radians: ", round(ThetA,3)
print fn, "Inclination angle from vertical at B in radians: ", round(ThetB,3)
print fn, "Inclination angle from vertical at A in degrees: ", round(ThetAd,3)
print fn, "Inclination angle from vertical at B in degrees: ", round(ThetBd,3)
#fn.close()

# graphing catenary curve - matplotlib & writting coordinates in file
#xinc=L/100
#y=[]
#xc=[]
#fncoords="catenary_coords.txt"
#fn=open(fncoords, "a")
#
#for x in np.arange (xb, xa+xinc, xinc):
#	ycal=a*math.cosh(x/a)
#	fn.write("\n")
#	fn.write(str(round(x,3)))
#	fn.write("\t")
#	fn.write(str(round(ycal[0],3)))
#	y.append(ycal)
#	xc.append(x)
#	fn.close()
#
#	# plotting, finally
#	plt.plot(xc,y)
#	plt.xlabel("X-distance [m]")
#	plt.ylabel("Y-distance [m]")
#	plt.grid()
#	plt.show()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from simulator import *
import time
import math

class PID(object):
    def __init__(self):
        self._kp = 790.0
        self._ki = 1.0
        self._kd = 3.0

        self._target_theta = 0.0
        self._sampling_time = 0.01

        self._theta0 = 0.0
        self._thetai = 0.0

    def init_status(self):
        self._theta0 = 0.0
        self._thetai = 0.0

    def get_velocity(self, theta):
        theta = self._target_theta - theta
        dtheta = (theta - self._theta0) / self._sampling_time
        self._theta0 = theta
        self._thetai += theta * self._sampling_time

        if self._thetai > 10000:
            self._thetai = 10000
        elif self._thetai < -10000:
            self._thetai = -10000

        duty_ratio = theta * self._kp + self._thetai * self._ki + dtheta * self._kd

        if duty_ratio > 1:
            duty_ratio = 1.0
        elif duty_ratio < -1:
            duty_ratio = -1.0

        return -duty_ratio

if __name__ == '__main__':
    pid = PID()

    clientID = start_simulator()

    res, objs = vrep.simxGetObjects(clientID, vrep.sim_handle_all, vrep.simx_opmode_oneshot_wait)
    error_check(clientID, res)

    time.sleep(1)

    res, JointDynamic = vrep.simxGetObjectHandle(clientID, "joint" , vrep.simx_opmode_oneshot_wait)
    error_check(clientID, res)
    res, Potentiometer = vrep.simxGetObjectHandle(clientID, "Potentiometer", vrep.simx_opmode_oneshot_wait)
    error_check(clientID, res)

    vrep.simxSetJointForce(clientID, JointDynamic, 0.384, vrep.simx_opmode_oneshot)

    while True:
        res, theta = vrep.simxGetJointPosition(clientID, Potentiometer, vrep.simx_opmode_oneshot)

        linearVelocity = pid.get_velocity(theta)

        vrep.simxSetJointTargetVelocity(clientID, JointDynamic, linearVelocity, vrep.simx_opmode_oneshot)

        #Set pendulum Position
        if math.fabs(theta) >= math.radians(90.0):
            print "Reset positions"
            pid.init_status()
            vrep.simxStopSimulation(clientID, vrep.simx_opmode_oneshot_wait)
            time.sleep(1)
            wait_until_simulator_started(clientID)

        time.sleep(0.01)

<commit_msg>update pid.py<commit_after># -*- coding: utf-8 -*-

from simulator import *
import time
import math

class PID(object):
    def __init__(self):
        self._kp = 400.0
        self._ki = 10.0
        self._kd = 40.0

        self._target_theta = 0.0
        self._sampling_time = 0.01

        self._theta0 = 0.0
        self._thetai = 0.0

    def init_status(self):
        self._theta0 = 0.0
        self._thetai = 0.0

    def get_velocity(self, theta):
        theta = self._target_theta - theta
        dtheta = (theta - self._theta0) / self._sampling_time
        self._theta0 = theta
        self._thetai += theta * self._sampling_time

        if self._thetai > 10000:
            self._thetai = 10000
        elif self._thetai < -10000:
            self._thetai = -10000

        duty_ratio = theta * self._kp + self._thetai * self._ki + dtheta * self._kd

        if duty_ratio > 100.0:
            duty_ratio = 100.0
        elif duty_ratio < -100.0:
            duty_ratio = -100.0

        return -duty_ratio

if __name__ == '__main__':
    pid = PID()

    clientID = start_simulator()

    res, objs = vrep.simxGetObjects(clientID, vrep.sim_handle_all, vrep.simx_opmode_oneshot_wait)
    error_check(clientID, res)

    time.sleep(1)

    res, JointDynamic = vrep.simxGetObjectHandle(clientID, "joint" , vrep.simx_opmode_oneshot_wait)
    error_check(clientID, res)
    res, Potentiometer = vrep.simxGetObjectHandle(clientID, "Potentiometer", vrep.simx_opmode_oneshot_wait)
    error_check(clientID, res)

    vrep.simxSetJointForce(clientID, JointDynamic, 0.384, vrep.simx_opmode_oneshot)

    while True:
        res, theta = vrep.simxGetJointPosition(clientID, Potentiometer, vrep.simx_opmode_oneshot)

        linearVelocity = pid.get_velocity(theta)

        vrep.simxSetJointTargetVelocity(clientID, JointDynamic, linearVelocity, vrep.simx_opmode_oneshot)

        #Set pendulum Position
        if math.fabs(theta) >= math.radians(90.0):
            print "Reset positions"
            pid.init_status()
            vrep.simxStopSimulation(clientID, vrep.simx_opmode_oneshot_wait)
            time.sleep(1)
            wait_until_simulator_started(clientID)

        time.sleep(0.01)

<|endoftext|>


<commit_before>#=======================================================================
# sim.py
#=======================================================================
# This is the common top-level simulator. ISA implementations can use
# various hooks to configure the behavior.

import os
import sys

# ensure we know where the pypy source code is
# XXX: removed the dependency to PYDGIN_PYPY_SRC_DIR because rpython
# libraries are much slower than native python when running on an
# interpreter. So unless the user have added rpython source to their
# PYTHONPATH, we should use native python.
#try:
#  sys.path.append( os.environ['PYDGIN_PYPY_SRC_DIR'] )
#except KeyError as e:
#  print "NOTE: PYDGIN_PYPY_SRC_DIR not defined, using pure python " \
#        "implementation"

from pydgin.debug import Debug, pad, pad_hex
from pydgin.misc  import FatalError, NotImplementedInstError
from pydgin.jit   import JitDriver, hint, set_user_param, set_param

def jitpolicy(driver):
  from rpython.jit.codewriter.policy import JitPolicy
  return JitPolicy()

#-------------------------------------------------------------------------
# Sim
#-------------------------------------------------------------------------
# Abstract simulator class

class Sim( object ):

  def __init__( self, arch_name_human, arch_name="", jit_enabled=False ):

    # the human-friendly architecture name can contain large caps, special
    # characters etc.

    self.arch_name_human = arch_name_human
    if arch_name == "":
      self.arch_name = arch_name_human.lower()
    else:
      self.arch_name = arch_name

    self.jit_enabled = jit_enabled

    if jit_enabled:
      self.jitdriver = JitDriver( greens =['pc',],
                                  reds   = ['max_insts', 'state', 'sim',],
                                  virtualizables  =['state',],
                                  get_printable_location=self.get_location,
                                )

      # Set the default trace limit here. Different ISAs can override this
      # value if necessary
      self.default_trace_limit = 400000

    self.max_insts = 0

  #-----------------------------------------------------------------------
  # decode
  #-----------------------------------------------------------------------
  # This needs to be implemented in the child class

  def decode( self, bits ):
    raise NotImplementedError()

  #-----------------------------------------------------------------------
  # hooks provided for isa implementations
  #-----------------------------------------------------------------------

  def pre_execute( self ):
    pass

  def post_execute( self ):
    pass

  #-----------------------------------------------------------------------
  # init_state
  #-----------------------------------------------------------------------
  # This needs to be implemented in the child class

  def init_state( self, exe_file, exe_name, run_argv, testbin ):
    raise NotImplementedError()

  #-----------------------------------------------------------------------
  # help message
  #-----------------------------------------------------------------------
  # the help message to display on --help

  help_message = """
  Pydgin %s Instruction Set Simulator
  usage: %s <args> <sim_exe> <sim_args>

  <sim_exe>  the executable to be simulated
  <sim_args> arguments to be passed to the simulated executable
  <args>     the following optional arguments are supported:

    --help,-h       Show this message and exit
    --test          Run in testing mode (for running asm tests)
    --env,-e <NAME>=<VALUE>
                    Set an environment variable to be passed to the
                    simulated program. Can use multiple --env flags to set
                    multiple environment variables.
    --debug,-d <flags>[:<start_after>]
                    Enable debug flags in a comma-separated form (e.g.
                    "--debug syscalls,insts"). If provided, debugs starts
                    after <start_after> cycles. The following flags are
                    supported:
         insts              cycle-by-cycle instructions
         rf                 register file accesses
         mem                memory accesses
         regdump            register dump
         syscalls           syscall information
         bootstrap          initial stack and register state

    --max-insts <i> Run until the maximum number of instructions
    --jit <flags>   Set flags to tune the JIT (see
                    rpython.rlib.jit.PARAMETER_DOCS)

  """

  #-----------------------------------------------------------------------
  # get_location
  #-----------------------------------------------------------------------
  # for debug printing in PYPYLOG

  @staticmethod
  def get_location( pc ):
    # TODO: add the disassembly of the instruction here as well
    return "pc: %x" % pc

  #-----------------------------------------------------------------------
  # run
  #-----------------------------------------------------------------------
  def run( self ):
    self = hint( self, promote=True )
    s = self.state

    max_insts = self.max_insts
    jitdriver = self.jitdriver

    while s.running:

      jitdriver.jit_merge_point(
        pc        = s.fetch_pc(),
        max_insts = max_insts,
        state     = s,
        sim       = self,
      )

      # constant-fold pc and mem
      pc  = hint( s.fetch_pc(), promote=True )
      old = pc
      mem = hint( s.mem, promote=True )

      if s.debug.enabled( "insts" ):
        print pad( "%x" % pc, 8, " ", False ),

      # the print statement in memcheck conflicts with @elidable in iread.
      # So we use normal read if memcheck is enabled which includes the
      # memory checks

      if s.debug.enabled( "memcheck" ):
        inst_bits = mem.read( pc, 4 )
      else:
        # we use trace elidable iread instead of just read
        inst_bits = mem.iread( pc, 4 )

      try:
        inst, exec_fun = self.decode( inst_bits )

        if s.debug.enabled( "insts" ):
          print "%s %s %s" % (
                  pad_hex( inst_bits ),
                  pad( inst.str, 12 ),
                  pad( "%d" % s.num_insts, 8 ), ),

        self.pre_execute()

        exec_fun( s, inst )
      except NotImplementedInstError:
        # re-decode instruction to get the instruction name
        inst, _ = self.decode( inst_bits )

        print "Instruction not implemented: %s (pc: 0x%s), aborting!" \
              % ( inst.str, pad_hex( pc ) )
        break
      except FatalError as error:
        print "Exception in execution (pc: 0x%s), aborting!" % pad_hex( pc )
        print "Exception message: %s" % error.msg
        break

      s.num_insts += 1    # TODO: should this be done inside instruction definition?
      if s.stats_en: s.stat_num_insts += 1

      self.post_execute()

      if s.debug.enabled( "insts" ):
        print
      if s.debug.enabled( "regdump" ):
        s.rf.print_regs( per_row=4 )

      # check if we have reached the end of the maximum instructions and
      # exit if necessary
      if max_insts != 0 and s.num_insts >= max_insts:
        print "Reached the max_insts (%d), exiting." % max_insts
        break

      if s.fetch_pc() < old:
        jitdriver.can_enter_jit(
          pc        = s.fetch_pc(),
          max_insts = max_insts,
          state     = s,
          sim       = self,
        )

    print 'DONE! Status =', s.status
    print 'Instructions Executed =', s.num_insts

  #-----------------------------------------------------------------------
  # get_entry_point
  #-----------------------------------------------------------------------
  # generates and returns the entry_point function used to start the
  # simulator

  def get_entry_point( self ):
    def entry_point( argv ):

      # set the trace_limit parameter of the jitdriver
      if self.jit_enabled:
        set_param( self.jitdriver, "trace_limit", self.default_trace_limit )

      filename_idx       = 0
      debug_flags        = []
      debug_starts_after = 0
      testbin            = False
      max_insts          = 0
      envp               = []

      # we're using a mini state machine to parse the args

      prev_token = ""

      # list of tokens that require an additional arg

      tokens_with_args = [ "-h", "--help",
                           "-e", "--env",
                           "-d", "--debug",
                           "--max-insts",
                           "--jit",
                         ]

      # go through the args one by one and parse accordingly

      for i in xrange( 1, len( argv ) ):
        token = argv[i]

        if prev_token == "":

          if token == "--help" or token == "-h":
            print self.help_message % ( self.arch_name_human, argv[0] )
            return 0

          elif token == "--test":
            testbin = True

          elif token == "--debug" or token == "-d":
            prev_token = token
            # warn the user if debugs are not enabled for this translation
            if not Debug.global_enabled:
              print "WARNING: debugs are not enabled for this translation. " + \
                    "To allow debugs, translate with --debug option."

          elif token in tokens_with_args:
            prev_token = token

          elif token[:1] == "-":
            # unknown option
            print "Unknown argument %s" % token
            return 1

          else:
            # this marks the start of the program name
            filename_idx = i
            break

        else:
          if prev_token == "--env" or prev_token == "-e":
            envp.append( token )

          elif prev_token == "--debug" or prev_token == "-d":
            # if debug start after provided (using a colon), parse it
            debug_tokens = token.split( ":" )
            if len( debug_tokens ) > 1:
              debug_starts_after = int( debug_tokens[1] )

            debug_flags = debug_tokens[0].split( "," )

          elif prev_token == "--max-insts":
            self.max_insts = int( token )

          elif prev_token == "--jit":
            # pass the jit flags to rpython.rlib.jit
            set_user_param( self.jitdriver, token )

          prev_token = ""

      if filename_idx == 0:
        print "You must supply a filename"
        return 1

      # create a Debug object which contains the debug flags

      self.debug = Debug( debug_flags, debug_starts_after )

      filename = argv[ filename_idx ]

      # args after program are args to the simulated program

      run_argv = argv[ filename_idx : ]

      # Open the executable for reading

      try:
        exe_file = open( filename, 'rb' )

      except IOError:
        print "Could not open file %s" % filename
        return 1

      # Call ISA-dependent init_state to load program, initialize memory
      # etc.

      self.init_state( exe_file, filename, run_argv, envp, testbin )

      # pass the state to debug for cycle-triggered debugging

      self.debug.set_state( self.state )

      # Close after loading

      exe_file.close()

      # Execute the program

      self.run()

      return 0

    return entry_point

  #-----------------------------------------------------------------------
  # target
  #-----------------------------------------------------------------------
  # Enables RPython translation of our interpreter.

  def target( self, driver, args ):

    # if --debug flag is provided in translation, we enable debug printing

    if "--debug" in args:
      print "Enabling debugging"
      Debug.global_enabled = True
    else:
      print "Disabling debugging"

    # form a name
    exe_name = "pydgin-%s" % self.arch_name
    if driver.config.translation.jit:
      exe_name += "-jit"
    else:
      exe_name += "-nojit"

    if Debug.global_enabled:
      exe_name += "-debug"

    print "Translated binary name:", exe_name
    driver.exe_name = exe_name

    # NOTE: RPython has an assertion to check the type of entry_point to
    # be function (not a bound method). So we use get_entry_point which
    # generates a function type
    #return self.entry_point, None
    return self.get_entry_point(), None

#-------------------------------------------------------------------------
# init_sim
#-------------------------------------------------------------------------
# Simulator implementations need to call this function at the top level.
# This takes care of adding target function to top level environment and
# running the simulation in interpreted mode if directly called
# ( __name__ == "__main__" )

def init_sim( sim ):

  # this is a bit hacky: we get the global variables of the caller from
  # the stack frame to determine if this is being run top level and add
  # target function required by rpython toolchain

  caller_globals = sys._getframe(1).f_globals
  caller_name    = caller_globals[ "__name__" ]

  # add target function to top level

  caller_globals[ "target" ] = sim.target

  #-----------------------------------------------------------------------
  # main
  #-----------------------------------------------------------------------
  # Enables CPython simulation of our interpreter.
  if caller_name == "__main__":
    # enable debug flags in interpreted mode
    Debug.global_enabled = True
    sim.get_entry_point()( sys.argv )

<commit_msg>Exit with correct code.<commit_after>#=======================================================================
# sim.py
#=======================================================================
# This is the common top-level simulator. ISA implementations can use
# various hooks to configure the behavior.

import os
import sys

# ensure we know where the pypy source code is
# XXX: removed the dependency to PYDGIN_PYPY_SRC_DIR because rpython
# libraries are much slower than native python when running on an
# interpreter. So unless the user have added rpython source to their
# PYTHONPATH, we should use native python.
#try:
#  sys.path.append( os.environ['PYDGIN_PYPY_SRC_DIR'] )
#except KeyError as e:
#  print "NOTE: PYDGIN_PYPY_SRC_DIR not defined, using pure python " \
#        "implementation"

from pydgin.debug import Debug, pad, pad_hex
from pydgin.misc  import FatalError, NotImplementedInstError
from pydgin.jit   import JitDriver, hint, set_user_param, set_param

def jitpolicy(driver):
  from rpython.jit.codewriter.policy import JitPolicy
  return JitPolicy()

#-------------------------------------------------------------------------
# Sim
#-------------------------------------------------------------------------
# Abstract simulator class

class Sim( object ):

  def __init__( self, arch_name_human, arch_name="", jit_enabled=False ):

    # the human-friendly architecture name can contain large caps, special
    # characters etc.

    self.arch_name_human = arch_name_human
    if arch_name == "":
      self.arch_name = arch_name_human.lower()
    else:
      self.arch_name = arch_name

    self.jit_enabled = jit_enabled

    if jit_enabled:
      self.jitdriver = JitDriver( greens =['pc',],
                                  reds   = ['max_insts', 'state', 'sim',],
                                  virtualizables  =['state',],
                                  get_printable_location=self.get_location,
                                )

      # Set the default trace limit here. Different ISAs can override this
      # value if necessary
      self.default_trace_limit = 400000

    self.max_insts = 0

  #-----------------------------------------------------------------------
  # decode
  #-----------------------------------------------------------------------
  # This needs to be implemented in the child class

  def decode( self, bits ):
    raise NotImplementedError()

  #-----------------------------------------------------------------------
  # hooks provided for isa implementations
  #-----------------------------------------------------------------------

  def pre_execute( self ):
    pass

  def post_execute( self ):
    pass

  #-----------------------------------------------------------------------
  # init_state
  #-----------------------------------------------------------------------
  # This needs to be implemented in the child class

  def init_state( self, exe_file, exe_name, run_argv, testbin ):
    raise NotImplementedError()

  #-----------------------------------------------------------------------
  # help message
  #-----------------------------------------------------------------------
  # the help message to display on --help

  help_message = """
  Pydgin %s Instruction Set Simulator
  usage: %s <args> <sim_exe> <sim_args>

  <sim_exe>  the executable to be simulated
  <sim_args> arguments to be passed to the simulated executable
  <args>     the following optional arguments are supported:

    --help,-h       Show this message and exit
    --test          Run in testing mode (for running asm tests)
    --env,-e <NAME>=<VALUE>
                    Set an environment variable to be passed to the
                    simulated program. Can use multiple --env flags to set
                    multiple environment variables.
    --debug,-d <flags>[:<start_after>]
                    Enable debug flags in a comma-separated form (e.g.
                    "--debug syscalls,insts"). If provided, debugs starts
                    after <start_after> cycles. The following flags are
                    supported:
         insts              cycle-by-cycle instructions
         rf                 register file accesses
         mem                memory accesses
         regdump            register dump
         syscalls           syscall information
         bootstrap          initial stack and register state

    --max-insts <i> Run until the maximum number of instructions
    --jit <flags>   Set flags to tune the JIT (see
                    rpython.rlib.jit.PARAMETER_DOCS)

  """

  #-----------------------------------------------------------------------
  # get_location
  #-----------------------------------------------------------------------
  # for debug printing in PYPYLOG

  @staticmethod
  def get_location( pc ):
    # TODO: add the disassembly of the instruction here as well
    return "pc: %x" % pc

  #-----------------------------------------------------------------------
  # run
  #-----------------------------------------------------------------------
  def run( self ):
    self = hint( self, promote=True )
    s = self.state

    max_insts = self.max_insts
    jitdriver = self.jitdriver

    while s.running:

      jitdriver.jit_merge_point(
        pc        = s.fetch_pc(),
        max_insts = max_insts,
        state     = s,
        sim       = self,
      )

      # constant-fold pc and mem
      pc  = hint( s.fetch_pc(), promote=True )
      old = pc
      mem = hint( s.mem, promote=True )

      if s.debug.enabled( "insts" ):
        print pad( "%x" % pc, 8, " ", False ),

      # the print statement in memcheck conflicts with @elidable in iread.
      # So we use normal read if memcheck is enabled which includes the
      # memory checks

      if s.debug.enabled( "memcheck" ):
        inst_bits = mem.read( pc, 4 )
      else:
        # we use trace elidable iread instead of just read
        inst_bits = mem.iread( pc, 4 )

      try:
        inst, exec_fun = self.decode( inst_bits )

        if s.debug.enabled( "insts" ):
          print "%s %s %s" % (
                  pad_hex( inst_bits ),
                  pad( inst.str, 12 ),
                  pad( "%d" % s.num_insts, 8 ), ),

        self.pre_execute()

        exec_fun( s, inst )
      except NotImplementedInstError:
        # re-decode instruction to get the instruction name
        inst, _ = self.decode( inst_bits )

        print "Instruction not implemented: %s (pc: 0x%s), aborting!" \
              % ( inst.str, pad_hex( pc ) )
        break
      except FatalError as error:
        print "Exception in execution (pc: 0x%s), aborting!" % pad_hex( pc )
        print "Exception message: %s" % error.msg
        break

      s.num_insts += 1    # TODO: should this be done inside instruction definition?
      if s.stats_en: s.stat_num_insts += 1

      self.post_execute()

      if s.debug.enabled( "insts" ):
        print
      if s.debug.enabled( "regdump" ):
        s.rf.print_regs( per_row=4 )

      # check if we have reached the end of the maximum instructions and
      # exit if necessary
      if max_insts != 0 and s.num_insts >= max_insts:
        print "Reached the max_insts (%d), exiting." % max_insts
        break

      if s.fetch_pc() < old:
        jitdriver.can_enter_jit(
          pc        = s.fetch_pc(),
          max_insts = max_insts,
          state     = s,
          sim       = self,
        )

    print 'DONE! Status =', s.status
    print 'Instructions Executed =', s.num_insts

  #-----------------------------------------------------------------------
  # get_entry_point
  #-----------------------------------------------------------------------
  # generates and returns the entry_point function used to start the
  # simulator

  def get_entry_point( self ):
    def entry_point( argv ):

      # set the trace_limit parameter of the jitdriver
      if self.jit_enabled:
        set_param( self.jitdriver, "trace_limit", self.default_trace_limit )

      filename_idx       = 0
      debug_flags        = []
      debug_starts_after = 0
      testbin            = False
      max_insts          = 0
      envp               = []

      # we're using a mini state machine to parse the args

      prev_token = ""

      # list of tokens that require an additional arg

      tokens_with_args = [ "-h", "--help",
                           "-e", "--env",
                           "-d", "--debug",
                           "--max-insts",
                           "--jit",
                         ]

      # go through the args one by one and parse accordingly

      for i in xrange( 1, len( argv ) ):
        token = argv[i]

        if prev_token == "":

          if token == "--help" or token == "-h":
            print self.help_message % ( self.arch_name_human, argv[0] )
            return 0

          elif token == "--test":
            testbin = True

          elif token == "--debug" or token == "-d":
            prev_token = token
            # warn the user if debugs are not enabled for this translation
            if not Debug.global_enabled:
              print "WARNING: debugs are not enabled for this translation. " + \
                    "To allow debugs, translate with --debug option."

          elif token in tokens_with_args:
            prev_token = token

          elif token[:1] == "-":
            # unknown option
            print "Unknown argument %s" % token
            return 1

          else:
            # this marks the start of the program name
            filename_idx = i
            break

        else:
          if prev_token == "--env" or prev_token == "-e":
            envp.append( token )

          elif prev_token == "--debug" or prev_token == "-d":
            # if debug start after provided (using a colon), parse it
            debug_tokens = token.split( ":" )
            if len( debug_tokens ) > 1:
              debug_starts_after = int( debug_tokens[1] )

            debug_flags = debug_tokens[0].split( "," )

          elif prev_token == "--max-insts":
            self.max_insts = int( token )

          elif prev_token == "--jit":
            # pass the jit flags to rpython.rlib.jit
            set_user_param( self.jitdriver, token )

          prev_token = ""

      if filename_idx == 0:
        print "You must supply a filename"
        return 1

      # create a Debug object which contains the debug flags

      self.debug = Debug( debug_flags, debug_starts_after )

      filename = argv[ filename_idx ]

      # args after program are args to the simulated program

      run_argv = argv[ filename_idx : ]

      # Open the executable for reading

      try:
        exe_file = open( filename, 'rb' )

      except IOError:
        print "Could not open file %s" % filename
        return 1

      # Call ISA-dependent init_state to load program, initialize memory
      # etc.

      self.init_state( exe_file, filename, run_argv, envp, testbin )

      # pass the state to debug for cycle-triggered debugging

      self.debug.set_state( self.state )

      # Close after loading

      exe_file.close()

      # Execute the program

      self.run()

      return 0

    return entry_point

  #-----------------------------------------------------------------------
  # target
  #-----------------------------------------------------------------------
  # Enables RPython translation of our interpreter.

  def target( self, driver, args ):

    # if --debug flag is provided in translation, we enable debug printing

    if "--debug" in args:
      print "Enabling debugging"
      Debug.global_enabled = True
    else:
      print "Disabling debugging"

    # form a name
    exe_name = "pydgin-%s" % self.arch_name
    if driver.config.translation.jit:
      exe_name += "-jit"
    else:
      exe_name += "-nojit"

    if Debug.global_enabled:
      exe_name += "-debug"

    print "Translated binary name:", exe_name
    driver.exe_name = exe_name

    # NOTE: RPython has an assertion to check the type of entry_point to
    # be function (not a bound method). So we use get_entry_point which
    # generates a function type
    #return self.entry_point, None
    return self.get_entry_point(), None

#-------------------------------------------------------------------------
# init_sim
#-------------------------------------------------------------------------
# Simulator implementations need to call this function at the top level.
# This takes care of adding target function to top level environment and
# running the simulation in interpreted mode if directly called
# ( __name__ == "__main__" )

def init_sim( sim ):

  # this is a bit hacky: we get the global variables of the caller from
  # the stack frame to determine if this is being run top level and add
  # target function required by rpython toolchain

  caller_globals = sys._getframe(1).f_globals
  caller_name    = caller_globals[ "__name__" ]

  # add target function to top level

  caller_globals[ "target" ] = sim.target

  #-----------------------------------------------------------------------
  # main
  #-----------------------------------------------------------------------
  # Enables CPython simulation of our interpreter.
  if caller_name == "__main__":
    # enable debug flags in interpreted mode
    Debug.global_enabled = True
    sys.exit(sim.get_entry_point()( sys.argv ))

<|endoftext|>