<commit_before># Copyright 2015 The Meson development team

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''This module provides helper functions for RPM related
functionality such as generating template RPM spec file.'''

import build
import compilers
import datetime
import mlog
import os

class RPMModule:

    def generate_spec_template(self, state, args, kwargs):
        compiler_deps = set()
        for compiler in state.compilers:
            if isinstance(compiler, compilers.ValaCompiler):
                compiler_deps.add('vala')
            elif isinstance(compiler, compilers.GnuFortranCompiler):
                compiler_deps.add('gcc-gfortran')
            elif isinstance(compiler, compilers.GnuObjCCompiler):
                compiler_deps.add('gcc-objc')
            elif compiler == compilers.GnuObjCPPCompiler:
                compiler_deps.add('gcc-objc++')
            elif isinstance(compiler, (compilers.GnuCCompiler, compilers.GnuCPPCompiler)):
                # Installed by default
                pass
            else:
                mlog.log('RPM spec file will not created, generating not allowed for:',
                         mlog.bold(compiler.get_id()))
                return
        proj = state.project_name.replace(' ', '_').replace('\t', '_')
        so_installed = False
        devel_subpkg = False
        files = set()
        files_devel = set()
        to_delete = set()
        for target in state.targets.values():
            if isinstance(target, build.Executable) and target.need_install:
                files.add('%%{_bindir}/%s' % target.get_filename())
            elif isinstance(target, build.SharedLibrary) and target.need_install:
                files.add('%%{_libdir}/%s' % target.get_filename())
                for alias in target.get_aliaslist():
                    if alias.endswith('.so'):
                        files_devel.add('%%{_libdir}/%s' % alias)
                    else:
                        files.add('%%{_libdir}/%s' % alias)
                so_installed = True
            elif isinstance(target, build.StaticLibrary) and target.need_install:
                to_delete.add('%%{buildroot}%%{_libdir}/%s' % target.get_filename())
                mlog.log('Warning, removing', mlog.bold(target.get_filename()),
                         'from package because packaging static libs not recommended')
        for header in state.headers:
            if len(header.get_install_subdir()) > 0:
                files_devel.add('%%{_includedir}/%s/' % header.get_install_subdir())
            else:
                for hdr_src in header.get_sources():
                    files_devel.add('%%{_includedir}/%s' % hdr_src)
        for man in state.man:
            for man_file in man.get_sources():
                files.add('%%{_mandir}/man%u/%s.*' % (int(man_file.split('.')[-1]), man_file))
        if len(files_devel) > 0:
            devel_subpkg = True
        fn = open('%s.spec' % os.path.join(state.environment.get_build_dir(), proj),
                  'w+')
        fn.write('Name: %s\n' % proj)
        fn.write('Version: # FIXME\n')
        fn.write('Release: 1%{?dist}\n')
        fn.write('Summary: # FIXME\n')
        fn.write('License: # FIXME\n')
        fn.write('\n')
        fn.write('Source0: %{name}-%{version}.tar.xz # FIXME\n')
        fn.write('\n')
        for compiler in compiler_deps:
            fn.write('BuildRequires: %s\n' % compiler)
        for dep in state.environment.coredata.deps:
            fn.write('BuildRequires: pkgconfig(%s)\n' % dep)
        for lib in state.environment.coredata.ext_libs.values():
            fn.write('BuildRequires: %s # FIXME\n' % lib.fullpath)
            mlog.log('Warning, replace', mlog.bold(lib.fullpath), 'with real package.',
                     'You can use following command to find package which contains this lib:',
                     mlog.bold('dnf provides %s' % lib.fullpath))
        for prog in state.environment.coredata.ext_progs.values():
            fn.write('BuildRequires: %s\n' % ' '.join(prog.fullpath))
        fn.write('BuildRequires: meson\n')
        fn.write('\n')
        fn.write('%description\n')
        fn.write('\n')
        if devel_subpkg:
            fn.write('%package devel\n')
            fn.write('Summary: Development files for %{name}\n')
            fn.write('Requires: %{name}%{?_isa} = %{version}-%{release}\n')
            fn.write('\n')
            fn.write('%description devel\n')
            fn.write('Development files for %{name}.\n')
            fn.write('\n')
        fn.write('%prep\n')
        fn.write('%autosetup\n')
        fn.write('rm -rf rpmbuilddir && mkdir rpmbuilddir\n')
        fn.write('echo "#!/bin/bash" > rpmbuilddir/configure && chmod +x rpmbuilddir/configure\n')
        fn.write('\n')
        fn.write('%build\n')
        fn.write('pushd rpmbuilddir\n')
        fn.write('  %configure\n')
        fn.write('  meson .. --buildtype=plain\n')
        fn.write('  ninja-build -v\n')
        fn.write('popd\n')
        fn.write('\n')
        fn.write('%install\n')
        fn.write('pushd rpmbuilddir\n')
        fn.write('  DESTDIR=%{buildroot} ninja-build -v install\n')
        fn.write('popd\n')
        if len(to_delete) > 0:
            fn.write('rm -rf %s\n' % ' '.join(to_delete))
        fn.write('\n')
        fn.write('%check\n')
        fn.write('pushd rpmbuilddir\n')
        fn.write('  ninja-build -v test\n')
        fn.write('popd\n')
        fn.write('\n')
        fn.write('%files\n')
        for f in files:
            fn.write('%s\n' % f)
        fn.write('\n')
        if devel_subpkg:
            fn.write('%files devel\n')
            for f in files_devel:
                fn.write('%s\n' % f)
        fn.write('\n')
        if so_installed:
            fn.write('%post -p /sbin/ldconfig\n')
            fn.write('\n')
            fn.write('%postun -p /sbin/ldconfig\n')
        fn.write('\n')
        fn.write('%changelog\n')
        fn.write('* %s meson <meson@example.com> - \n' % datetime.date.today().strftime('%a %b %d %Y'))
        fn.write('- \n')
        fn.write('\n')
        fn.close()
        mlog.log('Please, look at RPM spec file and fix FIXME\'s')

def initialize():
    return RPMModule()
<commit_msg>modules/rpm: use plain set up of compiler and linker flags<commit_after># Copyright 2015 The Meson development team

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''This module provides helper functions for RPM related
functionality such as generating template RPM spec file.'''

import build
import compilers
import datetime
import mlog
import os

class RPMModule:

    def generate_spec_template(self, state, args, kwargs):
        compiler_deps = set()
        for compiler in state.compilers:
            if isinstance(compiler, compilers.ValaCompiler):
                compiler_deps.add('vala')
            elif isinstance(compiler, compilers.GnuFortranCompiler):
                compiler_deps.add('gcc-gfortran')
            elif isinstance(compiler, compilers.GnuObjCCompiler):
                compiler_deps.add('gcc-objc')
            elif compiler == compilers.GnuObjCPPCompiler:
                compiler_deps.add('gcc-objc++')
            elif isinstance(compiler, (compilers.GnuCCompiler, compilers.GnuCPPCompiler)):
                # Installed by default
                pass
            else:
                mlog.log('RPM spec file will not created, generating not allowed for:',
                         mlog.bold(compiler.get_id()))
                return
        proj = state.project_name.replace(' ', '_').replace('\t', '_')
        so_installed = False
        devel_subpkg = False
        files = set()
        files_devel = set()
        to_delete = set()
        for target in state.targets.values():
            if isinstance(target, build.Executable) and target.need_install:
                files.add('%%{_bindir}/%s' % target.get_filename())
            elif isinstance(target, build.SharedLibrary) and target.need_install:
                files.add('%%{_libdir}/%s' % target.get_filename())
                for alias in target.get_aliaslist():
                    if alias.endswith('.so'):
                        files_devel.add('%%{_libdir}/%s' % alias)
                    else:
                        files.add('%%{_libdir}/%s' % alias)
                so_installed = True
            elif isinstance(target, build.StaticLibrary) and target.need_install:
                to_delete.add('%%{buildroot}%%{_libdir}/%s' % target.get_filename())
                mlog.log('Warning, removing', mlog.bold(target.get_filename()),
                         'from package because packaging static libs not recommended')
        for header in state.headers:
            if len(header.get_install_subdir()) > 0:
                files_devel.add('%%{_includedir}/%s/' % header.get_install_subdir())
            else:
                for hdr_src in header.get_sources():
                    files_devel.add('%%{_includedir}/%s' % hdr_src)
        for man in state.man:
            for man_file in man.get_sources():
                files.add('%%{_mandir}/man%u/%s.*' % (int(man_file.split('.')[-1]), man_file))
        if len(files_devel) > 0:
            devel_subpkg = True
        fn = open('%s.spec' % os.path.join(state.environment.get_build_dir(), proj),
                  'w+')
        fn.write('Name: %s\n' % proj)
        fn.write('Version: # FIXME\n')
        fn.write('Release: 1%{?dist}\n')
        fn.write('Summary: # FIXME\n')
        fn.write('License: # FIXME\n')
        fn.write('\n')
        fn.write('Source0: %{name}-%{version}.tar.xz # FIXME\n')
        fn.write('\n')
        for compiler in compiler_deps:
            fn.write('BuildRequires: %s\n' % compiler)
        for dep in state.environment.coredata.deps:
            fn.write('BuildRequires: pkgconfig(%s)\n' % dep)
        for lib in state.environment.coredata.ext_libs.values():
            fn.write('BuildRequires: %s # FIXME\n' % lib.fullpath)
            mlog.log('Warning, replace', mlog.bold(lib.fullpath), 'with real package.',
                     'You can use following command to find package which contains this lib:',
                     mlog.bold('dnf provides %s' % lib.fullpath))
        for prog in state.environment.coredata.ext_progs.values():
            fn.write('BuildRequires: %s\n' % ' '.join(prog.fullpath))
        fn.write('BuildRequires: meson\n')
        fn.write('\n')
        fn.write('%description\n')
        fn.write('\n')
        if devel_subpkg:
            fn.write('%package devel\n')
            fn.write('Summary: Development files for %{name}\n')
            fn.write('Requires: %{name}%{?_isa} = %{version}-%{release}\n')
            fn.write('\n')
            fn.write('%description devel\n')
            fn.write('Development files for %{name}.\n')
            fn.write('\n')
        fn.write('%prep\n')
        fn.write('%autosetup\n')
        fn.write('rm -rf rpmbuilddir && mkdir rpmbuilddir\n')
        fn.write('\n')
        fn.write('%build\n')
        fn.write('pushd rpmbuilddir\n')
        fn.write('  CFLAGS="${CFLAGS:-%optflags}" ; export CFLAGS\n')
        fn.write('  CXXFLAGS="${CXXFLAGS:-%optflags}" ; export CXXFLAGS\n')
        fn.write('  FFLAGS="${FFLAGS:-%optflags%{?_fmoddir: -I%_fmoddir}}" ; export FFLAGS\n')
        fn.write('  FCFLAGS="${FCFLAGS:-%optflags%{?_fmoddir: -I%_fmoddir}}" ; export FCFLAGS\n')
        fn.write('  %{?__global_ldflags:LDFLAGS="${LDFLAGS:-%__global_ldflags}" ; export LDFLAGS ;}\n')
        fn.write('  meson .. --buildtype=plain --prefix=%{_prefix}\n')
        fn.write('  ninja-build -v\n')
        fn.write('popd\n')
        fn.write('\n')
        fn.write('%install\n')
        fn.write('pushd rpmbuilddir\n')
        fn.write('  DESTDIR=%{buildroot} ninja-build -v install\n')
        fn.write('popd\n')
        if len(to_delete) > 0:
            fn.write('rm -rf %s\n' % ' '.join(to_delete))
        fn.write('\n')
        fn.write('%check\n')
        fn.write('pushd rpmbuilddir\n')
        fn.write('  ninja-build -v test\n')
        fn.write('popd\n')
        fn.write('\n')
        fn.write('%files\n')
        for f in files:
            fn.write('%s\n' % f)
        fn.write('\n')
        if devel_subpkg:
            fn.write('%files devel\n')
            for f in files_devel:
                fn.write('%s\n' % f)
        fn.write('\n')
        if so_installed:
            fn.write('%post -p /sbin/ldconfig\n')
            fn.write('\n')
            fn.write('%postun -p /sbin/ldconfig\n')
        fn.write('\n')
        fn.write('%changelog\n')
        fn.write('* %s meson <meson@example.com> - \n' % datetime.date.today().strftime('%a %b %d %Y'))
        fn.write('- \n')
        fn.write('\n')
        fn.close()
        mlog.log('Please, look at RPM spec file and fix FIXME\'s')

def initialize():
    return RPMModule()
<|endoftext|>


<commit_before># coding=utf-8

import re
from urllib.parse import urlparse
from urllib.request import urlopen

nemubotversion = 3.3

def help_tiny ():
  """Line inserted in the response to the command !help"""
  return "Gets YCC urls"

def help_full ():
  return "!ycc [<url>]: with an argument, reduce the given <url> thanks to ycc.fr; without argument, reduce the last URL said on the current channel."

def load(context):
    from hooks import Hook
    add_hook("cmd_hook", Hook(cmd_ycc, "ycc"))
    add_hook("all_post", Hook(parseresponse))

LAST_URLS = dict()

def gen_response(res, msg, srv):
    if res is None:
        return Response(msg.sender, "La situation est embarassante, il semblerait que YCC soit down :(", msg.channel)
    elif isinstance(res, str):
        return Response(msg.sender, "URL pour %s : %s" % (srv, res), msg.channel)
    else:
        return Response(msg.sender, "Mauvaise URL : %s" % srv, msg.channel)

def cmd_ycc(msg):
    if len(msg.cmds) == 1:
        global LAST_URLS
        if msg.channel in LAST_URLS and len(LAST_URLS[msg.channel]) > 0:
            msg.cmds.append(LAST_URLS[msg.channel].pop())
        else:
            return Response(msg.sender, "Je n'ai pas d'autre URL à réduire.", msg.channel)

    if len(msg.cmds) < 6:
        res = list()
        for url in msg.cmds[1:]:
            o = urlparse(url, "http")
            if o.scheme != "":
                raw = urlopen("http://ycc.fr/redirection/create/" + url,
                              timeout=10)
                if o.netloc == "":
                    res.append(gen_response(raw.read().decode(), msg, o.scheme))
                else:
                    res.append(gen_response(raw.read().decode(), msg, o.netloc))
            else:
                res.append(gen_response(False, msg, url))
        return res
    else:
        return Response(msg.sender, "je ne peux pas réduire autant d'URL "
                     "d'un seul coup.", msg.channel, nick=msg.nick)

def parselisten(msg):
    global LAST_URLS
    urls = re.findall("([a-zA-Z0-9+.-]+:(//)?[^ ]+)", msg.content)
    for (url, osef) in urls:
        o = urlparse(url)
        if o.scheme != "":
            if o.netloc == "ycc.fr" or (o.netloc == "" and len(o.path) < 10):
                continue
            if msg.channel not in LAST_URLS:
                LAST_URLS[msg.channel] = list()
            LAST_URLS[msg.channel].append(o.geturl())
    return False

def parseresponse(res):
    parselisten(res)
    return True
<commit_msg>YCC module: escape URL with unknown characters<commit_after># coding=utf-8

import re
from urllib.parse import urlparse
from urllib.parse import quote
from urllib.request import urlopen

nemubotversion = 3.3

def help_tiny ():
  """Line inserted in the response to the command !help"""
  return "Gets YCC urls"

def help_full ():
  return "!ycc [<url>]: with an argument, reduce the given <url> thanks to ycc.fr; without argument, reduce the last URL said on the current channel."

def load(context):
    from hooks import Hook
    add_hook("cmd_hook", Hook(cmd_ycc, "ycc"))
    add_hook("all_post", Hook(parseresponse))

LAST_URLS = dict()

def gen_response(res, msg, srv):
    if res is None:
        return Response(msg.sender, "La situation est embarassante, il semblerait que YCC soit down :(", msg.channel)
    elif isinstance(res, str):
        return Response(msg.sender, "URL pour %s : %s" % (srv, res), msg.channel)
    else:
        return Response(msg.sender, "Mauvaise URL : %s" % srv, msg.channel)

def cmd_ycc(msg):
    if len(msg.cmds) == 1:
        global LAST_URLS
        if msg.channel in LAST_URLS and len(LAST_URLS[msg.channel]) > 0:
            msg.cmds.append(LAST_URLS[msg.channel].pop())
        else:
            return Response(msg.sender, "Je n'ai pas d'autre URL à réduire.", msg.channel)

    if len(msg.cmds) < 6:
        res = list()
        for url in msg.cmds[1:]:
            o = urlparse(url, "http")
            if o.scheme != "":
                snd_url = "http://ycc.fr/redirection/create/" + quote(url, "/:%#@&=?")
                print_debug(snd_url)
                raw = urlopen(snd_url, timeout=10)
                if o.netloc == "":
                    res.append(gen_response(raw.read().decode(), msg, o.scheme))
                else:
                    res.append(gen_response(raw.read().decode(), msg, o.netloc))
            else:
                res.append(gen_response(False, msg, url))
        return res
    else:
        return Response(msg.sender, "je ne peux pas réduire autant d'URL "
                     "d'un seul coup.", msg.channel, nick=msg.nick)

def parselisten(msg):
    global LAST_URLS
    urls = re.findall("([a-zA-Z0-9+.-]+:(//)?[^ ]+)", msg.content)
    for (url, osef) in urls:
        o = urlparse(url)
        if o.scheme != "":
            if o.netloc == "ycc.fr" or (o.netloc == "" and len(o.path) < 10):
                continue
            if msg.channel not in LAST_URLS:
                LAST_URLS[msg.channel] = list()
            LAST_URLS[msg.channel].append(o.geturl())
    return False

def parseresponse(res):
    parselisten(res)
    return True
<|endoftext|>


<commit_before>import json
import Queue as queue
import socket
import select
import threading
import time
import sys

from processor import Session, Dispatcher
from utils import print_log, logger

READ_ONLY = select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR
READ_WRITE = READ_ONLY | select.POLLOUT
TIMEOUT = 100

import ssl

class TcpSession(Session):

    def __init__(self, dispatcher, connection, address, use_ssl, ssl_certfile, ssl_keyfile):
        Session.__init__(self, dispatcher)
        self.use_ssl = use_ssl
        self.raw_connection = connection
        if use_ssl:
            import ssl
            self._connection = ssl.wrap_socket(
                connection,
                server_side=True,
                certfile=ssl_certfile,
                keyfile=ssl_keyfile,
                ssl_version=ssl.PROTOCOL_SSLv23,
                do_handshake_on_connect=False)
        else:
            self._connection = connection

        self.address = address[0] + ":%d"%address[1]
        self.name = "TCP " if not use_ssl else "SSL "
        self.timeout = 1000
        self.dispatcher.add_session(self)
        self.response_queue = queue.Queue()
        self.message = ''
        self.retry_msg = ''
        self.handshake = not self.use_ssl
        self.need_write = True

    def connection(self):
        if self.stopped():
            raise Exception("Session was stopped")
        else:
            return self._connection

    def shutdown(self):
        try:
            self._connection.shutdown(socket.SHUT_RDWR)
        except:
            # print_log("problem shutting down", self.address)
            pass
        self._connection.close()

    def send_response(self, response):
        try:
            msg = json.dumps(response) + '\n'
        except BaseException as e:
            logger.error('send_response:' + str(e))
            return
        if self.response_queue.empty():
            self.need_write = True
        self.response_queue.put(msg)

    def parse_message(self):
        message = self.message
        self.time = time.time()
        raw_buffer = message.find('\n')
        if raw_buffer == -1:
            return False
        raw_command = message[0:raw_buffer].strip()
        self.message = message[raw_buffer + 1:]
        return raw_command

class TcpServer(threading.Thread):

    def __init__(self, dispatcher, host, port, use_ssl, ssl_certfile, ssl_keyfile):
        self.shared = dispatcher.shared
        self.dispatcher = dispatcher.request_dispatcher
        threading.Thread.__init__(self)
        self.daemon = True
        self.host = host
        self.port = port
        self.lock = threading.Lock()
        self.use_ssl = use_ssl
        self.ssl_keyfile = ssl_keyfile
        self.ssl_certfile = ssl_certfile

        self.fd_to_session = {}
        self.buffer_size = 4096

    def handle_command(self, raw_command, session):
        try:
            command = json.loads(raw_command)
        except:
            session.send_response({"error": "bad JSON"})
            return True

        try:
            # Try to load vital fields, and return an error if
            # unsuccessful.
            message_id = command['id']
            method = command['method']
        except:
            # Return an error JSON in response.
            session.send_response({"error": "syntax error", "request": raw_command})
        else:
            self.dispatcher.push_request(session, command)
            ## sleep a bit to prevent a single session from DOSing the queue
            #time.sleep(0.01)

    def run(self):

        for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM):
            af, socktype, proto, cannonname, sa = res
            try:
                sock = socket.socket(af, socktype, proto)
                sock.setblocking(0)
                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            except socket.error:
                sock = None
                continue
            try:
                sock.bind(sa)
                sock.listen(5)
            except socket.error:
                sock.close()
                sock = None
                continue
            break
        host = sa[0]
        if af == socket.AF_INET6:
            host = "[%s]" % host
        if sock is None:
            print_log( "could not open " + ("SSL" if self.use_ssl else "TCP") + " socket on %s:%d" % (host, self.port))
            return
        print_log( ("SSL" if self.use_ssl else "TCP") + " server started on %s:%d" % (host, self.port))

        sock_fd = sock.fileno()
        poller = select.poll()
        poller.register(sock)

        def stop_session(fd):
            try:
                # unregister before we close s
                poller.unregister(fd)
            except BaseException as e:
                logger.error('unregister error:' + str(e))
            session = self.fd_to_session.pop(fd)
            # this will close the socket
            session.stop()

        def check_do_handshake(session):
            if session.handshake:
                return
            t0 = time.time()
            try:
                timeout = session._connection.gettimeout()
                session._connection.settimeout(10)
                session._connection.do_handshake()
                session._connection.settimeout(timeout)
                t1 = time.time()
                if t1-t0>1:
                    print_log('do_handshake(success)', t1-t0,session.address)
            except ssl.SSLError as err:
                t1 = time.time()
                if t1-t0>1:
                    print_log('do_handshake(exception)', t1-t0,session.address,err.args[0])
                if err.args[0] == ssl.SSL_ERROR_WANT_READ:
                    return
                elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE:
                    poller.modify(session.raw_connection, READ_WRITE)
                    return
                else:
                    raise
            poller.modify(session.raw_connection, READ_ONLY)
            session.handshake = True

        redo = []

        while not self.shared.stopped():

            if self.shared.paused():
                sessions = self.fd_to_session.keys()
                if sessions:
                    logger.info("closing %d sessions"%len(sessions))
                for fd in sessions:
                    stop_session(fd)
                time.sleep(1)
                continue

            if redo:
                events = redo
                redo = []
            else:
                now = time.time()
                for fd, session in self.fd_to_session.items():
                    # check sessions that need to write
                    if session.need_write:
                        poller.modify(session.raw_connection, READ_WRITE)
                        session.need_write = False
                    # collect garbage
                    if now - session.time > session.timeout:
                        stop_session(fd)

                events = poller.poll(TIMEOUT)

            for fd, flag in events:

                if fd == sock_fd:
                    if flag & (select.POLLIN | select.POLLPRI):
                        connection, address = sock.accept()
                        try:
                            session = TcpSession(self.dispatcher, connection, address,
                                                 use_ssl=self.use_ssl, ssl_certfile=self.ssl_certfile, ssl_keyfile=self.ssl_keyfile)
                        except BaseException as e:
                            logger.error("cannot start TCP session" + str(e) + ' ' + repr(address))
                            connection.close()
                            continue
                        connection = session._connection
                        connection.setblocking(0)
                        self.fd_to_session[connection.fileno()] = session
                        poller.register(connection, READ_ONLY)
                        try:
                            check_do_handshake(session)
                        except BaseException as e:
                            logger.error('handshake failure:' + str(e) + ' ' + repr(address))
                            stop_session(connection.fileno())
                    continue

                session = self.fd_to_session[fd]
                s = session._connection
                try:
                    check_do_handshake(session)
                except:
                    stop_session(fd)
                    continue

                # handle inputs
                if flag & (select.POLLIN | select.POLLPRI):
                    try:
                        data = s.recv(self.buffer_size)
                    except ssl.SSLError as x:
                        if x.args[0] == ssl.SSL_ERROR_WANT_READ:
                            pass
                        elif x.args[0] == ssl.SSL_ERROR_SSL:
                            pass
                        else:
                            logger.error('SSL recv error:'+ repr(x))
                        continue
                    except socket.error as x:
                        if x.args[0] != 104:
                            logger.error('recv error: ' + repr(x) +' %d'%fd)
                        stop_session(fd)
                        continue
                    if data:
                        if len(data) == self.buffer_size:
                            redo.append( (fd, flag) )
                        session.message += data
                        while True:
                            cmd = session.parse_message()
                            if not cmd:
                                break
                            if cmd == 'quit':
                                data = False
                                break
                            self.handle_command(cmd, session)
                    if not data:
                        stop_session(fd)
                        continue

                elif flag & select.POLLHUP:
                    print_log('client hung up', address)
                    stop_session(fd)

                elif flag & select.POLLOUT:
                    # Socket is ready to send data, if there is any to send.
                    if session.retry_msg:
                        next_msg = session.retry_msg
                    else:
                        try:
                            next_msg = session.response_queue.get_nowait()
                        except queue.Empty:
                            # No messages waiting so stop checking for writability.
                            poller.modify(s, READ_ONLY)
                            continue
                    try:
                        sent = s.send(next_msg)
                    except socket.error as x:
                        logger.error("send error:" + str(x))
                        stop_session(fd)
                        continue
                    session.retry_msg = next_msg[sent:]

                elif flag & select.POLLERR:
                    print_log('handling exceptional condition for', session.address)
                    stop_session(fd)

                elif flag & select.POLLNVAL:
                    print_log('invalid request', session.address)
                    stop_session(fd)
<commit_msg>revert 8aba18aa2e5614d1ff21bdf5430536eb25bd7223 and clarify do_handshake<commit_after>import json
import Queue as queue
import socket
import select
import threading
import time
import sys

from processor import Session, Dispatcher
from utils import print_log, logger

READ_ONLY = select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR
READ_WRITE = READ_ONLY | select.POLLOUT
TIMEOUT = 100

import ssl

class TcpSession(Session):

    def __init__(self, dispatcher, connection, address, use_ssl, ssl_certfile, ssl_keyfile):
        Session.__init__(self, dispatcher)
        self.use_ssl = use_ssl
        self.raw_connection = connection
        if use_ssl:
            import ssl
            self._connection = ssl.wrap_socket(
                connection,
                server_side=True,
                certfile=ssl_certfile,
                keyfile=ssl_keyfile,
                ssl_version=ssl.PROTOCOL_SSLv23,
                do_handshake_on_connect=False)
        else:
            self._connection = connection

        self.address = address[0] + ":%d"%address[1]
        self.name = "TCP " if not use_ssl else "SSL "
        self.timeout = 1000
        self.dispatcher.add_session(self)
        self.response_queue = queue.Queue()
        self.message = ''
        self.retry_msg = ''
        self.handshake = not self.use_ssl
        self.need_write = True

    def connection(self):
        if self.stopped():
            raise Exception("Session was stopped")
        else:
            return self._connection

    def shutdown(self):
        try:
            self._connection.shutdown(socket.SHUT_RDWR)
        except:
            # print_log("problem shutting down", self.address)
            pass
        self._connection.close()

    def send_response(self, response):
        try:
            msg = json.dumps(response) + '\n'
        except BaseException as e:
            logger.error('send_response:' + str(e))
            return
        if self.response_queue.empty():
            self.need_write = True
        self.response_queue.put(msg)

    def parse_message(self):
        message = self.message
        self.time = time.time()
        raw_buffer = message.find('\n')
        if raw_buffer == -1:
            return False
        raw_command = message[0:raw_buffer].strip()
        self.message = message[raw_buffer + 1:]
        return raw_command

class TcpServer(threading.Thread):

    def __init__(self, dispatcher, host, port, use_ssl, ssl_certfile, ssl_keyfile):
        self.shared = dispatcher.shared
        self.dispatcher = dispatcher.request_dispatcher
        threading.Thread.__init__(self)
        self.daemon = True
        self.host = host
        self.port = port
        self.lock = threading.Lock()
        self.use_ssl = use_ssl
        self.ssl_keyfile = ssl_keyfile
        self.ssl_certfile = ssl_certfile

        self.fd_to_session = {}
        self.buffer_size = 4096

    def handle_command(self, raw_command, session):
        try:
            command = json.loads(raw_command)
        except:
            session.send_response({"error": "bad JSON"})
            return True

        try:
            # Try to load vital fields, and return an error if
            # unsuccessful.
            message_id = command['id']
            method = command['method']
        except:
            # Return an error JSON in response.
            session.send_response({"error": "syntax error", "request": raw_command})
        else:
            self.dispatcher.push_request(session, command)
            ## sleep a bit to prevent a single session from DOSing the queue
            #time.sleep(0.01)

    def run(self):

        for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM):
            af, socktype, proto, cannonname, sa = res
            try:
                sock = socket.socket(af, socktype, proto)
                sock.setblocking(0)
                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            except socket.error:
                sock = None
                continue
            try:
                sock.bind(sa)
                sock.listen(5)
            except socket.error:
                sock.close()
                sock = None
                continue
            break
        host = sa[0]
        if af == socket.AF_INET6:
            host = "[%s]" % host
        if sock is None:
            print_log( "could not open " + ("SSL" if self.use_ssl else "TCP") + " socket on %s:%d" % (host, self.port))
            return
        print_log( ("SSL" if self.use_ssl else "TCP") + " server started on %s:%d" % (host, self.port))

        sock_fd = sock.fileno()
        poller = select.poll()
        poller.register(sock)

        def stop_session(fd):
            try:
                # unregister before we close s
                poller.unregister(fd)
            except BaseException as e:
                logger.error('unregister error:' + str(e))
            session = self.fd_to_session.pop(fd)
            # this will close the socket
            session.stop()

        def try_do_handshake(session, fd):
            try:
                session._connection.do_handshake()
            except ssl.SSLError as err:
                if err.args[0] == ssl.SSL_ERROR_WANT_READ:
                    return
                elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE:
                    poller.modify(session.raw_connection, READ_WRITE)
                    return
                else:
                    logger.error('handshake failure:' + str(err) + ' ' + repr(session.address))
                    stop_session(fd)
                    return
            except BaseException as e:
                logger.error('handshake failure:' + str(e) + ' ' + repr(session.address))
                stop_session(fd)
                return
            poller.modify(session.raw_connection, READ_ONLY)
            session.handshake = True

        redo = []

        while not self.shared.stopped():

            if self.shared.paused():
                sessions = self.fd_to_session.keys()
                if sessions:
                    logger.info("closing %d sessions"%len(sessions))
                for fd in sessions:
                    stop_session(fd)
                time.sleep(1)
                continue

            if redo:
                events = redo
                redo = []
            else:
                now = time.time()
                for fd, session in self.fd_to_session.items():
                    # check sessions that need to write
                    if session.need_write:
                        poller.modify(session.raw_connection, READ_WRITE)
                        session.need_write = False
                    # collect garbage
                    if now - session.time > session.timeout:
                        stop_session(fd)

                events = poller.poll(TIMEOUT)

            for fd, flag in events:

                if fd == sock_fd:
                    if flag & (select.POLLIN | select.POLLPRI):
                        connection, address = sock.accept()
                        try:
                            session = TcpSession(self.dispatcher, connection, address,
                                                 use_ssl=self.use_ssl, ssl_certfile=self.ssl_certfile, ssl_keyfile=self.ssl_keyfile)
                        except BaseException as e:
                            logger.error("cannot start TCP session" + str(e) + ' ' + repr(address))
                            connection.close()
                            continue
                        connection = session._connection
                        connection.setblocking(False)
                        self.fd_to_session[connection.fileno()] = session
                        poller.register(connection, READ_ONLY)
                    continue

                session = self.fd_to_session[fd]
                s = session._connection

                # non-blocking handshake
                if not session.handshake:
                    try_do_handshake(session, fd)
                    continue

                # handle inputs
                if flag & (select.POLLIN | select.POLLPRI):
                    try:
                        data = s.recv(self.buffer_size)
                    except ssl.SSLError as x:
                        if x.args[0] == ssl.SSL_ERROR_WANT_READ:
                            pass
                        elif x.args[0] == ssl.SSL_ERROR_SSL:
                            pass
                        else:
                            logger.error('SSL recv error:'+ repr(x))
                        continue
                    except socket.error as x:
                        if x.args[0] != 104:
                            logger.error('recv error: ' + repr(x) +' %d'%fd)
                        stop_session(fd)
                        continue
                    if data:
                        if len(data) == self.buffer_size:
                            redo.append( (fd, flag) )
                        session.message += data
                        while True:
                            cmd = session.parse_message()
                            if not cmd:
                                break
                            if cmd == 'quit':
                                data = False
                                break
                            self.handle_command(cmd, session)
                    if not data:
                        stop_session(fd)
                        continue

                elif flag & select.POLLHUP:
                    print_log('client hung up', session.address)
                    stop_session(fd)

                elif flag & select.POLLOUT:
                    # Socket is ready to send data, if there is any to send.
                    if session.retry_msg:
                        next_msg = session.retry_msg
                    else:
                        try:
                            next_msg = session.response_queue.get_nowait()
                        except queue.Empty:
                            # No messages waiting so stop checking for writability.
                            poller.modify(s, READ_ONLY)
                            continue
                    try:
                        sent = s.send(next_msg)
                    except socket.error as x:
                        logger.error("send error:" + str(x))
                        stop_session(fd)
                        continue
                    session.retry_msg = next_msg[sent:]

                elif flag & select.POLLERR:
                    print_log('handling exceptional condition for', session.address)
                    stop_session(fd)

                elif flag & select.POLLNVAL:
                    print_log('invalid request', session.address)
                    stop_session(fd)
<|endoftext|>


<commit_before>import os
import sys
import argparse
import json

from importlib import import_module
from importlib.util import spec_from_file_location, module_from_spec

from molotov.fmwk import runner
from molotov.api import get_scenarios
from molotov import __version__
from molotov.util import log
from molotov import ui

def _parser():
    parser = argparse.ArgumentParser(description='Load test.')

    parser.add_argument('scenario', default="loadtest.py",
                        help="path or module name that contains scenarii",
                        nargs="?")

    parser.add_argument('--config', default=None, type=str,
                        help='Point to a JSON config file.')

    parser.add_argument('--version', action='store_true', default=False,
                        help='Displays version and exits.')

    parser.add_argument('--debug', action='store_true', default=False,
                        help='Run the event loop in debug mode.')

    parser.add_argument('-v', '--verbose', action='store_true', default=False,
                        help='Verbose')

    parser.add_argument('-w', '--workers', help='Number of workers',
                        type=int, default=1)

    parser.add_argument('-p', '--processes', help='Number of processes',
                        type=int, default=1)

    parser.add_argument('-d', '--duration', help='Duration in seconds',
                        type=int, default=10)

    parser.add_argument('-q', '--quiet', action='store_true', default=False,
                        help='Quiet')

    parser.add_argument('-x', '--exception', action='store_true',
                        default=False,
                        help='Stop on first failure.')

    parser.add_argument('-c', '--console', action='store_true',
                        default=False,
                        help='Use simple console for feedback')

    parser.add_argument('--statsd', help='Activates statsd',
                        action='store_true', default=False)

    parser.add_argument('--statsd-server', help='Statsd Server',
                        type=str, default="127.0.0.1")

    parser.add_argument('--statsd-port', help='Statsd Port',
                        type=int, default=8125)

    return parser

def _expand_args(args, options):
    for key, val in options.items():
        setattr(args, key, val)

def main():
    parser = _parser()
    args = parser.parse_args()

    if args.version:
        print(__version__)
        sys.exit(0)

    if args.scenario is None:
        print('You need to provide a scenario name or file.')
        parser.print_usage()
        sys.exit(0)

    if args.config:
        if not os.path.exists(args.config):
            print("Can't find %r" % args.config)
            sys.exit(0)

        with open(args.config) as f:
            try:
                config = json.loads(f.read())
            except ValueError:
                print("Can't parse %r" % args.config)
                sys.exit(0)

        if 'molotov' not in config:
            print("Bad config -- no molotov key")
            sys.exit(0)

        if args.scenario not in config['molotov']:
            print("Can't find %r in the config" % args.scenario)
            sys.exit(0)

        _expand_args(args, config['molotov'][args.scenario])

    if args.statsd:
        # early import to quit if no aiostatsd
        from aiostatsd.client import StatsdClient
        if StatsdClient is None:
            print('You need to install aiostatsd when using --statds')
            sys.exit(0)

    return run(args)

def run(args):
    if not args.quiet:
        log('**** Molotov v%s. Happy breaking! ****' % __version__, pid=None)
    if os.path.exists(args.scenario):
        spec = spec_from_file_location("loadtest", args.scenario)
        module = module_from_spec(spec)
        spec.loader.exec_module(module)
    else:
        try:
            import_module(args.scenario)
        except ImportError:
            print('Cannot import %r' % args.scenario)
            sys.exit(1)

    if len(get_scenarios()) == 0:
        print('You need at least one scenario. No scenario was found.')
        print('A scenario with a weight of 0 is ignored')
        sys.exit(1)

    if args.verbose and args.quiet:
        print("You can't use -q and -v at the same time")
        sys.exit(1)

    if args.verbose and not args.console:
        print("You have to be in console mode (-c) to use -v")
        sys.exit(1)

    res = runner(args, screen=ui.init_screen)
    if not args.quiet:
        log('', pid=False)
    log('%(OK)d OK, %(FAILED)d Failed' % res, pid=False)

if __name__ == '__main__':
    main()
<commit_msg>tweaked the config reader<commit_after>import os
import sys
import argparse
import json

from importlib import import_module
from importlib.util import spec_from_file_location, module_from_spec

from molotov.fmwk import runner
from molotov.api import get_scenarios
from molotov import __version__
from molotov.util import log
from molotov import ui

def _parser():
    parser = argparse.ArgumentParser(description='Load test.')

    parser.add_argument('scenario', default="loadtest.py",
                        help="path or module name that contains scenarii",
                        nargs="?")

    parser.add_argument('--config', default=None, type=str,
                        help='Point to a JSON config file.')

    parser.add_argument('--version', action='store_true', default=False,
                        help='Displays version and exits.')

    parser.add_argument('--debug', action='store_true', default=False,
                        help='Run the event loop in debug mode.')

    parser.add_argument('-v', '--verbose', action='store_true', default=False,
                        help='Verbose')

    parser.add_argument('-w', '--workers', help='Number of workers',
                        type=int, default=1)

    parser.add_argument('-p', '--processes', help='Number of processes',
                        type=int, default=1)

    parser.add_argument('-d', '--duration', help='Duration in seconds',
                        type=int, default=10)

    parser.add_argument('-q', '--quiet', action='store_true', default=False,
                        help='Quiet')

    parser.add_argument('-x', '--exception', action='store_true',
                        default=False,
                        help='Stop on first failure.')

    parser.add_argument('-c', '--console', action='store_true',
                        default=False,
                        help='Use simple console for feedback')

    parser.add_argument('--statsd', help='Activates statsd',
                        action='store_true', default=False)

    parser.add_argument('--statsd-server', help='Statsd Server',
                        type=str, default="127.0.0.1")

    parser.add_argument('--statsd-port', help='Statsd Port',
                        type=int, default=8125)

    return parser

def _expand_args(args, options):
    for key, val in options.items():
        setattr(args, key, val)

def main():
    parser = _parser()
    args = parser.parse_args()

    if args.version:
        print(__version__)
        sys.exit(0)

    if args.scenario is None:
        print('You need to provide a scenario name or file.')
        parser.print_usage()
        sys.exit(0)

    if args.config:
        if not os.path.exists(args.config):
            print("Can't find %r" % args.config)
            sys.exit(0)

        with open(args.config) as f:
            try:
                config = json.loads(f.read())
            except ValueError:
                print("Can't parse %r" % args.config)
                sys.exit(0)

        if 'molotov' not in config:
            print("Bad config -- no molotov key")
            sys.exit(0)

        if 'tests' not in config['molotov']:
            print("Bad config -- no molotov/tests key")
            sys.exit(0)

        if args.scenario == 'loadtest.py':
            args.scenario = 'test'

        if args.scenario not in config['molotov']['tests']:
            print("Can't find %r in the config" % args.scenario)
            sys.exit(0)

        _expand_args(args, config['molotov']['tests'][args.scenario])

    if args.statsd:
        # early import to quit if no aiostatsd
        from aiostatsd.client import StatsdClient
        if StatsdClient is None:
            print('You need to install aiostatsd when using --statds')
            sys.exit(0)

    return run(args)

def run(args):
    if not args.quiet:
        log('**** Molotov v%s. Happy breaking! ****' % __version__, pid=None)
    if os.path.exists(args.scenario):
        spec = spec_from_file_location("loadtest", args.scenario)
        module = module_from_spec(spec)
        spec.loader.exec_module(module)
    else:
        try:
            import_module(args.scenario)
        except ImportError:
            print('Cannot import %r' % args.scenario)
            sys.exit(1)

    if len(get_scenarios()) == 0:
        print('You need at least one scenario. No scenario was found.')
        print('A scenario with a weight of 0 is ignored')
        sys.exit(1)

    if args.verbose and args.quiet:
        print("You can't use -q and -v at the same time")
        sys.exit(1)

    if args.verbose and not args.console:
        print("You have to be in console mode (-c) to use -v")
        sys.exit(1)

    res = runner(args, screen=ui.init_screen)
    if not args.quiet:
        log('', pid=False)
    log('%(OK)d OK, %(FAILED)d Failed' % res, pid=False)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from __future__ import print_function
import xml.sax
import requests
import datetime
import sys
from fuzzywuzzy import fuzz

class MovieHandler( xml.sax.ContentHandler ):
   def __init__(self, endtag):
      self.CurrentData = ""
      self.movies = []
      self.current = {}
      self.endtag = endtag

   # Call when an element starts
   def startElement(self, tag, attributes):
      self.CurrentData = tag

   # Call when an elements ends
   def endElement(self, tag):
      if tag == self.endtag:
         self.movies.append(self.current)
         self.current = {}

      self.CurrentData = ""

   # Call when a character is read
   def characters(self, content):
      self.current[self.CurrentData] = content

def movies_place(place):
   # create an XMLReader
   parser = xml.sax.make_parser()
   # turn off namepsaces
   parser.setFeature(xml.sax.handler.feature_namespaces, 0)

   # override the default ContextHandler
   Handler = MovieHandler("Show")
   parser.setContentHandler( Handler )

   if place:
      r = requests.get("http://www.finnkino.fi/xml/Schedule/?area=%s" % place)
   else:
      r = requests.get("http://www.finnkino.fi/xml/Schedule/")
   xml.sax.parseString(r.text.encode("UTF-8"), Handler)

   dateformat = "%Y-%m-%dT%H:%M:%S"

   #datetime.datetime.strptime("2016-01-22T23:45:00", "%Y-%m-%dT%H:%M:%S")
   movies = [(datetime.datetime.strptime(x["dttmShowStart"], dateformat),datetime.datetime.strptime(x["dttmShowEnd"], dateformat),x["Title"],x["TheatreAndAuditorium"], x["LengthInMinutes"]) for x in Handler.movies]
   now = datetime.datetime.now()
   return filter(lambda x: x[1] > now,movies)

def areas():
   # create an XMLReader
   parser = xml.sax.make_parser()
   # turn off namepsaces
   parser.setFeature(xml.sax.handler.feature_namespaces, 0)

   # override the default ContextHandler
   Handler = MovieHandler("TheatreArea")
   parser.setContentHandler( Handler )

   r = requests.get("http://www.finnkino.fi/xml/TheatreAreas/")
   xml.sax.parseString(r.text.encode("UTF-8"), Handler)
   areas = [(x["ID"], x["Name"]) for x in Handler.movies]
   return areas

def nice_line(item):
   ts = datetime.datetime.strftime(item[0], "%H.%M")
   return "%s: %s, %s, kesto %s min" % (ts,item[2],item[3],item[4])

def fuzz_match(areas, place):
   return "1029"

if ( __name__ == "__main__"):
   places = areas()
   venues = dict(places)
   if len(sys.argv) < 2:
      print ("One argument stating area is needed.")
      for v in sorted(venues.keys()):
         print ("%s: %s" % (v, venues[v]))
      sys.exit(0)

   try:
      place = str(int(sys.argv[1]))
   except ValueError:
      place = sorted(map(lambda x: (fuzz.ratio(sys.argv[1],x[1]),x), areas()))[-1][1][0]

   l = movies_place(place)
   for item in l:
      print (nice_line(item))
<commit_msg>fuzzy matching any string to a place<commit_after>from __future__ import print_function
import xml.sax
import requests
import datetime
import sys
from fuzzywuzzy import fuzz

class MovieHandler( xml.sax.ContentHandler ):
   def __init__(self, endtag):
      self.CurrentData = ""
      self.movies = []
      self.current = {}
      self.endtag = endtag

   # Call when an element starts
   def startElement(self, tag, attributes):
      self.CurrentData = tag

   # Call when an elements ends
   def endElement(self, tag):
      if tag == self.endtag:
         self.movies.append(self.current)
         self.current = {}

      self.CurrentData = ""

   # Call when a character is read
   def characters(self, content):
      self.current[self.CurrentData] = content

def movies_place(place):
   # create an XMLReader
   parser = xml.sax.make_parser()
   # turn off namepsaces
   parser.setFeature(xml.sax.handler.feature_namespaces, 0)

   # override the default ContextHandler
   Handler = MovieHandler("Show")
   parser.setContentHandler( Handler )

   if place:
      r = requests.get("http://www.finnkino.fi/xml/Schedule/?area=%s" % place)
   else:
      r = requests.get("http://www.finnkino.fi/xml/Schedule/")
   xml.sax.parseString(r.text.encode("UTF-8"), Handler)

   dateformat = "%Y-%m-%dT%H:%M:%S"

   #datetime.datetime.strptime("2016-01-22T23:45:00", "%Y-%m-%dT%H:%M:%S")
   movies = [(datetime.datetime.strptime(x["dttmShowStart"], dateformat),datetime.datetime.strptime(x["dttmShowEnd"], dateformat),x["Title"],x["TheatreAndAuditorium"], x["LengthInMinutes"]) for x in Handler.movies]
   now = datetime.datetime.now()
   return filter(lambda x: x[1] > now,movies)

def areas():
   # create an XMLReader
   parser = xml.sax.make_parser()
   # turn off namepsaces
   parser.setFeature(xml.sax.handler.feature_namespaces, 0)

   # override the default ContextHandler
   Handler = MovieHandler("TheatreArea")
   parser.setContentHandler( Handler )

   r = requests.get("http://www.finnkino.fi/xml/TheatreAreas/")
   xml.sax.parseString(r.text.encode("UTF-8"), Handler)
   areas = [(x["ID"], x["Name"]) for x in Handler.movies]
   return areas

def nice_line(item):
   ts = datetime.datetime.strftime(item[0], "%H.%M")
   return "%s: %s, %s, kesto %s min" % (ts,item[2],item[3],item[4])

if ( __name__ == "__main__"):
   places = areas()
   venues = dict(places)
   if len(sys.argv) < 2:
      print ("One argument stating area is needed.")
      for v in sorted(venues.keys()):
         print ("%s: %s" % (v, venues[v]))
      sys.exit(0)

   try:
      place = str(int(sys.argv[1]))
   except ValueError:
      place = sorted(map(lambda x: (fuzz.ratio(sys.argv[1],x[1]),x), areas()))[-1][1][0]

   l = movies_place(place)
   for item in l:
      print (nice_line(item))
<|endoftext|>


<commit_before>from django.test import TestCase

# Create your tests here.
<commit_msg>[NEW]Dodati osnovni testovi<commit_after>from django.test import TestCase

# Create your tests here.
from django.contrib.auth.models import User, AnonymousUser
from django.core.urlresolvers import reverse
from django.test import Client
from django.test import TestCase
from django.test.client import RequestFactory
from django.utils import timezone

from tasks.forms import MilestoneDetail, TimelineList
from tasks.models import Milestone

# Create your tests here.
"""
    pokretanje:
        python manage.py test
        ili desni klik na projekat > Django > run Django Django Tests
"""

class PollsViewsTestCase(TestCase):
    def setUp(self):
        milestone = Milestone.objects.create(date_created=timezone.now(),name="Test",summary="Test adding new Milestone")

    def test_timeline_view(self):
        resp = self.client.get('/timeline/')
        self.assertEqual(resp.status_code, 200)

    def test_fake_url(self):
        resp = self.client.get('/jovan/')
        self.assertEqual(resp.status_code, 404)

    def test_fake_url_reverse(self):
        resp = self.client.get('/jovan/')
        self.assertNotEqual(resp.status_code, 200)

    def test_milestones_number(self):
        milestone = Milestone.objects.get(pk=1)

        self.assertEqual(milestone.name, "Test")
        self.assertEqual(Milestone.objects.all().count(), 1)

    def test_milestones_name(self):
        milestone = Milestone.objects.get(pk=1)

        self.assertEqual(milestone.name, "Test")

    def test_client(self):
        c = Client()
        resp =  c.get(reverse('mdetail', args=(2,)))
        self.assertEqual(resp.status_code, 404)
        <|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2015 Matt Martz
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from __future__ import (absolute_import, division, print_function)

import argparse
import base64
import hashlib
import json
import os
import re
import sys
import time
import uuid
import zipfile

from contextlib import contextmanager

from distutils import log

import boto3

from botocore.exceptions import ClientError

from py.error import ENOENT
from py.path import local

import setuptools
from setuptools import find_packages
from setuptools.command.build_py import build_py
from setuptools.command.egg_info import FileList
import setuptools.command.sdist
from setuptools.dist import Distribution

from tox.config import DepConfig, TestenvConfig
import tox.session

__version__ = '0.0.1'
__metaclass__ = type

class Mu:
    def __init__(self, config_file='lambda.json', zip_file=None,
                 with_pyc=False):
        self.config_file = config_file
        self.config_data = {}

        self.with_pyc = with_pyc

        self.venv = None

        self.build = None
        self.sdist = None

        self.files = []

        log.set_threshold(log.ERROR)

        self.read_config()

        if zip_file:
            self.zip_file = zip_file
        else:
            self.zip_file = '%s.zip' % self.config_data.get('name', 'lambda')
        self.zip = None

        self.prepare_tox()
        self.prepare_setuptools()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        try:
            self.venv.path.remove()
        except (ENOENT, AttributeError):
            pass

    @contextmanager
    def _zipper(self):
        z = zipfile.ZipFile(self.zip_file, 'w', zipfile.ZIP_DEFLATED)
        yield z
        z.close()

    def read_config(self):
        try:
            with open(self.config_file, 'rbU') as f:
                self.config_data = json.load(f)
        except (OSError, IOError) as e:
            raise SystemExit('Could not load "%s" (%s)' % (self.config_file,
                                                           e))
            self.config_data = {}

        return self.config_data

    def prepare_tox(self):
        toxini = os.path.join(os.path.dirname(__file__), 'tox.ini')
        config = tox.session.prepare(['-c', toxini])
        name = uuid.uuid4().hex
        envconfig = TestenvConfig(name, config, None, None)
        envconfig.envdir = local('%s/%s' % (config.toxworkdir, name))
        envconfig.envlogdir = local('%s/log-%s' % (config.toxworkdir, name))
        envconfig.basepython = sys.executable
        envconfig.sitepackages = False
        envconfig.downloadcache = False
        envconfig.pip_pre = False
        envconfig.setenv = []
        envconfig.install_command = 'pip install {opts} {packages}'.split()
        config.envconfigs[name] = envconfig

        session = tox.session.Session(config)
        self.venv = session.getvenv(name)
        # self.venv = session.getvenv('python')

    def prepare_setuptools(self):
        dist = Distribution()
        dist.script_name = 'mu'

        self.sdist = setuptools.command.sdist.sdist(dist)
        self.sdist.filelist = FileList()

        self.build = build_py(dist)
        self.build.packages = find_packages(**self.config_data.get('packages',
                                                                   {}))
        self.build.py_modules = self.config_data.get('py_modules', [])

    def get_file_list(self):
        for module in self.build.find_all_modules():
            self.files.append((os.path.abspath(module[2]), module[2]))

        deps = self.config_data.get('deps', [])
        if deps:
            ixserver = self.venv.envconfig.config.indexserver['default']
            self.venv.create()
            self.venv.envconfig.deps = [DepConfig(dep, ixserver)
                                        for dep in deps]
            site_packages = self.venv.envconfig.envsitepackagesdir()
            self.venv.install_deps()

            for root, _, files in os.walk(site_packages):
                for filename in files:
                    if not self.with_pyc and re.search('\.py[co]$', filename):
                        continue
                    filepath = os.path.abspath(os.path.join(root, filename))
                    arcname = filepath[len(site_packages) + 1:]
                    self.files.append((filepath, arcname))

        self.sdist.finalize_options()
        self.sdist.get_file_list()

        for _file in self.sdist.filelist.files:
            self.files.append((os.path.abspath(_file), _file))

        return self.files

    def create_zip(self):
        with self._zipper() as z:
            for _file in self.files:
                z.write(*_file)

    def _ensure_iam_default_role(self, session):
        iam = session.client('iam')
        try:
            role_resp = iam.get_role(RoleName='lambda_basic_execution')
        except ClientError:
            name = 'lambda_basic_execution'
            role_resp = iam.create_role(
                Path='/',
                RoleName=name,
                AssumeRolePolicyDocument=json.dumps({
                    'Statement': [
                        {
                            'Action': 'sts:AssumeRole',
                            'Effect': 'Allow',
                            'Principal': {
                                'Service': 'lambda.amazonaws.com'
                            },
                            'Sid': ''
                        }
                    ],
                    'Version': '2012-10-17'
                }, indent=2)
            )

            iam.put_role_policy(
                RoleName=name,
                PolicyName=name,
                PolicyDocument=json.dumps({
                    'Version': '2012-10-17',
                    'Statement': [
                        {
                            "Effect": "Allow",
                            "Action": [
                                "logs:CreateLogGroup",
                                "logs:CreateLogStream",
                                "logs:PutLogEvents"
                            ],
                            "Resource": "arn:aws:logs:*:*:*"
                        }
                    ]
                }, indent=2)
            )

        return role_resp['Role']['Arn']

    def _get_sha256(self):
        sha = hashlib.sha256()
        with open(self.zip_file, 'rb') as f:
            while True:
                chunk = f.read(1024)
                if not chunk:
                    break
                sha.update(chunk)
        return base64.b64encode(sha.digest())

    def upload(self, profile='default'):
        session = boto3.Session(
            region_name=self.config_data.get('region', 'us-east-1'),
            profile_name=profile
        )

        client = session.client('lambda')

        if self.config_data.get('role'):
            role = self.config_data.get('role')
        else:
            role = self._ensure_iam_default_role(session)
            # iam = session.client('iam')
            # try:
            #     role_resp = iam.get_role(RoleName='lambda_basic_execution')
            # except ClientError:
            #     name = 'lambda_basic_execution'
            #     role_resp = iam.create_role(
            #         Path='/',
            #         RoleName=name,
            #         AssumeRolePolicyDocument=json.dumps({
            #             'Statement': [
            #                 {
            #                     'Action': 'sts:AssumeRole',
            #                     'Effect': 'Allow',
            #                     'Principal': {
            #                         'Service': 'lambda.amazonaws.com'
            #                     },
            #                     'Sid': ''
            #                 }
            #             ],
            #             'Version': '2012-10-17'
            #         }, indent=2)
            #     )

            #     iam.put_role_policy(
            #         RoleName=name,
            #         PolicyName=name,
            #         PolicyDocument=json.dumps({
            #             'Version': '2012-10-17',
            #             'Statement': [
            #                 {
            #                     "Effect": "Allow",
            #                     "Action": [
            #                         "logs:CreateLogGroup",
            #                         "logs:CreateLogStream",
            #                         "logs:PutLogEvents"
            #                     ],
            #                     "Resource": "arn:aws:logs:*:*:*"
            #                 }
            #             ]
            #         }, indent=2)
            #     )

            # role = role_resp['Role']['Arn']

        try:
            func = client.get_function(FunctionName=self.config_data['name'])
        except ClientError:
            func = None
            for _ in range(5):
                try:
                    with open(self.zip_file, 'rb') as f:
                        func = client.create_function(
                            FunctionName=self.config_data['name'],
                            Runtime='python2.7',
                            Role=role,
                            Handler=self.config_data['handler'],
                            Code={'ZipFile': f.read()},
                            Description=self.config_data.get('description',
                                                             ''),
                            Timeout=self.config_data.get('timeout', 3),
                            MemorySize=self.config_data.get('memory', 128),
                            Publish=self.config_data.get('publish', True),
                        )
                except ClientError as e:
                    role_msg = ('The role defined for the task cannot be '
                                'assumed by Lambda.')
                    if e.response['Error']['Message'] == role_msg:
                        time.sleep(2)
                    else:
                        raise
                else:
                    break
            if not func:
                raise SystemExit('Error creating Lambda function: %s' %
                                 e.response['Error']['Message'])
        else:
            if self._get_sha256() != func['Configuration']['CodeSha256']:
                with open(self.zip_file, 'rb') as f:
                    client.update_function_code(
                        FunctionName=self.config_data['name'],
                        ZipFile=f.read(),
                        Publish=self.config_data.get('publish', True)
                    )

            client.update_function_configuration(
                FunctionName=self.config_data['name'],
                Role=role,
                Handler=self.config_data['handler'],
                Description=self.config_data.get('description', ''),
                Timeout=self.config_data.get('timeout', 3),
                MemorySize=self.config_data.get('memory', 128)
            )

def main():
    parser = argparse.ArgumentParser(prog='mu')
    parser.add_argument('config', nargs='?', default='lambda.json',
                        help='JSON file describing this lambda function. '
                             'Default %(default)s')
    parser.add_argument('--with-pyc', action='store_true',
                        help='Package pyc/pyo files')
    parser.add_argument('--zip-file', default='lambda.zip',
                        help='Name to give ZIP file. Default %(default)s')
    parser.add_argument('--profile', default='default',
                        help='boto/awscli profile name. Default %(default)s')
    parser.add_argument('--zip-only', action='store_true',
                        help='Only create the ZIP file, do not upload')
    args = parser.parse_args()
    with Mu(config_file=args.config, with_pyc=args.with_pyc) as mu:
        mu.get_file_list()
        mu.create_zip()
        if not args.zip_only:
            mu.upload(args.profile)

if __name__ == '__main__':
    main()
<commit_msg>Don't use find_packages, if packages wasn't defined in the json file<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2015 Matt Martz
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from __future__ import (absolute_import, division, print_function)

import argparse
import base64
import hashlib
import json
import os
import re
import sys
import time
import uuid
import zipfile

from contextlib import contextmanager

from distutils import log

import boto3

from botocore.exceptions import ClientError

from py.error import ENOENT
from py.path import local

import setuptools
from setuptools import find_packages
from setuptools.command.build_py import build_py
from setuptools.command.egg_info import FileList
import setuptools.command.sdist
from setuptools.dist import Distribution

from tox.config import DepConfig, TestenvConfig
import tox.session

__version__ = '0.0.1'
__metaclass__ = type

class Mu:
    def __init__(self, config_file='lambda.json', zip_file=None,
                 with_pyc=False):
        self.config_file = config_file
        self.config_data = {}

        self.with_pyc = with_pyc

        self.venv = None

        self.build = None
        self.sdist = None

        self.files = []

        log.set_threshold(log.ERROR)

        self.read_config()

        if zip_file:
            self.zip_file = zip_file
        else:
            self.zip_file = '%s.zip' % self.config_data.get('name', 'lambda')
        self.zip = None

        self.prepare_tox()
        self.prepare_setuptools()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        try:
            self.venv.path.remove()
        except (ENOENT, AttributeError):
            pass

    @contextmanager
    def _zipper(self):
        z = zipfile.ZipFile(self.zip_file, 'w', zipfile.ZIP_DEFLATED)
        yield z
        z.close()

    def read_config(self):
        try:
            with open(self.config_file, 'rbU') as f:
                self.config_data = json.load(f)
        except (OSError, IOError) as e:
            raise SystemExit('Could not load "%s" (%s)' % (self.config_file,
                                                           e))
            self.config_data = {}

        return self.config_data

    def prepare_tox(self):
        toxini = os.path.join(os.path.dirname(__file__), 'tox.ini')
        config = tox.session.prepare(['-c', toxini])
        name = uuid.uuid4().hex
        envconfig = TestenvConfig(name, config, None, None)
        envconfig.envdir = local('%s/%s' % (config.toxworkdir, name))
        envconfig.envlogdir = local('%s/log-%s' % (config.toxworkdir, name))
        envconfig.basepython = sys.executable
        envconfig.sitepackages = False
        envconfig.downloadcache = False
        envconfig.pip_pre = False
        envconfig.setenv = []
        envconfig.install_command = 'pip install {opts} {packages}'.split()
        config.envconfigs[name] = envconfig

        session = tox.session.Session(config)
        self.venv = session.getvenv(name)

    def prepare_setuptools(self):
        dist = Distribution()
        dist.script_name = 'mu.py'

        self.sdist = setuptools.command.sdist.sdist(dist)
        self.sdist.filelist = FileList()

        self.build = build_py(dist)
        self.build.packages = []
        packages = self.config_data.get('packages')
        if packages is not None:
            self.build.packages.extend(find_packages(**packages))
        self.build.py_modules = self.config_data.get('py_modules', [])

    def get_file_list(self):
        for module in self.build.find_all_modules():
            self.files.append((os.path.abspath(module[2]), module[2]))

        deps = self.config_data.get('deps', [])
        if deps:
            ixserver = self.venv.envconfig.config.indexserver['default']
            self.venv.create()
            self.venv.envconfig.deps = [DepConfig(dep, ixserver)
                                        for dep in deps]
            site_packages = self.venv.envconfig.envsitepackagesdir()
            self.venv.install_deps()

            for root, _, files in os.walk(site_packages):
                for filename in files:
                    if not self.with_pyc and re.search('\.py[co]$', filename):
                        continue
                    filepath = os.path.abspath(os.path.join(root, filename))
                    arcname = filepath[len(site_packages) + 1:]
                    self.files.append((filepath, arcname))

        self.sdist.finalize_options()
        self.sdist.get_file_list()

        for _file in self.sdist.filelist.files:
            self.files.append((os.path.abspath(_file), _file))

        return self.files

    def create_zip(self):
        with self._zipper() as z:
            for _file in self.files:
                z.write(*_file)

    def _ensure_iam_default_role(self, session):
        iam = session.client('iam')
        try:
            role_resp = iam.get_role(RoleName='lambda_basic_execution')
        except ClientError:
            name = 'lambda_basic_execution'
            role_resp = iam.create_role(
                Path='/',
                RoleName=name,
                AssumeRolePolicyDocument=json.dumps({
                    'Statement': [
                        {
                            'Action': 'sts:AssumeRole',
                            'Effect': 'Allow',
                            'Principal': {
                                'Service': 'lambda.amazonaws.com'
                            },
                            'Sid': ''
                        }
                    ],
                    'Version': '2012-10-17'
                }, indent=2)
            )

            iam.put_role_policy(
                RoleName=name,
                PolicyName=name,
                PolicyDocument=json.dumps({
                    'Version': '2012-10-17',
                    'Statement': [
                        {
                            "Effect": "Allow",
                            "Action": [
                                "logs:CreateLogGroup",
                                "logs:CreateLogStream",
                                "logs:PutLogEvents"
                            ],
                            "Resource": "arn:aws:logs:*:*:*"
                        }
                    ]
                }, indent=2)
            )

        return role_resp['Role']['Arn']

    def _get_sha256(self):
        sha = hashlib.sha256()
        with open(self.zip_file, 'rb') as f:
            while True:
                chunk = f.read(1024)
                if not chunk:
                    break
                sha.update(chunk)
        return base64.b64encode(sha.digest())

    def upload(self, profile='default'):
        session = boto3.Session(
            region_name=self.config_data.get('region', 'us-east-1'),
            profile_name=profile
        )

        client = session.client('lambda')

        if self.config_data.get('role'):
            role = self.config_data.get('role')
        else:
            role = self._ensure_iam_default_role(session)
            # iam = session.client('iam')
            # try:
            #     role_resp = iam.get_role(RoleName='lambda_basic_execution')
            # except ClientError:
            #     name = 'lambda_basic_execution'
            #     role_resp = iam.create_role(
            #         Path='/',
            #         RoleName=name,
            #         AssumeRolePolicyDocument=json.dumps({
            #             'Statement': [
            #                 {
            #                     'Action': 'sts:AssumeRole',
            #                     'Effect': 'Allow',
            #                     'Principal': {
            #                         'Service': 'lambda.amazonaws.com'
            #                     },
            #                     'Sid': ''
            #                 }
            #             ],
            #             'Version': '2012-10-17'
            #         }, indent=2)
            #     )

            #     iam.put_role_policy(
            #         RoleName=name,
            #         PolicyName=name,
            #         PolicyDocument=json.dumps({
            #             'Version': '2012-10-17',
            #             'Statement': [
            #                 {
            #                     "Effect": "Allow",
            #                     "Action": [
            #                         "logs:CreateLogGroup",
            #                         "logs:CreateLogStream",
            #                         "logs:PutLogEvents"
            #                     ],
            #                     "Resource": "arn:aws:logs:*:*:*"
            #                 }
            #             ]
            #         }, indent=2)
            #     )

            # role = role_resp['Role']['Arn']

        try:
            func = client.get_function(FunctionName=self.config_data['name'])
        except ClientError:
            func = None
            for _ in range(5):
                try:
                    with open(self.zip_file, 'rb') as f:
                        func = client.create_function(
                            FunctionName=self.config_data['name'],
                            Runtime='python2.7',
                            Role=role,
                            Handler=self.config_data['handler'],
                            Code={'ZipFile': f.read()},
                            Description=self.config_data.get('description',
                                                             ''),
                            Timeout=self.config_data.get('timeout', 3),
                            MemorySize=self.config_data.get('memory', 128),
                            Publish=self.config_data.get('publish', True),
                        )
                except ClientError as e:
                    role_msg = ('The role defined for the task cannot be '
                                'assumed by Lambda.')
                    if e.response['Error']['Message'] == role_msg:
                        time.sleep(2)
                    else:
                        raise
                else:
                    break
            if not func:
                raise SystemExit('Error creating Lambda function: %s' %
                                 e.response['Error']['Message'])
        else:
            if self._get_sha256() != func['Configuration']['CodeSha256']:
                with open(self.zip_file, 'rb') as f:
                    client.update_function_code(
                        FunctionName=self.config_data['name'],
                        ZipFile=f.read(),
                        Publish=self.config_data.get('publish', True)
                    )

            client.update_function_configuration(
                FunctionName=self.config_data['name'],
                Role=role,
                Handler=self.config_data['handler'],
                Description=self.config_data.get('description', ''),
                Timeout=self.config_data.get('timeout', 3),
                MemorySize=self.config_data.get('memory', 128)
            )

def main():
    parser = argparse.ArgumentParser(prog='mu')
    parser.add_argument('config', nargs='?', default='lambda.json',
                        help='JSON file describing this lambda function. '
                             'Default %(default)s')
    parser.add_argument('--with-pyc', action='store_true',
                        help='Package pyc/pyo files')
    parser.add_argument('--zip-file', default='lambda.zip',
                        help='Name to give ZIP file. Default %(default)s')
    parser.add_argument('--profile', default='default',
                        help='boto/awscli profile name. Default %(default)s')
    parser.add_argument('--zip-only', action='store_true',
                        help='Only create the ZIP file, do not upload')
    args = parser.parse_args()
    with Mu(config_file=args.config, with_pyc=args.with_pyc) as mu:
        mu.get_file_list()
        mu.create_zip()
        if not args.zip_only:
            mu.upload(args.profile)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from django.contrib import admin
from music.models import Music

class MusicAdmin(admin.ModelAdmin):
    list_display = ('name', 'count', 'music_id', 'source', 'duration', 'last_play', 'dead_link', 'thumbnail', 'room', 'timer_start', 'timer_end')
    actions = ('add_music',)

    def has_add_permission(self, request):
        return False

    def add_music(self, request, queryset):
        for music in queryset:
            music.room.play(music)
        return

admin.site.register(Music, MusicAdmin)
<commit_msg>fix add music<commit_after>from django.contrib import admin
from music.models import Music

from music.serializers import MusicSerializer

class MusicAdmin(admin.ModelAdmin):
    list_display = ('name', 'count', 'music_id', 'source', 'duration', 'last_play', 'dead_link', 'thumbnail', 'room', 'timer_start', 'timer_end')
    actions = ('add_music',)

    def has_add_permission(self, request):
        return False

    def add_music(self, request, queryset):
        for music in queryset:
            music.room.add_music(**MusicSerializer(music).data)
        return

admin.site.register(Music, MusicAdmin)
<|endoftext|>


<commit_before><commit_msg>implemented multithreading<commit_after><|endoftext|>