<commit_before>#!/usr/bin/env python
from __future__ import print_function
import re
import logging
import sys
import os
import codecs

from ycmd.completers.completer import Completer
from ycmd import responses
from ycmd import utils

"""
TODO- change approach, and use seperate objects for each
completer engine. Right now, they share a cache, which is silly,
to say the least.
"""

# To handle BibTeX properly
nobibparser = False
try:
    import bibtexparser
    from bibtexparser.bparser import BibTexParser
    from bibtexparser.customization import convert_to_unicode, author
except ImportError:
    nobibparser = True

def smart_truncate(content, length=30, suffix='...'):
    if len(content) <= length:
        return content
    else:
        return ' '.join(content[:length+1-len(suffix)].split(' ')[0:-1]) + suffix

def bib_customizations(record):
    def truncate_title(record):
        title = record['title'] if 'title' in record else ''
        title = smart_truncate(title)
        record['title'] = title
        return record

    def et_al(record):
        author = record['author'] if 'author' in record else []
        author = [a.replace(', ', ' ').replace(',', ' ') for a in author]
        if len(author) == 0:
            record['author'] = ''
        elif len(author) == 1:
            record['author'] = author[0]
        else:
            record['author'] = author[0] + ' et al.'
        return record

    record = convert_to_unicode(record)
    record = author(record)
    record = et_al(record)
    record = truncate_title(record)
    return record

LOG = logging.getLogger(__name__)

class LatexCompleter( Completer ):
    """
    Completer for LaTeX that takes into account BibTex entries
    for completion.
    """

    def __init__( self, user_options ):
        super( LatexCompleter, self ).__init__( user_options )
        self._completion_target = 'none'
        self._main_directory    = None
        self._cite_reg          = re.compile("\\\\[a-zA-Z]*cite[a-zA-Z]*\*?\{[^\s\}]*\}?")
        self._ref_reg           = re.compile("\\\\[a-zA-Z]*ref\{[^\s\}]*\}?")
        self._env_reg           = re.compile(r"\\(begin)|(end)\{")
        self._files             = {}
        self._cached_data       = {}
        self._d_cache_hits      = 0
        self._goto_labels       = {}

    def ShouldUseNowInner( self, request_data ):
        #q    = utils.ToUtf8IfNeeded(request_data['query'])
        #col  = request_data["column_codepoint"] - 1
        line = utils.ToUnicode(request_data["line_value"])

        if self._main_directory is None:
            self._ComputeMainDirectory(request_data)

        should_use = False
        line_splitted = line
        match = self._cite_reg.search(line_splitted)
        if match is not None:
            self._completion_target = 'cite'
            should_use = True

        match = self._ref_reg.search(line_splitted)
        if match is not None:
            if self._completion_target == 'cite':
                self._completion_target = 'all'
            else:
                self._completion_target = 'label'
            should_use = True

        match = self._env_reg.search(line_splitted)
        if match is not None:
            self._completion_target = 'environment'
            should_use = True

        return should_use

    def SupportedFiletypes( self ):
        """
        Determines which vim filetypes we support
        """
        return ['plaintex', 'tex']

    def _Walk( self, path, what ):
        for root, dirs, files in os.walk(path):
            for file in files:
                if file.endswith(what):
                    yield os.path.join(root, file)

    def _ComputeMainDirectory( self, request_data ):
        def FindMain(path, what):
            found = False
            for file in os.listdir(path):
                if file.endswith(what):
                    self._main_directory = path
                    found = True
                    break

            if not found:
                new_path = os.path.dirname(os.path.normpath(path))
                if new_path == path or new_path == "":
                    return False
                else:
                    return FindMain(new_path, what)
            return True

        filepath = request_data['filepath']
        path     = os.path.dirname(filepath)

        #old version which looks for the .bib. probably should
        #enable it for bibtex completion
        #if not FindMain(path, ".bib"):
        if not FindMain(path, ".latexmain"):
            self._main_directory = filepath
            print("Unable to set the main directory...", sys.stderr)
        else:
            print("Main directory successfully found at {}".format(self._main_directory), sys.stderr)

    def _CacheDataAndSkip(self, filename):
        last_modification = os.path.getmtime(filename)

        if filename not in self._files:
            self._files[filename] = last_modification
            return False, []
        if last_modification <= self._files[filename]:
            self._d_cache_hits += 1
            return True, self._cached_data[filename]

        self._files[filename] = last_modification
        return False, []

    def _FindBibEntriesRegex(self):
        """
        """
        ret = []
        for filename in self._Walk(self._main_directory, ".bib"):
            skip, cache = self._CacheDataAndSkip(filename)
            if skip:
                ret.extend(cache)
                continue

            resp = []
            for line in codecs.open(filename, 'r', 'utf-8'):
                line = line.rstrip()
                found = re.search(r"@(.*){([^,]*).*", line)
                if found is not None:
                    if found.group(1) != "string":
                        resp.append(responses.BuildCompletionData(
                            re.sub(r"@(.*){([^,]*).*", r"\2", line))
                        )

            ret.extend(resp)
            self._cached_data[filename] = resp
        return ret

    def _FindBibEntriesParser(self):
        """
        """
        ret = []
        parser = BibTexParser()
        parser.customization = bib_customizations
        for filename in self._Walk(self._main_directory, ".bib"):
            skip, cache = self._CacheDataAndSkip(filename)
            if skip:
                ret.extend(cache)
                continue

            resp = []
            with open(filename) as bibtex_file:
                bib_database = bibtexparser.load(bibtex_file, parser=parser)
                for entry in bib_database.entries:
                    if 'ID' not in entry:
                        continue
                    title = entry['title']
                    author = entry['author']
                    resp.append(responses.BuildCompletionData(
                        entry['ID'],
                        "%s (%s)" % (title, author)
                    ))

            ret.extend(resp)
            self._cached_data[filename] = resp
        return ret

    def _FindBibEntries(self):
        """
        Find BIBtex entries.

        Using a proper BibTeXParser to be able to retrieve field from the bib
        entry and add it as a help into YCM popup.

        If the BibTexParser module is not available, fallbacks to smart regexes
        to only acquire bibid
        """
        if nobibparser:
            return self._FindBibEntriesRegex()
        else:
            return self._FindBibEntriesParser()

    def _FindEnvironments(self):
        """
        Find LaTeX environments for \begin{} completion.

        This time we scan through all .tex files in the current
        directory and extract the content of all \begin{} commands
        as sources for completion.
        """
        ret = []
        for filename in self._Walk(self._main_directory, ".tex"):
            skip, cache = self._CacheDataAndSkip(filename)
            if skip:
                ret.extend(cache)
                continue

            resp = []
            for i, line in enumerate(codecs.open(filename, 'r', 'utf-8')):
                line = line.rstrip()
                match = re.search(r".*\\begin{(.*?)}.*", line)
                if match is not None:
                    lid = re.sub(r".*\\begin{(.*?)}.*", r"\1", line)
                    temp = responses.BuildCompletionData(lid)
                    if not temp in ret and not temp in resp:
                      resp.append( temp )

            self._cached_data[filename] = resp
            ret.extend(resp)
        return ret

    def _FindLabels(self):
        """
        Find LaTeX labels for \ref{} completion.

        This time we scan through all .tex files in the current
        directory and extract the content of all \label{} commands
        as sources for completion.
        """
        ret = []
        for filename in self._Walk(self._main_directory, ".tex"):
            skip, cache = self._CacheDataAndSkip(filename)
            if skip:
                ret.extend(cache)
                continue

            resp = []
            for i, line in enumerate(codecs.open(filename, 'r', 'utf-8')):
                line = line.rstrip()
                match = re.search(r".*\\\w*(?<!contents)label{(.*?)}.*", line)
                if match is not None:
                    lid = re.sub(r".*\\\w*label{(.*?)}.*", r"\1", line)
                    self._goto_labels[lid] = (filename, i+1, match.start(1))
                    resp.append( responses.BuildCompletionData(lid) )

            self._cached_data[filename] = resp
            ret.extend(resp)
        return ret

    def _GoToDefinition(self, request_data):
        def find_end_of_command(line, match):
            if match is None:
                return -1
            for i in range(match.start(), len(line)):
                e = line[i]
                if e == "}":
                    return i
            return -1

        line = utils.ToUnicode(request_data["line_value"])
        match = self._ref_reg.search(line)
        end_of_command = find_end_of_command(line, match)
        if end_of_command == -1:
            raise RuntimeError( 'Can\'t jump to definition or declaration: not implemented yet' )
        else:
            ref = line[match.end():end_of_command]
            if ref not in self._goto_labels:
                raise RuntimeError( 'Can\'t jump to definition or declaration: not implemented yet' )
            filename, line, col = self._goto_labels[ref]
            return responses.BuildGoToResponse( filename, line, col )

    def GetSubcommandsMap( self ):
        return {
        'GoToDefinition'           : ( lambda self, request_data, args:
            self._GoToDefinition( request_data ) ),
        'GoToDeclaration'          : ( lambda self, request_data, args:
            self._GoToDefinition( request_data ) ),
        'GoTo'                     : ( lambda self, request_data, args:
            self._GoToDefinition( request_data ) ),
        }

    def GetDetailedDiagnostic( self, request_data ):
        return responses.BuildDisplayMessageResponse(
      self.DebugInfo(request_data))

    def DebugInfo( self, request_data ):
        bib_dir = "Looking for *.bib in %s" % self._main_directory
        cache   = "Number of cached files: %i" % len(self._files)
        hits    = "Number of cache hits: %i" % self._d_cache_hits
        return "%s\n%s\n%s" % (bib_dir, cache, hits)

    def ComputeCandidatesInner( self, request_data ):
        """
        Worker function executed by the asynchronous
        completion thread.
        """
        candidates = []

        if self._main_directory is None:
            self._ComputeMainDirectory(request_data)

        if self._completion_target == 'cite':
            candidates = self._FindBibEntries()
        elif self._completion_target == 'label':
            candidates = self._FindLabels()
        elif self._completion_target == 'environment':
            candidates = self._FindEnvironments()
        elif self._completion_target == 'all':
            candidates = self._FindLabels() + self._FindBibEntries()

        print(request_data['query'], sys.stderr)

        return self.FilterAndSortCandidates( candidates, request_data[ 'query' ])
<commit_msg>Trying to limit completion to when you're in the braces.. not going so well<commit_after>#!/usr/bin/env python
from __future__ import print_function
import re
import logging
import sys
import os
import codecs

from ycmd.completers.completer import Completer
from ycmd import responses
from ycmd import utils

"""
TODO- change approach, and use seperate objects for each
completer engine. Right now, they share a cache, which is silly,
to say the least.
"""

# To handle BibTeX properly
nobibparser = False
try:
    import bibtexparser
    from bibtexparser.bparser import BibTexParser
    from bibtexparser.customization import convert_to_unicode, author
except ImportError:
    nobibparser = True

def smart_truncate(content, length=30, suffix='...'):
    if len(content) <= length:
        return content
    else:
        return ' '.join(content[:length+1-len(suffix)].split(' ')[0:-1]) + suffix

def bib_customizations(record):
    def truncate_title(record):
        title = record['title'] if 'title' in record else ''
        title = smart_truncate(title)
        record['title'] = title
        return record

    def et_al(record):
        author = record['author'] if 'author' in record else []
        author = [a.replace(', ', ' ').replace(',', ' ') for a in author]
        if len(author) == 0:
            record['author'] = ''
        elif len(author) == 1:
            record['author'] = author[0]
        else:
            record['author'] = author[0] + ' et al.'
        return record

    record = convert_to_unicode(record)
    record = author(record)
    record = et_al(record)
    record = truncate_title(record)
    return record

LOG = logging.getLogger(__name__)

class LatexCompleter( Completer ):
    """
    Completer for LaTeX that takes into account BibTex entries
    for completion.
    """

    def __init__( self, user_options ):
        super( LatexCompleter, self ).__init__( user_options )
        self._completion_target = 'none'
        self._main_directory    = None
        self._cite_reg          = re.compile("\\\\[a-zA-Z]*cite[a-zA-Z]*\*?\{[^\s\}]*\}?")
        self._ref_reg           = re.compile(r"\\[a-zA-Z]*ref\{")
        self._env_reg           = re.compile(r"\\(begin)|(end)\{")
        self.closebrace         = re.compile(r"\{?[a-zA-z:_]*\}?")
        self._files             = {}
        self._cached_data       = {}
        self._d_cache_hits      = 0
        self._goto_labels       = {}

    def ShouldUseNowInner( self, request_data ):
        #q    = utils.ToUtf8IfNeeded(request_data['query'])
        col  = request_data["column_codepoint"] - 1
        line = request_data["line_value"]

        if self._main_directory is None:
            self._ComputeMainDirectory(request_data)

        should_use = False
        line_splitted = line[ : col]
        line_left     = line[ col : ]

        if self.closebrace.match(line_left) is not None:
            match = self._cite_reg.search(line_splitted)
            if match is not None:
                self._completion_target = 'cite'
                should_use = True

            match = self._ref_reg.search(line_splitted)
            if match is not None:
                if self._completion_target == 'cite':
                    self._completion_target = 'all'
                else:
                    self._completion_target = 'label'
                should_use = True

            match = self._env_reg.search(line_splitted)
            if match is not None:
                self._completion_target = 'environment'
                should_use = True

        return should_use

    def SupportedFiletypes( self ):
        """
        Determines which vim filetypes we support
        """
        return ['plaintex', 'tex']

    def _Walk( self, path, what ):
        for root, dirs, files in os.walk(path):
            for file in files:
                if file.endswith(what):
                    yield os.path.join(root, file)

    def _ComputeMainDirectory( self, request_data ):
        def FindMain(path, what):
            found = False
            for file in os.listdir(path):
                if file.endswith(what):
                    self._main_directory = path
                    found = True
                    break

            if not found:
                new_path = os.path.dirname(os.path.normpath(path))
                if new_path == path or new_path == "":
                    return False
                else:
                    return FindMain(new_path, what)
            return True

        filepath = request_data['filepath']
        path     = os.path.dirname(filepath)

        #old version which looks for the .bib. probably should
        #enable it for bibtex completion
        #if not FindMain(path, ".bib"):
        if not FindMain(path, ".latexmain"):
            self._main_directory = filepath
            print("Unable to set the main directory...", sys.stderr)
        else:
            print("Main directory successfully found at {}".format(self._main_directory), sys.stderr)

    def _CacheDataAndSkip(self, filename):
        last_modification = os.path.getmtime(filename)

        if filename not in self._files:
            self._files[filename] = last_modification
            return False, []
        if last_modification <= self._files[filename]:
            self._d_cache_hits += 1
            return True, self._cached_data[filename]

        self._files[filename] = last_modification
        return False, []

    def _FindBibEntriesRegex(self):
        """
        """
        ret = []
        for filename in self._Walk(self._main_directory, ".bib"):
            skip, cache = self._CacheDataAndSkip(filename)
            if skip:
                ret.extend(cache)
                continue

            resp = []
            for line in codecs.open(filename, 'r', 'utf-8'):
                line = line.rstrip()
                found = re.search(r"@(.*){([^,]*).*", line)
                if found is not None:
                    if found.group(1) != "string":
                        resp.append(responses.BuildCompletionData(
                            re.sub(r"@(.*){([^,]*).*", r"\2", line))
                        )

            ret.extend(resp)
            self._cached_data[filename] = resp
        return ret

    def _FindBibEntriesParser(self):
        """
        """
        ret = []
        parser = BibTexParser()
        parser.customization = bib_customizations
        for filename in self._Walk(self._main_directory, ".bib"):
            skip, cache = self._CacheDataAndSkip(filename)
            if skip:
                ret.extend(cache)
                continue

            resp = []
            with open(filename) as bibtex_file:
                bib_database = bibtexparser.load(bibtex_file, parser=parser)
                for entry in bib_database.entries:
                    if 'ID' not in entry:
                        continue
                    title = entry['title']
                    author = entry['author']
                    resp.append(responses.BuildCompletionData(
                        entry['ID'],
                        "%s (%s)" % (title, author)
                    ))

            ret.extend(resp)
            self._cached_data[filename] = resp
        return ret

    def _FindBibEntries(self):
        """
        Find BIBtex entries.

        Using a proper BibTeXParser to be able to retrieve field from the bib
        entry and add it as a help into YCM popup.

        If the BibTexParser module is not available, fallbacks to smart regexes
        to only acquire bibid
        """
        if nobibparser:
            return self._FindBibEntriesRegex()
        else:
            return self._FindBibEntriesParser()

    def _FindEnvironments(self):
        """
        Find LaTeX environments for \begin{} completion.

        This time we scan through all .tex files in the current
        directory and extract the content of all \begin{} commands
        as sources for completion.
        """
        ret = []
        for filename in self._Walk(self._main_directory, ".tex"):
            skip, cache = self._CacheDataAndSkip(filename)
            if skip:
                ret.extend(cache)
                continue

            resp = []
            for i, line in enumerate(codecs.open(filename, 'r', 'utf-8')):
                line = line.rstrip()
                match = re.search(r".*\\begin{(.*?)}.*", line)
                if match is not None:
                    lid = re.sub(r".*\\begin{(.*?)}.*", r"\1", line)
                    temp = responses.BuildCompletionData(lid)
                    if not temp in ret and not temp in resp:
                      resp.append( temp )

            self._cached_data[filename] = resp
            ret.extend(resp)
        return ret

    def _FindLabels(self):
        """
        Find LaTeX labels for \ref{} completion.

        This time we scan through all .tex files in the current
        directory and extract the content of all \label{} commands
        as sources for completion.
        """
        ret = []
        for filename in self._Walk(self._main_directory, ".tex"):
            skip, cache = self._CacheDataAndSkip(filename)
            if skip:
                ret.extend(cache)
                continue

            resp = []
            for i, line in enumerate(codecs.open(filename, 'r', 'utf-8')):
                line = line.rstrip()
                match = re.search(r".*\\\w*(?<!contents)label{(.*?)}.*", line)
                if match is not None:
                    lid = re.sub(r".*\\\w*label{(.*?)}.*", r"\1", line)
                    self._goto_labels[lid] = (filename, i+1, match.start(1))
                    resp.append( responses.BuildCompletionData(lid) )

            self._cached_data[filename] = resp
            ret.extend(resp)
        return ret

    def _GoToDefinition(self, request_data):
        def find_end_of_command(line, match):
            if match is None:
                return -1
            for i in range(match.start(), len(line)):
                e = line[i]
                if e == "}":
                    return i
            return -1

        line = utils.ToUnicode(request_data["line_value"])
        match = self._ref_reg.search(line)
        end_of_command = find_end_of_command(line, match)
        if end_of_command == -1:
            raise RuntimeError( 'Can\'t jump to definition or declaration: not implemented yet' )
        else:
            ref = line[match.end():end_of_command]
            if ref not in self._goto_labels:
                raise RuntimeError( 'Can\'t jump to definition or declaration: not implemented yet' )
            filename, line, col = self._goto_labels[ref]
            return responses.BuildGoToResponse( filename, line, col )

    def GetSubcommandsMap( self ):
        return {
        'GoToDefinition'           : ( lambda self, request_data, args:
            self._GoToDefinition( request_data ) ),
        'GoToDeclaration'          : ( lambda self, request_data, args:
            self._GoToDefinition( request_data ) ),
        'GoTo'                     : ( lambda self, request_data, args:
            self._GoToDefinition( request_data ) ),
        }

    def GetDetailedDiagnostic( self, request_data ):
        return responses.BuildDisplayMessageResponse(
      self.DebugInfo(request_data))

    def DebugInfo( self, request_data ):
        bib_dir = "Looking for *.bib in %s" % self._main_directory
        cache   = "Number of cached files: %i" % len(self._files)
        hits    = "Number of cache hits: %i" % self._d_cache_hits
        return "%s\n%s\n%s" % (bib_dir, cache, hits)

    def ComputeCandidatesInner( self, request_data ):
        """
        Worker function executed by the asynchronous
        completion thread.
        """
        candidates = []

        if self._main_directory is None:
            self._ComputeMainDirectory(request_data)

        if self._completion_target == 'cite':
            candidates = self._FindBibEntries()
        elif self._completion_target == 'label':
            candidates = self._FindLabels()
        elif self._completion_target == 'environment':
            candidates = self._FindEnvironments()
        elif self._completion_target == 'all':
            candidates = self._FindLabels() + self._FindBibEntries()

        print(request_data['query'], sys.stderr)

        return self.FilterAndSortCandidates( candidates, request_data[ 'query' ] )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import http.server
import socket
import socketserver

from pathlib import Path
from shutil import copyfile
from urllib.parse import unquote, urlparse

from logya.core import Logya
from logya.content import add_collections, read, read_all, write_collection
from logya.util import load_settings

# Deprecated imports
from logya.writer import DocWriter

settings = load_settings()
site_index = read_all(settings)

L = Logya()
L.init_env()
L.build_index()
L.template.vars['debug'] = True

class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
    """SimpleHTTPRequestHandler based class to return resources."""

    def __init__(self, *args):
        super(HTTPRequestHandler, self).__init__(*args, directory=settings['paths']['public'].as_posix())

    def do_GET(self):
        update_resource(self.path)
        super(HTTPRequestHandler, self).do_GET()

def update_resource(url):
    """Update resource corresponding to given url.

    Resources that exist in the `static` directory are updated if they are newer than the destination file.
    For other HTML resources the whole `site_index` is updated and the destination is newly written."""

    # Use only the actual path and ignore possible query params (see issue #3).
    src_url = unquote(urlparse(url).path)
    src_name = src_url.lstrip('/')

    # If a static file is requested update it and return.
    src_static = Path(settings['paths']['static'], src_name)
    if src_static.is_file():
        dst_static = Path(settings['paths']['public'], src_name)
        dst_static.parent.mkdir(exist_ok=True)
        if not dst_static.exists() or src_static.stat().st_mtime > dst_static.stat().st_mtime:
            print(f'Update static resource: {dst_static}')
            copyfile(src_static, dst_static)
        return True

    # Rebuild index for HTML file requests which are not in index.
    if src_url.endswith(('/', '.html', '.htm')) and src_url not in site_index:
        print(f'Rebuild index for request URL: {src_url}')
        site_index.update(read_all(settings))

    content = site_index[src_url]
    path_dst = Path(settings['paths']['public'], src_name, 'index.html')

    # Update content document
    if 'doc' in content:
        content['doc'] = read(content['path'], settings)
        if 'collections' in settings:
            add_collections(content['doc'], site_index, settings['collections'])
        # Always write doc because of possible template changes.
        DocWriter(settings['paths']['public'], L.template).write(content['doc'], L.get_doc_template(content['doc']))
        print(f'Refreshed doc at URL: {src_url}')
        return

    # Update collection page
    if 'docs' in content:
        write_collection(path_dst, content, L.template, settings)
        print(f'Refreshed collection: {src_url}')

def serve(args):
    L.template.vars['base_url'] = f'http://{args.host}:{args.port}'
    with socketserver.TCPServer((args.host, args.port), HTTPRequestHandler) as httpd:
        # Avoid "OSError: [Errno 98] Address already in use"
        httpd.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        print(f'Serving on {L.template.vars["base_url"]}')
        httpd.serve_forever()<commit_msg>Set socketserver.TCPServer.allow_reuse_address to True<commit_after># -*- coding: utf-8 -*-
import http.server
import socket
import socketserver

from pathlib import Path
from shutil import copyfile
from urllib.parse import unquote, urlparse

from logya.core import Logya
from logya.content import add_collections, read, read_all, write_collection
from logya.util import load_settings

# Deprecated imports
from logya.writer import DocWriter

settings = load_settings()
site_index = read_all(settings)

L = Logya()
L.init_env()
L.build_index()
L.template.vars['debug'] = True

class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
    """SimpleHTTPRequestHandler based class to return resources."""

    def __init__(self, *args):
        super(HTTPRequestHandler, self).__init__(*args, directory=settings['paths']['public'].as_posix())

    def do_GET(self):
        update_resource(self.path)
        super(HTTPRequestHandler, self).do_GET()

def update_resource(url):
    """Update resource corresponding to given url.

    Resources that exist in the `static` directory are updated if they are newer than the destination file.
    For other HTML resources the whole `site_index` is updated and the destination is newly written."""

    # Use only the actual path and ignore possible query params (see issue #3).
    src_url = unquote(urlparse(url).path)
    src_name = src_url.lstrip('/')

    # If a static file is requested update it and return.
    src_static = Path(settings['paths']['static'], src_name)
    if src_static.is_file():
        dst_static = Path(settings['paths']['public'], src_name)
        dst_static.parent.mkdir(exist_ok=True)
        if not dst_static.exists() or src_static.stat().st_mtime > dst_static.stat().st_mtime:
            print(f'Update static resource: {dst_static}')
            copyfile(src_static, dst_static)
        return True

    # Rebuild index for HTML file requests which are not in index.
    if src_url.endswith(('/', '.html', '.htm')) and src_url not in site_index:
        print(f'Rebuild index for request URL: {src_url}')
        site_index.update(read_all(settings))

    content = site_index[src_url]
    path_dst = Path(settings['paths']['public'], src_name, 'index.html')

    # Update content document
    if 'doc' in content:
        content['doc'] = read(content['path'], settings)
        if 'collections' in settings:
            add_collections(content['doc'], site_index, settings['collections'])
        # Always write doc because of possible template changes.
        DocWriter(settings['paths']['public'], L.template).write(content['doc'], L.get_doc_template(content['doc']))
        print(f'Refreshed doc at URL: {src_url}')
        return

    # Update collection page
    if 'docs' in content:
        write_collection(path_dst, content, L.template, settings)
        print(f'Refreshed collection: {src_url}')

def serve(args):
    L.template.vars['base_url'] = f'http://{args.host}:{args.port}'
    # Avoid "OSError: [Errno 98] Address already in use"
    socketserver.TCPServer.allow_reuse_address = True
    with socketserver.TCPServer((args.host, args.port), HTTPRequestHandler) as httpd:
        print(f'Serving on {L.template.vars["base_url"]}')
        httpd.serve_forever()<|endoftext|>


<commit_before>from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models import query

get_for_model = ContentType.objects.get_for_model

class RelatedContentQuerySet(query.QuerySet):
    def filter(self, destination_object=None, source_object=None, **kwargs):
        if destination_object:
            kwargs.update({
                "destination_id": destination_object.pk,
                "destination_type": get_for_model(destination_object),
            })
        if source_object:
            kwargs.update({
                "source_id": source_object.pk,
                "source_type": get_for_model(source_object),
            })
        return super(RelatedContentQuerySet, self).filter(**kwargs)

    def by_destination(self, destination):
        return self.filter(destination_object=destination)

    def by_source(self, source):
        return self.filter(source_object=source)

class RelatedContentManager(models.Manager):
    def get_query_set(self):
        return RelatedContentQuerySet(self.model, using=self._db)

    def by_destination(self, destination):
        return self.get_query_set().by_destination(destination)

    def by_source(self, source):
        return self.get_query_set().by_source(source)

    def by_type(self, type):
        return self.filter(related_type__title=type)
<commit_msg>Add documentation for new methods<commit_after>from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models import query

get_for_model = ContentType.objects.get_for_model

class RelatedContentQuerySet(query.QuerySet):
    def filter(self, destination_object=None, source_object=None, **kwargs):
        if destination_object:
            kwargs.update({
                "destination_id": destination_object.pk,
                "destination_type": get_for_model(destination_object),
            })
        if source_object:
            kwargs.update({
                "source_id": source_object.pk,
                "source_type": get_for_model(source_object),
            })
        return super(RelatedContentQuerySet, self).filter(**kwargs)

    def by_destination(self, destination):
        """
        Retrieve any ``RelatedContent`` where the ``destination_object``
        is equal to the provided ``destination``.

        This is a shortcut for calling::

            RelatedContent.objects.filter(destination_object=destination)
        """
        return self.filter(destination_object=destination)

    def by_source(self, source):
        """
        Retrieve any ``RelatedContent`` where the ``source_object``
        is equal to the provided ``source``.

        This is a shortcut for calling::

            RelatedContent.objects.filter(source_object=source)
        """
        return self.filter(source_object=source)

class RelatedContentManager(models.Manager):
    def get_query_set(self):
        return RelatedContentQuerySet(self.model, using=self._db)

    def by_destination(self, destination):
        """
        See ``RelatedContentQuerySet.by_destination``
        """
        return self.get_query_set().by_destination(destination)

    def by_source(self, source):
        """
        See ``RelatedContentQuerySet.by_source``
        """
        return self.get_query_set().by_source(source)

    def by_type(self, type):
        return self.filter(related_type__title=type)
<|endoftext|>


<commit_before># -*- coding: UTF8

'''
Created on 30.08.2015

@author: mEDI
'''
from PySide import QtCore, QtGui
import PySide

import gui.guitools as guitools

__toolname__ = "Shipyard Finder"
__statusTip__ = "Open A %s Window" % __toolname__

class tool(QtGui.QWidget):
    main = None
    mydb = None
    route = None

    def __init__(self, main):
        super(tool, self).__init__(main)

        self.main = main
        self.mydb = main.mydb
        self.guitools = guitools.guitools(self)
        self.createActions()

    def getWideget(self):

        locationButton = QtGui.QToolButton()
        locationButton.setIcon(self.guitools.getIconFromsvg("img/location.svg"))
        locationButton.clicked.connect(self.setCurentLocation)
        locationButton.setToolTip("Current Location")

        locationLabel = QtGui.QLabel("Location:")
        self.locationlineEdit = guitools.LineEdit()
        self.locationlineEdit.setText( self.main.location.getLocation() )
        self.locationlineEdit.textChanged.connect(self.searchShip)

        ShipLabel = QtGui.QLabel("Ship:")
        self.shipComboBox = QtGui.QComboBox()

        ships = self.mydb.getAllShipnames()
        self.shipList = []
        for ship in ships:
            self.shipComboBox.addItem( ship["Name"] )
            self.shipList.append(ship["id"])
        self.shipComboBox.currentIndexChanged.connect(self.searchShip)

        self.searchbutton = QtGui.QPushButton("Search")
        self.searchbutton.clicked.connect(self.searchShip)

        layout = QtGui.QHBoxLayout()

        layout.addWidget(locationLabel)
        layout.addWidget(locationButton)

        layout.addWidget(self.locationlineEdit)
        layout.addWidget(ShipLabel)
        layout.addWidget(self.shipComboBox)

        layout.addWidget(self.searchbutton)

        locationGroupBox = QtGui.QGroupBox()
        #locationGroupBox.setFlat(True)
        locationGroupBox.setLayout(layout)

        self.listView = QtGui.QTreeView()

#        self.proxyModel = QtGui.QSortFilterProxyModel()
#        self.proxyModel.setDynamicSortFilter(True)

        self.listView.setRootIsDecorated(False)
        self.listView.setAlternatingRowColors(True)
#        self.listView.setModel(self.proxyModel)
        self.listView.setSortingEnabled(True)

        self.listView.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
        self.listView.setSelectionBehavior(QtGui.QAbstractItemView.SelectItems)

        self.listView.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
        self.listView.customContextMenuRequested.connect(self.myContextMenuEvent)

        vGroupBox = QtGui.QGroupBox()
        vGroupBox.setFlat(True)

        layout = QtGui.QVBoxLayout()

        layout.addWidget(locationGroupBox)
        layout.addWidget(self.listView)

        vGroupBox.setLayout(layout)

        self.guitools.setSystemComplete("", self.locationlineEdit)

        return vGroupBox

    def myContextMenuEvent(self, event):
        menu = QtGui.QMenu(self)

        menu.addAction(self.copyAct)

        menu.exec_(self.listView.viewport().mapToGlobal(event))

    def createActions(self):
        self.copyAct = QtGui.QAction("Copy", self, triggered=self.guitools.copyToClipboard, shortcut=QtGui.QKeySequence.Copy)

    def setCurentLocation(self):
        self.locationlineEdit.setText( self.main.location.getLocation() )

    def searchShip(self):

        firstrun = False
        if not self.listView.header().count():
            firstrun = True

        self.headerList = ["System","StarDist", "Station", "Distance", "Age", ""]

        model = QtGui.QStandardItemModel(0, len(self.headerList), self)
        for x,column in enumerate(self.headerList):
            model.setHeaderData(x, QtCore.Qt.Horizontal, column)

        location = self.locationlineEdit.text()
        systemID = self.mydb.getSystemIDbyName(location)

        shipID = self.shipList[ self.shipComboBox.currentIndex() ]

        shipyards = self.mydb.getShipyardWithShip(shipID, systemID)

        for shipyard in shipyards:
            model.insertRow(0)
            model.setData(model.index(0, self.headerList.index("System") ), shipyard["System"])
            model.setData(model.index(0, self.headerList.index("StarDist") ), shipyard["StarDist"])
            model.item(0, self.headerList.index("StarDist")).setTextAlignment(QtCore.Qt.AlignRight)

            model.setData(model.index(0, self.headerList.index("Station") ), shipyard["Station"])
            model.setData(model.index(0, self.headerList.index("Distance") ),  shipyard["dist"])
            model.item(0, self.headerList.index("Distance")).setTextAlignment(QtCore.Qt.AlignRight)

            model.setData(model.index(0, self.headerList.index("Age") ), guitools.convertDateimeToAgeStr(shipyard["age"]) )
            model.item(0, self.headerList.index("Age")).setTextAlignment(QtCore.Qt.AlignCenter)

        self.listView.setModel(model)

        if firstrun:
            self.listView.sortByColumn( self.headerList.index("Distance"), PySide.QtCore.Qt.SortOrder.AscendingOrder )

        for i in range(0, len(self.headerList) ):
            self.listView.resizeColumnToContents(i)
<commit_msg>add Permit<commit_after># -*- coding: UTF8

'''
Created on 30.08.2015

@author: mEDI
'''
from PySide import QtCore, QtGui
import PySide

import gui.guitools as guitools

__toolname__ = "Shipyard Finder"
__statusTip__ = "Open A %s Window" % __toolname__

class tool(QtGui.QWidget):
    main = None
    mydb = None
    route = None

    def __init__(self, main):
        super(tool, self).__init__(main)

        self.main = main
        self.mydb = main.mydb
        self.guitools = guitools.guitools(self)
        self.createActions()

    def getWideget(self):

        locationButton = QtGui.QToolButton()
        locationButton.setIcon(self.guitools.getIconFromsvg("img/location.svg"))
        locationButton.clicked.connect(self.setCurentLocation)
        locationButton.setToolTip("Current Location")

        locationLabel = QtGui.QLabel("Location:")
        self.locationlineEdit = guitools.LineEdit()
        self.locationlineEdit.setText( self.main.location.getLocation() )
        self.locationlineEdit.textChanged.connect(self.searchShip)

        ShipLabel = QtGui.QLabel("Ship:")
        self.shipComboBox = QtGui.QComboBox()

        ships = self.mydb.getAllShipnames()
        self.shipList = []
        for ship in ships:
            self.shipComboBox.addItem( ship["Name"] )
            self.shipList.append(ship["id"])
        self.shipComboBox.currentIndexChanged.connect(self.searchShip)

        self.searchbutton = QtGui.QPushButton("Search")
        self.searchbutton.clicked.connect(self.searchShip)

        layout = QtGui.QHBoxLayout()

        layout.addWidget(locationLabel)
        layout.addWidget(locationButton)

        layout.addWidget(self.locationlineEdit)
        layout.addWidget(ShipLabel)
        layout.addWidget(self.shipComboBox)

        layout.addWidget(self.searchbutton)

        locationGroupBox = QtGui.QGroupBox()
        #locationGroupBox.setFlat(True)
        locationGroupBox.setLayout(layout)

        self.listView = QtGui.QTreeView()

#        self.proxyModel = QtGui.QSortFilterProxyModel()
#        self.proxyModel.setDynamicSortFilter(True)

        self.listView.setRootIsDecorated(False)
        self.listView.setAlternatingRowColors(True)
#        self.listView.setModel(self.proxyModel)
        self.listView.setSortingEnabled(True)

        self.listView.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
        self.listView.setSelectionBehavior(QtGui.QAbstractItemView.SelectItems)

        self.listView.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
        self.listView.customContextMenuRequested.connect(self.myContextMenuEvent)

        vGroupBox = QtGui.QGroupBox()
        vGroupBox.setFlat(True)

        layout = QtGui.QVBoxLayout()

        layout.addWidget(locationGroupBox)
        layout.addWidget(self.listView)

        vGroupBox.setLayout(layout)

        self.guitools.setSystemComplete("", self.locationlineEdit)

        return vGroupBox

    def myContextMenuEvent(self, event):
        menu = QtGui.QMenu(self)

        menu.addAction(self.copyAct)

        menu.exec_(self.listView.viewport().mapToGlobal(event))

    def createActions(self):
        self.copyAct = QtGui.QAction("Copy", self, triggered=self.guitools.copyToClipboard, shortcut=QtGui.QKeySequence.Copy)

    def setCurentLocation(self):
        self.locationlineEdit.setText( self.main.location.getLocation() )

    def searchShip(self):

        firstrun = False
        if not self.listView.header().count():
            firstrun = True

        self.headerList = ["System", "Permit","StarDist", "Station", "Distance", "Age", ""]

        model = QtGui.QStandardItemModel(0, len(self.headerList), self)
        for x,column in enumerate(self.headerList):
            model.setHeaderData(x, QtCore.Qt.Horizontal, column)

        location = self.locationlineEdit.text()
        systemID = self.mydb.getSystemIDbyName(location)

        shipID = self.shipList[ self.shipComboBox.currentIndex() ]

        shipyards = self.mydb.getShipyardWithShip(shipID, systemID)

        for shipyard in shipyards:
            model.insertRow(0)
            model.setData(model.index(0, self.headerList.index("System") ), shipyard["System"])

            model.setData(model.index(0, self.headerList.index("Permit") ),  "No" if not shipyard["permit"] else "Yes" )
            model.item(0, self.headerList.index("Permit")).setTextAlignment(QtCore.Qt.AlignCenter)

            model.setData(model.index(0, self.headerList.index("StarDist") ), shipyard["StarDist"])
            model.item(0, self.headerList.index("StarDist")).setTextAlignment(QtCore.Qt.AlignRight)

            model.setData(model.index(0, self.headerList.index("Station") ), shipyard["Station"])
            model.setData(model.index(0, self.headerList.index("Distance") ),  shipyard["dist"])
            model.item(0, self.headerList.index("Distance")).setTextAlignment(QtCore.Qt.AlignRight)

            model.setData(model.index(0, self.headerList.index("Age") ), guitools.convertDateimeToAgeStr(shipyard["age"]) )
            model.item(0, self.headerList.index("Age")).setTextAlignment(QtCore.Qt.AlignCenter)

        self.listView.setModel(model)

        if firstrun:
            self.listView.sortByColumn( self.headerList.index("Distance"), PySide.QtCore.Qt.SortOrder.AscendingOrder )

        for i in range(0, len(self.headerList) ):
            self.listView.resizeColumnToContents(i)
<|endoftext|>


<commit_before># Copyright (c) 2012 Spotify AB
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

# Simple REST server that takes commands in a JSON payload
import json
import os
import atexit
import mimetypes
import posixpath
import tornado.ioloop
import tornado.netutil
import tornado.web
import tornado.httpclient
import tornado.httpserver
import configuration
import scheduler
import pkg_resources
import signal
from rpc import RemoteSchedulerResponder
import task_history
import logging
logger = logging.getLogger("luigi.server")

def _create_scheduler():
    config = configuration.get_config()
    retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0)
    remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0)
    worker_disconnect_delay = config.getfloat('scheduler', 'worker-disconnect-delay', 60.0)
    state_path = config.get('scheduler', 'state-path', '/var/lib/luigi-server/state.pickle')
    if config.getboolean('scheduler', 'record_task_history', False):
        import db_task_history  # Needs sqlalchemy, thus imported here
        task_history_impl = db_task_history.DbTaskHistory()
    else:
        task_history_impl = task_history.NopHistory()
    return scheduler.CentralPlannerScheduler(retry_delay, remove_delay, worker_disconnect_delay, state_path, task_history_impl)

class RPCHandler(tornado.web.RequestHandler):
    """ Handle remote scheduling calls using rpc.RemoteSchedulerResponder"""

    def initialize(self, api):
        self._api = api

    def get(self, method):
        payload = self.get_argument('data', default="{}")
        arguments = json.loads(payload)

        if hasattr(self._api, method):
            result = getattr(self._api, method)(**arguments)
            self.write({"response": result})  # wrap all json response in a dictionary
        else:
            self.send_error(404)

class BaseTaskHistoryHandler(tornado.web.RequestHandler):
    def initialize(self, api):
        self._api = api

    def get_template_path(self):
        return pkg_resources.resource_filename(__name__, 'templates')

class RecentRunHandler(BaseTaskHistoryHandler):
    def get(self):
        tasks = self._api.task_history.find_latest_runs()
        self.render("recent.html", tasks=tasks)

class ByNameHandler(BaseTaskHistoryHandler):
    def get(self, name):
        tasks = self._api.task_history.find_all_by_name(name)
        self.render("recent.html", tasks=tasks)

class ByIdHandler(BaseTaskHistoryHandler):
    def get(self, id):
        task = self._api.task_history.find_task_by_id(id)
        self.render("show.html", task=task)

class ByParamsHandler(BaseTaskHistoryHandler):
    def get(self, name):
        payload = self.get_argument('data', default="{}")
        arguments = json.loads(payload)
        tasks = self._api.task_history.find_all_by_parameters(name, session=None, **arguments)
        self.render("recent.html", tasks=tasks)

class StaticFileHandler(tornado.web.RequestHandler):
    def get(self, path):
        # Path checking taken from Flask's safe_join function:
        # https://github.com/mitsuhiko/flask/blob/1d55b8983/flask/helpers.py#L563-L587
        path = posixpath.normpath(path)
        if os.path.isabs(path) or path.startswith(".."):
            return self.send_error(404)

        extension = os.path.splitext(path)[1]
        if extension in mimetypes.types_map:
            self.set_header("Content-Type", mimetypes.types_map[extension])
        data = pkg_resources.resource_string(__name__, os.path.join("static", path))
        self.write(data)

class RootPathHandler(tornado.web.RequestHandler):
    def get(self):
        self.redirect("/static/visualiser/index.html")

def app(api):
    handlers = [
        (r'/api/(.*)', RPCHandler, {"api": api}),
        (r'/static/(.*)', StaticFileHandler),
        (r'/', RootPathHandler),
        (r'/history', RecentRunHandler, {'api': api}),
        (r'/history/by_name/(.*?)', ByNameHandler, {'api': api}),
        (r'/history/by_id/(.*?)', ByIdHandler, {'api': api}),
        (r'/history/by_params/(.*?)', ByParamsHandler, {'api': api})
    ]
    api_app = tornado.web.Application(handlers)
    return api_app

def _init_api(sched, responder, api_port, address):
    api = responder or RemoteSchedulerResponder(sched)
    api_app = app(api)
    api_sockets = tornado.netutil.bind_sockets(api_port, address=address)
    server = tornado.httpserver.HTTPServer(api_app, max_header_size=2097152)
    server.add_sockets(api_sockets)

    # Return the bound socket names.  Useful for connecting client in test scenarios.
    return [s.getsockname() for s in api_sockets]

def run(api_port=8082, address=None, scheduler=None, responder=None):
    """ Runs one instance of the API server """
    sched = scheduler or _create_scheduler()
    # load scheduler state
    sched.load()

    _init_api(sched, responder, api_port, address)

    # prune work DAG every 60 seconds
    pruner = tornado.ioloop.PeriodicCallback(sched.prune, 60000)
    pruner.start()

    def shutdown_handler(foo=None, bar=None):
        logger.info("Scheduler instance shutting down")
        sched.dump()
        os._exit(0)

    signal.signal(signal.SIGINT, shutdown_handler)
    signal.signal(signal.SIGTERM, shutdown_handler)
    signal.signal(signal.SIGQUIT, shutdown_handler)
    atexit.register(shutdown_handler)

    logger.info("Scheduler starting up")

    tornado.ioloop.IOLoop.instance().start()

def run_api_threaded(api_port=8082, address=None):
    ''' For integration tests'''
    sock_names = _init_api(_create_scheduler(), None, api_port, address)

    import threading
    def scheduler_thread():
        # this is wrapped in a function so we get the instance
        # from the scheduler thread and not from the main thread
        tornado.ioloop.IOLoop.instance().start()

    threading.Thread(target=scheduler_thread).start()
    return sock_names

def stop():
    tornado.ioloop.IOLoop.instance().stop()

if __name__ == "__main__":
    run()
<commit_msg>Fallback since older versions of Tornado can't handle max_hader_size<commit_after># Copyright (c) 2012 Spotify AB
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

# Simple REST server that takes commands in a JSON payload
import json
import os
import atexit
import mimetypes
import posixpath
import tornado.ioloop
import tornado.netutil
import tornado.web
import tornado.httpclient
import tornado.httpserver
import configuration
import scheduler
import pkg_resources
import signal
from rpc import RemoteSchedulerResponder
import task_history
import logging
import warnings
logger = logging.getLogger("luigi.server")

def _create_scheduler():
    config = configuration.get_config()
    retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0)
    remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0)
    worker_disconnect_delay = config.getfloat('scheduler', 'worker-disconnect-delay', 60.0)
    state_path = config.get('scheduler', 'state-path', '/var/lib/luigi-server/state.pickle')
    if config.getboolean('scheduler', 'record_task_history', False):
        import db_task_history  # Needs sqlalchemy, thus imported here
        task_history_impl = db_task_history.DbTaskHistory()
    else:
        task_history_impl = task_history.NopHistory()
    return scheduler.CentralPlannerScheduler(retry_delay, remove_delay, worker_disconnect_delay, state_path, task_history_impl)

class RPCHandler(tornado.web.RequestHandler):
    """ Handle remote scheduling calls using rpc.RemoteSchedulerResponder"""

    def initialize(self, api):
        self._api = api

    def get(self, method):
        payload = self.get_argument('data', default="{}")
        arguments = json.loads(payload)

        if hasattr(self._api, method):
            result = getattr(self._api, method)(**arguments)
            self.write({"response": result})  # wrap all json response in a dictionary
        else:
            self.send_error(404)

class BaseTaskHistoryHandler(tornado.web.RequestHandler):
    def initialize(self, api):
        self._api = api

    def get_template_path(self):
        return pkg_resources.resource_filename(__name__, 'templates')

class RecentRunHandler(BaseTaskHistoryHandler):
    def get(self):
        tasks = self._api.task_history.find_latest_runs()
        self.render("recent.html", tasks=tasks)

class ByNameHandler(BaseTaskHistoryHandler):
    def get(self, name):
        tasks = self._api.task_history.find_all_by_name(name)
        self.render("recent.html", tasks=tasks)

class ByIdHandler(BaseTaskHistoryHandler):
    def get(self, id):
        task = self._api.task_history.find_task_by_id(id)
        self.render("show.html", task=task)

class ByParamsHandler(BaseTaskHistoryHandler):
    def get(self, name):
        payload = self.get_argument('data', default="{}")
        arguments = json.loads(payload)
        tasks = self._api.task_history.find_all_by_parameters(name, session=None, **arguments)
        self.render("recent.html", tasks=tasks)

class StaticFileHandler(tornado.web.RequestHandler):
    def get(self, path):
        # Path checking taken from Flask's safe_join function:
        # https://github.com/mitsuhiko/flask/blob/1d55b8983/flask/helpers.py#L563-L587
        path = posixpath.normpath(path)
        if os.path.isabs(path) or path.startswith(".."):
            return self.send_error(404)

        extension = os.path.splitext(path)[1]
        if extension in mimetypes.types_map:
            self.set_header("Content-Type", mimetypes.types_map[extension])
        data = pkg_resources.resource_string(__name__, os.path.join("static", path))
        self.write(data)

class RootPathHandler(tornado.web.RequestHandler):
    def get(self):
        self.redirect("/static/visualiser/index.html")

def app(api):
    handlers = [
        (r'/api/(.*)', RPCHandler, {"api": api}),
        (r'/static/(.*)', StaticFileHandler),
        (r'/', RootPathHandler),
        (r'/history', RecentRunHandler, {'api': api}),
        (r'/history/by_name/(.*?)', ByNameHandler, {'api': api}),
        (r'/history/by_id/(.*?)', ByIdHandler, {'api': api}),
        (r'/history/by_params/(.*?)', ByParamsHandler, {'api': api})
    ]
    api_app = tornado.web.Application(handlers)
    return api_app

def _init_api(sched, responder, api_port, address):
    api = responder or RemoteSchedulerResponder(sched)
    api_app = app(api)
    api_sockets = tornado.netutil.bind_sockets(api_port, address=address)
    try:
        # TODO: move the max_header_size argument once we've moved over to POST
        server = tornado.httpserver.HTTPServer(api_app, max_header_size=2097152)
    except TypeError:
        warnings.warn('could not create HTTPServer, trying again without the max_header_size argument (are you running an older version of Tornado?)')
        server = tornado.httpserver.HTTPServer(api_app)

    server.add_sockets(api_sockets)

    # Return the bound socket names.  Useful for connecting client in test scenarios.
    return [s.getsockname() for s in api_sockets]

def run(api_port=8082, address=None, scheduler=None, responder=None):
    """ Runs one instance of the API server """
    sched = scheduler or _create_scheduler()
    # load scheduler state
    sched.load()

    _init_api(sched, responder, api_port, address)

    # prune work DAG every 60 seconds
    pruner = tornado.ioloop.PeriodicCallback(sched.prune, 60000)
    pruner.start()

    def shutdown_handler(foo=None, bar=None):
        logger.info("Scheduler instance shutting down")
        sched.dump()
        os._exit(0)

    signal.signal(signal.SIGINT, shutdown_handler)
    signal.signal(signal.SIGTERM, shutdown_handler)
    signal.signal(signal.SIGQUIT, shutdown_handler)
    atexit.register(shutdown_handler)

    logger.info("Scheduler starting up")

    tornado.ioloop.IOLoop.instance().start()

def run_api_threaded(api_port=8082, address=None):
    ''' For integration tests'''
    sock_names = _init_api(_create_scheduler(), None, api_port, address)

    import threading
    def scheduler_thread():
        # this is wrapped in a function so we get the instance
        # from the scheduler thread and not from the main thread
        tornado.ioloop.IOLoop.instance().start()

    threading.Thread(target=scheduler_thread).start()
    return sock_names

def stop():
    tornado.ioloop.IOLoop.instance().stop()

if __name__ == "__main__":
    run()
<|endoftext|>


<commit_before>'''
Created on Dec 26, 2012

__author__ = "Elizabeth 'pidge' Flanagan"
__copyright__ = "Copyright 2013, Intel Corp."
__credits__ = ["Elizabeth Flanagan"]
__license__ = "GPL"
__version__ = "2.0"
__maintainer__ = "Elizabeth Flanagan"
__email__ = "elizabeth.flanagan@intel.com"
'''

from buildbot.steps.shell import ShellCommand
from twisted.python import log
from autobuilder.config import *
import os

class CreateAutoConf(ShellCommand):
    haltOnFailure = False
    flunkOnFailure = True
    name = "Create Auto Configuration"
    def __init__(self, factory, argdict=None, **kwargs):
        self.machine=""
        self.buildapp=""
        self.distro="poky"
        self.buildhistory=False
        self.gplv3=True
        self.multilib=False
        self.swabber=False
        self.x32=False
        self.atext=""
        self.emgd=False
        self.pvr=False
        self.atextprepend=""
        self.atextappend=""
        self.SDKMACHINE="i686"
        self.adtdev=False
        self.factory = factory
        self.buildappsrcrev = "${AUTOREV}"
        self.slaveworkdir=os.path.join(os.path.join(YOCTO_ABBASE, "yocto-slave"))

        for k, v in argdict.iteritems():
            if type(v) is bool:
                setattr(self, k, str(v))
            else:
                setattr(self, k, v)
        self.description = "Create Auto Configuration"
        ShellCommand.__init__(self, **kwargs)

    def start(self):
        buildername=self.getProperty("buildername")
        try:
            distroversion=self.getProperty("distroversion")
        except:
            pass
        fout = ""
        #check to see if we have a prepend

        if self.atextprepend is not "":
            fout = fout + self.atextprepend
        #check to see if we have text override
        if self.atext is not "":
            fout = fout+self.atext
        else:
            fout = 'MACHINE = "' + self.machine + '"\n'
            fout = 'PACKAGE_CLASSES = "package_rpm package_deb package_ipk"\n'
            fout = fout + 'BB_NUMBER_THREADS = "' + os.environ.get('BB_NUMBER_THREADS') + '"\n'
            fout = fout + 'PARALLEL_MAKE = "-j ' + os.environ.get('PARALLEL_MAKE') + '"\n'
            fout = fout + 'SDKMACHINE ?= "' + self.SDKMACHINE + '"\n'
            buildprops = self.getProperties().asDict()
            for k, v in buildprops.iteritems():
                if "buildappsrcrev" in k:
                    self.buildappsrcrev=v[0]
            if self.buildapp == "True":
                if self.buildappsrcrev == "AUTOREV":
                    self.buildappsrcrev="${AUTOREV}"
                fout = fout + 'DL_DIR ?= "${TOPDIR}/downloads"\n'
                fout = fout + 'INHERIT += "own-mirrors"\n'
                fout = fout + 'SOURCE_MIRROR_URL = "file:///' + os.environ.get('SOURCE_DL_DIR') +'"\n'
                if self.buildappsrcrev != "DEFAULT":
                    fout = fout + 'SRCREV_pn-build-appliance-image = "' + self.buildappsrcrev + '"\n'
            else:
                fout = fout + 'DL_DIR = "' + os.environ.get('SOURCE_DL_DIR')+'"\n'
            if self.emgd == "True":
                fout = fout + 'LICENSE_FLAGS_WHITELIST = "license_emgd-driver-bin" \n'
            if self.pvr == "True":
                fout = fout + 'LICENSE_FLAGS_WHITELIST += "license_cdv-pvr-driver" \n'
                fout = fout + 'PVR_LICENSE = "yes" \n'
            if self.multilib == "True":
                fout = fout + 'require conf/multilib.conf \n'
                fout = fout + 'MULTILIBS = "multilib:lib32" \n'
                fout = fout + 'DEFAULTTUNE_virtclass-multilib-lib32 = "x86" \n'
                fout = fout + 'SSTATE_DIR ?= "' + os.environ.get("SSTATE_PUBLISH_DIR") + '/multilib" \n'
            else:
                fout = fout + 'SSTATE_DIR ?= "' + os.environ.get("SSTATE_PUBLISH_DIR") + '/" \n'
            try:
                if self.gplv3 == "False":
                    fout = fout + 'INCOMPATIBLE_LICENSE = "GPLv3" \n'
            except:
                pass
            if self.x32 == "True":
                    fout = fout + 'DEFAULTTUNE = "x86-64-x32"'
                    fout = fout + 'baselib = "${@d.getVar(\'BASE_LIB_tune-\' + (d.getVar(\'DEFAULTTUNE\', True) or \'INVALID\'), True) or \'lib\'} \n"
            if self.distro == "poky-rt":
                fout = fout + 'PREFERRED_PROVIDER_virtual/kernel="linux-yocto-rt" \n'
            fout = fout + 'MACHINE = "' + self.machine + '"\n'
            fout = fout + 'PREMIRRORS = ""\n'
            try:
                if self.swabber == "True":
                    fout = fout + 'USER_CLASSES += "image-prelink image-swab"\n'
            except:
                pass
            if os.environ.get("PUBLISH_SOURCE_MIRROR") == "True":
                fout = fout + 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
            try:
                if self.buildhistory == True and os.environ.get("BUILD_HISTORY_COLLECT") == "True" and self.getProperty("branch") == "master":
                    fout = fout + 'INHERIT += "buildhistory"\n'
                    fout = fout + 'BUILDHISTORY_COMMIT = "1"\n'
                    fout = fout + 'BUILDHISTORY_DIR = "' + os.environ.get('BUILD_HISTORY_DIR') + '/' + self.getProperty("buildername") + '/poky-buildhistory"\n'
                    fout = fout + 'BUILDHISTORY_PUSH_REPO = "' + os.environ.get('BUILD_HISTORY_REPO') + ' ' + self.getProperty("buildername") + ':' + self.getProperty("buildername") + '"\n'
            except:
                pass
            try:
                revision=self.getProperty("got_revision_poky")
            except:
                pass
            if revision and self.getProperty("distroversion"):
                if self.adtdev == "True":
                    adtrepo_url=os.environ.get("ADTREPO_DEV_URL")
                    fout=fout+'ADTREPO = "' + adtrepo_url + '/' + self.getProperty("distroversion") + '-' + revision + '/ \n'
        if self.atextappend is not "":
            fout = fout + self.atextappend
        self.command = "rm -rf ./build/conf/auto.conf; echo '" +  fout + "'>> ./build/conf/auto.conf"
        ShellCommand.start(self)

    def describe(self, done=False):
        description = ShellCommand.describe(self,done)
        return description

<commit_msg>CreateAutoConf.py: missing terminating quote<commit_after>'''
Created on Dec 26, 2012

__author__ = "Elizabeth 'pidge' Flanagan"
__copyright__ = "Copyright 2013, Intel Corp."
__credits__ = ["Elizabeth Flanagan"]
__license__ = "GPL"
__version__ = "2.0"
__maintainer__ = "Elizabeth Flanagan"
__email__ = "elizabeth.flanagan@intel.com"
'''

from buildbot.steps.shell import ShellCommand
from twisted.python import log
from autobuilder.config import *
import os

class CreateAutoConf(ShellCommand):
    haltOnFailure = False
    flunkOnFailure = True
    name = "Create Auto Configuration"
    def __init__(self, factory, argdict=None, **kwargs):
        self.machine=""
        self.buildapp=""
        self.distro="poky"
        self.buildhistory=False
        self.gplv3=True
        self.multilib=False
        self.swabber=False
        self.x32=False
        self.atext=""
        self.emgd=False
        self.pvr=False
        self.atextprepend=""
        self.atextappend=""
        self.SDKMACHINE="i686"
        self.adtdev=False
        self.factory = factory
        self.buildappsrcrev = "${AUTOREV}"
        self.slaveworkdir=os.path.join(os.path.join(YOCTO_ABBASE, "yocto-slave"))

        for k, v in argdict.iteritems():
            if type(v) is bool:
                setattr(self, k, str(v))
            else:
                setattr(self, k, v)
        self.description = "Create Auto Configuration"
        ShellCommand.__init__(self, **kwargs)

    def start(self):
        buildername=self.getProperty("buildername")
        try:
            distroversion=self.getProperty("distroversion")
        except:
            pass
        fout = ""
        #check to see if we have a prepend

        if self.atextprepend is not "":
            fout = fout + self.atextprepend
        #check to see if we have text override
        if self.atext is not "":
            fout = fout+self.atext
        else:
            fout = 'MACHINE = "' + self.machine + '"\n'
            fout = 'PACKAGE_CLASSES = "package_rpm package_deb package_ipk"\n'
            fout = fout + 'BB_NUMBER_THREADS = "' + os.environ.get('BB_NUMBER_THREADS') + '"\n'
            fout = fout + 'PARALLEL_MAKE = "-j ' + os.environ.get('PARALLEL_MAKE') + '"\n'
            fout = fout + 'SDKMACHINE ?= "' + self.SDKMACHINE + '"\n'
            buildprops = self.getProperties().asDict()
            for k, v in buildprops.iteritems():
                if "buildappsrcrev" in k:
                    self.buildappsrcrev=v[0]
            if self.buildapp == "True":
                if self.buildappsrcrev == "AUTOREV":
                    self.buildappsrcrev="${AUTOREV}"
                fout = fout + 'DL_DIR ?= "${TOPDIR}/downloads"\n'
                fout = fout + 'INHERIT += "own-mirrors"\n'
                fout = fout + 'SOURCE_MIRROR_URL = "file:///' + os.environ.get('SOURCE_DL_DIR') +'"\n'
                if self.buildappsrcrev != "DEFAULT":
                    fout = fout + 'SRCREV_pn-build-appliance-image = "' + self.buildappsrcrev + '"\n'
            else:
                fout = fout + 'DL_DIR = "' + os.environ.get('SOURCE_DL_DIR')+'"\n'
            if self.emgd == "True":
                fout = fout + 'LICENSE_FLAGS_WHITELIST = "license_emgd-driver-bin" \n'
            if self.pvr == "True":
                fout = fout + 'LICENSE_FLAGS_WHITELIST += "license_cdv-pvr-driver" \n'
                fout = fout + 'PVR_LICENSE = "yes" \n'
            if self.multilib == "True":
                fout = fout + 'require conf/multilib.conf \n'
                fout = fout + 'MULTILIBS = "multilib:lib32" \n'
                fout = fout + 'DEFAULTTUNE_virtclass-multilib-lib32 = "x86" \n'
                fout = fout + 'SSTATE_DIR ?= "' + os.environ.get("SSTATE_PUBLISH_DIR") + '/multilib" \n'
            else:
                fout = fout + 'SSTATE_DIR ?= "' + os.environ.get("SSTATE_PUBLISH_DIR") + '/" \n'
            try:
                if self.gplv3 == "False":
                    fout = fout + 'INCOMPATIBLE_LICENSE = "GPLv3" \n'
            except:
                pass
            if self.x32 == "True":
                    fout = fout + 'DEFAULTTUNE = "x86-64-x32"'
                    fout = fout + 'baselib = "${@d.getVar(\'BASE_LIB_tune-\' + (d.getVar(\'DEFAULTTUNE\', True) or \'INVALID\'), True) or \'lib\'}" \n'
            if self.distro == "poky-rt":
                fout = fout + 'PREFERRED_PROVIDER_virtual/kernel="linux-yocto-rt" \n'
            fout = fout + 'MACHINE = "' + self.machine + '"\n'
            fout = fout + 'PREMIRRORS = ""\n'
            try:
                if self.swabber == "True":
                    fout = fout + 'USER_CLASSES += "image-prelink image-swab"\n'
            except:
                pass
            if os.environ.get("PUBLISH_SOURCE_MIRROR") == "True":
                fout = fout + 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
            try:
                if self.buildhistory == True and os.environ.get("BUILD_HISTORY_COLLECT") == "True" and self.getProperty("branch") == "master":
                    fout = fout + 'INHERIT += "buildhistory"\n'
                    fout = fout + 'BUILDHISTORY_COMMIT = "1"\n'
                    fout = fout + 'BUILDHISTORY_DIR = "' + os.environ.get('BUILD_HISTORY_DIR') + '/' + self.getProperty("buildername") + '/poky-buildhistory"\n'
                    fout = fout + 'BUILDHISTORY_PUSH_REPO = "' + os.environ.get('BUILD_HISTORY_REPO') + ' ' + self.getProperty("buildername") + ':' + self.getProperty("buildername") + '"\n'
            except:
                pass
            try:
                revision=self.getProperty("got_revision_poky")
            except:
                pass
            if revision and self.getProperty("distroversion"):
                if self.adtdev == "True":
                    adtrepo_url=os.environ.get("ADTREPO_DEV_URL")
                    fout=fout+'ADTREPO = "' + adtrepo_url + '/' + self.getProperty("distroversion") + '-' + revision + '/ \n'
        if self.atextappend is not "":
            fout = fout + self.atextappend
        self.command = "rm -rf ./build/conf/auto.conf; echo '" +  fout + "'>> ./build/conf/auto.conf"
        ShellCommand.start(self)

    def describe(self, done=False):
        description = ShellCommand.describe(self,done)
        return description

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import json
from psycopg2 import connect
from vocs import TermVocabulary, DocVocabulary
from msg import Message
from pymystem3 import Mystem

def show_progress(message, current, total):
    print "\r%s: %.2f%% [%d/%d]"%(message, float(current)*100/total, current,
        total),
    # Going to the next line in case of Finish
    if (current == total):
        print ""

if len(sys.argv) < 4:
    print "usage: indexer.py <database> <out> <tablelist>"
    exit(0)

config = {'database':sys.argv[1], 'out':sys.argv[2], 'tables':sys.argv[3:]}

# Init configuration files
with open("conn.conf", "r") as f:
    conn_config = json.load(f, encoding='utf-8')
with open("indexer.conf", "r") as f:
    indexer_config = json.load(f, encoding='utf-8')

# Connect to a database
settings = """dbname=%s user=%s password=%s host=%s"""%(config['database'],
    conn_config["user"], conn_config["password"], conn_config["host"])
connection = connect(settings)
cursor = connection.cursor()

# Add all possible words terms
term_voc = TermVocabulary()
mystem = Mystem(entire_input=False)
for table in config['tables']:
    cursor.execute("""SELECT text FROM %s;"""%(table))
    row = cursor.fetchone()
    current_row = 0
    rowcount = cursor.rowcount
    while (row is not None):
        message = Message(text=row[0], mystem=mystem, configpath="msg.conf")
        message.process()
        terms = message.get_terms()
        for t in terms:
            term_voc.insert_term(t)
        row = cursor.fetchone()
        current_row += 1
        show_progress("Extracting terms from messages of '%s' table"%(table),
            current_row, rowcount)

# Add all possible features
print "features: %s"%(indexer_config['feature_names'])
for feature_name in indexer_config['feature_names']:
    term_voc.insert_term(feature_name)

# Show and save result
term_voc.top(50)
term_voc.save(config['out'])
<commit_msg>Убран вывод top(50).<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import json
from psycopg2 import connect
from vocs import TermVocabulary, DocVocabulary
from msg import Message
from pymystem3 import Mystem

def show_progress(message, current, total):
    print "\r%s: %.2f%% [%d/%d]"%(message, float(current)*100/total, current,
        total),
    # Going to the next line in case of Finish
    if (current == total):
        print ""

if len(sys.argv) < 4:
    print "usage: indexer.py <database> <out> <tablelist>"
    exit(0)

config = {'database':sys.argv[1], 'out':sys.argv[2], 'tables':sys.argv[3:]}

# Init configuration files
with open("conn.conf", "r") as f:
    conn_config = json.load(f, encoding='utf-8')
with open("indexer.conf", "r") as f:
    indexer_config = json.load(f, encoding='utf-8')

# Connect to a database
settings = """dbname=%s user=%s password=%s host=%s"""%(config['database'],
    conn_config["user"], conn_config["password"], conn_config["host"])
connection = connect(settings)
cursor = connection.cursor()

# Add all possible words terms
term_voc = TermVocabulary()
mystem = Mystem(entire_input=False)
for table in config['tables']:
    cursor.execute("""SELECT text FROM %s;"""%(table))
    row = cursor.fetchone()
    current_row = 0
    rowcount = cursor.rowcount
    while (row is not None):
        message = Message(text=row[0], mystem=mystem, configpath="msg.conf")
        message.process()
        terms = message.get_terms()
        for t in terms:
            term_voc.insert_term(t)
        row = cursor.fetchone()
        current_row += 1
        show_progress("Extracting terms from messages of '%s' table"%(table),
            current_row, rowcount)

# Add all possible features
print "features: %s"%(indexer_config['feature_names'])
for feature_name in indexer_config['feature_names']:
    term_voc.insert_term(feature_name)

# Show and save result
# term_voc.top(50)
term_voc.save(config['out'])
<|endoftext|>


<commit_before># Copyright 2014 DreamHost, LLC
#
# Author: DreamHost, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from unittest2 import TestCase

import mock

from akanda.router.drivers import hostname, ip

CONFIG = mock.Mock()
CONFIG.hostname = 'akanda'

class HostnameTestCase(TestCase):
    """
    """
    def setUp(self):
        self.mock_execute = mock.patch('akanda.router.utils.execute').start()
        self.mock_replace_file = mock.patch(
            'akanda.router.utils.replace_file'
        ).start()
        self.addCleanup(mock.patch.stopall)

        self.mgr = hostname.HostnameManager()

    def test_update_hostname(self):
        self.mgr.update_hostname(CONFIG)
        self.mock_execute.assert_has_calls([
            mock.call(['/bin/hostname', 'akanda'], 'sudo'),
            mock.call(['mv', '/tmp/hostname', '/etc/hostname'], 'sudo')
        ])

    @mock.patch.object(ip.IPManager, 'get_management_address')
    def test_update_hosts(self, addr):
        expected = [
            '127.0.0.1  localhost',
            '::1  localhost ip6-localhost ip6-loopback',
            'fdca:3ba5:a17a:acda:f816:3eff:fe66:33b6  akanda'
        ]
        addr.return_value = 'fdca:3ba5:a17a:acda:f816:3eff:fe66:33b6'
        self.mgr.update_hosts(CONFIG)
        self.mock_execute.assert_has_calls([
            mock.call(['mv', '/tmp/hosts', '/etc/hosts'], 'sudo')
        ])
        self.mock_replace_file.assert_has_calls(mock.call(
            '/tmp/hosts',
            '\n'.join(expected))
        )
<commit_msg>Fix a failing unit test for the hostname driver.<commit_after># Copyright 2014 DreamHost, LLC
#
# Author: DreamHost, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from unittest2 import TestCase

import mock

from akanda.router.drivers import hostname, ip

CONFIG = mock.Mock()
CONFIG.hostname = 'akanda'

class HostnameTestCase(TestCase):
    """
    """
    def setUp(self):
        self.mock_execute = mock.patch('akanda.router.utils.execute').start()
        self.mock_replace_file = mock.patch(
            'akanda.router.utils.replace_file'
        ).start()
        self.addCleanup(mock.patch.stopall)

        self.mgr = hostname.HostnameManager()

    def test_update_hostname(self):
        self.mgr.update_hostname(CONFIG)
        self.mock_execute.assert_has_calls([
            mock.call(['/bin/hostname', 'akanda'], 'sudo'),
            mock.call(['mv', '/tmp/hostname', '/etc/hostname'], 'sudo')
        ])

    @mock.patch.object(ip.IPManager, 'get_management_address')
    def test_update_hosts(self, addr):
        expected = [
            '127.0.0.1  localhost',
            '::1  localhost ip6-localhost ip6-loopback',
            'fdca:3ba5:a17a:acda:f816:3eff:fe66:33b6  akanda'
        ]
        addr.return_value = 'fdca:3ba5:a17a:acda:f816:3eff:fe66:33b6'
        self.mgr.update_hosts(CONFIG)
        self.mock_execute.assert_has_calls([
            mock.call(['mv', '/tmp/hosts', '/etc/hosts'], 'sudo')
        ])
        self.mock_replace_file.assert_has_calls([mock.call(
            '/tmp/hosts',
            '\n'.join(expected))
        ])
<|endoftext|>


<commit_before># Copyright 2008-2011 WebDriver committers
# Copyright 2008-2011 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import platform
import logging
from subprocess import Popen, PIPE
from extension_connection import ExtensionConnection
from selenium.common.exceptions import WebDriverException
import time
import socket
import signal

class FirefoxBinary(object):

    NO_FOCUS_LIBRARY_NAME = "x_ignore_nofocus.so"

    def __init__(self, firefox_path=None):
        self._start_cmd = firefox_path
        if self._start_cmd is None:
            self._start_cmd = self._get_firefox_start_cmd()

    def launch_browser(self, profile):
        """Launches the browser for the given profile name.
        It is assumed the profile already exists.
        """
        self.profile = profile

        self._start_from_profile_path(self.profile.path)
        self._wait_until_connectable()

    def kill(self):
        """Kill the browser.

        This is useful when the browser is stuck.
        """
        try:
            if self.process:
                os.kill(self.process.pid, signal.SIGTERM)
                os.wait()
        except AttributeError:
            # kill may not be available under windows environment
            pass

    def _start_from_profile_path(self, path):
        os.environ["XRE_PROFILE_PATH"] = path
        os.environ["MOZ_CRASHREPORTER_DISABLE"] = "1"
        os.environ["MOZ_NO_REMOTE"] = "1"
        os.environ["NO_EM_RESTART"] = "1"
        Popen([self._start_cmd, "-silent"], stdout=PIPE, stderr=PIPE).wait()
        self.process = Popen([self._start_cmd], stdout=PIPE, stderr=PIPE)

    def is_connectable(self):
        """Trys to connect to the extension but do not retrieve context."""
        try:
            socket_ = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            socket_.settimeout(1)
            socket_.connect(("127.0.0.1", self.profile.port))
            socket_.close()
            return True
        except socket.error:
            return False

    def _wait_until_connectable(self):
        """Blocks until the extension is connectable in the firefox."""
        count = 0
        while not self.is_connectable():
            #LOGGER.debug("Waiting for browser to launch...")
            if self.process.returncode:
                # Browser has exited
                return False
            if count == 30:
                raise WebDriverException("Can't load the profile. Profile Dir : %s" % self.profile.path)
            count += 1
            time.sleep(1)
        return True

    def _find_exe_in_registry(self):
        from _winreg import OpenKey, QueryValue, HKEY_LOCAL_MACHINE
        import shlex
        keys = (
           r"SOFTWARE\Classes\FirefoxHTML\shell\open\command",
           r"SOFTWARE\Classes\Applications\firefox.exe\shell\open\command"
        )
        command = ""
        for path in keys:
            try:
                key = OpenKey(HKEY_LOCAL_MACHINE, path)
                command = QueryValue(key, "")
                break
            except WindowsError:
                pass
        else:
            return ""

        return shlex.split(command)[0]

    def _get_firefox_start_cmd(self):
        """Return the command to start firefox."""
        start_cmd = ""
        if platform.system() == "Darwin":
            start_cmd = ("/Applications/Firefox.app/Contents/MacOS/firefox-bin")
        elif platform.system() == "Windows":
            start_cmd = self._find_exe_in_registry() or self._default_windows_location()
        else:
            # Maybe iceweasel (Debian) is another candidate...
            for ffname in ["firefox2", "firefox", "firefox-3.0", "firefox-4.0"]:
                start_cmd = self.which(ffname)
                if start_cmd is not None:
                    break
        return start_cmd

    def _default_windows_location(self):
        program_files = os.getenv("PROGRAMFILES", r"\Program Files")
        return os.path.join(program_files, "Mozilla Firefox\\firefox.exe")

    def which(self, fname):
        """Returns the fully qualified path by searching Path of the given name"""
        for pe in os.environ['PATH'].split(os.pathsep):
            checkname = os.path.join(pe, fname)
            if os.access(checkname, os.X_OK):
                return checkname
        return None
<commit_msg>DavidBurns, on behalf of AndiAlbrecht, fixing where which finds directories. Fixes issue 1663<commit_after># Copyright 2008-2011 WebDriver committers
# Copyright 2008-2011 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import platform
import logging
from subprocess import Popen, PIPE
from extension_connection import ExtensionConnection
from selenium.common.exceptions import WebDriverException
import time
import socket
import signal

class FirefoxBinary(object):

    NO_FOCUS_LIBRARY_NAME = "x_ignore_nofocus.so"

    def __init__(self, firefox_path=None):
        self._start_cmd = firefox_path
        if self._start_cmd is None:
            self._start_cmd = self._get_firefox_start_cmd()

    def launch_browser(self, profile):
        """Launches the browser for the given profile name.
        It is assumed the profile already exists.
        """
        self.profile = profile

        self._start_from_profile_path(self.profile.path)
        self._wait_until_connectable()

    def kill(self):
        """Kill the browser.

        This is useful when the browser is stuck.
        """
        try:
            if self.process:
                os.kill(self.process.pid, signal.SIGTERM)
                os.wait()
        except AttributeError:
            # kill may not be available under windows environment
            pass

    def _start_from_profile_path(self, path):
        os.environ["XRE_PROFILE_PATH"] = path
        os.environ["MOZ_CRASHREPORTER_DISABLE"] = "1"
        os.environ["MOZ_NO_REMOTE"] = "1"
        os.environ["NO_EM_RESTART"] = "1"
        Popen([self._start_cmd, "-silent"], stdout=PIPE, stderr=PIPE).wait()
        self.process = Popen([self._start_cmd], stdout=PIPE, stderr=PIPE)

    def is_connectable(self):
        """Trys to connect to the extension but do not retrieve context."""
        try:
            socket_ = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            socket_.settimeout(1)
            socket_.connect(("127.0.0.1", self.profile.port))
            socket_.close()
            return True
        except socket.error:
            return False

    def _wait_until_connectable(self):
        """Blocks until the extension is connectable in the firefox."""
        count = 0
        while not self.is_connectable():
            #LOGGER.debug("Waiting for browser to launch...")
            if self.process.returncode:
                # Browser has exited
                return False
            if count == 30:
                raise WebDriverException("Can't load the profile. Profile Dir : %s" % self.profile.path)
            count += 1
            time.sleep(1)
        return True

    def _find_exe_in_registry(self):
        from _winreg import OpenKey, QueryValue, HKEY_LOCAL_MACHINE
        import shlex
        keys = (
           r"SOFTWARE\Classes\FirefoxHTML\shell\open\command",
           r"SOFTWARE\Classes\Applications\firefox.exe\shell\open\command"
        )
        command = ""
        for path in keys:
            try:
                key = OpenKey(HKEY_LOCAL_MACHINE, path)
                command = QueryValue(key, "")
                break
            except WindowsError:
                pass
        else:
            return ""

        return shlex.split(command)[0]

    def _get_firefox_start_cmd(self):
        """Return the command to start firefox."""
        start_cmd = ""
        if platform.system() == "Darwin":
            start_cmd = ("/Applications/Firefox.app/Contents/MacOS/firefox-bin")
        elif platform.system() == "Windows":
            start_cmd = self._find_exe_in_registry() or self._default_windows_location()
        else:
            # Maybe iceweasel (Debian) is another candidate...
            for ffname in ["firefox2", "firefox", "firefox-3.0", "firefox-4.0"]:
                start_cmd = self.which(ffname)
                if start_cmd is not None:
                    break
        return start_cmd

    def _default_windows_location(self):
        program_files = os.getenv("PROGRAMFILES", r"\Program Files")
        return os.path.join(program_files, "Mozilla Firefox\\firefox.exe")

    def which(self, fname):
        """Returns the fully qualified path by searching Path of the given name"""
        for pe in os.environ['PATH'].split(os.pathsep):
            checkname = os.path.join(pe, fname)
            if os.access(checkname, os.X_OK) and not os.path.isdir(checkname):
                return checkname
        return None
<|endoftext|>