<commit_before>"""
gallerize.main
~~~~~~~~~~~~~~

:Copyright: 2007-2021 Jochen Kupperschmidt
:License: MIT, see LICENSE for details.
"""

from __future__ import annotations
import codecs
from collections import defaultdict
from dataclasses import dataclass
from itertools import islice
import os
import shutil
import subprocess
import sys
from typing import (
    Any,
    Dict,
    Iterable,
    Iterator,
    List,
    Optional,
    Sequence,
    Tuple,
)

from jinja2 import Environment, PackageLoader

IMAGE_CAPTION_EXTENSION: str = '.txt'
TEMPLATE_EXTENSION: str = '.html'
OUTPUT_HTML_EXTENSION: str = '.html'

PATH: str = os.path.dirname(os.path.abspath(__file__))
PATH_STATIC: str = os.path.join(PATH, 'static')
TEMPLATE_ENVIRONMENT: Environment = Environment(
    autoescape=True,
    loader=PackageLoader(__package__, 'templates'),
    trim_blocks=False,
)

@dataclass(frozen=True)
class Dimension:
    width: int
    height: int

def debug(message: str, *args: Any) -> None:
    print(message.format(*args))

def resize_image(
    src_filename: str, dst_filename: str, max_dimension: Dimension
) -> None:
    """Create a resized (and antialiased) version of an image."""
    dimension_str = f'{max_dimension.width:d}x{max_dimension.height:d}'
    cmd = ['convert', '-resize', dimension_str, src_filename, dst_filename]
    subprocess.check_call(cmd)

def render_html_to_file(
    template_name: str, context: Dict[str, Any], path: str, page_name: str
) -> None:
    """Render the template and write the result to the given file."""
    context['url_for_page'] = lambda page: page + OUTPUT_HTML_EXTENSION
    html = render_template(template_name + TEMPLATE_EXTENSION, **context)

    filename = os.path.join(path, page_name + OUTPUT_HTML_EXTENSION)
    write_file(filename, html)

def render_template(template_filename: str, **context: Dict[str, Any]) -> str:
    return TEMPLATE_ENVIRONMENT.get_template(template_filename).render(context)

def write_file(filename: str, content: str) -> None:
    with codecs.open(filename, 'w', 'utf-8') as f:
        debug('Writing "{}" ...', filename)
        f.write(content)

def create_gallery(
    title: str,
    destination_path: str,
    resize: bool,
    max_image_size: Dimension,
    max_thumbnail_size: Dimension,
    full_image_filenames: List[str],
) -> Gallery:
    gallery = Gallery(
        title=title,
        destination_path=destination_path,
        resize=resize,
        max_image_size=max_image_size,
        max_thumbnail_size=max_thumbnail_size,
    )

    gallery.images = [
        Image(gallery, image) for image in sorted(full_image_filenames)
    ]
    gallery.link_images()

    return gallery

class Gallery:
    def __init__(
        self,
        title: str,
        destination_path: str,
        resize: bool,
        max_image_size: Dimension,
        max_thumbnail_size: Dimension,
    ) -> None:
        self.images: List[Image] = []
        self.title = title
        self.destination_path = destination_path
        self.resize = resize
        self.max_image_size = max_image_size
        self.max_thumbnail_size = max_thumbnail_size

    def link_images(self) -> None:
        """Assign the predecessor and successor for every image."""
        for previous_image, image, next_image in window(self.images):
            image.previous_image = previous_image
            image.next_image = next_image

    def generate(self) -> None:
        # Create destination path if it doesn't exist.
        if not os.path.exists(self.destination_path):
            debug(
                'Destination path "{}" does not exist, creating it.',
                self.destination_path,
            )
            os.mkdir(self.destination_path)

        self.generate_images()
        for image in self.images:
            image.load_caption()
        self.render_html_pages()
        self.copy_additional_static_files()
        debug('Done.')

    def generate_images(self) -> None:
        for image in self.images:
            image.generate_image()
            image.generate_thumbnail()
            image.load_caption()

    def render_html_pages(self) -> None:
        for image in self.images:
            image.render_html_page()
        self.render_html_index_page()

    def render_html_index_page(self) -> None:
        """Create an HTML document of thumbnails that link to single image
        documents.
        """
        context = {
            'gallery': self,
        }
        render_html_to_file('index', context, self.destination_path, 'index')

    def copy_additional_static_files(self) -> None:
        if not os.path.exists(PATH_STATIC):
            debug(
                'Path "{}", does not exist; not copying any static files.',
                PATH_STATIC,
            )
            return

        filenames = list(sorted(os.listdir(PATH_STATIC)))
        if not filenames:
            debug('No static files to copy.')
        for filename in filenames:
            debug('Copying static file "{}" ...', filename)
            shutil.copy(
                os.path.join(PATH_STATIC, filename),
                os.path.join(self.destination_path, filename),
            )

def window(
    images: Iterable[Image],
) -> Iterator[Tuple[Optional[Image], Image, Optional[Image]]]:
    """Return a sliding window over the images.

    Returns a triple. Its first and third element can be `None`, but the
    second element never is.
    """
    n = 3
    padded_list: List[Optional[Image]] = list(images)
    padded_list.insert(0, None)
    padded_list.append(None)

    it = iter(padded_list)
    result = tuple(islice(it, n))

    if len(result) == n:
        yield result  # type: ignore

    for elem in it:
        result = result[1:] + (elem,)
        yield result  # type: ignore

class Image:
    def __init__(self, gallery: Gallery, full_filename: str) -> None:
        self.gallery = gallery
        self.full_filename = full_filename
        self.path, self.filename = os.path.split(full_filename)
        self.basename, self.extension = os.path.splitext(self.filename)
        self.thumbnail_filename = f'{self.basename}_t{self.extension}'
        self.page_name = self.basename
        self.previous_image: Optional[Image] = None
        self.next_image: Optional[Image] = None

    def generate_image(self) -> None:
        """Create a (optionally resized) copy of an image."""
        destination_filename = os.path.join(
            self.gallery.destination_path, self.filename
        )
        if self.gallery.resize:
            # Resize image.
            debug('Resizing image "{}" ...', self.full_filename)
            resize_image(
                self.full_filename,
                destination_filename,
                self.gallery.max_image_size,
            )
        else:
            # Copy image.
            debug('Copying image "{}" ...', self.full_filename)
            shutil.copy(self.full_filename, destination_filename)

    def generate_thumbnail(self) -> None:
        """Create a preview of an image."""
        debug('Creating thumbnail "{}" ...', self.thumbnail_filename)
        destination_filename = os.path.join(
            self.gallery.destination_path, self.thumbnail_filename
        )
        resize_image(
            self.full_filename,
            destination_filename,
            self.gallery.max_thumbnail_size,
        )

    def load_caption(self) -> None:
        """Load image caption from file."""
        caption_filename = os.path.join(
            self.path, self.filename + IMAGE_CAPTION_EXTENSION
        )
        self.caption = self._read_first_line(caption_filename)

    def _read_first_line(self, filename: str) -> Optional[str]:
        """Read the first line from the specified file."""
        try:
            with codecs.open(filename, 'rb', 'utf-8') as f:
                return next(f).strip()
        except IOError:
            # File does not exist (OK) or cannot be read (not really OK).
            return None

    def render_html_page(self) -> None:
        """Create an HTML document for a single image."""
        context = {
            'image': self,
        }
        render_html_to_file(
            'view', context, self.gallery.destination_path, self.page_name
        )

# -------------------------------------------------------------------- #

def handle_duplicate_filenames(paths) -> None:
    duplicates = find_duplicate_filenames(paths)
    if not duplicates:
        return

    print('Found duplicate filenames:')
    for filename, paths in duplicates:
        print(f'  + "{filename}" appears in the following paths:')
        for path in paths:
            print(f'    - {path}')

    sys.exit('Clashing target filenames, aborting.')

def find_duplicate_filenames(
    paths: Iterable[str],
) -> List[Tuple[str, List[str]]]:
    d = defaultdict(list)
    for path in paths:
        key = os.path.basename(path).lower()
        d[key].append(path)

    return [item for item in d.items() if len(item[1]) > 1]
<commit_msg>Use `pathlib.Path` to read/write text from/to files<commit_after>"""
gallerize.main
~~~~~~~~~~~~~~

:Copyright: 2007-2021 Jochen Kupperschmidt
:License: MIT, see LICENSE for details.
"""

from __future__ import annotations
from collections import defaultdict
from dataclasses import dataclass
from itertools import islice
import os
from pathlib import Path
import shutil
import subprocess
import sys
from typing import (
    Any,
    Dict,
    Iterable,
    Iterator,
    List,
    Optional,
    Sequence,
    Tuple,
)

from jinja2 import Environment, PackageLoader

IMAGE_CAPTION_EXTENSION: str = '.txt'
TEMPLATE_EXTENSION: str = '.html'
OUTPUT_HTML_EXTENSION: str = '.html'

PATH: str = os.path.dirname(os.path.abspath(__file__))
PATH_STATIC: str = os.path.join(PATH, 'static')
TEMPLATE_ENVIRONMENT: Environment = Environment(
    autoescape=True,
    loader=PackageLoader(__package__, 'templates'),
    trim_blocks=False,
)

@dataclass(frozen=True)
class Dimension:
    width: int
    height: int

def debug(message: str, *args: Any) -> None:
    print(message.format(*args))

def resize_image(
    src_filename: str, dst_filename: str, max_dimension: Dimension
) -> None:
    """Create a resized (and antialiased) version of an image."""
    dimension_str = f'{max_dimension.width:d}x{max_dimension.height:d}'
    cmd = ['convert', '-resize', dimension_str, src_filename, dst_filename]
    subprocess.check_call(cmd)

def render_html_to_file(
    template_name: str, context: Dict[str, Any], path: str, page_name: str
) -> None:
    """Render the template and write the result to the given file."""
    context['url_for_page'] = lambda page: page + OUTPUT_HTML_EXTENSION
    html = render_template(template_name + TEMPLATE_EXTENSION, **context)

    filename = os.path.join(path, page_name + OUTPUT_HTML_EXTENSION)
    write_file(filename, html)

def render_template(template_filename: str, **context: Dict[str, Any]) -> str:
    return TEMPLATE_ENVIRONMENT.get_template(template_filename).render(context)

def write_file(filename: str, content: str) -> None:
    debug('Writing "{}" ...', filename)
    Path(filename).write_text(content, encoding='utf-8')

def create_gallery(
    title: str,
    destination_path: str,
    resize: bool,
    max_image_size: Dimension,
    max_thumbnail_size: Dimension,
    full_image_filenames: List[str],
) -> Gallery:
    gallery = Gallery(
        title=title,
        destination_path=destination_path,
        resize=resize,
        max_image_size=max_image_size,
        max_thumbnail_size=max_thumbnail_size,
    )

    gallery.images = [
        Image(gallery, image) for image in sorted(full_image_filenames)
    ]
    gallery.link_images()

    return gallery

class Gallery:
    def __init__(
        self,
        title: str,
        destination_path: str,
        resize: bool,
        max_image_size: Dimension,
        max_thumbnail_size: Dimension,
    ) -> None:
        self.images: List[Image] = []
        self.title = title
        self.destination_path = destination_path
        self.resize = resize
        self.max_image_size = max_image_size
        self.max_thumbnail_size = max_thumbnail_size

    def link_images(self) -> None:
        """Assign the predecessor and successor for every image."""
        for previous_image, image, next_image in window(self.images):
            image.previous_image = previous_image
            image.next_image = next_image

    def generate(self) -> None:
        # Create destination path if it doesn't exist.
        if not os.path.exists(self.destination_path):
            debug(
                'Destination path "{}" does not exist, creating it.',
                self.destination_path,
            )
            os.mkdir(self.destination_path)

        self.generate_images()
        for image in self.images:
            image.load_caption()
        self.render_html_pages()
        self.copy_additional_static_files()
        debug('Done.')

    def generate_images(self) -> None:
        for image in self.images:
            image.generate_image()
            image.generate_thumbnail()
            image.load_caption()

    def render_html_pages(self) -> None:
        for image in self.images:
            image.render_html_page()
        self.render_html_index_page()

    def render_html_index_page(self) -> None:
        """Create an HTML document of thumbnails that link to single image
        documents.
        """
        context = {
            'gallery': self,
        }
        render_html_to_file('index', context, self.destination_path, 'index')

    def copy_additional_static_files(self) -> None:
        if not os.path.exists(PATH_STATIC):
            debug(
                'Path "{}", does not exist; not copying any static files.',
                PATH_STATIC,
            )
            return

        filenames = list(sorted(os.listdir(PATH_STATIC)))
        if not filenames:
            debug('No static files to copy.')
        for filename in filenames:
            debug('Copying static file "{}" ...', filename)
            shutil.copy(
                os.path.join(PATH_STATIC, filename),
                os.path.join(self.destination_path, filename),
            )

def window(
    images: Iterable[Image],
) -> Iterator[Tuple[Optional[Image], Image, Optional[Image]]]:
    """Return a sliding window over the images.

    Returns a triple. Its first and third element can be `None`, but the
    second element never is.
    """
    n = 3
    padded_list: List[Optional[Image]] = list(images)
    padded_list.insert(0, None)
    padded_list.append(None)

    it = iter(padded_list)
    result = tuple(islice(it, n))

    if len(result) == n:
        yield result  # type: ignore

    for elem in it:
        result = result[1:] + (elem,)
        yield result  # type: ignore

class Image:
    def __init__(self, gallery: Gallery, full_filename: str) -> None:
        self.gallery = gallery
        self.full_filename = full_filename
        self.path, self.filename = os.path.split(full_filename)
        self.basename, self.extension = os.path.splitext(self.filename)
        self.thumbnail_filename = f'{self.basename}_t{self.extension}'
        self.page_name = self.basename
        self.previous_image: Optional[Image] = None
        self.next_image: Optional[Image] = None

    def generate_image(self) -> None:
        """Create a (optionally resized) copy of an image."""
        destination_filename = os.path.join(
            self.gallery.destination_path, self.filename
        )
        if self.gallery.resize:
            # Resize image.
            debug('Resizing image "{}" ...', self.full_filename)
            resize_image(
                self.full_filename,
                destination_filename,
                self.gallery.max_image_size,
            )
        else:
            # Copy image.
            debug('Copying image "{}" ...', self.full_filename)
            shutil.copy(self.full_filename, destination_filename)

    def generate_thumbnail(self) -> None:
        """Create a preview of an image."""
        debug('Creating thumbnail "{}" ...', self.thumbnail_filename)
        destination_filename = os.path.join(
            self.gallery.destination_path, self.thumbnail_filename
        )
        resize_image(
            self.full_filename,
            destination_filename,
            self.gallery.max_thumbnail_size,
        )

    def load_caption(self) -> None:
        """Load image caption from file."""
        caption_filename = os.path.join(
            self.path, self.filename + IMAGE_CAPTION_EXTENSION
        )
        self.caption = self._read_first_line(caption_filename)

    def _read_first_line(self, filename: str) -> Optional[str]:
        """Read the first line from the specified file."""
        try:
            text = Path(filename).read_text(encoding='utf-8')
            first_line = text.splitlines()[0]
            return first_line.strip()
        except IOError:
            # File does not exist (OK) or cannot be read (not really OK).
            return None

    def render_html_page(self) -> None:
        """Create an HTML document for a single image."""
        context = {
            'image': self,
        }
        render_html_to_file(
            'view', context, self.gallery.destination_path, self.page_name
        )

# -------------------------------------------------------------------- #

def handle_duplicate_filenames(paths) -> None:
    duplicates = find_duplicate_filenames(paths)
    if not duplicates:
        return

    print('Found duplicate filenames:')
    for filename, paths in duplicates:
        print(f'  + "{filename}" appears in the following paths:')
        for path in paths:
            print(f'    - {path}')

    sys.exit('Clashing target filenames, aborting.')

def find_duplicate_filenames(
    paths: Iterable[str],
) -> List[Tuple[str, List[str]]]:
    d = defaultdict(list)
    for path in paths:
        key = os.path.basename(path).lower()
        d[key].append(path)

    return [item for item in d.items() if len(item[1]) > 1]
<|endoftext|>


<commit_before>import ast
import os
from .context import Context

class BaseAnalyzer(object):

    surround_by = 2

    def __init__(self, code_dict, repo_path):
        self._file_lines = None
        self.code_dict = code_dict
        self.repo_path = repo_path

    def get_file_lines(self, filepath, lineno):
        if self._file_lines is None:
            with open(os.path.join(self.repo_path, filepath)) as f:
                self._file_lines = f.readlines()
        start = max(0, lineno - self.surround_by - 1)
        stop = lineno + self.surround_by
        for i, line in enumerate(self._file_lines[start:stop], start=start):
            yield (i + 1, line.rstrip())

    def clear_file_lines_cache(self):
        self._file_lines = None

    def analyze_file(self, filepath, code):
        raise NotImplementedError

    def analyze(self):
        for filepath, code in self.code_dict.items():
            for result in self.analyze_file(filepath, code):
                yield result
            self.clear_file_lines_cache()

class Code(list):

    def add_line(self, line, text, important=True):
        self.append((line, important, text))

class Result(object):

    def __init__(self, description, path, line):
        self.description = description
        self.path = path
        self.line = line
        self.source = Code()
        self.solution = Code()

class AttributeVisitor(ast.NodeVisitor):
    """
    Process attribute node and build the name of the attribute if possible.

    Currently only simple expressions are supported (like `foo.bar.baz`).
    If it is not possible to get attribute name as string `is_usable` is
    set to `True`.

    After `visit()` method call `get_name()` method can be used to get
    attribute name if `is_usable` == `True`.
    """

    def __init__(self):
        self.is_usable = True
        self.name = []

    def get_name(self):
        return '.'.join(self.name)

    def visit_Attribute(self, node):
        self.generic_visit(node)
        self.name.append(node.attr)

    def visit_Name(self, node):
        self.name.append(node.id)

    def visit_Load(self, node):
        pass

    def generic_visit(self, node):
        """
        If attribute node consists not only from nodes of types `Attribute`
        and `Name` mark it as unusable.
        """
        if not isinstance(node, ast.Attribute):
            self.is_usable = False
        ast.NodeVisitor.generic_visit(self, node)

class ModuleVisitor(ast.NodeVisitor):
    """
    Collect interesting imported names during module nodes visiting.
    """

    interesting = {}

    def __init__(self):
        self.names = Context()

    def update_names(self, aliases, get_path):
        for alias in aliases:
            path = get_path(alias.name)
            if path not in self.interesting:
                continue
            if self.interesting[path]:
                for attr in self.interesting[path]:
                    name = '.'.join((alias.asname or alias.name, attr))
                    self.names[name] = '.'.join((path, attr))
            else:
                name = alias.asname or alias.name
                self.names[name] = path

    def visit_Import(self, node):
        self.update_names(node.names, lambda x: x)

    def visit_ImportFrom(self, node):
        self.update_names(node.names, lambda x: '.'.join((node.module, x)))

    def visit_FunctionDef(self, node):
        self.names.push()
        self.generic_visit(node)
        self.names.pop()

    def visit_Assign(self, node):
        visitor = AttributeVisitor()
        visitor.visit(node.value)
        if not visitor.is_usable:
            return
        name = visitor.get_name()
        if name not in self.names:
            return
        for target in node.targets:
            visitor = AttributeVisitor()
            visitor.visit(target)
            if not visitor.is_usable:
                continue
            target = visitor.get_name()
            self.names[target] = self.names[name]
<commit_msg>Add docstrings to BaseAnalyzer<commit_after>import ast
import os
from .context import Context

class BaseAnalyzer(object):
    """
    Base code analyzer class. Takes dict `file path => ast node` as first
    param and path to repository as second.

    Subclass this class and implement `analyze_file` method if you want to
    create new code analyzer.
    """

    surround_by = 2

    def __init__(self, code_dict, repo_path):
        self._file_lines = None
        self.code_dict = code_dict
        self.repo_path = repo_path

    def get_file_lines(self, filepath, lineno):
        """
        Yield code snippet from file `filepath` for line number `lineno`
        as tuples `(<line number>, <text>)` extending it by `surround_by`
        lines up and down if possible.
        """
        if self._file_lines is None:
            with open(os.path.join(self.repo_path, filepath)) as f:
                self._file_lines = f.readlines()
        start = max(0, lineno - self.surround_by - 1)
        stop = lineno + self.surround_by
        for i, line in enumerate(self._file_lines[start:stop], start=start):
            yield (i + 1, line.rstrip())

    def clear_file_lines_cache(self):
        self._file_lines = None

    def analyze_file(self, filepath, code):
        raise NotImplementedError

    def analyze(self):
        """
        Iterate over `code_dict` and yield all results from every file.
        """
        for filepath, code in self.code_dict.items():
            for result in self.analyze_file(filepath, code):
                yield result
            self.clear_file_lines_cache()

class Code(list):

    def add_line(self, line, text, important=True):
        self.append((line, important, text))

class Result(object):

    def __init__(self, description, path, line):
        self.description = description
        self.path = path
        self.line = line
        self.source = Code()
        self.solution = Code()

class AttributeVisitor(ast.NodeVisitor):
    """
    Process attribute node and build the name of the attribute if possible.

    Currently only simple expressions are supported (like `foo.bar.baz`).
    If it is not possible to get attribute name as string `is_usable` is
    set to `True`.

    After `visit()` method call `get_name()` method can be used to get
    attribute name if `is_usable` == `True`.
    """

    def __init__(self):
        self.is_usable = True
        self.name = []

    def get_name(self):
        return '.'.join(self.name)

    def visit_Attribute(self, node):
        self.generic_visit(node)
        self.name.append(node.attr)

    def visit_Name(self, node):
        self.name.append(node.id)

    def visit_Load(self, node):
        pass

    def generic_visit(self, node):
        """
        If attribute node consists not only from nodes of types `Attribute`
        and `Name` mark it as unusable.
        """
        if not isinstance(node, ast.Attribute):
            self.is_usable = False
        ast.NodeVisitor.generic_visit(self, node)

class ModuleVisitor(ast.NodeVisitor):
    """
    Collect interesting imported names during module nodes visiting.
    """

    interesting = {}

    def __init__(self):
        self.names = Context()

    def update_names(self, aliases, get_path):
        for alias in aliases:
            path = get_path(alias.name)
            if path not in self.interesting:
                continue
            if self.interesting[path]:
                for attr in self.interesting[path]:
                    name = '.'.join((alias.asname or alias.name, attr))
                    self.names[name] = '.'.join((path, attr))
            else:
                name = alias.asname or alias.name
                self.names[name] = path

    def visit_Import(self, node):
        self.update_names(node.names, lambda x: x)

    def visit_ImportFrom(self, node):
        self.update_names(node.names, lambda x: '.'.join((node.module, x)))

    def visit_FunctionDef(self, node):
        self.names.push()
        self.generic_visit(node)
        self.names.pop()

    def visit_Assign(self, node):
        visitor = AttributeVisitor()
        visitor.visit(node.value)
        if not visitor.is_usable:
            return
        name = visitor.get_name()
        if name not in self.names:
            return
        for target in node.targets:
            visitor = AttributeVisitor()
            visitor.visit(target)
            if not visitor.is_usable:
                continue
            target = visitor.get_name()
            self.names[target] = self.names[name]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import os
import sys
import signal
import tempfile

import requests

import pytest
import time
from multiprocessing import Semaphore

from rasa_nlu.config import RasaNLUConfig
import json
import io

from utilities import ResponseTest
from rasa_nlu.server import RasaNLU

@pytest.fixture(scope="module")
def http_test_server(tmpdir_factory):
    """
    Launches a Rasa HTTP server instance on a subprocess for testing.
    This is necessary as Klein HTTP application's endpoints cannot by tested directly.
    The twisted/treq library could do that but it is not compatible with pytest
    and pytest's plugins related to treq are no more maintained.
    """
    sem = Semaphore(1)
    _, nlu_log_file = tempfile.mkstemp(suffix="_rasa_nlu_logs.json")
    _config = {
        'write': nlu_log_file,
        'port': -1,  # unused in test app
        "backend": "mitie",
        "path": tmpdir_factory.mktemp("models").strpath,
        "server_model_dirs": {},
        "data": "./data/demo-restaurants.json",
        "emulate": "wit",
    }
    url = '127.0.0.1'
    port = 5000
    pid = os.fork()
    if pid == 0:
        sem.acquire()

        config = RasaNLUConfig(cmdline_args=_config)
        rasa = RasaNLU(config)
        sem.release()
        rasa.app.run(url, port)
        rasa.data_router.shutdown()
        sys.exit(0)

    else:
        time.sleep(3)
        sem.acquire()
        sem.release()

    def with_base_url(method):
        """Save some typing and ensure we always use our own pool."""

        def request(path, *args, **kwargs):
            return method("http://{}:{}".format(url, port) + path, *args, **kwargs)

        return request

    yield with_base_url

    if pid != 0:
        os.kill(pid, signal.SIGTERM)

@pytest.fixture
def rasa_default_train_data():
    with io.open('data/examples/rasa/demo-rasa.json',
                 encoding='utf-8') as train_file:
        return json.loads(train_file.read())

def test_root(http_test_server):
    response = http_test_server(requests.get)("/")
    content = response.content
    assert response.status_code == 200 and content.startswith(b"hello")

def test_status(http_test_server):
    response = http_test_server(requests.get)("/status")
    rjs = response.json()
    assert response.status_code == 200 and ('training_queued' in rjs and 'training_workers' in rjs)
    assert rjs['training_workers'] > 0

def test_config(http_test_server):
    response = http_test_server(requests.get)("/config")
    assert response.status_code == 200

def test_version(http_test_server):
    response = http_test_server(requests.get)("/version")
    rjs = response.json()
    assert response.status_code == 200 and ("version" in rjs)

@pytest.mark.parametrize("response_test", [
    ResponseTest(
        "/parse?q=hello",
        [{"entities": {}, "confidence": 1.0, "intent": "greet", "_text": "hello"}]
    ),
    ResponseTest(
        "/parse?q=hello ńöñàśçií",
        [{"entities": {}, "confidence": 1.0, "intent": "greet", "_text": "hello ńöñàśçií"}]
    ),
    ResponseTest(
        "/parse?q=",
        [{"entities": {}, "confidence": 0.0, "intent": None, "_text": ""}]
    ),
])
def test_get_parse(http_test_server, response_test):
    response = http_test_server(requests.get)(response_test.endpoint)
    rjs = response.json()
    assert response.status_code == 200
    assert len(rjs) == 1
    assert all(prop in rjs[0] for prop in ['entities', 'intent', '_text', 'confidence'])

@pytest.mark.parametrize("response_test", [
    ResponseTest(
        "/parse",
        [{"entities": {}, "confidence": 1.0, "intent": "greet", "_text": "hello"}],
        payload={"q": "hello"}
    ),
    ResponseTest(
        "/parse",
        [{"entities": {}, "confidence": 1.0, "intent": "greet", "_text": "hello ńöñàśçií"}],
        payload={"q": "hello ńöñàśçií"}
    ),
])
def test_post_parse(http_test_server, response_test):
    response = http_test_server(requests.post)(response_test.endpoint, json=response_test.payload)
    rjs = response.json()
    assert response.status_code == 200
    assert len(rjs) == 1
    assert all(prop in rjs[0] for prop in ['entities', 'intent', '_text', 'confidence'])

@utilities.slowtest
def test_post_train(http_test_server, rasa_default_train_data):
    response = http_test_server(requests.post)("/train", json=rasa_default_train_data)
    rjs = response.json()
    assert response.status_code == 200
    assert rjs["info"] == "training started."

def test_model_hot_reloading(http_test_server, rasa_default_train_data):
    query = "/parse?q=hello&model=my_keyword_model"
    response = http_test_server(requests.get)(query)
    assert response.status_code == 404, "Model should not exist yet"
    response = http_test_server(requests.post)("/train?name=my_keyword_model&pipeline=keyword",
                                               json=rasa_default_train_data)
    assert response.status_code == 200, "Training should start successfully"
    time.sleep(5)  # training should be quick as the keyword model doesn't do any training
    response = http_test_server(requests.get)(query)
    assert response.status_code == 200, "Model should now exist after it got trained"
<commit_msg>Fixed missing import<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import os
import sys
import signal
import tempfile

import requests

import pytest
import time
from multiprocessing import Semaphore

from rasa_nlu.config import RasaNLUConfig
import json
import io

import utilities
from utilities import ResponseTest
from rasa_nlu.server import RasaNLU

@pytest.fixture(scope="module")
def http_test_server(tmpdir_factory):
    """
    Launches a Rasa HTTP server instance on a subprocess for testing.
    This is necessary as Klein HTTP application's endpoints cannot by tested directly.
    The twisted/treq library could do that but it is not compatible with pytest
    and pytest's plugins related to treq are no more maintained.
    """
    sem = Semaphore(1)
    _, nlu_log_file = tempfile.mkstemp(suffix="_rasa_nlu_logs.json")
    _config = {
        'write': nlu_log_file,
        'port': -1,  # unused in test app
        "backend": "mitie",
        "path": tmpdir_factory.mktemp("models").strpath,
        "server_model_dirs": {},
        "data": "./data/demo-restaurants.json",
        "emulate": "wit",
    }
    url = '127.0.0.1'
    port = 5000
    pid = os.fork()
    if pid == 0:
        sem.acquire()

        config = RasaNLUConfig(cmdline_args=_config)
        rasa = RasaNLU(config)
        sem.release()
        rasa.app.run(url, port)
        rasa.data_router.shutdown()
        sys.exit(0)

    else:
        time.sleep(3)
        sem.acquire()
        sem.release()

    def with_base_url(method):
        """Save some typing and ensure we always use our own pool."""

        def request(path, *args, **kwargs):
            return method("http://{}:{}".format(url, port) + path, *args, **kwargs)

        return request

    yield with_base_url

    if pid != 0:
        os.kill(pid, signal.SIGTERM)

@pytest.fixture
def rasa_default_train_data():
    with io.open('data/examples/rasa/demo-rasa.json',
                 encoding='utf-8') as train_file:
        return json.loads(train_file.read())

def test_root(http_test_server):
    response = http_test_server(requests.get)("/")
    content = response.content
    assert response.status_code == 200 and content.startswith(b"hello")

def test_status(http_test_server):
    response = http_test_server(requests.get)("/status")
    rjs = response.json()
    assert response.status_code == 200 and ('training_queued' in rjs and 'training_workers' in rjs)
    assert rjs['training_workers'] > 0

def test_config(http_test_server):
    response = http_test_server(requests.get)("/config")
    assert response.status_code == 200

def test_version(http_test_server):
    response = http_test_server(requests.get)("/version")
    rjs = response.json()
    assert response.status_code == 200 and ("version" in rjs)

@pytest.mark.parametrize("response_test", [
    ResponseTest(
        "/parse?q=hello",
        [{"entities": {}, "confidence": 1.0, "intent": "greet", "_text": "hello"}]
    ),
    ResponseTest(
        "/parse?q=hello ńöñàśçií",
        [{"entities": {}, "confidence": 1.0, "intent": "greet", "_text": "hello ńöñàśçií"}]
    ),
    ResponseTest(
        "/parse?q=",
        [{"entities": {}, "confidence": 0.0, "intent": None, "_text": ""}]
    ),
])
def test_get_parse(http_test_server, response_test):
    response = http_test_server(requests.get)(response_test.endpoint)
    rjs = response.json()
    assert response.status_code == 200
    assert len(rjs) == 1
    assert all(prop in rjs[0] for prop in ['entities', 'intent', '_text', 'confidence'])

@pytest.mark.parametrize("response_test", [
    ResponseTest(
        "/parse",
        [{"entities": {}, "confidence": 1.0, "intent": "greet", "_text": "hello"}],
        payload={"q": "hello"}
    ),
    ResponseTest(
        "/parse",
        [{"entities": {}, "confidence": 1.0, "intent": "greet", "_text": "hello ńöñàśçií"}],
        payload={"q": "hello ńöñàśçií"}
    ),
])
def test_post_parse(http_test_server, response_test):
    response = http_test_server(requests.post)(response_test.endpoint, json=response_test.payload)
    rjs = response.json()
    assert response.status_code == 200
    assert len(rjs) == 1
    assert all(prop in rjs[0] for prop in ['entities', 'intent', '_text', 'confidence'])

@utilities.slowtest
def test_post_train(http_test_server, rasa_default_train_data):
    response = http_test_server(requests.post)("/train", json=rasa_default_train_data)
    rjs = response.json()
    assert response.status_code == 200
    assert rjs["info"] == "training started."

def test_model_hot_reloading(http_test_server, rasa_default_train_data):
    query = "/parse?q=hello&model=my_keyword_model"
    response = http_test_server(requests.get)(query)
    assert response.status_code == 404, "Model should not exist yet"
    response = http_test_server(requests.post)("/train?name=my_keyword_model&pipeline=keyword",
                                               json=rasa_default_train_data)
    assert response.status_code == 200, "Training should start successfully"
    time.sleep(5)  # training should be quick as the keyword model doesn't do any training
    response = http_test_server(requests.get)(query)
    assert response.status_code == 200, "Model should now exist after it got trained"
<|endoftext|>


<commit_before># Authors: Daichi Yoshikawa <daichi.yoshikawa@gmail.com>
# License: BSD 3 clause

import numpy as np
from enum import Enum

from dnn.layers.layer import Layer
from dnn.utils.nn_utils import is_multi_channels_image
from dnn.utils.nn_utils import flatten, unflatten

class ActivationLayer(Layer):
    """Implements layer which convert values by activation function.

    Parameters
    ----------
    activation : Derived class of Activation
        Activation function to use.
    """
    def __init__(self, activation):
        """
        Arguments
        ---------
        activation : Activation.Type
           Name of activation function to use.
        """
        self.activation = ActivationFactory.get(activation)

    def get_type(self):
        return 'activation'

    def set_parent(self, parent):
        Layer.set_parent(self, parent)
        self.output_shape = self.input_shape

    def forward(self, x):
        self.__forward(x)
        self.child.forward(self.fire)

    def backward(self, dy):
        self.__backward(dy)
        self.parent.backward(self.backfire)

    def predict(self, x):
        self.__forward(x)
        return self.child.predict(self.fire)

    def __forward(self, x):
        if is_multi_channels_image(self.input_shape):
            x = flatten(x, self.input_shape)

        self.fire = self.activation.activate(x)

        if is_multi_channels_image(self.output_shape):
            self.fire = unflatten(self.fire, self.input_shape)

    def __backward(self, dy):
        if is_multi_channels_image(self.output_shape):
            dy = flatten(dy, self.input_shape)

        self.backfire = dy * self.activation.grad(self.fire)

        if is_multi_channels_image(self.input_shape):
            self.backfire = unflatten(self.backfire, self.input_shape)

# Implement Activation Functions
class Activation:
    """Base class for activation functions.

    Warning
    -------
    This class should not be used directly.
    Use derived classes instead.
    """
    Type = Enum('Type', 'sigmoid, relu, elu, srrelu, tanh, softmax')

    def get_type(self):
        """Interface to get type of activation in string."""
        raise NotImplementedError('Activation.get_type')

    def activate(self, x):
        """Interface to get f(x), where f(x) is activation function."""
        raise NotImplementedError('Activation.activate')

    def grad(self, x):
        """Interface to get df(x)/dx, where f(x) is activation function."""
        raise NotImplementedError('Activation.grad')

class Sigmoid(Activation):
    """Sigmoid function.

    f(x) = 1 / (1 + exp(-a*x)).
    df(x)/dx = (1 - f(x)) * f(x)
    This class implements aboves with a == 1.
    """
    def get_type(self):
        return 'sigmoid'

    def activate(self, x):
        return 1. / (1. + np.exp(-x))

    def grad(self, x):
        return (1. - x) * x

class ReLU(Activation):
    """Rectified linear function.

    f(x) = x when x > 0 and f(x) = 0 when x <= 0.
    df(x)/dx = 1 when x > 0 and df(x)/dx = 0 when x <= 0.

    Parameters
    ----------
    __mask : np.array
        2d matrix whose entry(i, j) is
        1 when x(i, j) > 0 and 0 when x(i, j) == 0.
    """
    def get_type(self):
        return 'relu'

    def activate(self, x):
        self.__mask = (x > 0.).astype(x.dtype)
        return x * self.__mask

    def grad(self, x):
        return self.__mask

class ELU(Activation):
    """Exponential Linear Units.

    Parameters
    ----------
    alpha : float, default 1.0
        Controls the value to which an ELU saturates for negative net inputs.
    __mask : np.array
        2d matrix whose entry(i, j) is
        1 when x(i, j) > 0 and 0 when x(i, j) == 0.

    References
    ----------
    Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)
    https://arxiv.org/pdf/1511.07289.pdf
    """
    def get_type(self):
        return 'elu'

    def activate(self, x):
        self.__mask = (x > 0.)
        return self.__mask * x + ~(self.__mask) * (np.exp(x) - 1)

    def grad(self, x):
        return self.__mask + ~(self.__mask) * (x + 1)

class SRReLU(Activation):
    """Square Root version of Rectified Linear.

    This is an activation function the author, Daichi Yoshikawa, proposes.
    To reduce bias shift and enhance nonlinearity, use square root of input
    instead of original value.

    Parameters
    ----------
    __mask : np.array
        2d matrix whose entry(i, j) is
        1 when x(i, j) > 0 and 0 when x(i, j) == 0.
    """
    def get_type(self):
        return 'selu'

    def activate(self, x):
        self.__mask = (x > 0.)
        return (np.sqrt(self.__mask * x + 1) - 1)

    def grad(self, x):
        return self.__mask * 0.5 / (x + 1)

class Tanh(Activation):
    """Tanh function.

    f(x) = tanh(x) = (exp(x) - exp(-x)) / (exp(x) + exp(-x))
    df(x)/dx = 1 - f(x)^2
    """
    def get_type(self):
        return 'tanh'

    def activate(self, x):
        return np.tanh(x)

    def grad(self, x):
        return 1. - np.power(x, 2, dtype=x.dtype)

class Softmax(Activation):
    """Softmax function to convert values into probabilities.

    f(x) = exp(x) / exp(x).sum()
         = exp(x) * exp(-c) / (exp(x).sum()) * exp(-c)
         = exp(x - c) / exp(x - c).sum()
    where c == x.max(), to avoid overflow of exp calculation.
    """
    def get_type(self):
        return 'softmax'

    def activate(self, x):
        var = np.exp(x - x.max())
        return var / var.sum(axis=1).reshape(var.shape[0], 1)

    def grad(self, x):
        return 1.

class ActivationFactory:
    """Factory class to get activation's instance.

    Warning
    -------
    Get activation's instance through this class.
    """
    @classmethod
    def get(cls, activation):
        """Returns instance of selected activation function.

        Arguments
        ---------
        activation : Activation.Type
            Name of activation function to use.

        Returns
        -------
        Derived class of Activation
            Instance of selected activation function.
        """
        if activation is Activation.Type.sigmoid:
            return Sigmoid()
        elif activation is Activation.Type.relu:
            return ReLU()
        elif activation is Activation.Type.elu:
            return ELU()
        elif activation is Activation.Type.srrelu:
            return SRReLU()
        elif activation is Activation.Type.tanh:
            return Tanh()
        elif activation is Activation.Type.softmax:
            return Softmax()

<commit_msg>Deleted flatten and unflatten because these cause crash when input_shape is like (3, 28, 28).<commit_after># Authors: Daichi Yoshikawa <daichi.yoshikawa@gmail.com>
# License: BSD 3 clause

import numpy as np
from enum import Enum

from dnn.layers.layer import Layer
from dnn.utils.nn_utils import is_multi_channels_image
from dnn.utils.nn_utils import flatten, unflatten

class ActivationLayer(Layer):
    """Implements layer which convert values by activation function.

    Parameters
    ----------
    activation : Derived class of Activation
        Activation function to use.
    """
    def __init__(self, activation):
        """
        Arguments
        ---------
        activation : Activation.Type
           Name of activation function to use.
        """
        self.activation = ActivationFactory.get(activation)

    def get_type(self):
        return 'activation'

    def set_parent(self, parent):
        Layer.set_parent(self, parent)
        self.output_shape = self.input_shape

    def forward(self, x):
        self.__forward(x)
        self.child.forward(self.fire)

    def backward(self, dy):
        self.__backward(dy)
        self.parent.backward(self.backfire)

    def predict(self, x):
        self.__forward(x)
        return self.child.predict(self.fire)

    def __forward(self, x):
        self.fire = self.activation.activate(x)

    def __backward(self, dy):
        self.backfire = dy * self.activation.grad(self.fire)

# Implement Activation Functions
class Activation:
    """Base class for activation functions.

    Warning
    -------
    This class should not be used directly.
    Use derived classes instead.
    """
    Type = Enum('Type', 'sigmoid, relu, elu, srrelu, tanh, softmax')

    def get_type(self):
        """Interface to get type of activation in string."""
        raise NotImplementedError('Activation.get_type')

    def activate(self, x):
        """Interface to get f(x), where f(x) is activation function."""
        raise NotImplementedError('Activation.activate')

    def grad(self, x):
        """Interface to get df(x)/dx, where f(x) is activation function."""
        raise NotImplementedError('Activation.grad')

class Sigmoid(Activation):
    """Sigmoid function.

    f(x) = 1 / (1 + exp(-a*x)).
    df(x)/dx = (1 - f(x)) * f(x)
    This class implements aboves with a == 1.
    """
    def get_type(self):
        return 'sigmoid'

    def activate(self, x):
        return 1. / (1. + np.exp(-x))

    def grad(self, x):
        return (1. - x) * x

class ReLU(Activation):
    """Rectified linear function.

    f(x) = x when x > 0 and f(x) = 0 when x <= 0.
    df(x)/dx = 1 when x > 0 and df(x)/dx = 0 when x <= 0.

    Parameters
    ----------
    __mask : np.array
        2d matrix whose entry(i, j) is
        1 when x(i, j) > 0 and 0 when x(i, j) == 0.
    """
    def get_type(self):
        return 'relu'

    def activate(self, x):
        self.__mask = (x > 0.).astype(x.dtype)
        return x * self.__mask

    def grad(self, x):
        return self.__mask

class ELU(Activation):
    """Exponential Linear Units.

    Parameters
    ----------
    alpha : float, default 1.0
        Controls the value to which an ELU saturates for negative net inputs.
    __mask : np.array
        2d matrix whose entry(i, j) is
        1 when x(i, j) > 0 and 0 when x(i, j) == 0.

    References
    ----------
    Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)
    https://arxiv.org/pdf/1511.07289.pdf
    """
    def get_type(self):
        return 'elu'

    def activate(self, x):
        self.__mask = (x > 0.)
        return self.__mask * x + ~(self.__mask) * (np.exp(x) - 1)

    def grad(self, x):
        return self.__mask + ~(self.__mask) * (x + 1)

class SRReLU(Activation):
    """Square Root version of Rectified Linear.

    This is an activation function the author, Daichi Yoshikawa, proposes.
    To reduce bias shift and enhance nonlinearity, use square root of input
    instead of original value.

    Parameters
    ----------
    __mask : np.array
        2d matrix whose entry(i, j) is
        1 when x(i, j) > 0 and 0 when x(i, j) == 0.
    """
    def get_type(self):
        return 'selu'

    def activate(self, x):
        self.__mask = (x > 0.)
        return (np.sqrt(self.__mask * x + 1) - 1)

    def grad(self, x):
        return self.__mask * 0.5 / (x + 1)

class Tanh(Activation):
    """Tanh function.

    f(x) = tanh(x) = (exp(x) - exp(-x)) / (exp(x) + exp(-x))
    df(x)/dx = 1 - f(x)^2
    """
    def get_type(self):
        return 'tanh'

    def activate(self, x):
        return np.tanh(x)

    def grad(self, x):
        return 1. - np.power(x, 2, dtype=x.dtype)

class Softmax(Activation):
    """Softmax function to convert values into probabilities.

    f(x) = exp(x) / exp(x).sum()
         = exp(x) * exp(-c) / (exp(x).sum()) * exp(-c)
         = exp(x - c) / exp(x - c).sum()
    where c == x.max(), to avoid overflow of exp calculation.
    """
    def get_type(self):
        return 'softmax'

    def activate(self, x):
        var = np.exp(x - x.max())
        return var / var.sum(axis=1).reshape(var.shape[0], 1)

    def grad(self, x):
        return 1.

class ActivationFactory:
    """Factory class to get activation's instance.

    Warning
    -------
    Get activation's instance through this class.
    """
    @classmethod
    def get(cls, activation):
        """Returns instance of selected activation function.

        Arguments
        ---------
        activation : Activation.Type
            Name of activation function to use.

        Returns
        -------
        Derived class of Activation
            Instance of selected activation function.
        """
        if activation is Activation.Type.sigmoid:
            return Sigmoid()
        elif activation is Activation.Type.relu:
            return ReLU()
        elif activation is Activation.Type.elu:
            return ELU()
        elif activation is Activation.Type.srrelu:
            return SRReLU()
        elif activation is Activation.Type.tanh:
            return Tanh()
        elif activation is Activation.Type.softmax:
            return Softmax()

<|endoftext|>


<commit_before>from django.conf.urls.defaults import *
from django.conf import settings
# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    (r'^$', 'lingcod.common.views.map', {'template_name': 'common/map-ext.html'}),
    (r'^tests/', 'django.views.generic.simple.direct_to_template', {'template': 'common/tests.html', 'extra_context': {'api_key': settings.GOOGLE_API_KEY}}),
    (r'^layers/', include('lingcod.layers.urls')),
    (r'^studyregion/', include('lingcod.studyregion.urls')),
    (r'^faq/', include('lingcod.simplefaq.urls')),
    (r'^staticmap/', include('lingcod.staticmap.urls')),
    (r'^kml/', include('lingcod.kmlapp.urls')),
    (r'^wave/', include('lingcod.wave.urls')),
    (r'^manipulators/', include('lingcod.manipulators.urls')),
    #(r'^manipulators-list/', 'simple.views.manipulatorList'),
    #(r'^mpa/', include('lingcod.mpa.urls')),
    # (r'^mpa/$', 'simple.views.simpleCommit'),
    # (r'^mpa/save/form/$', 'simple.views.simpleCommit'),
    # (r'^mpa/load/$', 'simple.views.simpleLoad'),
    # (r'^mpa/load/form/$', 'simple.views.simpleLoadForm'),
    (r'^simple-app/', include('simple.simple_app.urls')),

    (r'^mpas/', include('lingcod.mpa.urls')),
    (r'^arrays/', include('lingcod.array.urls')),
    # (r'^data_distributor/', include('lingcod.data_distributor.urls')),
    url(r'^login/$', 'django.contrib.auth.views.login', name='login'),
    url(r'^logout/$', 'django.contrib.auth.views.logout', name='logout'),
    # Uncomment the admin/doc line below and add 'django.contrib.admindocs'
    # to INSTALLED_APPS to enable admin documentation:
    # (r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    (r'^admin/', include(admin.site.urls)),
)

# Useful for serving files when using the django dev server
urlpatterns += patterns('',
    (r'^media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT, 'show_indexes': True }),
)
<commit_msg>Fixed the url pointing to the map template, added url patterns required by that template to the simple app's url.py file<commit_after>from django.conf.urls.defaults import *
from django.conf import settings
# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    (r'^$', 'lingcod.common.views.map', {'template_name': 'common/map.html'}),
    (r'^tests/', 'django.views.generic.simple.direct_to_template', {'template': 'common/tests.html', 'extra_context': {'api_key': settings.GOOGLE_API_KEY}}),
    (r'^layers/', include('lingcod.layers.urls')),
    (r'^studyregion/', include('lingcod.studyregion.urls')),
    (r'^faq/', include('lingcod.simplefaq.urls')),
    (r'^staticmap/', include('lingcod.staticmap.urls')),
    (r'^kml/', include('lingcod.kmlapp.urls')),
    (r'^wave/', include('lingcod.wave.urls')),
    (r'^news/', include('lingcod.news.urls')),
    (r'^help/', include('lingcod.help.urls')),
    (r'^accounts/', include('lingcod.openid.urls')),
    (r'^accounts/profile/', include('lingcod.user_profile.urls')),
    (r'^manipulators/', include('lingcod.manipulators.urls')),
    #(r'^manipulators-list/', 'simple.views.manipulatorList'),
    #(r'^mpa/', include('lingcod.mpa.urls')),
    # (r'^mpa/$', 'simple.views.simpleCommit'),
    # (r'^mpa/save/form/$', 'simple.views.simpleCommit'),
    # (r'^mpa/load/$', 'simple.views.simpleLoad'),
    # (r'^mpa/load/form/$', 'simple.views.simpleLoadForm'),
    (r'^simple-app/', include('simple.simple_app.urls')),

    (r'^mpas/', include('lingcod.mpa.urls')),
    (r'^arrays/', include('lingcod.array.urls')),
    # (r'^data_distributor/', include('lingcod.data_distributor.urls')),
    url(r'^login/$', 'django.contrib.auth.views.login', name='login'),
    url(r'^logout/$', 'django.contrib.auth.views.logout', name='logout'),
    # Uncomment the admin/doc line below and add 'django.contrib.admindocs'
    # to INSTALLED_APPS to enable admin documentation:
    # (r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    (r'^admin/', include(admin.site.urls)),
)

# Useful for serving files when using the django dev server
urlpatterns += patterns('',
    (r'^media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT, 'show_indexes': True }),
)
<|endoftext|>


<commit_before>import os
import re
import shutil
from functools import wraps
from unittest import TestCase
from unittesting.utils import UTSetting
from unittesting import DeferrableTestCase
import sublime

version = sublime.version()

__dir__ = os.path.dirname(os.path.abspath(__file__))
UUT_dir = os.path.join(
    sublime.packages_path(), 'User', 'UnitTesting')

def set_package(package):
    try:
        shutil.rmtree(os.path.join(sublime.packages_path(), package))
    except:
        pass
    try:
        shutil.copytree(
            os.path.join(__dir__, package),
            os.path.join(sublime.packages_path(), package))
    except:
        pass
    try:
        shutil.rmtree(os.path.join(UUT_dir, package))
    except:
        pass

def cleanup_package(package):
    try:
        shutil.rmtree(os.path.join(sublime.packages_path(), package))
    except:
        pass

def perpare_package(package, output=None, syntax_test=False, delay=None):
    def wrapper(func):
        @wraps(func)
        def real_wrapper(self):
            set_package(package)
            if output:
                # set by _Ooutput/unittesting.json
                outfile = None
                result_file = os.path.join(sublime.packages_path(), package, output)
            else:
                outfiledir = os.path.join(UUT_dir, package)
                outfile = os.path.join(outfiledir, "result")
                result_file = outfile
                if not os.path.isdir(outfiledir):
                    os.makedirs(outfiledir)

            if syntax_test:
                yield 1000
                sublime.run_command(
                    "unit_testing_syntax", {"package": package, "output": outfile})
            else:
                sublime.run_command(
                    "unit_testing", {"package": package, "output": outfile})

            if delay:
                yield delay
            with open(result_file, 'r') as f:
                txt = f.read()
            m = re.search('^UnitTesting: Done\\.', txt, re.MULTILINE)
            self.assertTrue(hasattr(m, "group"))
            func(self, txt)
            cleanup_package(package)
        return real_wrapper
    return wrapper

class TestUnitTesting(DeferrableTestCase):

    def tearDown(self):
        UTSetting.set("recent-package", "UnitTesting")

    @perpare_package("_Success")
    def test_success(self, txt):
        m = re.search('^OK', txt, re.MULTILINE)
        self.assertTrue(hasattr(m, "group"))

    @perpare_package("_Failure")
    def test_failure(self, txt):
        m = re.search('^FAILED \(failures=1\)', txt, re.MULTILINE)
        self.assertTrue(hasattr(m, "group"))

    @perpare_package("_Error")
    def test_error(self, txt):
        m = re.search('^ERROR', txt, re.MULTILINE)
        self.assertTrue(hasattr(m, "group"))

    @perpare_package("_Output", "tests/result")
    def test_output(self, txt):
        m = re.search('^OK', txt, re.MULTILINE)
        self.assertTrue(hasattr(m, "group"))

    @perpare_package("_Deferred", delay=1000)
    def test_deferred(self, txt):
        m = re.search('^OK', txt, re.MULTILINE)
        self.assertTrue(hasattr(m, "group"))

    if version >= '3000':
        @perpare_package("_Async", delay=2000)
        def test_async(self, txt):
            m = re.search('^OK', txt, re.MULTILINE)
            self.assertTrue(hasattr(m, "group"))

if version >= '3103':
    class TestSyntax(DeferrableTestCase):

        def tearDown(self):
            UTSetting.set("recent-package", "UnitTesting")

        @perpare_package("_Syntax_Failure", syntax_test=True)
        def test_fail_syntax(self, txt):
            m = re.search('^FAILED: 1 of 21 assertions in 1 files failed$', txt, re.MULTILINE)
            self.assertTrue(hasattr(m, "group"))

        @perpare_package("_Syntax_Success", syntax_test=True)
        def test_success_syntax(self, txt):
            m = re.search('^OK', txt, re.MULTILINE)
            self.assertTrue(hasattr(m, "group"))

        @perpare_package("_Syntax_Error", syntax_test=True)
        def test_error_syntax(self, txt):
            m = re.search('^ERROR: No syntax_test', txt, re.MULTILINE)
            self.assertTrue(hasattr(m, "group"))
<commit_msg>add more delays<commit_after>import os
import re
import shutil
from functools import wraps
from unittest import TestCase
from unittesting.utils import UTSetting
from unittesting import DeferrableTestCase
import sublime

version = sublime.version()

__dir__ = os.path.dirname(os.path.abspath(__file__))
UUT_dir = os.path.join(
    sublime.packages_path(), 'User', 'UnitTesting')

def set_package(package):
    try:
        shutil.rmtree(os.path.join(sublime.packages_path(), package))
    except:
        pass
    try:
        shutil.copytree(
            os.path.join(__dir__, package),
            os.path.join(sublime.packages_path(), package))
    except:
        pass
    try:
        shutil.rmtree(os.path.join(UUT_dir, package))
    except:
        pass

def cleanup_package(package):
    try:
        shutil.rmtree(os.path.join(sublime.packages_path(), package))
    except:
        pass

def perpare_package(package, output=None, syntax_test=False, delay=None):
    def wrapper(func):
        @wraps(func)
        def real_wrapper(self):
            set_package(package)
            if output:
                # set by _Ooutput/unittesting.json
                outfile = None
                result_file = os.path.join(sublime.packages_path(), package, output)
            else:
                outfiledir = os.path.join(UUT_dir, package)
                outfile = os.path.join(outfiledir, "result")
                result_file = outfile
                if not os.path.isdir(outfiledir):
                    os.makedirs(outfiledir)

            if syntax_test:
                yield 1000
                sublime.run_command(
                    "unit_testing_syntax", {"package": package, "output": outfile})
            else:
                sublime.run_command(
                    "unit_testing", {"package": package, "output": outfile})

            if delay:
                yield delay
            with open(result_file, 'r') as f:
                txt = f.read()
            m = re.search('^UnitTesting: Done\\.', txt, re.MULTILINE)
            self.assertTrue(hasattr(m, "group"))
            func(self, txt)
            cleanup_package(package)
        return real_wrapper
    return wrapper

class TestUnitTesting(DeferrableTestCase):

    def tearDown(self):
        UTSetting.set("recent-package", "UnitTesting")

    @perpare_package("_Success")
    def test_success(self, txt):
        m = re.search('^OK', txt, re.MULTILINE)
        self.assertTrue(hasattr(m, "group"))

    @perpare_package("_Failure")
    def test_failure(self, txt):
        m = re.search('^FAILED \(failures=1\)', txt, re.MULTILINE)
        self.assertTrue(hasattr(m, "group"))

    @perpare_package("_Error")
    def test_error(self, txt):
        m = re.search('^ERROR', txt, re.MULTILINE)
        self.assertTrue(hasattr(m, "group"))

    @perpare_package("_Output", "tests/result")
    def test_output(self, txt):
        m = re.search('^OK', txt, re.MULTILINE)
        self.assertTrue(hasattr(m, "group"))

    @perpare_package("_Deferred", delay=2000)
    def test_deferred(self, txt):
        m = re.search('^OK', txt, re.MULTILINE)
        self.assertTrue(hasattr(m, "group"))
        yield 1000

    if version >= '3000':
        @perpare_package("_Async", delay=2000)
        def test_async(self, txt):
            m = re.search('^OK', txt, re.MULTILINE)
            self.assertTrue(hasattr(m, "group"))
            yield 1000

if version >= '3103':
    class TestSyntax(DeferrableTestCase):

        def tearDown(self):
            UTSetting.set("recent-package", "UnitTesting")

        @perpare_package("_Syntax_Failure", syntax_test=True)
        def test_fail_syntax(self, txt):
            m = re.search('^FAILED: 1 of 21 assertions in 1 files failed$', txt, re.MULTILINE)
            self.assertTrue(hasattr(m, "group"))

        @perpare_package("_Syntax_Success", syntax_test=True)
        def test_success_syntax(self, txt):
            m = re.search('^OK', txt, re.MULTILINE)
            self.assertTrue(hasattr(m, "group"))

        @perpare_package("_Syntax_Error", syntax_test=True)
        def test_error_syntax(self, txt):
            m = re.search('^ERROR: No syntax_test', txt, re.MULTILINE)
            self.assertTrue(hasattr(m, "group"))
<|endoftext|>


<commit_before>import os
import pytest

from aligner.aligner import BaseAligner

def test_sick_mono(sick_dict, sick_corpus, generated_dir):
    a = BaseAligner(sick_corpus, sick_dict, os.path.join(generated_dir,'sick_output'),
                        temp_directory = os.path.join(generated_dir,'sickcorpus'))
    a.train_mono()

@pytest.mark.xfail
def test_sick_tri(sick_dict, sick_corpus, generated_dir):
    a = BaseAligner(sick_corpus, sick_dict, os.path.join(generated_dir,'sick_output'),
                        temp_directory = os.path.join(generated_dir,'sickcorpus'))
    a.train_tri()

@pytest.mark.xfail
def test_sick_tri_fmllr(sick_dict, sick_corpus, generated_dir):
    a = BaseAligner(sick_corpus, sick_dict, os.path.join(generated_dir,'sick_output'),
                        temp_directory = os.path.join(generated_dir,'sickcorpus'))
    a.train_tri_fmllr()
    a.export_textgrids()

<commit_msg>Fix imports<commit_after>import os
import pytest

from aligner.aligner import TrainableAligner

def test_sick_mono(sick_dict, sick_corpus, generated_dir):
    a = TrainableAligner(sick_corpus, sick_dict, os.path.join(generated_dir,'sick_output'),
                        temp_directory = os.path.join(generated_dir,'sickcorpus'))
    a.train_mono()

@pytest.mark.xfail
def test_sick_tri(sick_dict, sick_corpus, generated_dir):
    a = TrainableAligner(sick_corpus, sick_dict, os.path.join(generated_dir,'sick_output'),
                        temp_directory = os.path.join(generated_dir,'sickcorpus'))
    a.train_tri()

@pytest.mark.xfail
def test_sick_tri_fmllr(sick_dict, sick_corpus, generated_dir):
    a = TrainableAligner(sick_corpus, sick_dict, os.path.join(generated_dir,'sick_output'),
                        temp_directory = os.path.join(generated_dir,'sickcorpus'))
    a.train_tri_fmllr()
    a.export_textgrids()

<|endoftext|>


<commit_before>from __future__ import unicode_literals

import mock

from mopidy import backend as backend_api

from pandora import APIClient, BaseAPIClient

from mopidy_pandora import client, library, playback
from tests.conftest import get_backend, get_station_list_mock, request_exception_mock

def test_uri_schemes(config):
    backend = get_backend(config)

    assert 'pandora' in backend.uri_schemes

def test_init_sets_up_the_providers(config):
    backend = get_backend(config)

    assert isinstance(backend.api, client.MopidyAPIClient)

    assert isinstance(backend.library, library.PandoraLibraryProvider)
    assert isinstance(backend.library, backend_api.LibraryProvider)

    assert isinstance(backend.playback, playback.PandoraPlaybackProvider)
    assert isinstance(backend.playback, backend_api.PlaybackProvider)

def test_init_sets_preferred_audio_quality(config):
    config['pandora']['preferred_audio_quality'] = 'lowQuality'
    backend = get_backend(config)

    assert backend.api.default_audio_quality == BaseAPIClient.LOW_AUDIO_QUALITY

def test_playback_provider_selection_events_disabled(config):
    config['pandora']['event_support_enabled'] = 'false'
    backend = get_backend(config)

    assert isinstance(backend.playback, playback.PandoraPlaybackProvider)

def test_playback_provider_selection_events_default(config):
    config['pandora']['event_support_enabled'] = ''
    backend = get_backend(config)

    assert isinstance(backend.playback, playback.PandoraPlaybackProvider)

def test_playback_provider_selection_events_enabled(config):
    config['pandora']['event_support_enabled'] = 'true'
    backend = get_backend(config)

    assert isinstance(backend.playback, playback.EventSupportPlaybackProvider)

def test_on_start_logs_in(config):
    backend = get_backend(config)

    login_mock = mock.PropertyMock()
    backend.api.login = login_mock
    backend.on_start()

    backend.api.login.assert_called_once_with('john', 'doe')

def test_on_start_pre_fetches_lists(config):
    with mock.patch.object(APIClient, 'get_station_list', get_station_list_mock):
        backend = get_backend(config)

        backend.api.login = mock.PropertyMock()
        backend.api.get_genre_stations = mock.PropertyMock()

        assert backend.api._station_list_cache.currsize == 0
        assert backend.api._genre_stations_cache.currsize == 0

        t = backend.on_start()
        t.join()

        assert backend.api._station_list_cache.currsize == 1
        assert backend.api.get_genre_stations.called

def test_on_start_handles_request_exception(config, caplog):
    backend = get_backend(config, True)

    backend.api.login = request_exception_mock
    t = backend.on_start()
    t.join()

    # Check that request exceptions are caught and logged
    assert 'Error logging in to Pandora' in caplog.text()
<commit_msg>Make unit test thread safe.<commit_after>from __future__ import unicode_literals

import mock

from mopidy import backend as backend_api

from pandora import APIClient, BaseAPIClient

from mopidy_pandora import client, library, playback
from tests.conftest import get_backend, get_station_list_mock, request_exception_mock

def test_uri_schemes(config):
    backend = get_backend(config)

    assert 'pandora' in backend.uri_schemes

def test_init_sets_up_the_providers(config):
    backend = get_backend(config)

    assert isinstance(backend.api, client.MopidyAPIClient)

    assert isinstance(backend.library, library.PandoraLibraryProvider)
    assert isinstance(backend.library, backend_api.LibraryProvider)

    assert isinstance(backend.playback, playback.PandoraPlaybackProvider)
    assert isinstance(backend.playback, backend_api.PlaybackProvider)

def test_init_sets_preferred_audio_quality(config):
    config['pandora']['preferred_audio_quality'] = 'lowQuality'
    backend = get_backend(config)

    assert backend.api.default_audio_quality == BaseAPIClient.LOW_AUDIO_QUALITY

def test_playback_provider_selection_events_disabled(config):
    config['pandora']['event_support_enabled'] = 'false'
    backend = get_backend(config)

    assert isinstance(backend.playback, playback.PandoraPlaybackProvider)

def test_playback_provider_selection_events_default(config):
    config['pandora']['event_support_enabled'] = ''
    backend = get_backend(config)

    assert isinstance(backend.playback, playback.PandoraPlaybackProvider)

def test_playback_provider_selection_events_enabled(config):
    config['pandora']['event_support_enabled'] = 'true'
    backend = get_backend(config)

    assert isinstance(backend.playback, playback.EventSupportPlaybackProvider)

def test_on_start_logs_in(config):
    backend = get_backend(config)

    login_mock = mock.PropertyMock()
    backend.api.login = login_mock
    t = backend.on_start()
    t.join()

    backend.api.login.assert_called_once_with('john', 'doe')

def test_on_start_pre_fetches_lists(config):
    with mock.patch.object(APIClient, 'get_station_list', get_station_list_mock):
        backend = get_backend(config)

        backend.api.login = mock.PropertyMock()
        backend.api.get_genre_stations = mock.PropertyMock()

        assert backend.api._station_list_cache.currsize == 0
        assert backend.api._genre_stations_cache.currsize == 0

        t = backend.on_start()
        t.join()

        assert backend.api._station_list_cache.currsize == 1
        assert backend.api.get_genre_stations.called

def test_on_start_handles_request_exception(config, caplog):
    backend = get_backend(config, True)

    backend.api.login = request_exception_mock
    t = backend.on_start()
    t.join()

    # Check that request exceptions are caught and logged
    assert 'Error logging in to Pandora' in caplog.text()
<|endoftext|>


<commit_before># The MIT License (MIT)

# Copyright (c) 2015 Breschine Cummins

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import itertools
import subprocess
import patternmatch
import sys,time,traceback
import pp,os
import fileparsers
from math import ceil

# Call this module as python useDSGRN.py

# Given a Morse graph number MGN for a database DATABASEFILE (.db), do the following database searches:
#
# The following gives the list of parameters in the file PARAMFILE (.txt)
# sqlite3 /share/data/CHomP/Projects/DSGRN/DB/data/DATABASEFILE.db 'select ParameterIndex from Signatures where MorseGraphIndex=MGN' > ./PARAMFILE
#
# The following finds the Morse set number:
# sqlite3 /share/data/CHomP/Projects/DSGRN/DB/data/DATABASEFILE.db 'select * from MorseGraphAnnotations where MorseGraphIndex=MGN'
#
# The following is a search for stable FCs:
# sqlite3 /share/data/CHomP/Projects/DSGRN/DB/data/DATABASEFILE.db 'select MorseGraphIndex,Vertex from MorseGraphAnnotations where Label="FC" except select MorseGraphIndex,Source from MorseGraphEdges'
#

def patternSearch(networkfile,paramfile,resultsfile,patternstr,printtoscreen,findallmatches):
    # had to completely rewrite this function from scratch (copying it), in order to get
    # subprocess.call to redirect to a file. I wrote other short functions with the same
    # commands and didn't have this issue. Could there have been invisible characters?
    # At any rate, this is a potential source of failure in the future, since I have no
    # idea what happened.
    domaincells_jsonstr=getJSONstring(networkfile,['domaingraph'])
    results_list=[]
    with open(paramfile,'r') as P:
        for info in P.readlines():
            morsegraph,morseset,param = info.replace('|',' ').split()
            domaingraph_jsonstr=getJSONstring(networkfile,['domaingraph', 'json',param])
            morseset_jsonstr=getJSONstring(networkfile,['analyze', 'morseset', morseset, param])
            try:
                patterns,matches=patternmatch.callPatternMatch(morseset_jsonstr,domaingraph_jsonstr,domaincells_jsonstr,patternstr,resultsfile,writetofile=0,returnmatches=1,printtoscreen=printtoscreen,findallmatches=findallmatches)
                for pat,match in zip(patterns,matches):
                    if findallmatches:
                        results_list.append("Parameter: {}, Morse Graph: {}, Morse Set: {}, Pattern: {}, Results: {}".format(param,morsegraph,morseset,pat,match)+'\n')
                    else:
                        results_list.append("Parameter: {}, Morse Graph: {}, Morse Set: {}, Pattern: {}".format(param,morsegraph,morseset,pat)+'\n')
            except Exception as e:
                print 'Problem parameter is {}'.format(param)
                print traceback.format_exception_only(type(e),e)
                sys.stdout.flush()
    with open(resultsfile,'w',0) as R:
        for line in results_list:
            R.write(line)
    removeFiles([paramfile])

def getJSONstring(networkfile,command=['analyze', 'morseset', '0', '0']):
    return subprocess.check_output(['dsgrn','network', networkfile]+command,shell=False)

def splitParamFile(paramfile,subparamfilestart,numparams,paramsperslice):
    numjobs=int(ceil(float(numparams)/paramsperslice))
    with open(paramfile,'r') as apf:
        for n in range(numjobs):
            subparamfile=subparamfilestart+'_{:04d}.txt'.format(n)
            with open(subparamfile,'w',0) as subapf:
                for line in [apf.readline() for _ in range(paramsperslice)]:
                    if len(line):
                        subapf.write(line)
    return numjobs

def removeFiles(listoffiles):
    for f in listoffiles:
        subprocess.call(['rm '+f],shell=True)

def mergeFiles(mergefile,splitfiles):
    with open(mergefile,'w') as m:
        for sfile in splitfiles:
            with open(sfile,'r') as f:
                for l in f.readlines():
                    m.write(l)
    removeFiles(splitfiles)

def parallelrun(job_server,numjobs,subparamfilestart,resultsfile,allresultsfile,patternstr,networkfile,printtoscreen=0,findallmatches=0):
    jobs=[]
    allsubresultsfiles=[]
    for n in range(numjobs):
        unique_identifier='{:04d}'.format(n)
        subparamfile=subparamfilestart+'_'+unique_identifier+'.txt'
        subresultsfile=resultsfile+unique_identifier+'.txt'
        allsubresultsfiles.append(subresultsfile)
        jobs.append(job_server.submit(patternSearch,(networkfile,subparamfile,subresultsfile,patternstr,printtoscreen,findallmatches), depfuncs=(),modules = ("subprocess","patternmatch", "preprocess","fileparsers","walllabels","itertools","numpy","json","traceback","sys"),globals=globals()))
    print "{} jobs starting.".format(numjobs)
    sys.stdout.flush()
    for job in jobs:
        job()
    job_server.destroy()
    print "All jobs ended."
    sys.stdout.flush()
    mergeFiles(allresultsfile,allsubresultsfiles)
    print "Results files merged."
    sys.stdout.flush()

def initServer():
    secret = os.environ["PPSERVERSECRET"]
    ppservers = ("*",)
    job_server = pp.Server(ncpus=0,ppservers=ppservers,secret=secret)
    time.sleep(30)
    N = sum(job_server.get_active_nodes().values())
    print job_server.get_active_nodes()
    print "There are", N, " cores in total."
    return job_server

def concatenateParams(allparamsfile,morse_graphs_and_sets):
    numparams=0
    with open(allparamsfile,'w') as apf:
        for (mgraph,mset) in morse_graphs_and_sets:
            with open('param_temp.txt','w',0) as p:
                subprocess.call(["sqlite3 /share/data/CHomP/Projects/DSGRN/DB/data/{}.db 'select ParameterIndex from Signatures where MorseGraphIndex={}'".format(networkfilename,mgraph)],shell=True,stdout=p)
            with open('param_temp.txt','r') as p:
                for ms in mset:
                    for param in p.readlines():
                        numparams+=1
                        apf.write('{}|{}|{}'.format(mgraph,ms,param))
            removeFiles(['param_temp.txt'])
    return numparams

def main_conley3_filesystem(patternsetter,getMorseGraphs,networkfilename="5D_2015_09_11",morsegraphselection="stableFCs",morsegraph=0,morseset=0,paramsperslice=500,printtoscreen=0,findallmatches=0):
    # find all morse graphs with desired characteristic
    networkfilebasedir="/share/data/bcummins/DSGRN/networks/"
    morsegraphfile="/share/data/bcummins/morsegraphs/"+networkfilename+'_'+morsegraphselection+'_listofmorsegraphs.txt'
    getMorseGraphs(networkfilename,morsegraphfile,morsegraph,morseset)
    # set patterns
    patternstr=patternsetter()
    # set other file names and paths
    allparamsfile="/share/data/bcummins/parameterfiles/"+networkfilename+'_'+morsegraphselection+'_concatenatedparams.txt'
    resultsbasedir='/share/data/bcummins/parameterresults/'
    resultsfile=resultsbasedir+networkfilename
    allresultsfile=resultsfile+'_'+morsegraphselection+'_allresults.txt'
    # parse morse graphs
    morse_graphs_and_sets=fileparsers.parseMorseGraphs(morsegraphfile)
    # get all parameters from all morse graphs
    numparams=concatenateParams(allparamsfile,morse_graphs_and_sets)
    subparamfilestart=allparamsfile[:-4]
    numjobs = splitParamFile(allparamsfile,subparamfilestart,numparams,paramsperslice)
    return [numjobs,subparamfilestart,resultsfile+'_results_',allresultsfile,patternstr,networkfilebasedir+networkfilename+".txt",printtoscreen,findallmatches]

def main_local_filesystem(patternsetter,allparamsfile,networkfilename="5D_2015_09_11",morsegraphselection="stableFCs",paramsperslice=500,printtoscreen=0,findallmatches=0):
    # set patterns
    patternstr=patternsetter()
    # set file names and paths
    networkfilebasedir="/Users/bcummins/GIT/DSGRN/networks/"
    resultsbasedir='/Users/bcummins/patternmatch_helper_files/parameterresults/'
    resultsfile=resultsbasedir+networkfilename
    allresultsfile=resultsfile+'_'+morsegraphselection+'_allresults.txt'
    # get all parameters from all morse graphs
    with open(allparamsfile,'r') as apf:
        numparams=len(list(apf.readlines()))
    subparamfilestart=allparamsfile[:-4]
    numjobs = splitParamFile(allparamsfile,subparamfilestart,numparams,paramsperslice)
    return [numjobs,subparamfilestart,resultsfile+'_results_',allresultsfile,patternstr,networkfilebasedir+networkfilename+".txt",printtoscreen,findallmatches]

def setPattern_Malaria_20hr_2015_09_11():
    patternstr=[]
    for s1 in itertools.permutations(['x3 max','x5 max','x1 max', 'x4 max']):
        patternstr1=', '.join(s1) + ', x2 max, '
        for s2 in itertools.permutations(['x1 min', 'x3 min','x5  min', 'x2 min','x4 min']):
            patternstr2=patternstr1 + ', '.join(s2) + ', ' + s1[0] + '\n'
            patternstr.append(patternstr2)
    return ' '.join(patternstr)

def setPattern_5D_Cycle():
    return 'X3 max, X4 max, X3 min, X4 min\n X3 max, X4 min, X3 min, X4 max'

def setPattern_3D_Example():
    return 'Z min, X min, Y min, Z max, X max, Y max\n X max, Y max, Z max, X min, Y min, Z min\n X min, Y max, Z min, X max, Y min, Z max\n X max, Y min, Z max, X min, Y max, Z min'

def setPattern_3D_Cycle():
    return 'X3 min, X1 min, X2 min, X3 max, X1 max, X2 max\n X1 max, X2 max, X3 max, X1 min, X2 min, X3 min\n X1 min, X2 max, X3 min, X1 max, X2 min, X3 max\n X1 max, X2 min, X3 max, X1 min, X2 max, X3 min \n X1 max, X3 max, X1 min, X3 min\n X1 max, X3 max, X2 max, X1 min, X3 min, X2 min'

def selectStableFC(networkfile,morsegraphfile,junk1,junk2):
    with open(morsegraphfile,'w',0) as m:
        subprocess.call(['''sqlite3 /share/data/CHomP/Projects/DSGRN/DB/data/{}.db 'select MorseGraphIndex,Vertex from MorseGraphAnnotations where Label="FC" except select MorseGraphIndex,Source from MorseGraphEdges' '''.format(networkfile)],shell=True,stdout=m)

def selectAnyFC(networkfile,morsegraphfile,junk1,junk2):
    with open(morsegraphfile,'w',0) as m:
        subprocess.call(['''sqlite3 /share/data/CHomP/Projects/DSGRN/DB/data/{}.db 'select MorseGraphIndex,Vertex from MorseGraphAnnotations where Label="FC"' '''.format(networkfile)],shell=True,stdout=m)

def selectOneMorseGraph(networkfile,morsegraphfile,morsegraph,morseset):
    with open(morsegraphfile,'w',0) as m:
        m.write(str(morsegraph)+'|'+str(morseset))

if __name__=='__main__':
    # # Select which network to analyze
    # morsegraphselection="stableFCs"
    # getMorseGraphs=selectStableFC
    # networkfilename="5D_Cycle"
    # patternsetter=setPattern_5D_Cycle
    # getMorseGraphs=selectOneMorseGraph
    # morsegraph=38564
    # morseset=0
    # morsegraphselection="MG38564FC"
    networkfilename="5D_2015_09_11"
    patternsetter=setPattern_Malaria_20hr_2015_09_11
    # morsegraph=565
    # morseset=0
    morsegraph=None
    morseset=None
    morsegraphselection="anyFCs"
    getMorseGraphs=selectAnyFC
    # networkfilename="3D_Cycle"
    # patternsetter=setPattern_3D_Cycle
    paramsperslice=100

    # # Run on local file system
    # allparamsfile='/Users/bcummins/patternmatch_helper_files/parameterfiles/5D_2015_09_11_MG565_param1196132.txt'
    # with open(allparamsfile,'w') as ap:
    #     ap.write('565|0|1196132')
    # listofargs=main_local_filesystem(patternsetter,allparamsfile,networkfilename,morsegraphselection,paramsperslice,printtoscreen=0,findallmatches=0)
    # paramfile=listofargs[1]+'_0000.txt'
    # patternSearch(listofargs[5],paramfile,listofargs[2],listofargs[4],1,0)
    # # job_server = pp.Server(ppservers=("*",))
    # # N = sum(job_server.get_active_nodes().values())
    # # print "There are", N, " cores in total."
    # # parallelrun(job_server,*listofargs)

    # Run on conley3
    listofargs=main_conley3_filesystem(patternsetter,getMorseGraphs,networkfilename,morsegraphselection,morsegraph,morseset,paramsperslice,printtoscreen=0,findallmatches=0)
    job_server=initServer()
    parallelrun(job_server,*listofargs)

<commit_msg>added search for unstable FCs<commit_after># The MIT License (MIT)

# Copyright (c) 2015 Breschine Cummins

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import itertools
import subprocess
import patternmatch
import sys,time,traceback
import pp,os
import fileparsers
from math import ceil

# Call this module as python useDSGRN.py

# Given a Morse graph number MGN for a database DATABASEFILE (.db), do the following database searches:
#
# The following gives the list of parameters in the file PARAMFILE (.txt)
# sqlite3 /share/data/CHomP/Projects/DSGRN/DB/data/DATABASEFILE.db 'select ParameterIndex from Signatures where MorseGraphIndex=MGN' > ./PARAMFILE
#
# The following finds the Morse set number:
# sqlite3 /share/data/CHomP/Projects/DSGRN/DB/data/DATABASEFILE.db 'select * from MorseGraphAnnotations where MorseGraphIndex=MGN'
#

def patternSearch(networkfile,paramfile,resultsfile,patternstr,printtoscreen,findallmatches):
    # had to completely rewrite this function from scratch (copying it), in order to get
    # subprocess.call to redirect to a file. I wrote other short functions with the same
    # commands and didn't have this issue. Could there have been invisible characters?
    # At any rate, this is a potential source of failure in the future, since I have no
    # idea what happened.
    domaincells_jsonstr=getJSONstring(networkfile,['domaingraph'])
    results_list=[]
    with open(paramfile,'r') as P:
        for info in P.readlines():
            morsegraph,morseset,param = info.replace('|',' ').split()
            domaingraph_jsonstr=getJSONstring(networkfile,['domaingraph', 'json',param])
            morseset_jsonstr=getJSONstring(networkfile,['analyze', 'morseset', morseset, param])
            try:
                patterns,matches=patternmatch.callPatternMatch(morseset_jsonstr,domaingraph_jsonstr,domaincells_jsonstr,patternstr,resultsfile,writetofile=0,returnmatches=1,printtoscreen=printtoscreen,findallmatches=findallmatches)
                for pat,match in zip(patterns,matches):
                    if findallmatches:
                        results_list.append("Parameter: {}, Morse Graph: {}, Morse Set: {}, Pattern: {}, Results: {}".format(param,morsegraph,morseset,pat,match)+'\n')
                    else:
                        results_list.append("Parameter: {}, Morse Graph: {}, Morse Set: {}, Pattern: {}".format(param,morsegraph,morseset,pat)+'\n')
            except Exception as e:
                print 'Problem parameter is {}'.format(param)
                print traceback.format_exception_only(type(e),e)
                sys.stdout.flush()
    with open(resultsfile,'w',0) as R:
        for line in results_list:
            R.write(line)
    removeFiles([paramfile])

def getJSONstring(networkfile,command=['analyze', 'morseset', '0', '0']):
    return subprocess.check_output(['dsgrn','network', networkfile]+command,shell=False)

def splitParamFile(paramfile,subparamfilestart,numparams,paramsperslice):
    numjobs=int(ceil(float(numparams)/paramsperslice))
    with open(paramfile,'r') as apf:
        for n in range(numjobs):
            subparamfile=subparamfilestart+'_{:04d}.txt'.format(n)
            with open(subparamfile,'w',0) as subapf:
                for line in [apf.readline() for _ in range(paramsperslice)]:
                    if len(line):
                        subapf.write(line)
    return numjobs

def removeFiles(listoffiles):
    for f in listoffiles:
        subprocess.call(['rm '+f],shell=True)

def mergeFiles(mergefile,splitfiles):
    with open(mergefile,'w') as m:
        for sfile in splitfiles:
            with open(sfile,'r') as f:
                for l in f.readlines():
                    m.write(l)
    removeFiles(splitfiles)

def parallelrun(job_server,numjobs,subparamfilestart,resultsfile,allresultsfile,patternstr,networkfile,printtoscreen=0,findallmatches=0):
    jobs=[]
    allsubresultsfiles=[]
    for n in range(numjobs):
        unique_identifier='{:04d}'.format(n)
        subparamfile=subparamfilestart+'_'+unique_identifier+'.txt'
        subresultsfile=resultsfile+unique_identifier+'.txt'
        allsubresultsfiles.append(subresultsfile)
        jobs.append(job_server.submit(patternSearch,(networkfile,subparamfile,subresultsfile,patternstr,printtoscreen,findallmatches), depfuncs=(),modules = ("subprocess","patternmatch", "preprocess","fileparsers","walllabels","itertools","numpy","json","traceback","sys"),globals=globals()))
    print "{} jobs starting.".format(numjobs)
    sys.stdout.flush()
    for job in jobs:
        job()
    job_server.destroy()
    print "All jobs ended."
    sys.stdout.flush()
    mergeFiles(allresultsfile,allsubresultsfiles)
    print "Results files merged."
    sys.stdout.flush()

def initServer():
    secret = os.environ["PPSERVERSECRET"]
    ppservers = ("*",)
    job_server = pp.Server(ncpus=0,ppservers=ppservers,secret=secret)
    time.sleep(30)
    N = sum(job_server.get_active_nodes().values())
    print job_server.get_active_nodes()
    print "There are", N, " cores in total."
    return job_server

def concatenateParams(allparamsfile,morse_graphs_and_sets):
    numparams=0
    with open(allparamsfile,'w') as apf:
        for (mgraph,mset) in morse_graphs_and_sets:
            with open('param_temp.txt','w',0) as p:
                subprocess.call(["sqlite3 /share/data/CHomP/Projects/DSGRN/DB/data/{}.db 'select ParameterIndex from Signatures where MorseGraphIndex={}'".format(networkfilename,mgraph)],shell=True,stdout=p)
            with open('param_temp.txt','r') as p:
                for ms in mset:
                    for param in p.readlines():
                        numparams+=1
                        apf.write('{}|{}|{}'.format(mgraph,ms,param))
            removeFiles(['param_temp.txt'])
    return numparams

def main_conley3_filesystem(patternsetter,getMorseGraphs,networkfilename="5D_2015_09_11",morsegraphselection="stableFCs",morsegraph=0,morseset=0,paramsperslice=500,printtoscreen=0,findallmatches=0):
    # find all morse graphs with desired characteristic
    networkfilebasedir="/share/data/bcummins/DSGRN/networks/"
    morsegraphfile="/share/data/bcummins/morsegraphs/"+networkfilename+'_'+morsegraphselection+'_listofmorsegraphs.txt'
    getMorseGraphs(networkfilename,morsegraphfile,morsegraph,morseset)
    # set patterns
    patternstr=patternsetter()
    # set other file names and paths
    allparamsfile="/share/data/bcummins/parameterfiles/"+networkfilename+'_'+morsegraphselection+'_concatenatedparams.txt'
    resultsbasedir='/share/data/bcummins/parameterresults/'
    resultsfile=resultsbasedir+networkfilename
    allresultsfile=resultsfile+'_'+morsegraphselection+'_allresults.txt'
    # parse morse graphs
    morse_graphs_and_sets=fileparsers.parseMorseGraphs(morsegraphfile)
    # get all parameters from all morse graphs
    numparams=concatenateParams(allparamsfile,morse_graphs_and_sets)
    subparamfilestart=allparamsfile[:-4]
    numjobs = splitParamFile(allparamsfile,subparamfilestart,numparams,paramsperslice)
    return [numjobs,subparamfilestart,resultsfile+'_results_',allresultsfile,patternstr,networkfilebasedir+networkfilename+".txt",printtoscreen,findallmatches]

def main_local_filesystem(patternsetter,allparamsfile,networkfilename="5D_2015_09_11",morsegraphselection="stableFCs",paramsperslice=500,printtoscreen=0,findallmatches=0):
    # set patterns
    patternstr=patternsetter()
    # set file names and paths
    networkfilebasedir="/Users/bcummins/GIT/DSGRN/networks/"
    resultsbasedir='/Users/bcummins/patternmatch_helper_files/parameterresults/'
    resultsfile=resultsbasedir+networkfilename
    allresultsfile=resultsfile+'_'+morsegraphselection+'_allresults.txt'
    # get all parameters from all morse graphs
    with open(allparamsfile,'r') as apf:
        numparams=len(list(apf.readlines()))
    subparamfilestart=allparamsfile[:-4]
    numjobs = splitParamFile(allparamsfile,subparamfilestart,numparams,paramsperslice)
    return [numjobs,subparamfilestart,resultsfile+'_results_',allresultsfile,patternstr,networkfilebasedir+networkfilename+".txt",printtoscreen,findallmatches]

def setPattern_Malaria_20hr_2015_09_11():
    patternstr=[]
    for s1 in itertools.permutations(['x3 max','x5 max','x1 max', 'x4 max']):
        patternstr1=', '.join(s1) + ', x2 max, '
        for s2 in itertools.permutations(['x1 min', 'x3 min','x5  min', 'x2 min','x4 min']):
            patternstr2=patternstr1 + ', '.join(s2) + ', ' + s1[0] + '\n'
            patternstr.append(patternstr2)
    return ' '.join(patternstr)

def setPattern_5D_Cycle():
    return 'X3 max, X4 max, X3 min, X4 min\n X3 max, X4 min, X3 min, X4 max'

def setPattern_3D_Example():
    return 'Z min, X min, Y min, Z max, X max, Y max\n X max, Y max, Z max, X min, Y min, Z min\n X min, Y max, Z min, X max, Y min, Z max\n X max, Y min, Z max, X min, Y max, Z min'

def setPattern_3D_Cycle():
    return 'X3 min, X1 min, X2 min, X3 max, X1 max, X2 max\n X1 max, X2 max, X3 max, X1 min, X2 min, X3 min\n X1 min, X2 max, X3 min, X1 max, X2 min, X3 max\n X1 max, X2 min, X3 max, X1 min, X2 max, X3 min \n X1 max, X3 max, X1 min, X3 min\n X1 max, X3 max, X2 max, X1 min, X3 min, X2 min'

def selectStableFC(networkfile,morsegraphfile,junk1,junk2):
    with open(morsegraphfile,'w',0) as m:
        subprocess.call(['''sqlite3 /share/data/CHomP/Projects/DSGRN/DB/data/{}.db 'select MorseGraphIndex,Vertex from MorseGraphAnnotations where Label="FC" except select MorseGraphIndex,Source from MorseGraphEdges' '''.format(networkfile)],shell=True,stdout=m)

def selectUnstableFC(networkfile,morsegraphfile,junk1,junk2):
    with open(morsegraphfile,'w',0) as m:
        subprocess.call(['''sqlite3 /share/data/CHomP/Projects/DSGRN/DB/data/{}.db 'select MorseGraphIndex,Vertex from MorseGraphAnnotations where Label="FC" INTERSECT select MorseGraphIndex,Source from MorseGraphEdges' '''.format(networkfile)],shell=True,stdout=m)

def selectAnyFC(networkfile,morsegraphfile,junk1,junk2):
    with open(morsegraphfile,'w',0) as m:
        subprocess.call(['''sqlite3 /share/data/CHomP/Projects/DSGRN/DB/data/{}.db 'select MorseGraphIndex,Vertex from MorseGraphAnnotations where Label="FC"' '''.format(networkfile)],shell=True,stdout=m)

def selectOneMorseGraph(networkfile,morsegraphfile,morsegraph,morseset):
    with open(morsegraphfile,'w',0) as m:
        m.write(str(morsegraph)+'|'+str(morseset))

if __name__=='__main__':
    # # Select which network to analyze
    # morsegraphselection="stableFCs"
    # getMorseGraphs=selectStableFC
    # networkfilename="5D_Cycle"
    # patternsetter=setPattern_5D_Cycle
    # getMorseGraphs=selectOneMorseGraph
    # morsegraph=38564
    # morseset=0
    # morsegraphselection="MG38564FC"
    networkfilename="5D_2015_09_11"
    patternsetter=setPattern_Malaria_20hr_2015_09_11
    # morsegraph=565
    # morseset=0
    morsegraph=None
    morseset=None
    morsegraphselection="unstableFCs"
    getMorseGraphs=selectUnstableFC
    # networkfilename="3D_Cycle"
    # patternsetter=setPattern_3D_Cycle
    paramsperslice=100

    # # Run on local file system
    # allparamsfile='/Users/bcummins/patternmatch_helper_files/parameterfiles/5D_2015_09_11_MG565_param1196132.txt'
    # with open(allparamsfile,'w') as ap:
    #     ap.write('565|0|1196132')
    # listofargs=main_local_filesystem(patternsetter,allparamsfile,networkfilename,morsegraphselection,paramsperslice,printtoscreen=0,findallmatches=0)
    # paramfile=listofargs[1]+'_0000.txt'
    # patternSearch(listofargs[5],paramfile,listofargs[2],listofargs[4],1,0)
    # # job_server = pp.Server(ppservers=("*",))
    # # N = sum(job_server.get_active_nodes().values())
    # # print "There are", N, " cores in total."
    # # parallelrun(job_server,*listofargs)

    # Run on conley3
    listofargs=main_conley3_filesystem(patternsetter,getMorseGraphs,networkfilename,morsegraphselection,morsegraph,morseset,paramsperslice,printtoscreen=0,findallmatches=0)
    job_server=initServer()
    parallelrun(job_server,*listofargs)

<|endoftext|>