<commit_before>#!/usr/bin/env python
"""Find pull request from given commit hash and open it in a Web browser.

Depends on: git
Supported Services: GitHub, Bitbucket
"""

import argparse
import re
import subprocess
import sys
import webbrowser

_pull_request_url = {
    'github.com': 'https://github.com/{module}/pull/{number}',
    'bitbucket.org': 'https://bitbucket.org/{module}/pull-requests/{number}',
}

def extract_service_and_module(repo_url):
    """Extract service and module from repository url.

    :param str repo_url: repository url

    :return (service, module)
    :rtype (str, str)
    """
    m = re.match('.+[/@]([^\.]+\.[^\.]+)[:/]([^/]+/[^/]+)\.git$', repo_url)
    if not m:
        raise Exception(
            'cannot detect service and module from {}'.format(repo_url))
    service = m.group(1)
    module = m.group(2)
    if service not in _pull_request_url.keys():
        raise Exception(
            'service not supported: {}'.format(service))
    return (service, module)

def get_remote_url(remote):
    """Get remote repository url of current git repository.

    :param str remote: remote name against which pull requests are created

    :return: remote repository url
    :rtype: str
    """
    args = ['git', 'config', 'remote.{}.url'.format(remote)]
    return subprocess.check_output(args).strip().decode('utf-8')

def extract_pull_request_number(commit_logs):
    """Extract first occurance of pull request number from commit logs.

    :param str commit_logs: oneline commit logs

    :return: pull request number
    :rtype: str
    """
    m = re.search('pull request #(\d+)', commit_logs)
    if not m:
        raise Exception(
            'cannot detect pull request number from\n{}'.format(commit_logs))
    return m.group(1)

def get_pull_request_number(revision, remote, base_branch):
    """Get pull request number from commit messages.

    :param str revision: revision string
    :param str remote: remote name against which pull requests are created
    :param str base_branch: branch against which pull request are merged

    :return: pull request number
    :rtype: str
    """
    if not re.match('^[0-9a-f]+$', revision):
        raise Exception('invalid revision: {}'.format(revision))
    tracking_branch = '{}/{}'.format(remote, base_branch)
    args = ['git', 'merge-base',
            '--is-ancestor',
            revision,
            tracking_branch]
    try:
        subprocess.check_output(args)
    except subprocess.CalledProcessError as e:
        if e.returncode == 1:
            raise Exception(
                '{revision} is not merged to {tracking_branch} yet'.format(
                    **{'revision': revision,
                       'tracking_branch': tracking_branch}))
        raise
    args = ['git', 'log',
            '--merges', '--oneline', '--reverse', '--ancestry-path',
            '{revision}...{tracking_branch}'.format(
                **{'revision': revision, 'tracking_branch': tracking_branch})]
    output = subprocess.check_output(args).decode('utf-8')
    return extract_pull_request_number(output)

def get_pull_request_url(service, module, number):
    """Get pull-request URL.

    :param str service: service name
    :param str module: module path ('owner/repo')
    :param int number: pull-request identification number
    """
    if service not in _pull_request_url:
        raise Exception(
            'service not supported: {}'.format(service))
    url = _pull_request_url[service]
    return url.format(**{'module': module, 'number': number})

def openpr(revision, base_branch, remote, print_only):
    """Find pull request from given commit hash and open it in a Web browser.

    :param str revision: revision string of the target commit
    :param str remote: remote name against which pull requests are created
    :param str base_branch: branch against which pull requests are merged
    :param bool print_only: print pull request url instead of opening it
    """
    number = get_pull_request_number(revision, remote, base_branch)
    repo_url = get_remote_url(remote)
    (service, module) = extract_service_and_module(repo_url)
    pr_url = get_pull_request_url(service, module, number)
    if print_only:
        print(pr_url)
    else:
        webbrowser.open(pr_url)

def main():
    parser = argparse.ArgumentParser(
        description='Find pull request from given commit hash and open it in a Web browser.')
    parser.add_argument(
        'revision',
        help='revision string of the target commit')
    parser.add_argument(
        '-b', '--base-branch',
        metavar='BRANCH',
        default='master',
        help='branch against which pull requests are merged (default: master)')
    parser.add_argument(
        '-r', '--remote',
        default='origin',
        help='remote name against which pull requests are created (default: origin)')
    parser.add_argument(
        '-p', '--print-only',
        action='store_true',
        help='print pull request url instead of opening it')
    args = parser.parse_args()
    try:
        openpr(args.revision, args.base_branch, args.remote, args.print_only)
        sys.exit(0)
    except Exception as e:
        print(e)
        sys.exit(-1)

if __name__ == '__main__':
    main()
<commit_msg>detect default branch of cloned repository<commit_after>#!/usr/bin/env python
"""Find pull request from given commit hash and open it in a Web browser.

Depends on: git
Supported Services: GitHub, Bitbucket
"""

import argparse
import os
import re
import subprocess
import sys
import webbrowser

_pull_request_url = {
    'github.com': 'https://github.com/{module}/pull/{number}',
    'bitbucket.org': 'https://bitbucket.org/{module}/pull-requests/{number}',
}

def extract_service_and_module(repo_url):
    """Extract service and module from repository url.

    :param str repo_url: repository url

    :return (service, module)
    :rtype (str, str)
    """
    m = re.match('.+[/@]([^\.]+\.[^\.]+)[:/]([^/]+/[^/]+)\.git$', repo_url)
    if not m:
        raise Exception(
            'cannot detect service and module from {}'.format(repo_url))
    service = m.group(1)
    module = m.group(2)
    if service not in _pull_request_url.keys():
        raise Exception(
            'service not supported: {}'.format(service))
    return (service, module)

def _run_command(args):
    """Run command and get its output.

    :param list[str] args: command and arguments

    :return: output of the command
    :rtype: str

    :raise subprocess.CalledProcessError: command failed
    """
    # don't use subprocess.DEVNULL for python2.7
    with open(os.devnull, 'w') as devnull:
        return subprocess.check_output(
            args, stderr=devnull).strip().decode('utf-8')

def get_remote_url(remote):
    """Get remote repository url of current git repository.

    :param str remote: remote name against which pull requests are created

    :return: remote repository url
    :rtype: str
    """
    args = ['git', 'config', 'remote.{}.url'.format(remote)]
    return _run_command(args)

def extract_pull_request_number(commit_logs):
    """Extract first occurance of pull request number from commit logs.

    :param str commit_logs: oneline commit logs

    :return: pull request number
    :rtype: str
    """
    m = re.search('pull request #(\d+)', commit_logs)
    if not m:
        raise Exception(
            'cannot detect pull request number from\n{}'.format(commit_logs))
    return m.group(1)

def get_default_tracking_branch(remote):
    """Get default remote branch of the current repository.

    detect by remote HEAD if present, or use "master" otherwise

    :param str remote: remote name

    :return: branch name
    :rtype: str
    """
    head = 'remotes/{}/HEAD'.format(remote)
    args = ['git', 'rev-parse', '--abbrev-ref', head]
    try:
        ref = _run_command(args)
        if ref == head:
            raise Exception
        return ref
    except Exception:
        return '{}/master'.format(remote)

def get_pull_request_number(revision, remote, base_branch):
    """Get pull request number from commit messages.

    :param str revision: revision string
    :param str remote: remote name against which pull requests are created
    :param str base_branch: branch against which pull request are merged

    :return: pull request number
    :rtype: str
    """
    if not re.match('^[0-9a-f]+$', revision):
        raise Exception('invalid revision: {}'.format(revision))
    if not base_branch:
        tracking_branch = get_default_tracking_branch(remote)
    else:
        tracking_branch = '{}/{}'.format(remote, base_branch)
    args = ['git', 'merge-base',
            '--is-ancestor',
            revision,
            tracking_branch]
    try:
        _run_command(args)
    except subprocess.CalledProcessError as e:
        if e.returncode == 1:
            raise Exception(
                '{revision} is not merged to {tracking_branch} yet'.format(
                    **{'revision': revision,
                       'tracking_branch': tracking_branch}))
        raise
    args = ['git', 'log',
            '--merges', '--oneline', '--reverse', '--ancestry-path',
            '{revision}...{tracking_branch}'.format(
                **{'revision': revision, 'tracking_branch': tracking_branch})]
    output = subprocess.check_output(args).decode('utf-8')
    return extract_pull_request_number(output)

def get_pull_request_url(service, module, number):
    """Get pull-request URL.

    :param str service: service name
    :param str module: module path ('owner/repo')
    :param int number: pull-request identification number
    """
    if service not in _pull_request_url:
        raise Exception(
            'service not supported: {}'.format(service))
    url = _pull_request_url[service]
    return url.format(**{'module': module, 'number': number})

def openpr(revision, base_branch, remote, print_only):
    """Find pull request from given commit hash and open it in a Web browser.

    :param str revision: revision string of the target commit
    :param str remote: remote name against which pull requests are created
    :param str base_branch: branch against which pull requests are merged
    :param bool print_only: print pull request url instead of opening it
    """
    number = get_pull_request_number(revision, remote, base_branch)
    repo_url = get_remote_url(remote)
    (service, module) = extract_service_and_module(repo_url)
    pr_url = get_pull_request_url(service, module, number)
    if print_only:
        print(pr_url)
    else:
        webbrowser.open(pr_url)

def main():
    parser = argparse.ArgumentParser(
        description='Find pull request from given commit hash and open it in a Web browser.')
    parser.add_argument(
        'revision',
        help='revision string of the target commit')
    parser.add_argument(
        '-b', '--base-branch',
        metavar='BRANCH',
        default='',
        help='branch against which pull requests are merged (default: detect by remote HEAD if present, or use "master" otherwise')
    parser.add_argument(
        '-r', '--remote',
        default='origin',
        help='remote name against which pull requests are created (default: origin)')
    parser.add_argument(
        '-p', '--print-only',
        action='store_true',
        help='print pull request url instead of opening it')
    args = parser.parse_args()
    try:
        openpr(args.revision, args.base_branch, args.remote, args.print_only)
        sys.exit(0)
    except Exception as e:
        print(e)
        sys.exit(-1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Slugsin parser with BDD backend.

This module translates Boolean formulas in prefix format
to binary decision diagrams.

Reference
=========

Slugsin syntax:
    https://github.com/LTLMoP/slugs/blob/master/doc/input_formats.md
"""
from __future__ import absolute_import
import logging
import ply.lex
from logic.ast import Nodes as _Nodes

logger = logging.getLogger(__name__)
slugs_log = logging.getLogger(__name__ + '.slugs')

class SlugsLexer(object):
    """Token rules for slugsin lexer."""

    operators = ['NOT', 'AND', 'OR', 'XOR', 'DOLLAR', 'QUESTION']
    identifiers = ['NAME', 'NUMBER']

    t_NUMBER = r'\d+'
    t_NAME = r"[A-Za-z_][A-Za-z0-9_']*"
    t_NOT = r'\!'
    t_AND = r'\&'
    t_OR = r'\|'
    t_XOR = r'\^'
    t_DOLLAR = r'\$'
    t_QUESTION = r'\?'
    t_ignore = ' \t'

    def __init__(self):
        self.tokens = self.operators + self.identifiers
        self.build()

    def build(self):
        log = logging.getLogger(__name__ + '.slugs_logger')
        self.lexer = ply.lex.lex(module=self, debug=True, debuglog=log)

    def t_error(self, t):
        raise Exception('Illegal character "{t}"'.format(t=t.value[0]))

class PrefixParser(object):
    """Parser for prefix syntax with buffers."""
    # Context-sensitive grammar, so cannot use PLY

    def __init__(self, nodes=None):
        self.lexer = SlugsLexer()
        self.tokens = self.lexer.tokens
        self._binary = {'AND', 'OR', 'XOR'}
        if nodes is None:
            nodes = MemoryNodes()
        self._ast = nodes

    def parse(self, data):
        self.lexer.lexer.input(data)
        r = self._recurse()
        # empty stack ?
        tok = self.lexer.lexer.token()
        if not tok:
            return r
        # error
        s = tok.value
        while True:
            tok = self.lexer.lexer.token()
            if not tok:
                break
            s += tok.value
        raise Exception(
            'syntax error: remaining characters: {s}'.format(
                s=s))

    def _recurse(self):
        tok = self.lexer.lexer.token()
        if not tok:
            raise Exception('syntax error: stream ended early')
        t = tok.type
        if t == 'NOT':
            x = self._recurse()
            return self._ast.Operator('!', x)
        elif t in self._binary:
            op = tok.value
            x = self._recurse()
            y = self._recurse()
            return self._ast.Operator(op, x, y)
        elif t == 'DOLLAR':
            u = self._recurse()
            assert u.type == 'num', u.type
            n = int(u.value)
            mem = [self._recurse() for i in xrange(n)]
            return self._ast.Buffer(mem)
        elif t == 'QUESTION':
            u = self._recurse()
            assert u.type == 'num', u.type
            return self._ast.Register(u.value)
        elif t == 'NAME':
            return self._ast.Var(tok.value)
        elif t == 'NUMBER':
            return self._ast.Num(tok.value)
        else:
            raise Exception('Unknown token "{t}"'.format(t=tok))

class MemoryNodes(_Nodes):

    # difference with slugs parser:
    # cyclic references are possible, but assumed absent
    # any cyclic reference would introduce infinite recursion,
    # so a new variable
    class Buffer(object):
        def __init__(self, memory):
            self.memory = memory
            self.type = 'buffer'

        def __repr__(self):
            return 'Memory({c})'.format(
                c=', '.join(repr(u) for u in self.memory))

        def flatten(self, indent=0, *arg, **kw):
            indent += 1
            sep = ',\n{s}'.format(s=' ' * indent)
            mem = sep.join(
                u.flatten(indent=indent)
                for u in self.memory)
            return '\n{s}buffer[{i}](\n{s1}{mem})'.format(
                i=len(self.memory),
                mem=mem,
                s=' ' * (indent - 1),
                s1=' ' * indent)

    class Register(_Nodes.Terminal):
        def __init__(self, value):
            super(MemoryNodes.Register, self).__init__(value)
            self.type = 'register'

        def flatten(self, *arg, **kw):
            return 'reg[{i}]'.format(i=self.value)

    # infix Binary flattening
    class Operator(_Nodes.Operator):
        def flatten(self, *arg, **kw):
            if len(self.operands) == 2:
                return MemoryNodes.Binary.flatten(self)
            else:
                return super(MemoryNodes.Operator, self).flatten()

class BDDNodes(MemoryNodes):
    """Factory of AST to flatten prefix syntax to a BDD."""

    class Buffer(MemoryNodes.Buffer):
        def flatten(self, bdd, *arg, **kw):
            mem = list()
            for u in self.memory:
                s = u.flatten(bdd=bdd, mem=mem)
                mem.append(s)
            r = mem[-1]
            # print 'mem: ', mem, ', res: ', r
            return r

    class Operator(MemoryNodes.Operator):
        def flatten(self, bdd, *arg, **kw):
            operands = [
                u.flatten(bdd=bdd, *arg, **kw)
                for u in self.operands]
            u = bdd.apply(self.operator, *operands)
            # print 'op: ', self.operator, operands, u
            return u

    class Register(MemoryNodes.Register):
        def flatten(self, bdd, mem, *arg, **kw):
            i = int(self.value)
            # no circular refs
            assert 0 <= i < len(mem), (i, mem)
            r = mem[i]
            # print 'reg: ', i, ', of mem: ', mem, ', contains: ', r
            return r

    class Var(MemoryNodes.Var):
        def flatten(self, bdd, *arg, **kw):
            u = bdd.add_ast(self)
            # print 'add var: ', self.value, u
            return u

    class Num(MemoryNodes.Num):
        def flatten(self, bdd, *arg, **kw):
            # the only registers in the tree are Boolean constants
            assert self.value in ('0', '1'), self.value
            u = int(self.value)
            if u == 0:
                u = -1
            return u
<commit_msg>refactor prefix parser<commit_after>#!/usr/bin/env python
"""Slugsin parser with BDD backend.

This module translates Boolean formulas in prefix format
to binary decision diagrams.

Reference
=========

Slugsin syntax:
    https://github.com/LTLMoP/slugs/blob/master/doc/input_formats.md
"""
from __future__ import absolute_import
import logging
import ply.lex
from logic.ast import Nodes as _Nodes

logger = logging.getLogger(__name__)
slugs_log = logging.getLogger(__name__ + '.slugs')

class Lexer(object):
    """Token rules for slugsin lexer."""

    operators = ['NOT', 'AND', 'OR', 'XOR', 'DOLLAR', 'QUESTION']
    identifiers = ['NAME', 'NUMBER']

    t_NUMBER = r'\d+'
    t_NAME = r"[A-Za-z_][A-Za-z0-9_']*"
    t_NOT = r'\!'
    t_AND = r'\&'
    t_OR = r'\|'
    t_XOR = r'\^'
    t_DOLLAR = r'\$'
    t_QUESTION = r'\?'
    t_ignore = ' \t'

    def __init__(self):
        self.tokens = self.operators + self.identifiers
        self.build()

    def build(self):
        log = logging.getLogger(__name__ + '.slugs_logger')
        self.lexer = ply.lex.lex(module=self, debug=True, debuglog=log)

    def t_error(self, t):
        raise Exception('Illegal character "{t}"'.format(t=t.value[0]))

class Parser(object):
    """Parser for prefix syntax with buffers."""
    # Context-sensitive grammar, so cannot use PLY

    def __init__(self, nodes=None):
        if nodes is None:
            nodes = Nodes()
        self._nodes = nodes
        self.lexer = Lexer()
        self.tokens = self.lexer.tokens
        self._binary = {'AND', 'OR', 'XOR'}

    def parse(self, data):
        self.lexer.lexer.input(data)
        r = self._recurse()
        # empty stack ?
        tok = self.lexer.lexer.token()
        if not tok:
            return r
        # error
        s = tok.value
        while True:
            tok = self.lexer.lexer.token()
            if not tok:
                break
            s += tok.value
        raise Exception(
            'syntax error: remaining characters: {s}'.format(
                s=s))

    def _recurse(self):
        tok = self.lexer.lexer.token()
        if not tok:
            raise Exception('syntax error: stream ended early')
        t = tok.type
        if t == 'NOT':
            x = self._recurse()
            return self._nodes.Operator('!', x)
        elif t in self._binary:
            op = tok.value
            x = self._recurse()
            y = self._recurse()
            return self._nodes.Operator(op, x, y)
        elif t == 'DOLLAR':
            u = self._recurse()
            assert u.type == 'num', u.type
            n = int(u.value)
            mem = [self._recurse() for i in xrange(n)]
            return self._nodes.Buffer(mem)
        elif t == 'QUESTION':
            u = self._recurse()
            assert u.type == 'num', u.type
            return self._nodes.Register(u.value)
        elif t == 'NAME':
            return self._nodes.Var(tok.value)
        elif t == 'NUMBER':
            return self._nodes.Num(tok.value)
        else:
            raise Exception('Unknown token "{t}"'.format(t=tok))

class Nodes(_Nodes):
    """AST that that evaluates memory buffers."""

    # difference with slugs parser:
    # cyclic references are possible, but assumed absent
    # any cyclic reference would introduce infinite recursion,
    # so a new variable
    class Buffer(object):
        def __init__(self, memory):
            self.memory = memory
            self.type = 'buffer'

        def __repr__(self):
            return 'Memory({c})'.format(
                c=', '.join(repr(u) for u in self.memory))

        def flatten(self, mem=None, *arg, **kw):
            mem = list()
            for u in self.memory:
                s = u.flatten(mem=mem, *arg, **kw)
                mem.append(s)
            r = mem[-1]
            # print 'mem: ', mem, ', res: ', r
            return r

    class Register(_Nodes.Terminal):
        def __init__(self, value):
            super(Nodes.Register, self).__init__(value)
            self.type = 'register'

        def flatten(self, mem, *arg, **kw):
            i = int(self.value)
            # no circular refs
            assert 0 <= i < len(mem), (i, mem)
            r = mem[i]
            # print 'reg: ', i, ', of mem: ', mem, ', contains: ', r
            return r

    # infix Binary flattening
    class Operator(_Nodes.Operator):
        def flatten(self, *arg, **kw):
            if len(self.operands) == 1:
                return ' '.join([
                    '(',
                    self.operator,
                    self.operands[0].flatten(*arg, **kw),
                    ')'])
            assert len(self.operands) == 2, self.operands
            return ' '.join([
                '(',
                self.operands[0].flatten(*arg, **kw),
                self.operator,
                self.operands[1].flatten(*arg, **kw),
                ')'])

class DebugNodes(Nodes):
    """AST to expand memory buffers."""

    class Buffer(Nodes.Buffer):
        def flatten(self, indent=0, *arg, **kw):
            indent += 1
            sep = ',\n{s}'.format(s=' ' * indent)
            mem = sep.join(
                u.flatten(indent=indent)
                for u in self.memory)
            return '\n{s}buffer[{i}](\n{s1}{mem})'.format(
                i=len(self.memory),
                mem=mem,
                s=' ' * (indent - 1),
                s1=' ' * indent)

    class Register(Nodes.Register):
        def flatten(self, *arg, **kw):
            return 'reg[{i}]'.format(i=self.value)

class BDDNodes(Nodes):
    """AST to flatten prefix syntax to a BDD."""

    class Operator(Nodes.Operator):
        def flatten(self, bdd, *arg, **kw):
            operands = [
                u.flatten(bdd=bdd, *arg, **kw)
                for u in self.operands]
            u = bdd.apply(self.operator, *operands)
            # print 'op: ', self.operator, operands, u
            return u

    class Var(Nodes.Var):
        def flatten(self, bdd, *arg, **kw):
            u = bdd.add_ast(self)
            # print 'add var: ', self.value, u
            return u

    class Num(Nodes.Num):
        def flatten(self, bdd, *arg, **kw):
            # the only numbers in the tree are Boolean constants
            assert self.value in ('0', '1'), self.value
            u = int(self.value)
            if u == 0:
                u = -1
            return u
<|endoftext|>


<commit_before># This tests the label inference pipeline. It uses fake data and placeholder inference algorithms
# and thus intentionally does not test the real inference algorithms themselves.
import unittest
import numpy as np
import time

import emission.analysis.classification.inference.labels.pipeline as eacilp
import emission.core.wrapper.labelprediction as ecwl
import emission.storage.decorations.analysis_timeseries_queries as esda
import emission.storage.timeseries.timequery as estt
import emission.storage.decorations.trip_queries as esdt
import emission.core.get_database as edb
import emission.tests.common as etc

class TestLabelInferencePipeline(unittest.TestCase):
    # It is important that these functions be deterministic
    test_algorithms = {
            ecwl.AlgorithmTypes.PLACEHOLDER_0: eacilp.placeholder_predictor_0,
            ecwl.AlgorithmTypes.PLACEHOLDER_2: eacilp.placeholder_predictor_2
    }

    def setUp(self):
        np.random.seed(61297777)
        self.reset_all()
        # Note that shankari_2015-07-22 is different from shankari_2015-jul-22 and that if you use shankari_2015-jul-22 things will fail silently!!! Took me a lot of time to find that bug.
        etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2015-07-22")
        self.run_pipeline(self.test_algorithms)
        time_range = estt.TimeQuery("metadata.write_ts", None, time.time())
        self.cleaned_trips = esda.get_entries(esda.CLEANED_TRIP_KEY, self.testUUID, time_query=time_range)
        self.cleaned_id_to_trip = {trip.get_id(): trip for trip in self.cleaned_trips}
        self.inferred_trips = esda.get_entries(esda.INFERRED_TRIP_KEY, self.testUUID, time_query=time_range)

    def tearDown(self):
        self.reset_all()

    def run_pipeline(self, algorithms):
        default_primary_algorithms = eacilp.primary_algorithms
        eacilp.primary_algorithms = algorithms
        etc.runIntakePipeline(self.testUUID)
        eacilp.primary_algorithms = default_primary_algorithms

    def reset_all(self):
        etc.dropAllCollections(edb._get_current_db())

    # Tests that the fields from the cleaned trip are carried over into the inferred trip correctly
    def testPipelineIntegrity(self):
        self.assertGreaterEqual(len(self.inferred_trips), 1)  # Make sure we've actually loaded trips
        self.assertEqual(len(self.inferred_trips), len(self.cleaned_trips))
        for inferred_trip in self.inferred_trips:
            cleaned_id = inferred_trip["data"]["cleaned_trip"]
            self.assertIn(cleaned_id, self.cleaned_id_to_trip.keys())
            cleaned_trip = self.cleaned_id_to_trip[cleaned_id]
            self.assertEqual(inferred_trip["data"]["raw_trip"], cleaned_trip["data"]["raw_trip"])
            self.assertTrue(inferred_trip["data"]["inferred_labels"])  # Check for existence here, check for congruence later

    # Tests that each of the (test) algorithms runs and saves to the database correctly
    def testIndividualAlgorithms(self):
        for trip in self.inferred_trips:
            entries = esdt.get_sections_for_trip("inference/labels", self.testUUID, trip.get_id())
            self.assertEqual(len(entries), len(self.test_algorithms))
            for entry in entries:
                self.assertEqual(entry["data"]["trip_id"], trip.get_id())
                this_algorithm = ecwl.AlgorithmTypes(entry["data"]["algorithm_id"])
                self.assertIn(this_algorithm, self.test_algorithms)
                self.assertEqual(entry["data"]["prediction"], self.test_algorithms[this_algorithm](trip))
                self.assertEqual(entry["data"]["start_ts"], trip["data"]["start_ts"])
                self.assertEqual(entry["data"]["end_ts"], trip["data"]["end_ts"])

    # Tests that the ensemble algorithm runs and saves to the database correctly
    def testEnsemble(self):
        for trip in self.inferred_trips:
            entries = esdt.get_sections_for_trip("analysis/inferred_labels", self.testUUID, trip.get_id())
            self.assertEqual(len(entries), 1)
            entry = entries[0]
            # TODO: when we have a real ensemble implemented:
            # self.assertEqual(entry["data"]["algorithm_id"], ecwl.AlgorithmTypes.ENSEMBLE)
            # TODO: perhaps assert something about the prediction when we have a real ensemble
            self.assertEqual(entry["data"]["start_ts"], trip["data"]["start_ts"])
            self.assertEqual(entry["data"]["end_ts"], trip["data"]["end_ts"])

    # Tests that the final inferred labels in the inferred trip are the same as those given by the ensemble algorithm
    def testInferredTrip(self):
        for trip in self.inferred_trips:
            entry = esdt.get_sections_for_trip("analysis/inferred_labels", self.testUUID, trip.get_id())[0]
            self.assertEqual(trip["data"]["inferred_labels"], entry["data"]["prediction"])

def main():
    etc.configLogging()
    unittest.main()

if __name__ == "__main__":
    main()
<commit_msg>Edit test to account for refactoring<commit_after># This tests the label inference pipeline. It uses fake data and placeholder inference algorithms
# and thus intentionally does not test the real inference algorithms themselves.
import unittest
import numpy as np
import time

import emission.analysis.classification.inference.labels.pipeline as eacilp
import emission.analysis.classification.inference.labels.inferrers as eacili
import emission.core.wrapper.labelprediction as ecwl
import emission.storage.decorations.analysis_timeseries_queries as esda
import emission.storage.timeseries.timequery as estt
import emission.storage.decorations.trip_queries as esdt
import emission.core.get_database as edb
import emission.tests.common as etc

class TestLabelInferencePipeline(unittest.TestCase):
    # It is important that these functions be deterministic
    test_algorithms = {
            ecwl.AlgorithmTypes.PLACEHOLDER_0: eacili.placeholder_predictor_0,
            ecwl.AlgorithmTypes.PLACEHOLDER_2: eacili.placeholder_predictor_2
    }

    def setUp(self):
        np.random.seed(61297777)
        self.reset_all()
        # Note that shankari_2015-07-22 is different from shankari_2015-jul-22 and that if you use shankari_2015-jul-22 things will fail silently!!! Took me a lot of time to find that bug.
        etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2015-07-22")
        self.run_pipeline(self.test_algorithms)
        time_range = estt.TimeQuery("metadata.write_ts", None, time.time())
        self.cleaned_trips = esda.get_entries(esda.CLEANED_TRIP_KEY, self.testUUID, time_query=time_range)
        self.cleaned_id_to_trip = {trip.get_id(): trip for trip in self.cleaned_trips}
        self.inferred_trips = esda.get_entries(esda.INFERRED_TRIP_KEY, self.testUUID, time_query=time_range)

    def tearDown(self):
        self.reset_all()

    def run_pipeline(self, algorithms):
        default_primary_algorithms = eacilp.primary_algorithms
        eacilp.primary_algorithms = algorithms
        etc.runIntakePipeline(self.testUUID)
        eacilp.primary_algorithms = default_primary_algorithms

    def reset_all(self):
        etc.dropAllCollections(edb._get_current_db())

    # Tests that the fields from the cleaned trip are carried over into the inferred trip correctly
    def testPipelineIntegrity(self):
        self.assertGreaterEqual(len(self.inferred_trips), 1)  # Make sure we've actually loaded trips
        self.assertEqual(len(self.inferred_trips), len(self.cleaned_trips))
        for inferred_trip in self.inferred_trips:
            cleaned_id = inferred_trip["data"]["cleaned_trip"]
            self.assertIn(cleaned_id, self.cleaned_id_to_trip.keys())
            cleaned_trip = self.cleaned_id_to_trip[cleaned_id]
            self.assertEqual(inferred_trip["data"]["raw_trip"], cleaned_trip["data"]["raw_trip"])
            self.assertTrue(inferred_trip["data"]["inferred_labels"])  # Check for existence here, check for congruence later

    # Tests that each of the (test) algorithms runs and saves to the database correctly
    def testIndividualAlgorithms(self):
        for trip in self.inferred_trips:
            entries = esdt.get_sections_for_trip("inference/labels", self.testUUID, trip.get_id())
            self.assertEqual(len(entries), len(self.test_algorithms))
            for entry in entries:
                self.assertEqual(entry["data"]["trip_id"], trip.get_id())
                this_algorithm = ecwl.AlgorithmTypes(entry["data"]["algorithm_id"])
                self.assertIn(this_algorithm, self.test_algorithms)
                self.assertEqual(entry["data"]["prediction"], self.test_algorithms[this_algorithm](trip))
                self.assertEqual(entry["data"]["start_ts"], trip["data"]["start_ts"])
                self.assertEqual(entry["data"]["end_ts"], trip["data"]["end_ts"])

    # Tests that the ensemble algorithm runs and saves to the database correctly
    def testEnsemble(self):
        for trip in self.inferred_trips:
            entries = esdt.get_sections_for_trip("analysis/inferred_labels", self.testUUID, trip.get_id())
            self.assertEqual(len(entries), 1)
            entry = entries[0]
            # TODO: when we have a real ensemble implemented:
            # self.assertEqual(entry["data"]["algorithm_id"], ecwl.AlgorithmTypes.ENSEMBLE)
            # TODO: perhaps assert something about the prediction when we have a real ensemble
            self.assertEqual(entry["data"]["start_ts"], trip["data"]["start_ts"])
            self.assertEqual(entry["data"]["end_ts"], trip["data"]["end_ts"])

    # Tests that the final inferred labels in the inferred trip are the same as those given by the ensemble algorithm
    def testInferredTrip(self):
        for trip in self.inferred_trips:
            entry = esdt.get_sections_for_trip("analysis/inferred_labels", self.testUUID, trip.get_id())[0]
            self.assertEqual(trip["data"]["inferred_labels"], entry["data"]["prediction"])

def main():
    etc.configLogging()
    unittest.main()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>
"""Resource manager for datablox-fileserver 1.0
"""

# Common stdlib imports
import sys
import os
import os.path
import random
## import commands

# fix path if necessary (if running from source or running as test)
try:
    import engage.utils
except:
    sys.exc_clear()
    dir_to_add_to_python_path = os.path.abspath((os.path.join(os.path.dirname(__file__), "../../../..")))
    sys.path.append(dir_to_add_to_python_path)

import engage.drivers.service_manager as service_manager
import engage.drivers.utils
# Drivers compose *actions* to implement their methods.
from engage.drivers.action import *

# setup errors
from engage.utils.user_error import UserError, EngageErrInf
import gettext
_ = gettext.gettext

errors = { }

def define_error(error_code, msg):
    global errors
    error_info = EngageErrInf(__name__, error_code, msg)
    errors[error_info.error_code] = error_info

# error codes
# FILL IN
ERR_BAD_PW_FILE = 1

define_error(ERR_BAD_PW_FILE,
             _("Master password file path seems to be changed from the default, which is not supported. Expecting %(exp)s, got %(act)s"))

# setup logging
from engage.utils.log_setup import setup_engage_logger
logger = setup_engage_logger(__name__)

# this is used by the package manager to locate the packages.json
# file associated with the driver
def get_packages_filename():
    return engage.drivers.utils.get_packages_filename(__file__)

def make_context(resource_json, sudo_password_fn, dry_run=False):
    """Create a Context object (defined in engage.utils.action). This contains
    the resource's metadata in ctx.props, references to the logger and sudo
    password function, and various helper functions. The context object is used
    by individual actions.

    If your resource does not need the sudo password, you can just pass in
    None for sudo_password_fn.
    """
    ctx = Context(resource_json, logger, __file__,
                  sudo_password_fn=sudo_password_fn,
                  dry_run=dry_run)
    ctx.check_port('config_port',
                  pid_file=unicode,
                  config_dir=unicode,
                  log_file=unicode)
    ctx.check_port('input_ports.datablox_framework',
                  BLOXPATH=unicode)
    ctx.check_port('input_ports.gunicorn',
                  gunicorn_exe=unicode)
    ctx.check_port('input_ports.host',
                   genforma_home=unicode,
                   log_directory=unicode)

    # add any extra computed properties here using the ctx.add() method.
    ctx.add('server_key_file',
            os.path.join(ctx.props.input_ports.host.genforma_home,
                         "datablox_file_server_key"))
    return ctx

def np(path):
    return os.path.abspath(os.path.expanduser(path))

def gen_random(length, chars=string.letters+string.digits):
    return ''.join([ random.choice(chars) for i in range(length) ])

@make_action
def write_key_file(self, path):
    deskey = gen_random(8)
    with open(path, 'w') as f:
        f.write(deskey)
    os.chmod(path, 0600)

# Now, define the main resource manager class for the driver. If this driver is
# a service, inherit from service_manager.Manager instead of
# resource_manager.Manager. If you need the sudo password, add
# PasswordRepoMixin to the inheritance list.
#
class Manager(service_manager.Manager):
    def __init__(self, metadata, dry_run=False):
        package_name = "%s %s" % (metadata.key["name"],
                                  metadata.key["version"])
        service_manager.Manager.__init__(self, metadata, package_name)
        self.ctx = make_context(metadata.to_json(),
                                None, # self._get_sudo_password,
                                dry_run=dry_run)

    def validate_pre_install(self):
        pass

    def is_installed(self):
        return os.path.exists(self.ctx.props.config_port.config_dir)

    def install(self, package):
        p = self.ctx.props
        r = self.ctx.r
        r(ensure_dir_exists, p.config_port.config_dir)
        r(write_key_file, p.server_key_file)

    def validate_post_install(self):
        pass

    def start(self):
        p = self.ctx.props
        cmd = [p.input_ports.gunicorn.gunicorn_exe,
               "--daemon", "--log-file=%s" % p.config_port.log_file,
               "--pid=%s" % p.config_port.pid_file, "-w", "4",
               "--bind=%s:%d" % ("0.0.0.0", 4990),
               "datablox_framework.fileserver_wsgi:app"]
        self.ctx.r(run_program, cmd,
                   cwd=os.path.dirname(p.input_ports.gunicorn.gunicorn_exe))
        self.ctx.check_poll(10, 1.0, lambda x: x!=None,
                            get_server_status, p.config_port.pid_file)

    def stop(self):
        p = self.ctx.props
        self.ctx.r(stop_server,
                   p.config_port.pid_file)
        self.ctx.check_poll(10, 1.0, lambda x: x!=None,
                            get_server_status, p.config_port.pid_file)

    def is_running(self):
        p = self.ctx.props
        return self.ctx.rv(get_server_status,
                           p.config_port.pid_file) != None

    def get_pid_file_path(self):
        return self.ctx.props.config_port.pid_file
<commit_msg>Fix bug in shutdown<commit_after>
"""Resource manager for datablox-fileserver 1.0
"""

# Common stdlib imports
import sys
import os
import os.path
import random
## import commands

# fix path if necessary (if running from source or running as test)
try:
    import engage.utils
except:
    sys.exc_clear()
    dir_to_add_to_python_path = os.path.abspath((os.path.join(os.path.dirname(__file__), "../../../..")))
    sys.path.append(dir_to_add_to_python_path)

import engage.drivers.service_manager as service_manager
import engage.drivers.utils
# Drivers compose *actions* to implement their methods.
from engage.drivers.action import *

# setup errors
from engage.utils.user_error import UserError, EngageErrInf
import gettext
_ = gettext.gettext

errors = { }

def define_error(error_code, msg):
    global errors
    error_info = EngageErrInf(__name__, error_code, msg)
    errors[error_info.error_code] = error_info

# error codes
# FILL IN
ERR_BAD_PW_FILE = 1

define_error(ERR_BAD_PW_FILE,
             _("Master password file path seems to be changed from the default, which is not supported. Expecting %(exp)s, got %(act)s"))

# setup logging
from engage.utils.log_setup import setup_engage_logger
logger = setup_engage_logger(__name__)

# this is used by the package manager to locate the packages.json
# file associated with the driver
def get_packages_filename():
    return engage.drivers.utils.get_packages_filename(__file__)

def make_context(resource_json, sudo_password_fn, dry_run=False):
    """Create a Context object (defined in engage.utils.action). This contains
    the resource's metadata in ctx.props, references to the logger and sudo
    password function, and various helper functions. The context object is used
    by individual actions.

    If your resource does not need the sudo password, you can just pass in
    None for sudo_password_fn.
    """
    ctx = Context(resource_json, logger, __file__,
                  sudo_password_fn=sudo_password_fn,
                  dry_run=dry_run)
    ctx.check_port('config_port',
                  pid_file=unicode,
                  config_dir=unicode,
                  log_file=unicode)
    ctx.check_port('input_ports.datablox_framework',
                  BLOXPATH=unicode)
    ctx.check_port('input_ports.gunicorn',
                  gunicorn_exe=unicode)
    ctx.check_port('input_ports.host',
                   genforma_home=unicode,
                   log_directory=unicode)

    # add any extra computed properties here using the ctx.add() method.
    ctx.add('server_key_file',
            os.path.join(ctx.props.input_ports.host.genforma_home,
                         "datablox_file_server_key"))
    return ctx

def np(path):
    return os.path.abspath(os.path.expanduser(path))

def gen_random(length, chars=string.letters+string.digits):
    return ''.join([ random.choice(chars) for i in range(length) ])

@make_action
def write_key_file(self, path):
    deskey = gen_random(8)
    with open(path, 'w') as f:
        f.write(deskey)
    os.chmod(path, 0600)

# Now, define the main resource manager class for the driver. If this driver is
# a service, inherit from service_manager.Manager instead of
# resource_manager.Manager. If you need the sudo password, add
# PasswordRepoMixin to the inheritance list.
#
class Manager(service_manager.Manager):
    def __init__(self, metadata, dry_run=False):
        package_name = "%s %s" % (metadata.key["name"],
                                  metadata.key["version"])
        service_manager.Manager.__init__(self, metadata, package_name)
        self.ctx = make_context(metadata.to_json(),
                                None, # self._get_sudo_password,
                                dry_run=dry_run)

    def validate_pre_install(self):
        pass

    def is_installed(self):
        return os.path.exists(self.ctx.props.config_port.config_dir)

    def install(self, package):
        p = self.ctx.props
        r = self.ctx.r
        r(ensure_dir_exists, p.config_port.config_dir)
        r(write_key_file, p.server_key_file)

    def validate_post_install(self):
        pass

    def start(self):
        p = self.ctx.props
        cmd = [p.input_ports.gunicorn.gunicorn_exe,
               "--daemon", "--log-file=%s" % p.config_port.log_file,
               "--pid=%s" % p.config_port.pid_file, "-w", "4",
               "--bind=%s:%d" % ("0.0.0.0", 4990),
               "datablox_framework.fileserver_wsgi:app"]
        self.ctx.r(run_program, cmd,
                   cwd=os.path.dirname(p.input_ports.gunicorn.gunicorn_exe))
        self.ctx.check_poll(10, 1.0, lambda x: x!=None,
                            get_server_status, p.config_port.pid_file)

    def stop(self):
        p = self.ctx.props
        self.ctx.r(stop_server,
                   p.config_port.pid_file)
        self.ctx.check_poll(10, 1.0, lambda x: x==None,
                            get_server_status, p.config_port.pid_file)

    def is_running(self):
        p = self.ctx.props
        return self.ctx.rv(get_server_status,
                           p.config_port.pid_file) != None

    def get_pid_file_path(self):
        return self.ctx.props.config_port.pid_file
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import json
import errno
import numpy as np
import networkx as nx
import yaml
import cv2

class DataSet:
    """
    Dataset representing directory with images, extracted EXIF, SIFT descriptors, etc.

    Methods to retrieve *base directory* for data file(s) have suffix ``_path``, methods to retrieve path of specified
    data file have suffix ``_file``.
    """
    def __init__(self, data_path):
        """
        Create dataset instance. Empty directories (for EXIF, robust matches, etc) will be created if they don't exist
        already.

        :param data_path: Path to directory containing dataset
        """
        self.data_path = data_path

        with open(os.path.join(self.data_path, 'config.yaml')) as fin:
            self.config = yaml.load(fin)

        for p in [self.exif_path(),
                  self.sift_path(),
                  self.robust_matches_path()]:
            try:
                os.makedirs(p)
            except os.error as exc:
                if exc.errno == errno.EEXIST and os.path.isdir(p):
                    pass
                else:
                    raise

    def is_image_file(self, file):
        return file.split('.')[-1].lower() in ['jpg', 'jpeg', 'png', 'tif', 'tiff', 'pgm', 'pnm', 'gif']

    def images(self):
        """Return list of file paths of all images in this dataset"""
        return [i for i in os.listdir(self.image_path()) if self.is_image_file(i)]

    def image_path(self):
        """Return path of images directory"""
        return os.path.join(self.data_path, 'images')

    def image_file(self, image):
        """
        Return path of image with given name
        :param image: Image file name (**with extension**)
        """
        return os.path.join(self.image_path(), image)

    def image_as_array(self, image):
        """Return image pixels as 3-dimensional OpenCV matrix (R G B order)"""
        return cv2.imread(self.image_file(image))[:,:,::-1]  # Turn BGR to RGB

    def exif_path(self):
        """Return path of extracted exif directory"""
        return os.path.join(self.data_path, 'exif')

    def exif_file(self, image):
        """
        Return path of exif information for given image
        :param image: Image name, with extension (i.e. 123.jpg)
        """
        return os.path.join(self.exif_path(), image + '.exif')

    def exif_data(self, image):
        """
        Return extracted exif information, as dictionary, usually with fields:

        ================  =====  ===================================
        Field             Type   Description
        ================  =====  ===================================
        width             int    Width of image, in pixels
        height            int    Height of image, in pixels
        focal_ratio       float  Focal length (real) / sensor width
        focal_35mm_equiv  float  Focal length in 35 mm equivalent
        ================  =====  ===================================

        :param image: Image name, with extension (i.e. 123.jpg)
        """
        with open(self.exif_file(image), 'r') as fin:
            return json.load(fin)

    def sift_path(self):
        """Return path of SIFT descriptors and FLANN indices directory"""
        return os.path.join(self.data_path, 'sift')

    def sift_file(self, image):
        """
        Return path of SIFT file for specified image
        :param image: Image name, with extension (i.e. 123.jpg)
        """
        return os.path.join(self.sift_path(), image + '.sift')

    def sift_index_file(self, image):
        """
        Return path of FLANN index file for specified image
        :param image: Image name, with extension (i.e. 123.jpg)
        """
        return os.path.join(self.sift_path(), image + '.flann')

    def matches_path(self):
        """Return path of matches directory"""
        return os.path.join(self.data_path, 'matches')

    def matches_file(self, image1, image2):
        """
        Return path of matches file for pair of specified images
        :param image1: Image name, with extension (i.e. 123.jpg)
        :param image2: Image name, with extension (i.e. 123.jpg)
        """
        return os.path.join(self.matches_path(), '%s_%s_matches.csv' % (image1, image2))

    def robust_matches_path(self):
        """Return path of robust matches directory"""
        return os.path.join(self.data_path, 'robust_matches')

    def robust_matches_file(self, image1, image2):
        """
        Return path of *robust* matches file for pair of specified images
        :param image1: Image name, with extension (i.e. 123.jpg)
        :param image2: Image name, with extension (i.e. 123.jpg)
        """
        return os.path.join(self.robust_matches_path(), '%s_%s_matches.csv' % (image1, image2))

    def tracks_file(self):
        """Return path of tracks file"""
        return os.path.join(self.data_path, 'tracks.csv')

    def tracks_graph(self):
        """Return graph (networkx data structure) of tracks"""
        with open(self.tracks_file()) as fin:
            g = nx.Graph()
            for line in fin:
                image, track, observation, x, y = line.split('\t')
                g.add_node(image, bipartite=0)
                g.add_node(track, bipartite=1)
                g.add_edge(image, track, feature=(float(x), float(y)))
            return g

    def reconstruction_file(self):
        """Return path of reconstruction file"""
        return os.path.join(self.data_path, 'reconstruction.json')

def common_tracks(g, im1, im2):
    """
    Return the list of tracks observed in both images
    :param g: Graph structure (networkx) as returned by :method:`DataSet.tracks_graph`
    :param im1: Image name, with extension (i.e. 123.jpg)
    :param im2: Image name, with extension (i.e. 123.jpg)
    :return: tuple: track, feature from first image, feature from second image
    """
    # TODO: return value is unclear
    # TODO: maybe make as method of DataSet?
    t1, t2 = g[im1], g[im2]
    tracks, p1, p2 = [], [], []
    for track in t1:
        if track in t2:
            p1.append(t1[track]['feature'])
            p2.append(t2[track]['feature'])
            tracks.append(track)
    p1 = np.array(p1)
    p2 = np.array(p2)
    return tracks, p1, p2

<commit_msg>Made is_image_file private and static (not really needed outside)<commit_after># -*- coding: utf-8 -*-

import os
import json
import errno
import numpy as np
import networkx as nx
import yaml
import cv2

class DataSet:
    """
    Dataset representing directory with images, extracted EXIF, SIFT descriptors, etc.

    Methods to retrieve *base directory* for data file(s) have suffix ``_path``, methods to retrieve path of specified
    data file have suffix ``_file``.
    """
    def __init__(self, data_path):
        """
        Create dataset instance. Empty directories (for EXIF, robust matches, etc) will be created if they don't exist
        already.

        :param data_path: Path to directory containing dataset
        """
        self.data_path = data_path

        with open(os.path.join(self.data_path, 'config.yaml')) as fin:
            self.config = yaml.load(fin)

        for p in [self.exif_path(),
                  self.sift_path(),
                  self.robust_matches_path()]:
            try:
                os.makedirs(p)
            except os.error as exc:
                if exc.errno == errno.EEXIST and os.path.isdir(p):
                    pass
                else:
                    raise

    @staticmethod
    def _is_image_file(filename):
        return filename.split('.')[-1].lower() in {'jpg', 'jpeg', 'png', 'tif', 'tiff', 'pgm', 'pnm', 'gif'}

    def images(self):
        """Return list of file paths of all images in this dataset"""
        return [i for i in os.listdir(self.image_path()) if self._is_image_file(i)]

    def image_path(self):
        """Return path of images directory"""
        return os.path.join(self.data_path, 'images')

    def image_file(self, image):
        """
        Return path of image with given name
        :param image: Image file name (**with extension**)
        """
        return os.path.join(self.image_path(), image)

    def image_as_array(self, image):
        """Return image pixels as 3-dimensional OpenCV matrix (R G B order)"""
        return cv2.imread(self.image_file(image))[:,:,::-1]  # Turn BGR to RGB

    def exif_path(self):
        """Return path of extracted exif directory"""
        return os.path.join(self.data_path, 'exif')

    def exif_file(self, image):
        """
        Return path of exif information for given image
        :param image: Image name, with extension (i.e. 123.jpg)
        """
        return os.path.join(self.exif_path(), image + '.exif')

    def exif_data(self, image):
        """
        Return extracted exif information, as dictionary, usually with fields:

        ================  =====  ===================================
        Field             Type   Description
        ================  =====  ===================================
        width             int    Width of image, in pixels
        height            int    Height of image, in pixels
        focal_ratio       float  Focal length (real) / sensor width
        focal_35mm_equiv  float  Focal length in 35 mm equivalent
        ================  =====  ===================================

        :param image: Image name, with extension (i.e. 123.jpg)
        """
        with open(self.exif_file(image), 'r') as fin:
            return json.load(fin)

    def sift_path(self):
        """Return path of SIFT descriptors and FLANN indices directory"""
        return os.path.join(self.data_path, 'sift')

    def sift_file(self, image):
        """
        Return path of SIFT file for specified image
        :param image: Image name, with extension (i.e. 123.jpg)
        """
        return os.path.join(self.sift_path(), image + '.sift')

    def sift_index_file(self, image):
        """
        Return path of FLANN index file for specified image
        :param image: Image name, with extension (i.e. 123.jpg)
        """
        return os.path.join(self.sift_path(), image + '.flann')

    def matches_path(self):
        """Return path of matches directory"""
        return os.path.join(self.data_path, 'matches')

    def matches_file(self, image1, image2):
        """
        Return path of matches file for pair of specified images
        :param image1: Image name, with extension (i.e. 123.jpg)
        :param image2: Image name, with extension (i.e. 123.jpg)
        """
        return os.path.join(self.matches_path(), '%s_%s_matches.csv' % (image1, image2))

    def robust_matches_path(self):
        """Return path of robust matches directory"""
        return os.path.join(self.data_path, 'robust_matches')

    def robust_matches_file(self, image1, image2):
        """
        Return path of *robust* matches file for pair of specified images
        :param image1: Image name, with extension (i.e. 123.jpg)
        :param image2: Image name, with extension (i.e. 123.jpg)
        """
        return os.path.join(self.robust_matches_path(), '%s_%s_matches.csv' % (image1, image2))

    def tracks_file(self):
        """Return path of tracks file"""
        return os.path.join(self.data_path, 'tracks.csv')

    def tracks_graph(self):
        """Return graph (networkx data structure) of tracks"""
        with open(self.tracks_file()) as fin:
            g = nx.Graph()
            for line in fin:
                image, track, observation, x, y = line.split('\t')
                g.add_node(image, bipartite=0)
                g.add_node(track, bipartite=1)
                g.add_edge(image, track, feature=(float(x), float(y)))
            return g

    def reconstruction_file(self):
        """Return path of reconstruction file"""
        return os.path.join(self.data_path, 'reconstruction.json')

def common_tracks(g, im1, im2):
    """
    Return the list of tracks observed in both images
    :param g: Graph structure (networkx) as returned by :method:`DataSet.tracks_graph`
    :param im1: Image name, with extension (i.e. 123.jpg)
    :param im2: Image name, with extension (i.e. 123.jpg)
    :return: tuple: track, feature from first image, feature from second image
    """
    # TODO: return value is unclear
    # TODO: maybe make as method of DataSet?
    t1, t2 = g[im1], g[im2]
    tracks, p1, p2 = [], [], []
    for track in t1:
        if track in t2:
            p1.append(t1[track]['feature'])
            p2.append(t2[track]['feature'])
            tracks.append(track)
    p1 = np.array(p1)
    p2 = np.array(p2)
    return tracks, p1, p2

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    openslides.main
    ~~~~~~~~~~~~~~~

    Main script to start and set up OpenSlides.

    :copyright: 2011–2013 by OpenSlides team, see AUTHORS.
    :license: GNU GPL, see LICENSE for more details.
"""

import base64
import ctypes
import optparse
import os
import socket
import sys
import tempfile
import threading
import time
import webbrowser

from django.conf import ENVIRONMENT_VARIABLE
from django.core.management import execute_from_command_line

from openslides import get_version
from openslides.utils.tornado_webserver import run_tornado

CONFIG_TEMPLATE = """#!/usr/bin/env python
# -*- coding: utf-8 -*-

import openslides.main
from openslides.global_settings import *

# Use 'DEBUG = True' to get more details for server errors
# (Default for releases: 'False')
DEBUG = False
TEMPLATE_DEBUG = DEBUG

DBPATH = %(dbpath)s

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': DBPATH,
        'USER': '',
        'PASSWORD': '',
        'HOST': '',
        'PORT': '',
    }
}

# Set timezone
TIME_ZONE = 'Europe/Berlin'

# Make this unique, and don't share it with anybody.
SECRET_KEY = %(default_key)r

# Add OpenSlides plugins to this list (see example entry in comment)
INSTALLED_PLUGINS = (
#    'pluginname',
)

INSTALLED_APPS += INSTALLED_PLUGINS

# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = %(media_root_path)s

"""

KEY_LENGTH = 30

# sentinel used to signal that the database ought to be stored
# relative to the portable's directory
_portable_db_path = object()

def process_options(argv=None, manage_runserver=False):
    if argv is None:
        argv = sys.argv[1:]

    parser = optparse.OptionParser(
        description="Run openslides using the tornado webserver")
    parser.add_option(
        "-a", "--address",
        help="IP Address to listen on. Default: 0.0.0.0")
    parser.add_option(
        "-p", "--port", type="int",
        help="Port to listen on. Default: 8000 (start as admin/root: 80)")
    parser.add_option(
        "--syncdb", action="store_true",
        help="Update/create database before starting the server.")
    parser.add_option(
        "--backupdb", action="store", metavar="BACKUP_PATH",
        help="Make a backup copy of the database to BACKUP_PATH.")
    parser.add_option(
        "--reset-admin", action="store_true",
        help="Make sure the user 'admin' exists and uses 'admin' as password.")
    parser.add_option(
        "-s", "--settings", help="Set the path to the settings file.")
    parser.add_option(
        "--no-browser",
        action="store_false", dest="start_browser", default=True,
        help="Do not automatically start the web browser.")
    parser.add_option(
        "--no-reload", action="store_true",
        help="Do not reload the web server.")
    parser.add_option(
        "--no-run", action="store_true",
        help="Do not start the web server.")
    parser.add_option(
        "--version", action="store_true",
        help="Show version and exit.")

    opts, args = parser.parse_args(argv)

    # Do not parse any argv if the script is started via manage.py runserver.
    # This simulates the django runserver command
    if manage_runserver:
        opts.start_browser = False
        opts.no_reload = False
        return opts

    if opts.version:
        print get_version()
        exit(0)

    if args:
        sys.stderr.write("This command does not take arguments!\n\n")
        parser.print_help()
        sys.exit(1)

    return opts

def main(argv=None, manage_runserver=False):
    opts = process_options(argv, manage_runserver)
    _main(opts)

def win32_portable_main(argv=None):
    """special entry point for the win32 portable version"""

    opts = process_options(argv)

    database_path = None

    if opts.settings is None:
        portable_dir = get_portable_path()
        try:
            fd, test_file = tempfile.mkstemp(dir=portable_dir)
        except OSError:
            portable_dir_writeable = False
        else:
            portable_dir_writeable = True
            os.close(fd)
            os.unlink(test_file)

        if portable_dir_writeable:
            opts.settings = os.path.join(
                portable_dir, "openslides", "settings.py")
            database_path = _portable_db_path

    _main(opts, database_path=database_path)

def _main(opts, database_path=None):
    # Find the path to the settings
    settings_path = opts.settings
    if settings_path is None:
        settings_path = get_user_config_path('openslides', 'settings.py')

    # Create settings if necessary
    if not os.path.exists(settings_path):
        create_settings(settings_path, database_path)

    # Set the django environment to the settings
    setup_django_environment(settings_path)

    # Find url to openslides
    addr, port = detect_listen_opts(opts.address, opts.port)

    # Create Database if necessary
    if not database_exists() or opts.syncdb:
        run_syncdb()

    # Reset Admin
    elif opts.reset_admin:
        reset_admin_user()

    if opts.backupdb:
        backup_database(opts.backupdb)

    if opts.no_run:
        return

    # Start OpenSlides
    reload = True
    if opts.no_reload:
        reload = False

    if opts.start_browser:
        if port == 80:
            suffix = ""
        else:
            suffix = ":%d" % port
        start_browser("http://localhost%s" % suffix)

    # Start the server
    run_tornado(addr, port, reload)

def create_settings(settings_path, database_path=None):
    settings_module = os.path.dirname(settings_path)

    if database_path is _portable_db_path:
        database_path = get_portable_db_path()
        dbpath_value = 'openslides.main.get_portable_db_path()'
        media_root_path_value = 'openslides.main.get_portable_media_root_path()'
    else:
        if database_path is None:
            database_path = get_user_data_path('openslides', 'database.sqlite')
        dbpath_value = repr(fs2unicode(database_path))
        media_root_path_value = repr(fs2unicode(get_user_data_path('openslides', 'media', '')))

    settings_content = CONFIG_TEMPLATE % dict(
        default_key=base64.b64encode(os.urandom(KEY_LENGTH)),
        dbpath=dbpath_value,
        media_root_path=media_root_path_value)

    if not os.path.exists(settings_module):
        os.makedirs(settings_module)

    if not os.path.exists(os.path.dirname(database_path)):
        os.makedirs(os.path.dirname(database_path))

    with open(settings_path, 'w') as file:
        file.write(settings_content)

def setup_django_environment(settings_path):
    settings_file = os.path.basename(settings_path)
    settings_module_name = "".join(settings_file.split('.')[:-1])
    if '.' in settings_module_name:
        print "'.' is not an allowed character in the settings-file"
        sys.exit(1)
    settings_module_dir = os.path.dirname(settings_path)
    sys.path.append(settings_module_dir)
    os.environ[ENVIRONMENT_VARIABLE] = '%s' % settings_module_name

def detect_listen_opts(address=None, port=None):
    if address is None:
        address = "0.0.0.0"

    if port is None:
        # test if we can use port 80
        s = socket.socket()
        port = 80
        try:
            s.bind((address, port))
            s.listen(-1)
        except socket.error:
            port = 8000
        finally:
            s.close()

    return address, port

def database_exists():
    """Detect if database exists"""
    # can't be imported in global scope as they already require
    # the settings module during import
    from django.db import DatabaseError
    from django.core.exceptions import ImproperlyConfigured
    from openslides.participant.models import User

    try:
        # TODO: Use another model, the User could be deactivated
        User.objects.count()
    except DatabaseError:
        return False
    except ImproperlyConfigured:
        print "Your settings file seems broken"
        sys.exit(0)
    else:
        return True

def run_syncdb():
    # now initialize the database
    argv = ["", "syncdb", "--noinput"]
    execute_from_command_line(argv)

def reset_admin_user():
    # can't be imported in global scope as it already requires
    # the settings module during import
    from openslides.participant.api import create_or_reset_admin_user
    create_or_reset_admin_user()

def backup_database(dest_path):
    argv = ["", "backupdb", "--destination={0}".format(dest_path)]
    execute_from_command_line(argv)

def start_browser(url):
    browser = webbrowser.get()

    def f():
        time.sleep(1)
        browser.open(url)

    t = threading.Thread(target=f)
    t.start()

def fs2unicode(s):
    if isinstance(s, unicode):
        return s
    fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
    return s.decode(fs_encoding)

def get_user_config_path(*args):
    if sys.platform == "win32":
        return win32_get_app_data_path(*args)

    config_home = os.environ.get(
        'XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config'))

    return os.path.join(fs2unicode(config_home), *args)

def get_user_data_path(*args):
    if sys.platform == "win32":
        return win32_get_app_data_path(*args)

    data_home = os.environ.get(
        'XDG_DATA_HOME', os.path.join(
            os.path.expanduser('~'), '.local', 'share'))

    return os.path.join(fs2unicode(data_home), *args)

def is_portable():
    """Return True if openslides is run as portable version"""

    # NOTE: sys.executable is the path of the *interpreter*
    #       the portable version embeds python so it *is* the interpreter.
    #       The wrappers generated by pip and co. will spawn
    #       the usual python(w).exe, so there is no danger of mistaking
    #       them for the portable even though they may also be called
    #       openslides.exe
    exename = os.path.basename(sys.executable).lower()
    return exename == "openslides.exe"

def get_portable_path(*args):
    # NOTE: sys.executable will be the path to openslides.exe
    #       since it is essentially a small wrapper that embeds the
    #       python interpreter

    if not is_portable():
        raise Exception(
            "Cannot determine portable path when "
            "not running as portable")

    portable_dir = fs2unicode(os.path.dirname(os.path.abspath(sys.executable)))
    return os.path.join(portable_dir, *args)

def get_portable_db_path():
    return get_portable_path('openslides', 'database.sqlite')

def get_portable_media_root_path():
    return get_portable_path('openslides', 'media', '')

def win32_get_app_data_path(*args):
    shell32 = ctypes.WinDLL("shell32.dll")
    SHGetFolderPath = shell32.SHGetFolderPathW
    SHGetFolderPath.argtypes = (
        ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_uint32,
        ctypes.c_wchar_p)
    SHGetFolderPath.restype = ctypes.c_uint32

    CSIDL_LOCAL_APPDATA = 0x001c
    MAX_PATH = 260

    buf = ctypes.create_unicode_buffer(MAX_PATH)
    res = SHGetFolderPath(0, CSIDL_LOCAL_APPDATA, 0, 0, buf)
    if res != 0:
        raise Exception("Could not deterime APPDATA path")

    return os.path.join(buf.value, *args)

if __name__ == "__main__":
    if is_portable():
        win32_portable_main()
    else:
        main()
<commit_msg>Start Browser on custom IP, fixed #827<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    openslides.main
    ~~~~~~~~~~~~~~~

    Main script to start and set up OpenSlides.

    :copyright: 2011–2013 by OpenSlides team, see AUTHORS.
    :license: GNU GPL, see LICENSE for more details.
"""

import base64
import ctypes
import optparse
import os
import socket
import sys
import tempfile
import threading
import time
import webbrowser

from django.conf import ENVIRONMENT_VARIABLE
from django.core.management import execute_from_command_line

from openslides import get_version
from openslides.utils.tornado_webserver import run_tornado

CONFIG_TEMPLATE = """#!/usr/bin/env python
# -*- coding: utf-8 -*-

import openslides.main
from openslides.global_settings import *

# Use 'DEBUG = True' to get more details for server errors
# (Default for releases: 'False')
DEBUG = False
TEMPLATE_DEBUG = DEBUG

DBPATH = %(dbpath)s

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': DBPATH,
        'USER': '',
        'PASSWORD': '',
        'HOST': '',
        'PORT': '',
    }
}

# Set timezone
TIME_ZONE = 'Europe/Berlin'

# Make this unique, and don't share it with anybody.
SECRET_KEY = %(default_key)r

# Add OpenSlides plugins to this list (see example entry in comment)
INSTALLED_PLUGINS = (
#    'pluginname',
)

INSTALLED_APPS += INSTALLED_PLUGINS

# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = %(media_root_path)s

"""

KEY_LENGTH = 30

# sentinel used to signal that the database ought to be stored
# relative to the portable's directory
_portable_db_path = object()

def process_options(argv=None, manage_runserver=False):
    if argv is None:
        argv = sys.argv[1:]

    parser = optparse.OptionParser(
        description="Run openslides using the tornado webserver")
    parser.add_option(
        "-a", "--address",
        help="IP Address to listen on. Default: 0.0.0.0")
    parser.add_option(
        "-p", "--port", type="int",
        help="Port to listen on. Default: 8000 (start as admin/root: 80)")
    parser.add_option(
        "--syncdb", action="store_true",
        help="Update/create database before starting the server.")
    parser.add_option(
        "--backupdb", action="store", metavar="BACKUP_PATH",
        help="Make a backup copy of the database to BACKUP_PATH.")
    parser.add_option(
        "--reset-admin", action="store_true",
        help="Make sure the user 'admin' exists and uses 'admin' as password.")
    parser.add_option(
        "-s", "--settings", help="Set the path to the settings file.")
    parser.add_option(
        "--no-browser",
        action="store_false", dest="start_browser", default=True,
        help="Do not automatically start the web browser.")
    parser.add_option(
        "--no-reload", action="store_true",
        help="Do not reload the web server.")
    parser.add_option(
        "--no-run", action="store_true",
        help="Do not start the web server.")
    parser.add_option(
        "--version", action="store_true",
        help="Show version and exit.")

    opts, args = parser.parse_args(argv)

    # Do not parse any argv if the script is started via manage.py runserver.
    # This simulates the django runserver command
    if manage_runserver:
        opts.start_browser = False
        opts.no_reload = False
        return opts

    if opts.version:
        print get_version()
        exit(0)

    if args:
        sys.stderr.write("This command does not take arguments!\n\n")
        parser.print_help()
        sys.exit(1)

    return opts

def main(argv=None, manage_runserver=False):
    opts = process_options(argv, manage_runserver)
    _main(opts)

def win32_portable_main(argv=None):
    """special entry point for the win32 portable version"""

    opts = process_options(argv)

    database_path = None

    if opts.settings is None:
        portable_dir = get_portable_path()
        try:
            fd, test_file = tempfile.mkstemp(dir=portable_dir)
        except OSError:
            portable_dir_writeable = False
        else:
            portable_dir_writeable = True
            os.close(fd)
            os.unlink(test_file)

        if portable_dir_writeable:
            opts.settings = os.path.join(
                portable_dir, "openslides", "settings.py")
            database_path = _portable_db_path

    _main(opts, database_path=database_path)

def _main(opts, database_path=None):
    # Find the path to the settings
    settings_path = opts.settings
    if settings_path is None:
        settings_path = get_user_config_path('openslides', 'settings.py')

    # Create settings if necessary
    if not os.path.exists(settings_path):
        create_settings(settings_path, database_path)

    # Set the django environment to the settings
    setup_django_environment(settings_path)

    # Find url to openslides
    addr, port = detect_listen_opts(opts.address, opts.port)

    # Create Database if necessary
    if not database_exists() or opts.syncdb:
        run_syncdb()

    # Reset Admin
    elif opts.reset_admin:
        reset_admin_user()

    if opts.backupdb:
        backup_database(opts.backupdb)

    if opts.no_run:
        return

    # Start OpenSlides
    reload = True
    if opts.no_reload:
        reload = False

    if opts.start_browser:
        if opts.address:
            prefix = opts.address
        else:
            prefix = 'localhost'
        if port == 80:
            suffix = ""
        else:
            suffix = ":%d" % port
        start_browser("http://%s%s" % (prefix, suffix))

    # Start the server
    run_tornado(addr, port, reload)

def create_settings(settings_path, database_path=None):
    settings_module = os.path.dirname(settings_path)

    if database_path is _portable_db_path:
        database_path = get_portable_db_path()
        dbpath_value = 'openslides.main.get_portable_db_path()'
        media_root_path_value = 'openslides.main.get_portable_media_root_path()'
    else:
        if database_path is None:
            database_path = get_user_data_path('openslides', 'database.sqlite')
        dbpath_value = repr(fs2unicode(database_path))
        media_root_path_value = repr(fs2unicode(get_user_data_path('openslides', 'media', '')))

    settings_content = CONFIG_TEMPLATE % dict(
        default_key=base64.b64encode(os.urandom(KEY_LENGTH)),
        dbpath=dbpath_value,
        media_root_path=media_root_path_value)

    if not os.path.exists(settings_module):
        os.makedirs(settings_module)

    if not os.path.exists(os.path.dirname(database_path)):
        os.makedirs(os.path.dirname(database_path))

    with open(settings_path, 'w') as file:
        file.write(settings_content)

def setup_django_environment(settings_path):
    settings_file = os.path.basename(settings_path)
    settings_module_name = "".join(settings_file.split('.')[:-1])
    if '.' in settings_module_name:
        print "'.' is not an allowed character in the settings-file"
        sys.exit(1)
    settings_module_dir = os.path.dirname(settings_path)
    sys.path.append(settings_module_dir)
    os.environ[ENVIRONMENT_VARIABLE] = '%s' % settings_module_name

def detect_listen_opts(address=None, port=None):
    if address is None:
        address = "0.0.0.0"

    if port is None:
        # test if we can use port 80
        s = socket.socket()
        port = 80
        try:
            s.bind((address, port))
            s.listen(-1)
        except socket.error:
            port = 8000
        finally:
            s.close()

    return address, port

def database_exists():
    """Detect if database exists"""
    # can't be imported in global scope as they already require
    # the settings module during import
    from django.db import DatabaseError
    from django.core.exceptions import ImproperlyConfigured
    from openslides.participant.models import User

    try:
        # TODO: Use another model, the User could be deactivated
        User.objects.count()
    except DatabaseError:
        return False
    except ImproperlyConfigured:
        print "Your settings file seems broken"
        sys.exit(0)
    else:
        return True

def run_syncdb():
    # now initialize the database
    argv = ["", "syncdb", "--noinput"]
    execute_from_command_line(argv)

def reset_admin_user():
    # can't be imported in global scope as it already requires
    # the settings module during import
    from openslides.participant.api import create_or_reset_admin_user
    create_or_reset_admin_user()

def backup_database(dest_path):
    argv = ["", "backupdb", "--destination={0}".format(dest_path)]
    execute_from_command_line(argv)

def start_browser(url):
    browser = webbrowser.get()

    def f():
        time.sleep(1)
        browser.open(url)

    t = threading.Thread(target=f)
    t.start()

def fs2unicode(s):
    if isinstance(s, unicode):
        return s
    fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
    return s.decode(fs_encoding)

def get_user_config_path(*args):
    if sys.platform == "win32":
        return win32_get_app_data_path(*args)

    config_home = os.environ.get(
        'XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config'))

    return os.path.join(fs2unicode(config_home), *args)

def get_user_data_path(*args):
    if sys.platform == "win32":
        return win32_get_app_data_path(*args)

    data_home = os.environ.get(
        'XDG_DATA_HOME', os.path.join(
            os.path.expanduser('~'), '.local', 'share'))

    return os.path.join(fs2unicode(data_home), *args)

def is_portable():
    """Return True if openslides is run as portable version"""

    # NOTE: sys.executable is the path of the *interpreter*
    #       the portable version embeds python so it *is* the interpreter.
    #       The wrappers generated by pip and co. will spawn
    #       the usual python(w).exe, so there is no danger of mistaking
    #       them for the portable even though they may also be called
    #       openslides.exe
    exename = os.path.basename(sys.executable).lower()
    return exename == "openslides.exe"

def get_portable_path(*args):
    # NOTE: sys.executable will be the path to openslides.exe
    #       since it is essentially a small wrapper that embeds the
    #       python interpreter

    if not is_portable():
        raise Exception(
            "Cannot determine portable path when "
            "not running as portable")

    portable_dir = fs2unicode(os.path.dirname(os.path.abspath(sys.executable)))
    return os.path.join(portable_dir, *args)

def get_portable_db_path():
    return get_portable_path('openslides', 'database.sqlite')

def get_portable_media_root_path():
    return get_portable_path('openslides', 'media', '')

def win32_get_app_data_path(*args):
    shell32 = ctypes.WinDLL("shell32.dll")
    SHGetFolderPath = shell32.SHGetFolderPathW
    SHGetFolderPath.argtypes = (
        ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_uint32,
        ctypes.c_wchar_p)
    SHGetFolderPath.restype = ctypes.c_uint32

    CSIDL_LOCAL_APPDATA = 0x001c
    MAX_PATH = 260

    buf = ctypes.create_unicode_buffer(MAX_PATH)
    res = SHGetFolderPath(0, CSIDL_LOCAL_APPDATA, 0, 0, buf)
    if res != 0:
        raise Exception("Could not deterime APPDATA path")

    return os.path.join(buf.value, *args)

if __name__ == "__main__":
    if is_portable():
        win32_portable_main()
    else:
        main()
<|endoftext|>