<commit_before>#!/usr/bin/env python3

# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Builds this project.

Mimics a trivial version of Make."""

import argparse
import functools
import io
import os
import pathlib
import platform
import shlex
import shutil
import subprocess
import sys
from typing import (Callable, Dict, FrozenSet, Iterable, Mapping, Optional,
                    Sequence)

_Target = Callable[['Builder'], None]
_targets: Dict[str, _Target] = {}

def target(func: _Target) -> _Target:
    """Decorator to mark a function as a build target."""
    name = func.__name__
    @functools.wraps(func)
    def wrapper(self: 'Builder') -> None:
        print(f'building target {name}')
        func(self)
    _targets[name] = wrapper
    return wrapper

class Builder:
    """Builds the project."""

    def __init__(self, *,
                 bazel: pathlib.Path,
                 action_cache: Optional[pathlib.Path] = None,
                 repository_cache: Optional[pathlib.Path] = None) -> None:
        self._kernel = platform.system()
        self._cwd = pathlib.Path(os.getcwd())
        self._env = dict(os.environ)
        self._github = self._env.get('CI') == 'true'
        self._bazel_program = bazel
        self._action_cache = action_cache
        self._repository_cache = repository_cache

    def build(self, goals: Sequence[str]) -> None:
        """Builds the specified goals."""
        if not goals:
            raise ValueError('no goals provided')
        funcs = []
        for goal in goals:
            func = _targets.get(goal)
            if not func:
                raise KeyError(f'unknown goal {goal}')
            funcs.append(func)
        for func in funcs:
            func(self)

    @target
    def all(self) -> None:
        """Builds all targets."""
        self.generate()
        self.check()

    @target
    def check(self) -> None:
        """Builds and tests the project."""
        self.buildifier()
        self.nogo()
        # Test both default toolchain and versioned toolchains.
        self.test()
        self.versions()
        self.ext()

    @target
    def generate(self) -> None:
        """Generates files to be checked in."""
        self.docs()
        self.compdb()

    @target
    def buildifier(self) -> None:
        """Checks that all BUILD files are formatted correctly."""
        self._bazel('run',
                    ['@com_github_bazelbuild_buildtools//buildifier',
                     '--mode=check', '--lint=warn', '-r', '--', str(self._cwd)])

    @target
    def nogo(self) -> None:
        """Checks that there are no unwanted Go targets in public packages.

        We don’t want any Go rules in the public packages, as our users would
        have to depend on the Go rules then as well.
        """
        print('looking for unwanted Go targets in public packages')
        for directory in ('elisp', 'emacs'):
            for dirpath, _, filenames in os.walk(self._cwd / directory):
                for file in filenames:
                    file = pathlib.Path(dirpath) / file
                    text = file.read_text(encoding='utf-8')
                    if '@io_bazel_rules_go' in text:
                        raise ValueError('unwanted Go targets in {file} found')

    @target
    def test(self) -> None:
        """Runs the Bazel tests."""
        self._test()

    @target
    def versions(self) -> None:
        """Runs the Bazel tests for all supported Emacs versions."""
        for version in sorted(_versions()):
            self._test(f'--extra_toolchains=//elisp:emacs_{version}_toolchain')

    def _test(self, *args: str) -> None:
        self._bazel('test', ['//...'], options=['--test_output=errors'],
                    postfix_options=args)

    @target
    def ext(self) -> None:
        """Run the tests in the example workspace."""
        self._bazel('test', ['//...'], options=['--test_output=errors'],
                    cwd=self._cwd / 'examples' / 'ext')

    @target
    def docs(self) -> None:
        """Build the generated documentation files."""
        output = self._run(
            [str(self._bazel_program), 'query', '--output=label',
             r'filter("\.md\.generated$", kind("generated file", //...:*))'],
            capture_stdout=True)
        targets = output.splitlines()
        bazel_bin = self._cwd / 'bazel-bin'
        self._bazel('build', targets)
        for tgt in targets:
            gen = bazel_bin / tgt.lstrip('/').replace(':', os.sep)
            src = self._cwd / gen.with_suffix('').relative_to(bazel_bin)
            shutil.copyfile(gen, src)

    @target
    def compdb(self) -> None:
        """Generates a compilation database for clangd."""
        self._bazel('build', ['@com_grail_bazel_compdb//:files'])
        output = self._run([str(self._bazel_program), 'info', 'execution_root'],
                           capture_stdout=True)
        execroot = pathlib.Path(output.rstrip('\n'))
        generator = (execroot / 'external' / 'com_grail_bazel_compdb' /
                     'generate.py')
        self._run([sys.executable, str(generator)])

    def _bazel(self, command: str, targets: Iterable[str], *,
               options: Iterable[str] = (), postfix_options: Iterable[str] = (),
               cwd: Optional[pathlib.Path] = None) -> None:
        args = [str(self._bazel_program), command]
        args.extend(options)
        if self._action_cache:
            args.append('--disk_cache=' + str(self._action_cache))
        if self._repository_cache:
            args.append('--repository_cache=' + str(self._repository_cache))
        if self._kernel == 'Windows':
            # We only support compilation using MinGW-64 at the moment.
            # Binaries linked with the MinGW-64 linker will depend on a few
            # libraries that reside in C:\msys64\mingw64\bin, e.g.,
            # libwinpthread-1.dll or libstdc++-6.dll.  These libraries won’t be
            # found because C:\msys64\mingw64\bin is normally not in the search
            # path.  We therefore enforce static linking.  We need to add the
            # -static option to both the target and host options.  We need to
            # add it to the compiler option as well, otherwise the option comes
            # too late, after the -l… options.  Also see
            # https://stackoverflow.com/a/14033674/.
            args += ['--compiler=mingw-gcc',
                     '--linkopt=-static', '--host_linkopt=-static',
                     '--copt=-static', '--host_copt=-static']
        args.extend(postfix_options)
        args.append('--')
        args.extend(targets)
        env = dict(self._env)
        if self._kernel == 'Darwin':
            # We don’t need XCode, and using the Unix toolchain tends to be less
            # flaky.  See
            # https://github.com/bazelbuild/bazel/issues/14113#issuecomment-999794586.
            env['BAZEL_USE_CPP_ONLY_TOOLCHAIN'] = '1'
        if self._github:
            # Hacks so that Bazel finds the right binaries on GitHub.  See
            # https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables.
            if self._kernel in ('Linux', 'Darwin'):
                tool_cache = pathlib.Path(os.getenv('RUNNER_TOOL_CACHE'))
                # Remove Python versions set up by the setup-python action.  For
                # some reason they don’t work.
                for var in ('PATH', 'LD_LIBRARY_PATH'):
                    path = env.get(var, '').split(os.pathsep)
                    prefix = str(tool_cache / 'Python') + '/'
                    env[var] = os.pathsep.join(
                        d for d in path if not d.startswith(prefix))
            elif self._kernel == 'Windows':
                env['BAZEL_SH'] = r'C:\msys64\usr\bin\bash.exe'
        self._run(args, cwd=cwd, env=env)

    def _run(self, args: Sequence[str], *,
             cwd: Optional[pathlib.Path] = None,
             env: Optional[Mapping[str, str]] = None,
             capture_stdout: bool = False) -> Optional[str]:
        print(*map(shlex.quote, args))
        result = subprocess.run(
            args, check=True, cwd=cwd or self._cwd, env=env or self._env,
            stdout=subprocess.PIPE if capture_stdout else None,
            encoding='utf-8')
        return result.stdout

def _versions() -> FrozenSet[str]:
    # All potentially supported Emacs versions.
    ret = {'26.1', '26.2', '26.3', '27.1', '27.2'}
    uname = platform.uname()
    if uname.system == 'Linux':
        # GNU/Linux supports all Emacs versions.
        pass
    elif uname.system == 'Darwin':
        # macOS only supports Emacs 27.
        ret -= {'26.1', '26.2', '26.3'}
        if uname.machine != 'x86_64':
            # Apple Silicon doesn’t support Emacs 27.1.
            ret.remove('27.1')
    elif uname.system == 'Windows':
        # Windows only supports Emacs 27.
        ret -= {'26.1', '26.2', '26.3'}
    else:
        raise ValueError(f'unsupported kernel {uname.system}')
    return frozenset(ret)

def main() -> None:
    """Builds the project."""
    if isinstance(sys.stdout, io.TextIOWrapper):
        sys.stdout.reconfigure(encoding='utf-8', line_buffering=True)
    parser = argparse.ArgumentParser(allow_abbrev=False)
    parser.add_argument('--bazel', type=_program, default='bazel')
    parser.add_argument('--action_cache', type=_cache_directory)
    parser.add_argument('--repository_cache', type=_cache_directory)
    parser.add_argument('goals', nargs='*', choices=sorted(_targets))
    args = parser.parse_args(sys.argv[1:] or ['all'])
    builder = Builder(bazel=args.bazel,
                      action_cache=args.action_cache,
                      repository_cache=args.repository_cache)
    try:
        builder.build(args.goals)
    except subprocess.CalledProcessError as ex:
        print(*map(shlex.quote, ex.cmd), 'failed with exit code', ex.returncode)
        sys.exit(ex.returncode)

def _program(name: str) -> pathlib.Path:
    file = shutil.which(name)
    if not file:
        raise FileNotFoundError(f'program {name} not found')
    return pathlib.Path(file)

def _cache_directory(arg: str) -> pathlib.Path:
    return pathlib.Path(arg).expanduser()

if __name__ == '__main__':
    main()
<commit_msg>Use Clang to generate the compilation database<commit_after>#!/usr/bin/env python3

# Copyright 2021, 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Builds this project.

Mimics a trivial version of Make."""

import argparse
import functools
import io
import os
import pathlib
import platform
import shlex
import shutil
import subprocess
import sys
from typing import (Callable, Dict, FrozenSet, Iterable, Mapping, Optional,
                    Sequence)

_Target = Callable[['Builder'], None]
_targets: Dict[str, _Target] = {}

def target(func: _Target) -> _Target:
    """Decorator to mark a function as a build target."""
    name = func.__name__
    @functools.wraps(func)
    def wrapper(self: 'Builder') -> None:
        print(f'building target {name}')
        func(self)
    _targets[name] = wrapper
    return wrapper

class Builder:
    """Builds the project."""

    def __init__(self, *,
                 bazel: pathlib.Path,
                 action_cache: Optional[pathlib.Path] = None,
                 repository_cache: Optional[pathlib.Path] = None) -> None:
        self._kernel = platform.system()
        self._cwd = pathlib.Path(os.getcwd())
        self._env = dict(os.environ)
        self._github = self._env.get('CI') == 'true'
        self._bazel_program = bazel
        self._action_cache = action_cache
        self._repository_cache = repository_cache

    def build(self, goals: Sequence[str]) -> None:
        """Builds the specified goals."""
        if not goals:
            raise ValueError('no goals provided')
        funcs = []
        for goal in goals:
            func = _targets.get(goal)
            if not func:
                raise KeyError(f'unknown goal {goal}')
            funcs.append(func)
        for func in funcs:
            func(self)

    @target
    def all(self) -> None:
        """Builds all targets."""
        self.generate()
        self.check()

    @target
    def check(self) -> None:
        """Builds and tests the project."""
        self.buildifier()
        self.nogo()
        # Test both default toolchain and versioned toolchains.
        self.test()
        self.versions()
        self.ext()

    @target
    def generate(self) -> None:
        """Generates files to be checked in."""
        self.docs()
        self.compdb()

    @target
    def buildifier(self) -> None:
        """Checks that all BUILD files are formatted correctly."""
        self._bazel('run',
                    ['@com_github_bazelbuild_buildtools//buildifier',
                     '--mode=check', '--lint=warn', '-r', '--', str(self._cwd)])

    @target
    def nogo(self) -> None:
        """Checks that there are no unwanted Go targets in public packages.

        We don’t want any Go rules in the public packages, as our users would
        have to depend on the Go rules then as well.
        """
        print('looking for unwanted Go targets in public packages')
        for directory in ('elisp', 'emacs'):
            for dirpath, _, filenames in os.walk(self._cwd / directory):
                for file in filenames:
                    file = pathlib.Path(dirpath) / file
                    text = file.read_text(encoding='utf-8')
                    if '@io_bazel_rules_go' in text:
                        raise ValueError('unwanted Go targets in {file} found')

    @target
    def test(self) -> None:
        """Runs the Bazel tests."""
        self._test()

    @target
    def versions(self) -> None:
        """Runs the Bazel tests for all supported Emacs versions."""
        for version in sorted(_versions()):
            self._test(f'--extra_toolchains=//elisp:emacs_{version}_toolchain')

    def _test(self, *args: str) -> None:
        self._bazel('test', ['//...'], options=['--test_output=errors'],
                    postfix_options=args)

    @target
    def ext(self) -> None:
        """Run the tests in the example workspace."""
        self._bazel('test', ['//...'], options=['--test_output=errors'],
                    cwd=self._cwd / 'examples' / 'ext')

    @target
    def docs(self) -> None:
        """Build the generated documentation files."""
        output = self._run(
            [str(self._bazel_program), 'query', '--output=label',
             r'filter("\.md\.generated$", kind("generated file", //...:*))'],
            capture_stdout=True)
        targets = output.splitlines()
        bazel_bin = self._cwd / 'bazel-bin'
        self._bazel('build', targets)
        for tgt in targets:
            gen = bazel_bin / tgt.lstrip('/').replace(':', os.sep)
            src = self._cwd / gen.with_suffix('').relative_to(bazel_bin)
            shutil.copyfile(gen, src)

    @target
    def compdb(self) -> None:
        """Generates a compilation database for clangd."""
        self._bazel('build', ['@com_grail_bazel_compdb//:files'])
        output = self._run([str(self._bazel_program), 'info', 'execution_root'],
                           capture_stdout=True)
        execroot = pathlib.Path(output.rstrip('\n'))
        generator = (execroot / 'external' / 'com_grail_bazel_compdb' /
                     'generate.py')
        self._run([sys.executable, str(generator)],
                  # Need to compile with Clang for clangd to work.
                  env=dict(self._env, CC='clang'))

    def _bazel(self, command: str, targets: Iterable[str], *,
               options: Iterable[str] = (), postfix_options: Iterable[str] = (),
               cwd: Optional[pathlib.Path] = None) -> None:
        args = [str(self._bazel_program), command]
        args.extend(options)
        if self._action_cache:
            args.append('--disk_cache=' + str(self._action_cache))
        if self._repository_cache:
            args.append('--repository_cache=' + str(self._repository_cache))
        if self._kernel == 'Windows':
            # We only support compilation using MinGW-64 at the moment.
            # Binaries linked with the MinGW-64 linker will depend on a few
            # libraries that reside in C:\msys64\mingw64\bin, e.g.,
            # libwinpthread-1.dll or libstdc++-6.dll.  These libraries won’t be
            # found because C:\msys64\mingw64\bin is normally not in the search
            # path.  We therefore enforce static linking.  We need to add the
            # -static option to both the target and host options.  We need to
            # add it to the compiler option as well, otherwise the option comes
            # too late, after the -l… options.  Also see
            # https://stackoverflow.com/a/14033674/.
            args += ['--compiler=mingw-gcc',
                     '--linkopt=-static', '--host_linkopt=-static',
                     '--copt=-static', '--host_copt=-static']
        args.extend(postfix_options)
        args.append('--')
        args.extend(targets)
        env = dict(self._env)
        if self._kernel == 'Darwin':
            # We don’t need XCode, and using the Unix toolchain tends to be less
            # flaky.  See
            # https://github.com/bazelbuild/bazel/issues/14113#issuecomment-999794586.
            env['BAZEL_USE_CPP_ONLY_TOOLCHAIN'] = '1'
        if self._github:
            # Hacks so that Bazel finds the right binaries on GitHub.  See
            # https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables.
            if self._kernel in ('Linux', 'Darwin'):
                tool_cache = pathlib.Path(os.getenv('RUNNER_TOOL_CACHE'))
                # Remove Python versions set up by the setup-python action.  For
                # some reason they don’t work.
                for var in ('PATH', 'LD_LIBRARY_PATH'):
                    path = env.get(var, '').split(os.pathsep)
                    prefix = str(tool_cache / 'Python') + '/'
                    env[var] = os.pathsep.join(
                        d for d in path if not d.startswith(prefix))
            elif self._kernel == 'Windows':
                env['BAZEL_SH'] = r'C:\msys64\usr\bin\bash.exe'
        self._run(args, cwd=cwd, env=env)

    def _run(self, args: Sequence[str], *,
             cwd: Optional[pathlib.Path] = None,
             env: Optional[Mapping[str, str]] = None,
             capture_stdout: bool = False) -> Optional[str]:
        print(*map(shlex.quote, args))
        result = subprocess.run(
            args, check=True, cwd=cwd or self._cwd, env=env or self._env,
            stdout=subprocess.PIPE if capture_stdout else None,
            encoding='utf-8')
        return result.stdout

def _versions() -> FrozenSet[str]:
    # All potentially supported Emacs versions.
    ret = {'26.1', '26.2', '26.3', '27.1', '27.2'}
    uname = platform.uname()
    if uname.system == 'Linux':
        # GNU/Linux supports all Emacs versions.
        pass
    elif uname.system == 'Darwin':
        # macOS only supports Emacs 27.
        ret -= {'26.1', '26.2', '26.3'}
        if uname.machine != 'x86_64':
            # Apple Silicon doesn’t support Emacs 27.1.
            ret.remove('27.1')
    elif uname.system == 'Windows':
        # Windows only supports Emacs 27.
        ret -= {'26.1', '26.2', '26.3'}
    else:
        raise ValueError(f'unsupported kernel {uname.system}')
    return frozenset(ret)

def main() -> None:
    """Builds the project."""
    if isinstance(sys.stdout, io.TextIOWrapper):
        sys.stdout.reconfigure(encoding='utf-8', line_buffering=True)
    parser = argparse.ArgumentParser(allow_abbrev=False)
    parser.add_argument('--bazel', type=_program, default='bazel')
    parser.add_argument('--action_cache', type=_cache_directory)
    parser.add_argument('--repository_cache', type=_cache_directory)
    parser.add_argument('goals', nargs='*', choices=sorted(_targets))
    args = parser.parse_args(sys.argv[1:] or ['all'])
    builder = Builder(bazel=args.bazel,
                      action_cache=args.action_cache,
                      repository_cache=args.repository_cache)
    try:
        builder.build(args.goals)
    except subprocess.CalledProcessError as ex:
        print(*map(shlex.quote, ex.cmd), 'failed with exit code', ex.returncode)
        sys.exit(ex.returncode)

def _program(name: str) -> pathlib.Path:
    file = shutil.which(name)
    if not file:
        raise FileNotFoundError(f'program {name} not found')
    return pathlib.Path(file)

def _cache_directory(arg: str) -> pathlib.Path:
    return pathlib.Path(arg).expanduser()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
 # -*- coding: utf-8 -*-

import unittest
from nose.tools import nottest
from nose.tools import assert_equals
from nose.tools import assert_not_equals
from nose.tools import assert_true
import requests
from ddt import ddt, data
import requests_cache

from software import Software
from app import libraries_to_mum

requests_cache.install_cache('my_requests_cache', expire_after=60*60*24*7)  # expire_after is in seconds

# run it like this:
# nosetests --processes=10 test/

test_urls = [
    ("http://cnn.com", "", u'Anon, CNN - Breaking News, Latest News and Videos. Available at: http://cnn.com.'),
    ("https://github.com/pvlib/pvlib-python", "10.5281/zenodo.50141", "Holmgren, W. et al., 2016. pvlib-python: 0.3.1. Available at: https://doi.org/10.5281/zenodo.50141."),
    ("https://github.com/gcowan/hyperk", "10.5281/zenodo.160400", "Cowan, G., 2016. Gcowan/Hyperk: Mcp Data Processing Code. Available at: https://doi.org/10.5281/zenodo.160400."),
    ("https://github.com/NSLS-II-XPD/xpdView", "10.5281/zenodo.60479", "Duff, C. & Kaming-Thanassi, J., 2016. xpdView: xpdView initial release. Available at: https://doi.org/10.5281/zenodo.60479."),
    ("https://github.com/impactstory/depsy", "", "Impactstory, 2015. depsy. Available at: https://github.com/Impactstory/depsy."),
    ("https://github.com/abianchetti/blick", "", u'Bianchetti, A., 2012. Determinación del diámetro pupilar ocular en tiempo real.'),
    ("https://github.com/jasonpriem/FeedVis", "", u"Priem, J., 2011. FeedVis. Available at: https://github.com/jasonpriem/FeedVis."),
    ("https://github.com/vahtras/loprop", "", u'Vahtras, O., 2014. Loprop For Dalton. Available at: https://doi.org/10.5281/zenodo.13276.'),
    ("https://github.com/cvitolo/r_BigDataAnalytics", "", u'Vitolo, C., 2015. R_Bigdataanalytics V.0.0.1. Available at: https://doi.org/10.5281/zenodo.15722.'),
    ("https://github.com/nicholasricci/DDM_Framework", "", u"Marzolla, M., D'Angelo, G. & Mandrioli, M., 2014. A Parallel Data Distribution Management Algorithm. In Proc. IEEE/ACM International Symposium on Distributed Simulation and Real Time Applications (DS-RT 2013).  Delft, the Netherlands."),
    ("http://yt-project.org", "", "Turk, M.J. et al., 2010. Yt: A Multi-Code Analysis Toolkit For Astrophysical Simulation Data. The Astrophysical Journal Supplement Series, 192(1), p.9. Available at: http://dx.doi.org/10.1088/0067-0049/192/1/9."),
    # ("https://github.com/alvarag/LSH-IS", "", u'Arnaiz-Gonz\xe1lez, \xc1. et al., 2016. Instance selection of linear complexity for big data. Knowledge-Based Systems, 107, pp.83\u201395.'),
    # ("https://github.com/sanger-pathogens/mlst_check", "", """"Multilocus sequence typing by blast from de novo assemblies against PubMLST", Andrew J. Page, Ben Taylor, Jacqueline A. Keane, The Journal of Open Source Software, (2016). doi: http://dx.doi.org/10.21105/joss.00118"""),
    # ("https://github.com/AndrasHartmann/rtfractools", "", "Hartmann, A., Mukli, P., Nagy, Z., Kocsis, L., Hermán, P., & Eke, A. (2013). Real-time fractal signal processing in the time domain. Physica A: Statistical Mechanics and Its Applications, 392(1), 89–102. doi:10.1016/j.physa.2012.08.002"),
    # ("https://github.com/magitz/1001", "", "Mavrodiev EV. (2015) 1001 - A tool for binary representations of unordered multistate characters (with examples from genomic data) PeerJ PrePrints 3:e1403 https://dx.doi.org/10.7287/peerj.preprints.1153v1"),
    # ("https://github.com/gregmacfarlane/trucksim_disagg", "", """Macfarlane, G. and Donnelly, R. (2014). A national simulation of freight truck flows."""),
    # ("https://github.com/aimalz/chippr", "", "alz et al, in preparation\footnote{\texttt{https://github.com/aimalz/chippr}}"),
    # ("https://github.com/happynotes/NLDAS2TS", "", """PIHM Analysis Suite developed by Lele Shu, contributions by PIHM and http://www.pihm.psu.edu"""),
    ]

# We request that applications and derivative work cite this as:
# Cite this as
# If you end up using any of the code or ideas you find here in your academic research, please cite me as
# How to cite this software
# To cite this software please use the following BibTex information
# Please cite this software as
# To cite this software:
# To cite this software use:
# Please cite this software:
# Cite this software as:
# If you use this software please, cite the following paper:
# Citation: if you want to cite this software use the following bibtex entry.

@ddt
class MyTestCase(unittest.TestCase):
    _multiprocess_can_split_ = True

    @data(*test_urls)
    def test_the_urls(self, test_data):
        (url, doi, expected) = test_data
        my_software = Software(url)
        my_software.find_metadata()
        assert_equals(my_software.citation_plain, expected)

    @data(*test_urls)
    def test_the_dois(self, test_data):
        (url, doi, expected) = test_data
        if doi:
            my_software = Software(doi)
            my_software.find_metadata()
            assert_equals(my_software.citation_plain, expected)
<commit_msg>all unit tests passing<commit_after>#!/usr/bin/python
 # -*- coding: utf-8 -*-

import unittest
from nose.tools import nottest
from nose.tools import assert_equals
from nose.tools import assert_not_equals
from nose.tools import assert_true
import requests
from ddt import ddt, data
import requests_cache

from software import Software
from app import libraries_to_mum

requests_cache.install_cache('my_requests_cache', expire_after=60*60*24*7)  # expire_after is in seconds

# run it like this:
# nosetests --processes=10 test/

test_urls = [
    ("http://cnn.com", "", u'Anon, CNN - Breaking News, Latest News and Videos. Available at: http://cnn.com.'),
    ("https://github.com/pvlib/pvlib-python", "10.5281/zenodo.50141", "Holmgren, W. et al., 2016. pvlib-python: 0.3.1. Available at: https://doi.org/10.5281/zenodo.50141."),
    ("https://github.com/gcowan/hyperk", "10.5281/zenodo.160400", "Cowan, G., 2016. Gcowan/Hyperk: Mcp Data Processing Code. Available at: https://doi.org/10.5281/zenodo.160400."),
    ("https://github.com/NSLS-II-XPD/xpdView", "10.5281/zenodo.60479", "Duff, C. & Kaming-Thanassi, J., 2016. xpdView: xpdView initial release. Available at: https://doi.org/10.5281/zenodo.60479."),
    ("https://github.com/impactstory/depsy", "", "Impactstory, 2015. depsy. Available at: https://github.com/Impactstory/depsy."),
    ("https://github.com/abianchetti/blick", "", u'Bianchetti, A., 2012. blick. Available at: https://github.com/abianchetti/blick.'),
    ("https://github.com/jasonpriem/FeedVis", "", u"Priem, J., 2011. FeedVis. Available at: https://github.com/jasonpriem/FeedVis."),
    ("https://github.com/vahtras/loprop", "", u'Vahtras, O., 2014. Loprop For Dalton. Available at: https://doi.org/10.5281/zenodo.13276.'),
    ("https://github.com/cvitolo/r_BigDataAnalytics", "", u'Vitolo, C., 2015. R_Bigdataanalytics V.0.0.1. Available at: https://doi.org/10.5281/zenodo.15722.'),
    ("https://github.com/nicholasricci/DDM_Framework", "", u"Marzolla, M., D'Angelo, G. & Mandrioli, M., 2013. A Parallel Data Distribution Management Algorithm."),
    ("http://yt-project.org", "", "Turk, M.J. et al., 2011. yt: A Multi-code Analysis Toolkit for Astrophysical Simulation Data. , 192."),

    # getting the number and page wrong at the moment
    # ("http://yt-project.org", "", "Turk, M.J. et al., 2010. Yt: A Multi-Code Analysis Toolkit For Astrophysical Simulation Data. The Astrophysical Journal Supplement Series, 192(1), p.9. Available at: http://dx.doi.org/10.1088/0067-0049/192/1/9."),

    # bug.  doesn't include the proceedings title at the moment.
    # ("https://github.com/nicholasricci/DDM_Framework", "", u"Marzolla, M., D'Angelo, G. & Mandrioli, M., 2014. A Parallel Data Distribution Management Algorithm. In Proc. IEEE/ACM International Symposium on Distributed Simulation and Real Time Applications (DS-RT 2013).  Delft, the Netherlands."),

    # ("https://github.com/alvarag/LSH-IS", "", u'Arnaiz-Gonz\xe1lez, \xc1. et al., 2016. Instance selection of linear complexity for big data. Knowledge-Based Systems, 107, pp.83\u201395.'),
    # ("https://github.com/sanger-pathogens/mlst_check", "", """"Multilocus sequence typing by blast from de novo assemblies against PubMLST", Andrew J. Page, Ben Taylor, Jacqueline A. Keane, The Journal of Open Source Software, (2016). doi: http://dx.doi.org/10.21105/joss.00118"""),
    # ("https://github.com/AndrasHartmann/rtfractools", "", "Hartmann, A., Mukli, P., Nagy, Z., Kocsis, L., Hermán, P., & Eke, A. (2013). Real-time fractal signal processing in the time domain. Physica A: Statistical Mechanics and Its Applications, 392(1), 89–102. doi:10.1016/j.physa.2012.08.002"),
    # ("https://github.com/magitz/1001", "", "Mavrodiev EV. (2015) 1001 - A tool for binary representations of unordered multistate characters (with examples from genomic data) PeerJ PrePrints 3:e1403 https://dx.doi.org/10.7287/peerj.preprints.1153v1"),
    # ("https://github.com/gregmacfarlane/trucksim_disagg", "", """Macfarlane, G. and Donnelly, R. (2014). A national simulation of freight truck flows."""),
    # ("https://github.com/aimalz/chippr", "", "alz et al, in preparation\footnote{\texttt{https://github.com/aimalz/chippr}}"),
    # ("https://github.com/happynotes/NLDAS2TS", "", """PIHM Analysis Suite developed by Lele Shu, contributions by PIHM and http://www.pihm.psu.edu"""),
    ]

# We request that applications and derivative work cite this as:
# Cite this as
# If you end up using any of the code or ideas you find here in your academic research, please cite me as
# How to cite this software
# To cite this software please use the following BibTex information
# Please cite this software as
# To cite this software:
# To cite this software use:
# Please cite this software:
# Cite this software as:
# If you use this software please, cite the following paper:
# Citation: if you want to cite this software use the following bibtex entry.

@ddt
class MyTestCase(unittest.TestCase):
    _multiprocess_can_split_ = True

    @data(*test_urls)
    def test_the_urls(self, test_data):
        (url, doi, expected) = test_data
        my_software = Software(url)
        my_software.find_metadata()
        assert_equals(my_software.citation_plain, expected)

    @data(*test_urls)
    def test_the_dois(self, test_data):
        (url, doi, expected) = test_data
        if doi:
            my_software = Software(doi)
            my_software.find_metadata()
            assert_equals(my_software.citation_plain, expected)
<|endoftext|>


<commit_before>import sched, time
import os
import subprocess
import twittercron

s = sched.scheduler(time.time, time.sleep)

def runevery():
    print("inside .py")
    #twittercron.luigi.task
    command = 'python -m luigi --module twittercron Elasticsearch --index {index} --doctype {doctype} --searchquery {searchquery}'.format(index=os.environ.get('INDEX'),searchquery=os.environ.get('SEARCH_QUERY'),doctype=os.environ.get('DOC_TYPE'))
    subprocess.call(command.split(), shell= False)
    s.enter(860400, 1, runevery)
s.enter(10, 1, runevery)
s.run()<commit_msg>cron job time set to 1 day<commit_after>import sched, time
import os
import subprocess
import twittercron

s = sched.scheduler(time.time, time.sleep)

def runevery():
    print("inside .py")
    #twittercron.luigi.task
    command = 'python -m luigi --module twittercron Elasticsearch --index {index} --doctype {doctype} --searchquery {searchquery}'.format(index=os.environ.get('INDEX'),searchquery=os.environ.get('SEARCH_QUERY'),doctype=os.environ.get('DOC_TYPE'))
    subprocess.call(command.split(), shell= False)
    s.enter(86400, 1, runevery) #Change here your cron time
s.enter(10, 1, runevery)
s.run()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''Code for generating the AION static website.
'''

from __future__ import unicode_literals
import markdown
import os
import jinja2

PROJECTS_MD_DIR = './projects'
HTML_DIR = './website'
TEMPLATES_DIR = './templates'

jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATES_DIR))

def parse_project_file(fpath):
    f = open(fpath)
    metadata = {}
    headers = {'title', 'date', 'category', 'contact', 'tagline', 'mailing_list', 'repository'}
    for line in f:
        line = line.decode('utf-8')
        if not line.strip().split():
            break
        if ':' in line:
            split_index = line.find(':')
            header = line[:split_index].strip().lower()
            header = header.replace(' ', '_')
            header_content = line[split_index + 1:].strip()
            if header in headers:
                metadata[header] = header_content
    content_lines = []
    for line in f:
        content_lines.append(line.decode('utf-8'))
    content = ''.join(content_lines)
    f.close()
    return content, metadata

def render_project_pages():
    fnames = [n for n in os.listdir(PROJECTS_MD_DIR) if n.endswith('.md')]
    template = jinja_env.get_template('project.html')
    folder = os.path.join(HTML_DIR, 'projects')
    if not os.path.exists(folder):
        os.makedirs(folder)

    research_projects = []
    applied_projects = []

    for fname in fnames:
        fpath = os.path.join(PROJECTS_MD_DIR, fname)
        # parse project file into metadata and content
        md_content, metadata = parse_project_file(fpath)
        # render content as HTML
        html_content = mdown_to_html(md_content)
        # render project HTML page using Jinja
        page = template.render(content=html_content, **metadata)
        # save project page to website/projects
        page_fname = fname.replace('.md', '.html')
        page_fpath = os.path.join(folder, page_fname)
        metadata['fname'] = page_fname
        f = open(page_fpath, 'w')
        f.write(page.encode('utf-8'))
        f.close()

        if 'category' in metadata:
            category = metadata['category'].lower()
            if category == 'applied research':
                applied_projects.append(metadata)
            if category == 'fundamental research':
                research_projects.append(metadata)

    return research_projects, applied_projects

def mdown_to_html(mdown):
    return markdown.markdown(mdown, extensions=[
        'fenced_code',
        'codehilite(css_class=highlight)',
        'toc',
        'tables',
        'sane_lists',
    ])

def render_page(template, title, tab, fpath, **kwargs):
    fpath = os.path.join(HTML_DIR, fpath)
    folder = os.path.dirname(fpath)
    if folder and not os.path.exists(folder):
        os.makedirs(folder)
    head = head_template.render(title=title)
    header = header_template.render(tab=tab)
    template = jinja_env.get_template(template)
    page = template.render(head=head, header=header, **kwargs)
    f = open(fpath, 'w')
    f.write(page.encode('utf-8'))
    f.close()

if __name__ == '__main__':
    head_template = jinja_env.get_template('head.html')
    header_template = jinja_env.get_template('header.html')

    # renders core website
    render_page(template='home.html', title=None, tab='home', fpath='index.html')
    render_page(template='process.html', title='Process', tab='process', fpath='process/index.html')
    render_page(template='about.html', title='About', tab='about', fpath='about/index.html')
    render_page(template='error.html', title=None, tab=None, fpath='error.html')

    # render projects page
    research_projects, applied_projects = render_project_pages()
    projects_list_template = jinja_env.get_template('projects_list.html')
    applied = projects_list_template.render(projects=applied_projects)
    fundamental = projects_list_template.render(projects=research_projects)
    render_page(template='projects.html', title='Projects', tab='projects', fpath='projects/index.html',
                applied=applied, fundamental=fundamental)
<commit_msg>Update build script<commit_after># -*- coding: utf-8 -*-
'''Code for generating the AION static website.
'''
from __future__ import print_function
from __future__ import unicode_literals
import markdown
import os
import jinja2

PROJECTS_MD_DIR = './projects'
HTML_DIR = './website'
TEMPLATES_DIR = './templates'

jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATES_DIR))

def parse_project_file(fpath):
    f = open(fpath)
    metadata = {}
    headers = {'title',
               'date',
               'category',
               'contact',
               'tagline',
               'mailing_list',
               'repository'}
    for line in f:
        line = line.decode('utf-8')
        if not line.strip().split():
            break
        if ':' in line:
            split_index = line.find(':')
            header = line[:split_index].strip().lower()
            header = header.replace(' ', '_')
            header_content = line[split_index + 1:].strip()
            if header in headers:
                metadata[header] = header_content
    content_lines = []
    for line in f:
        content_lines.append(line.decode('utf-8'))
    content = ''.join(content_lines)
    f.close()
    return content, metadata

def render_project_pages():
    fnames = [n for n in os.listdir(PROJECTS_MD_DIR) if n.endswith('.md')]
    template = jinja_env.get_template('project.html')
    folder = os.path.join(HTML_DIR, 'projects')
    if not os.path.exists(folder):
        os.makedirs(folder)

    research_projects = []
    applied_projects = []

    for fname in fnames:
        print('Processing project:', fname)
        fpath = os.path.join(PROJECTS_MD_DIR, fname)
        # parse project file into metadata and content
        md_content, metadata = parse_project_file(fpath)
        # render content as HTML
        html_content = mdown_to_html(md_content)
        # render project HTML page using Jinja
        page = template.render(content=html_content, **metadata)
        # save project page to website/projects
        page_fname = fname.replace('.md', '.html')
        page_fpath = os.path.join(folder, page_fname)
        metadata['fname'] = page_fname
        f = open(page_fpath, 'w')
        f.write(page.encode('utf-8'))
        f.close()

        if 'category' in metadata:
            category = metadata['category'].lower()
            if category == 'applied research':
                applied_projects.append(metadata)
            if category == 'fundamental research':
                research_projects.append(metadata)

    return research_projects, applied_projects

def mdown_to_html(mdown):
    return markdown.markdown(mdown, extensions=[
        'fenced_code',
        'codehilite(css_class=highlight)',
        'toc',
        'tables',
        'sane_lists',
    ])

def render_page(template, title, tab, fpath, **kwargs):
    fpath = os.path.join(HTML_DIR, fpath)
    folder = os.path.dirname(fpath)
    if folder and not os.path.exists(folder):
        os.makedirs(folder)
    head = head_template.render(title=title)
    header = header_template.render(tab=tab)
    template = jinja_env.get_template(template)
    page = template.render(head=head, header=header, **kwargs)
    f = open(fpath, 'w')
    f.write(page.encode('utf-8'))
    f.close()

if __name__ == '__main__':
    head_template = jinja_env.get_template('head.html')
    header_template = jinja_env.get_template('header.html')

    # renders core website
    render_page(template='home.html', title=None, tab='home', fpath='index.html')
    render_page(template='process.html', title='Process', tab='process', fpath='process/index.html')
    render_page(template='about.html', title='About', tab='about', fpath='about/index.html')
    render_page(template='error.html', title=None, tab=None, fpath='error.html')

    # render projects page
    research_projects, applied_projects = render_project_pages()
    projects_list_template = jinja_env.get_template('projects_list.html')
    applied = projects_list_template.render(projects=applied_projects)
    fundamental = projects_list_template.render(projects=research_projects)
    render_page(template='projects.html', title='Projects', tab='projects', fpath='projects/index.html',
                applied=applied, fundamental=fundamental)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import os
import socket

from argparse import ArgumentParser
from subprocess import check_call, check_output
from binstar_client.utils import get_config, get_binstar, store_token

def ensure_tool(name):
    check_call(['which', name])

def build_and_publish(path, binstar, username):
    binfile = check_output(['conda', 'build', '--output', path])
    binfile = binfile.strip()

    print >>sys.stderr, "conda build {}".format(path)
    check_call(['conda', 'build', path])

    print >>sys.stderr, "binstar upload --force {}".format(binfile)
    check_call(['binstar', 'upload', '--force', binfile])

def binstar_login(args):
    # login to binstar
    # this code is taken from:
    #   binstar_client/commands/login.py
    bs = get_binstar()
    config = get_config()
    url = config.get('url', 'https://api.binstar.org')
    token = bs.authenticate(
        args.username, args.password,
        'binstar_client:{}'.format(socket.gethostname()),
        url, created_with='')
    if token is None:
        print >>sys.stderr, 'could not login'
        return 1
    store_token(token, args)

def main():
    parser = ArgumentParser()
    parser.add_argument('-u', '--username', required=True)
    parser.add_argument('-P', '--password', required=True)
    parser.add_argument('-p', '--project', required=True)
    parser.add_argument('-s', '--site', required=False, default=None)
    args = parser.parse_args()

    # make sure we have a conda environment
    ensure_tool('conda')
    ensure_tool('binstar')

    # make sure the project has a conda recipes folder
    conda_recipes_dir = os.path.join(args.project, 'conda')
    if not os.path.isdir(conda_recipes_dir):
        print >>sys.stderr, 'no such dir: {}'.format(conda_recipes_dir)
        return 1

    binstar_login()

    for name in sorted(os.listdir(conda_recipes_dir)):
        build_and_publish(
            os.path.join(conda_recipes_dir, name),
            None,
            args.username)

    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Add missing argument<commit_after>#!/usr/bin/env python

import sys
import os
import socket

from argparse import ArgumentParser
from subprocess import check_call, check_output
from binstar_client.utils import get_config, get_binstar, store_token

def ensure_tool(name):
    check_call(['which', name])

def build_and_publish(path, binstar, username):
    binfile = check_output(['conda', 'build', '--output', path])
    binfile = binfile.strip()

    print >>sys.stderr, "conda build {}".format(path)
    check_call(['conda', 'build', path])

    print >>sys.stderr, "binstar upload --force {}".format(binfile)
    check_call(['binstar', 'upload', '--force', binfile])

def binstar_login(args):
    # login to binstar
    # this code is taken from:
    #   binstar_client/commands/login.py
    bs = get_binstar()
    config = get_config()
    url = config.get('url', 'https://api.binstar.org')
    token = bs.authenticate(
        args.username, args.password,
        'binstar_client:{}'.format(socket.gethostname()),
        url, created_with='')
    if token is None:
        print >>sys.stderr, 'could not login'
        return 1
    store_token(token, args)

def main():
    parser = ArgumentParser()
    parser.add_argument('-u', '--username', required=True)
    parser.add_argument('-P', '--password', required=True)
    parser.add_argument('-p', '--project', required=True)
    parser.add_argument('-s', '--site', required=False, default=None)
    args = parser.parse_args()

    # make sure we have a conda environment
    ensure_tool('conda')
    ensure_tool('binstar')

    # make sure the project has a conda recipes folder
    conda_recipes_dir = os.path.join(args.project, 'conda')
    if not os.path.isdir(conda_recipes_dir):
        print >>sys.stderr, 'no such dir: {}'.format(conda_recipes_dir)
        return 1

    binstar_login(args)

    for name in sorted(os.listdir(conda_recipes_dir)):
        build_and_publish(
            os.path.join(conda_recipes_dir, name),
            None,
            args.username)

    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''

Creative Limitation

http://en.wikipedia.org/wiki/Creative_limitation

For songwriting / production. This script should print
a bunch of ideas for a song. The goal would then to be
to create a track with the proscibed features.

'''

import random

#set up data

spice = ('key change', 'bonus bar', 'claps',
'drums + melody breakdown', 'Guitar! shout',
'double time', 'syncopated hits')

feel = ('16th Hats', 'the gallops', 'slow burner',
'4 on the floor', 'synopated kick')

intro = ('don’t bore us, get to the chorus',
'dark and brooding','percussive','hits',
'lo-fi fade in','lick')

tonality = ('major', 'minor', 'dorian', 'phrygian')

words = ('laser', 'force', 'glitch', 'wave',
'cyber', 'thrust', 'machine', 'eagle', 'hawk', 'shark',
'bionic', 'eclipse', 'diamond', 'rain', 'chill',
'fire', 'obey', 'hip', 'ray', 'electron', 'beast', 'escape',
'future', 'battle', 'tomb', 'lost', 'party', 'disco',
'light', 'lawless', 'axe', 'streets', 'japan', 'speed',
'cocain', 'neon', 'OD', 'time', 'freeway', 'muscle', 'car',
'panther', 'epoch', 'midnight', 'euro', 'wilderness',
'bleak', 'gem', 'quest', 'run', 'error', 'free')

# Helper functions

def rand_list(alist):
    #returns a random member of list
    x = random.choice(alist)
    return x

# make a song "report"

def song():
    i = rand_list(intro)
    f = rand_list(feel)
    t = rand_list(tonality)
    s = rand_list(spice)
    w = rand_list(words)
    w2 = rand_list(words)
    print('____________________')
    print('')
    print('Intro: \t %s' % i)
    print('Feel: \t %s' % f)
    print('Using a %s scale' % t)
    print('For added spice, throw in a %s' % s)
    print('Your words of wisdom are: %s and %s' %(w, w2))
    print('Rock on. \m/')
    print('')
    print('____________________')

<commit_msg>Fix some pep8 errors<commit_after># -*- coding: utf-8 -*-
'''

Creative Limitation

http://en.wikipedia.org/wiki/Creative_limitation

For songwriting / production. This script should print
a bunch of ideas for a song. The goal would then to be
to create a track with the proscibed features.

'''

import random

#set up data

spice = ('key change', 'bonus bar', 'claps',
         'drums + melody breakdown', 'Guitar! shout',
         'double time', 'syncopated hits')

feel = ('16th Hats', 'the gallops', 'slow burner',
'4 on the floor', 'synopated kick')

intro = ('don’t bore us, get to the chorus',
'dark and brooding','percussive','hits',
'lo-fi fade in','lick')

tonality = ('major', 'minor', 'dorian', 'phrygian')

words = ('laser', 'force', 'glitch', 'wave',
'cyber', 'thrust', 'machine', 'eagle', 'hawk', 'shark',
'bionic', 'eclipse', 'diamond', 'rain', 'chill',
'fire', 'obey', 'hip', 'ray', 'electron', 'beast', 'escape',
'future', 'battle', 'tomb', 'lost', 'party', 'disco',
'light', 'lawless', 'axe', 'streets', 'japan', 'speed',
'cocain', 'neon', 'OD', 'time', 'freeway', 'muscle', 'car',
'panther', 'epoch', 'midnight', 'euro', 'wilderness',
'bleak', 'gem', 'quest', 'run', 'error', 'free')

# Helper functions

def rand_list(alist):
    #returns a random member of list
    x = random.choice(alist)
    return x

# make a song "report"

def song():
    i = rand_list(intro)
    f = rand_list(feel)
    t = rand_list(tonality)
    s = rand_list(spice)
    w = rand_list(words)
    w2 = rand_list(words)
    print('____________________')
    print('')
    print('Intro: \t %s' % i)
    print('Feel: \t %s' % f)
    print('Using a %s scale' % t)
    print('For added spice, throw in a %s' % s)
    print('Your words of wisdom are: %s and %s' %(w, w2))
    print('Rock on. \m/')
    print('')
    print('____________________')

<|endoftext|>


<commit_before>__author__ = 'heni'

import os.path
import pickle
import numpy
import datetime
import math
from preprocess.wordemb import WordEmbeddings,create_word_index
from preprocess.preprocess import get_iob
from rnn.elman_model import Elman
from preprocess.labeledText import LabeledText
from is13.utils.tools import contextwin,minibatch,shuffle

# this function aims to train a RNN
# it takes as input:
#   network_path: path to folder containing specific RNN configuration
#   articles: list of articles on which the network will make his training
def create_network(settings,classes_number,vocab_size,folder):
    print('Building RNN model...')
    numpy.random.seed(settings['seed'])
    rnn=Elman(nh=settings['nhidden'],
              nc=classes_number,
              ne=vocab_size,
              de=settings['emb_dimension'],
              cs=settings['win'])
    rnn.setup()
    rnn_fold=os.path.join('./data/rnnElman',folder)
    rnn.save(rnn_fold)

    return settings

def get_data_from_iob(article_name,labeled_data):
    try:
        article = open('./data/iob/' + str(article_name) + '.iob','r')
    except:
        print('No .iob file found for: %s .' % article_name)
    # labeled_data=[[],[]]
    lines=article.readlines()
    # beg=0
    # end=lines.index('\n')
    labeled_data[0].append([])
    labeled_data[1].append([])
    for i in range(0,len(lines)):
        line=lines[i]
        if line=='\n': # new sentence
            labeled_data[0].append([])
            labeled_data[1].append([])
            assert len(labeled_data[0])==len(labeled_data[1])
            continue
        if '\t\t' not in line:
            continue
        w_l=line.split('\t\t')
        labeled_data[0][len(labeled_data[0])-1].append(w_l[0])
        labeled_data[1][len(labeled_data[1])-1].append(w_l[1].replace('\n',''))
        assert len(labeled_data[0])==len(labeled_data[1])

    assert len(labeled_data[0])==len(labeled_data[1])
    return labeled_data

# This function allows to create a dictionary from an already generated .iob file
def get_dict_from_iob(article_name,dictionary):
    try:
        article = open('./data/iob/' + str(article_name) + '.iob','r')
    except:
        print('No .iob file found for: %s .' % article_name)

    print('The .iob file for %s has been found and a word2index dictionary will be created' % article_name)
    lines=article.readlines()
    i=0
    while i<len(lines):
        line=lines[i]
        if '\t\t' in line:
            w_l=line.split('\t\t')
            dictionary['wordIndex'].add_word(w_l[0])
            dictionary['labelIndex'].add_word(w_l[1].replace('\n',''))
            i+=1
        else:
            i+=1

    return dictionary

# main process: taking as input a list of articles to be processed
def run_process(articles):
    settings = {'partial_training':0.95,
                'partial_testing':0.05,
                'fold':3, # 5 folds 0,1,2,3,4
                'lr':0.0627142536696559,
                'verbose':1,
                'decay':False, # decay on the learning rate if improvement stops
                'win':7, # number of words in the context window
                'bs':9, # number of backprop through time steps
                'nhidden':100, # number of hidden units
                'seed':345,
                'emb_dimension':100, # dimension of word embedding
                'nepochs':50}
    indices={
        'wordIndex':WordEmbeddings(),
        'labelIndex':WordEmbeddings()
    }
    indices_dict_path='./data/word2indices/indices.pickle'
    labeled_data=LabeledText()
    # In this part, we will check if a word dictionary is already created.
    # Otherwise, we create a new one
    # PS: a new dictionary should be created only for the first article.
    # For further articles, we use a merge function to append the dictionary created while processing the 1st article
    for i in range(0,len(articles)):
        article=articles[i]
        get_iob(article)
        if os.path.isfile(indices_dict_path):
            print('Loading indices dictionary from %s' % indices_dict_path)
            existent_indices_dict=pickle.load(open(indices_dict_path,'rb'))
            indices['wordIndex'].merge(existent_indices_dict['wordIndex'])
            indices['labelIndex'].merge(existent_indices_dict['labelIndex'])
            indices['wordIndex'].merge(get_dict_from_iob(article,indices)['wordIndex'])
            indices['labelIndex'].merge(get_dict_from_iob(article,indices)['labelIndex'])
        else:
            if i>0:
                raise Exception('INDICES DICTIONARY ALREADY CREATED. NO NEED TO CREATE IT AGAIN !!!!!!')
            print('No indices dictionary found. This will be created based on the first article: %s' % str(article))
            indices['wordIndex'].merge(get_dict_from_iob(article,indices)['wordIndex'])
            indices['labelIndex'].merge(get_dict_from_iob(article,indices)['labelIndex'])
            print('The created dictionary is going to be stored.')
            with open(indices_dict_path,'wb') as indices_dict:
                pickle.dump(indices,indices_dict)
            print('The dictionary was successfully saved. Path: %s' % indices_dict_path)

        word_index=indices['wordIndex']
        label_index=indices['labelIndex']
        word2index=word_index.getCurrentIndex()
        index2word=word_index.getIndex2Word()
        label2index=label_index.getCurrentIndex()
        index2label=label_index.getIndex2Word()

        vocsize = len(word2index)
        nclasses = len(label2index)

        # In this part, we check if a RNN is already created.
        # Otherwise, we create a new one

        rnn_folder='./data/rnnElman'
        if not os.listdir(rnn_folder):
            if i>0:
                raise Exception('RNN MODEL ALREADY CREATED. NO NEED TO CREATE IT AGAIN !!!!!')
            new_network_folder=datetime.datetime.now().strftime('%Y-%m-%d_%Hh%M')
            create_network(settings,nclasses,vocsize,new_network_folder)

        #rnn=None
        model_folder=os.path.join(rnn_folder,os.listdir(rnn_folder)[0])
        print('Loading RNN model from "%s" ' % model_folder)
        rnn=Elman.load(model_folder)
        print('RNN model successfully loaded from "%s" ' % model_folder)

        # create labeled data
        labeled_data.addData(get_data_from_iob(article,labeled_data))
        print('Labeled data successfully generated')
        sentences_list,labels_list=labeled_data.getData()
        number_labeled_sentences=len(sentences_list)
        # shuffle before splitting up to train & test set
        shuffle([sentences_list,labels_list],settings['seed'])

        training_size=int(math.floor(settings['partial_training']*number_labeled_sentences))
        testing_size=int(math.floor(settings['partial_testing']*number_labeled_sentences))
        print('Training size: [0:{0}] = {0}'.format(training_size))
        train_sentences=sentences_list[0:training_size]
        train_labels=labels_list[0:training_size]
        print('Testing size: [{0}:{1}] = {2}'.format(training_size, training_size+testing_size, testing_size))
        test_sentences=sentences_list[training_size:training_size+testing_size]
        test_labels=sentences_list[training_size:training_size+testing_size]

        ####################
        # training process #
        ####################
        number_train_sentences=len(train_sentences)

<commit_msg>Words' dictionary, rnn and labeled data generated. Work on training started<commit_after>__author__ = 'heni'

import os.path
import pickle
import numpy
import datetime
import math
import time
import sys
from preprocess.wordemb import WordEmbeddings,create_word_index
from preprocess.preprocess import get_iob
from rnn.elman_model import Elman
from preprocess.labeledText import LabeledText
from is13.utils.tools import contextwin,minibatch,shuffle

# this function aims to train a RNN
# it takes as input:
#   network_path: path to folder containing specific RNN configuration
#   articles: list of articles on which the network will make his training
def create_network(settings,classes_number,vocab_size,folder):
    print('Building RNN model...')
    numpy.random.seed(settings['seed'])
    rnn=Elman(nh=settings['nhidden'],
              nc=classes_number,
              ne=vocab_size,
              de=settings['emb_dimension'],
              cs=settings['win'])
    rnn.setup()
    rnn_fold=os.path.join('./data/rnnElman',folder)
    rnn.save(rnn_fold)

    return settings

def get_data_from_iob(article_name,labeled_data):
    try:
        article = open('./data/iob/' + str(article_name) + '.iob','r')
    except:
        print('No .iob file found for: %s .' % article_name)
    # labeled_data=[[],[]]
    lines=article.readlines()
    # beg=0
    # end=lines.index('\n')
    labeled_data[0].append([])
    labeled_data[1].append([])
    for i in range(0,len(lines)):
        line=lines[i]
        if line=='\n': # new sentence
            labeled_data[0].append([])
            labeled_data[1].append([])
            assert len(labeled_data[0])==len(labeled_data[1])
            continue
        if '\t\t' not in line:
            continue
        w_l=line.split('\t\t')
        labeled_data[0][len(labeled_data[0])-1].append(w_l[0])
        labeled_data[1][len(labeled_data[1])-1].append(w_l[1].replace('\n',''))
        assert len(labeled_data[0])==len(labeled_data[1])

    assert len(labeled_data[0])==len(labeled_data[1])
    return labeled_data

# This function allows to create a dictionary from an already generated .iob file
def get_dict_from_iob(article_name,dictionary):
    try:
        article = open('./data/iob/' + str(article_name) + '.iob','r')
    except:
        print('No .iob file found for: %s .' % article_name)

    print('The .iob file for %s has been found and a word2index dictionary will be created' % article_name)
    lines=article.readlines()
    i=0
    while i<len(lines):
        line=lines[i]
        if '\t\t' in line:
            w_l=line.split('\t\t')
            dictionary['wordIndex'].add_word(w_l[0])
            dictionary['labelIndex'].add_word(w_l[1].replace('\n',''))
            i+=1
        else:
            i+=1

    return dictionary

# main process: taking as input a list of articles to be processed
def run_process(articles):
    settings = {'partial_training':0.7,
                'partial_testing':0.3,
                'fold':3, # 5 folds 0,1,2,3,4
                'lr':0.0627142536696559,
                'verbose':1,
                'decay':False, # decay on the learning rate if improvement stops
                'win':7, # number of words in the context window
                'bs':9, # number of backprop through time steps
                'nhidden':100, # number of hidden units
                'seed':345,
                'emb_dimension':100, # dimension of word embedding
                'nepochs':50}
    indices={
        'wordIndex':WordEmbeddings(),
        'labelIndex':WordEmbeddings()
    }
    indices_dict_path='./data/word2indices/indices.pickle'
    labeled_data=LabeledText()
    # In this part, we will check if a word dictionary is already created.
    # Otherwise, we create a new one
    # PS: a new dictionary should be created only for the first article.
    # For further articles, we use a merge function to append the dictionary created while processing the 1st article
    for i in range(0,len(articles)):
        article=articles[i]
        get_iob(article)
        if os.path.isfile(indices_dict_path):
            print('Loading indices dictionary from %s' % indices_dict_path)
            existent_indices_dict=pickle.load(open(indices_dict_path,'rb'))
            indices['wordIndex'].merge(existent_indices_dict['wordIndex'])
            indices['labelIndex'].merge(existent_indices_dict['labelIndex'])
            indices['wordIndex'].merge(get_dict_from_iob(article,indices)['wordIndex'])
            indices['labelIndex'].merge(get_dict_from_iob(article,indices)['labelIndex'])
        else:
            if i>0:
                raise Exception('INDICES DICTIONARY ALREADY CREATED. NO NEED TO CREATE IT AGAIN !!!!!!')
            print('No indices dictionary found. This will be created based on the first article: %s' % str(article))
            indices['wordIndex'].merge(get_dict_from_iob(article,indices)['wordIndex'])
            indices['labelIndex'].merge(get_dict_from_iob(article,indices)['labelIndex'])
            print('The created dictionary is going to be stored.')
            with open(indices_dict_path,'wb') as indices_dict:
                pickle.dump(indices,indices_dict)
            print('The dictionary was successfully saved. Path: %s' % indices_dict_path)

        word_index=indices['wordIndex']
        label_index=indices['labelIndex']
        word2index=word_index.getCurrentIndex()
        index2word=word_index.getIndex2Word()
        label2index=label_index.getCurrentIndex()
        index2label=label_index.getIndex2Word()

        vocsize = len(word2index)
        nclasses = len(label2index)

        # In this part, we check if a RNN is already created.
        # Otherwise, we create a new one

        rnn_folder='./data/rnnElman'
        if not os.listdir(rnn_folder):
            if i>0:
                raise Exception('RNN MODEL ALREADY CREATED. NO NEED TO CREATE IT AGAIN !!!!!')
            new_network_folder=datetime.datetime.now().strftime('%Y-%m-%d_%Hh%M')
            create_network(settings,nclasses,vocsize,new_network_folder)

        #rnn=None
        model_folder=os.path.join(rnn_folder,os.listdir(rnn_folder)[0])
        print('Loading RNN model from "%s" ' % model_folder)
        rnn=Elman.load(model_folder)
        print('RNN model successfully loaded from "%s" ' % model_folder)

        # create labeled data
        labeled_data.addData(get_data_from_iob(article,labeled_data))
        print('Labeled data successfully generated')
        sentences_list,labels_list=labeled_data.getData()
        number_labeled_sentences=len(sentences_list)
        # shuffle before splitting up to train & test set
        shuffle([sentences_list,labels_list],settings['seed'])

        training_size=int(math.floor(settings['partial_training']*number_labeled_sentences))
        testing_size=int(math.floor(settings['partial_testing']*number_labeled_sentences))
        print('Training size: [0:{0}] = {0}'.format(training_size))
        train_sentences=sentences_list[0:training_size]
        train_labels=labels_list[0:training_size]
        print('Testing size: [{0}:{1}] = {2}'.format(training_size, training_size+testing_size, testing_size))
        test_sentences=sentences_list[training_size:training_size+testing_size]
        test_labels=sentences_list[training_size:training_size+testing_size]

        ####################
        # training process #
        ####################
        number_train_sentences=len(train_sentences)
        best_f1=-numpy.inf
        current_learning_rate=settings['lr']
        for e in range(0,settings['nepochs']):
            print('Epoch {0}'.format(e))
            print('----------------------------------------------')
            shuffle([train_sentences,train_labels],settings['seed'])
            tic=time.time()
            for i in range(0,len(train_sentences)):
                indexed_sentence=[word2index[w] for w in train_sentences[i]]
                indexed_labels=[label2index[l] for l in train_labels[i]]
                cs_window=contextwin(indexed_sentence,settings['win'])
                words=map(lambda x:numpy.asarray(x).astype('int32'),minibatch(cs_window,settings['bs']))
                for word,label in zip(words,indexed_labels):
                    rnn.train(word,label,current_learning_rate)
                    rnn.normalize()
                if settings['verbose']:
                    print('[learning] epoch %i >> %2.2f%%' % (e,(i+1)*100./number_train_sentences),
                          'completed in %.2f (sec) <<\r' %(time.time()-tic),sys.stdout.flush())
            print('\n')

            # evaluation // back into the real world : idx -> words
            predictions_test=[map(lambda x:index2label[x],
                                  rnn.classify(numpy.asarray(cs_window(x,settings['win'])).astype('int32')))
                              for x in [[word2index[word] for word in sentence] for sentence in test_sentences]]
            for sen, pred, lab in zip(test_sentences,predictions_test,test_labels):
                print(sen)
                print('Pred: %s' %pred)
                print('Orig: %s' %lab)

<|endoftext|>


<commit_before>import sys
import argparse
import logging
import pathlib
import json

import cv2

from blur_detection import estimate_blur
from blur_detection import fix_image_size
from blur_detection import pretty_blur_map

def parse_args():
    parser = argparse.ArgumentParser(description='run blur detection on a single image')
    parser.add_argument('-i', '--images', type=str, nargs='+', required=True, help='directory of images')
    parser.add_argument('-s', '--save-path', type=str, default=None, help='path to save output')

    parser.add_argument('-t', '--threshold', type=float, default=100.0, help='blurry threshold')
    parser.add_argument('-f', '--variable-size', action='store_true', help='fix the image size')

    parser.add_argument('-v', '--verbose', action='store_true', help='set logging level to debug')
    parser.add_argument('-d', '--display', action='store_true', help='display images')

    return parser.parse_args()

def find_images(image_paths, img_extensions=['.jpg', '.png', '.jpeg']):
    img_extensions += [i.upper() for i in img_extensions]

    for path in image_paths:
        path = pathlib.Path(path)

        if path.is_file():
            if path.suffix not in img_extensions:
                logging.info(f'{path.suffix} is not an image extension! skipping {path}')
                continue
            else:
                yield path

        if path.is_dir():
            for img_ext in img_extensions:
                yield from path.rglob(f'*{img_ext}')

if __name__ == '__main__':
    assert sys.version_info >= (3, 7), sys.version_info
    args = parse_args()

    level = logging.DEBUG if args.verbose else logging.INFO
    logging.basicConfig(level=level)

    fix_size = not args.variable_size
    logging.info(f'fix_size: {fix_size}')

    if args.save_path is not None:
        save_path = pathlib.Path(args.save_path)
        assert save_path.suffix == '.json', save_path.suffix
    else:
        save_path = None

    results = []

    for image_path in find_images(args.images):
        image = cv2.imread(str(image_path))
        if image is None:
            logging.warning(f'warning! failed to read image from {image_path}; skipping!')
            continue

        logging.info(f'processing {image_path}')

        if fix_size:
            image = fix_image_size(image)
        else:
            logging.warning('not normalizing image size for consistent scoring!')

        blur_map, score, blurry = estimate_blur(image)

        logging.info(f'image_path: {image_path} score: {score} blurry: {blurry}')
        results.append({'input_path': str(image_path), 'score': score, 'blurry': blurry})

        if args.display:
            cv2.imshow('input', image)
            cv2.imshow('result', pretty_blur_map(blur_map))

            if cv2.waitKey(0) == ord('q'):
                logging.info('exiting...')
                exit()

    if save_path is not None:
        logging.info(f'saving json to {save_path}')

        with open(save_path, 'w') as result_file:
            data = {'images': args.images, 'threshold': args.threshold, 'fix_size': fix_size, 'results': results}
            json.dump(data, result_file, indent=4)
<commit_msg>blur_detection - decrease to minimum required python version<commit_after>import sys
import argparse
import logging
import pathlib
import json

import cv2

from blur_detection import estimate_blur
from blur_detection import fix_image_size
from blur_detection import pretty_blur_map

def parse_args():
    parser = argparse.ArgumentParser(description='run blur detection on a single image')
    parser.add_argument('-i', '--images', type=str, nargs='+', required=True, help='directory of images')
    parser.add_argument('-s', '--save-path', type=str, default=None, help='path to save output')

    parser.add_argument('-t', '--threshold', type=float, default=100.0, help='blurry threshold')
    parser.add_argument('-f', '--variable-size', action='store_true', help='fix the image size')

    parser.add_argument('-v', '--verbose', action='store_true', help='set logging level to debug')
    parser.add_argument('-d', '--display', action='store_true', help='display images')

    return parser.parse_args()

def find_images(image_paths, img_extensions=['.jpg', '.png', '.jpeg']):
    img_extensions += [i.upper() for i in img_extensions]

    for path in image_paths:
        path = pathlib.Path(path)

        if path.is_file():
            if path.suffix not in img_extensions:
                logging.info(f'{path.suffix} is not an image extension! skipping {path}')
                continue
            else:
                yield path

        if path.is_dir():
            for img_ext in img_extensions:
                yield from path.rglob(f'*{img_ext}')

if __name__ == '__main__':
    assert sys.version_info >= (3, 6), sys.version_info
    args = parse_args()

    level = logging.DEBUG if args.verbose else logging.INFO
    logging.basicConfig(level=level)

    fix_size = not args.variable_size
    logging.info(f'fix_size: {fix_size}')

    if args.save_path is not None:
        save_path = pathlib.Path(args.save_path)
        assert save_path.suffix == '.json', save_path.suffix
    else:
        save_path = None

    results = []

    for image_path in find_images(args.images):
        image = cv2.imread(str(image_path))
        if image is None:
            logging.warning(f'warning! failed to read image from {image_path}; skipping!')
            continue

        logging.info(f'processing {image_path}')

        if fix_size:
            image = fix_image_size(image)
        else:
            logging.warning('not normalizing image size for consistent scoring!')

        blur_map, score, blurry = estimate_blur(image)

        logging.info(f'image_path: {image_path} score: {score} blurry: {blurry}')
        results.append({'input_path': str(image_path), 'score': score, 'blurry': blurry})

        if args.display:
            cv2.imshow('input', image)
            cv2.imshow('result', pretty_blur_map(blur_map))

            if cv2.waitKey(0) == ord('q'):
                logging.info('exiting...')
                exit()

    if save_path is not None:
        logging.info(f'saving json to {save_path}')

        with open(save_path, 'w') as result_file:
            data = {'images': args.images, 'threshold': args.threshold, 'fix_size': fix_size, 'results': results}
            json.dump(data, result_file, indent=4)
<|endoftext|>


<commit_before>import ConfigParser
import pickle
import re
import sys
import threading
import time

from clientmedia import *
import clientgame
import clientmenu
import clientstyle
import clientworld
import config
from constants import *
from mapfile import *
from paths import *
import res
import tts
from world import *
from worldclient import *

class _Game(object):

    default_triggers = () # empty tuple; a tuple is immutable
    seed = None
    game_type_name = None
    alliances = ()
    races = ()

    def _game_type(self):
        return "%s/%s/%s" % (VERSION,
                             self.game_type_name + "-" + self.map.get_name(),
                             self.nb_human_players)

    def _record_stats(self, world):
        s = stats.Stats(STATS_PATH, METASERVER_URL)
        s.add(self._game_type(), int(world.time / 1000))

    def run(self, speed=config.speed):
        self.world = World(self.default_triggers, self.seed)
        if self.world.load_and_build_map(self.map):
            load_style(res.get_text("ui/style", append=True, locale=True),
                       self.map.campaign_style,
                       self.map.additional_style)
            sounds.enter_map(self.map.mapfile)
            clientworld.update_orders_list() # when style has changed
            self.pre_run()
            self.interface = clientgame.GameInterface(self.me, speed=speed)
            self.interface.load_bindings(
                res.get_text("ui/bindings", append=True, locale=True) + "\n" +
                self.map.get_campaign("ui/bindings.txt") + "\n" +
                self.map.get_additional("ui/bindings.txt"))
            self.world.populate_map(self.players, self.alliances, self.races)
            self.nb_human_players = self.world.current_nb_human_players()
            threading.Thread(target=self.world.loop).start()
            self.interface.loop()
            self._record_stats(self.world)
            self.post_run()
            sounds.exit_map()
            self.world.clean()
        else:
            voice.alert([1029]) # hostile sound
            if tts.is_available:
                voice.alert([self.world.map_error])

    def pre_run(self):
        pass

    def post_run(self):
        self.say_score()

    def say_score(self):
        for msg in self.interface.get_score_msgs():
            voice.info(msg)
        voice.flush()

class _MultiplayerGame(_Game):

    default_triggers = (
        ["players", ["no_enemy_player_left"], ["victory"]],
        ["players", ["no_building_left"], ["defeat"]],
        ["computers", ["no_unit_left"], ["defeat"]],
        ) # a tuple is immutable

class MultiplayerGame(_MultiplayerGame):

    game_type_name = "multiplayer"

    def __init__(self, map, players, my_login, main_server, seed, speed):
        self.map = map
        computers, humans = self._computers_and_humans(players, my_login)
        self.me = Coordinator(my_login, main_server, humans)
        humans[humans.index(None)] = self.me
        self.players = humans + computers # humans first because the first in the list is the game admin
        self.seed = seed
        self.speed = speed
        self.main_server = main_server

    def run(self):
        _MultiplayerGame.run(self, speed=self.speed)

    def _countdown(self):
        voice.important([4062]) # "the game starts in 5 seconds"
        for n in [5, 4, 3, 2, 1, 0]:
            voice.item(nombre(n))
            time.sleep(1)
        pygame.event.clear(KEYDOWN)

    def pre_run(self):
        nb_human_players = len([p for p in self.players if p.login != "ai"])
        if nb_human_players > 1:
            if compatibility_version() != COMPATIBILITY_VERSION:
                warning("rules.txt or ai.txt has been modified"
                        " after the program started: exit...")
                sys.exit()
            send_platform_version_to_metaserver(self.map.get_name(), nb_human_players)
            self._countdown()

    def post_run(self):
        # alert the server of the exit from the game interface
        if self.interface.forced_quit:
            self.main_server.write_line("abort_game")
        else:
            self.main_server.write_line("quit_game")
        self.say_score() # say score only after quit_game to avoid blocking the main server
        voice.menu([4010, 4030]) # "menu" "please make a selection" (long enough to allow history navigation)

    def _computers_and_humans(self, players, my_login):
        computers = []
        humans = []
        for p in players:
            if p in ["ai_aggressive", "ai_easy"]:
                computers.append(DummyClient(p[3:]))
            else:
                if p != my_login:
                    humans.append(HalfDummyClient(p))
                else:
                    humans.append(None) # marked for further replacement, because the order must be the same (the worlds must be the same)
        return computers, humans

class _Savable(object):

    def save(self):
        f = open(SAVE_PATH, "w")
        i = stats.Stats(None, None)._get_weak_user_id()
        f.write("%s\n" % i)
        self.world.remove_links_for_savegame()
        self._rules = clientstyle._rules
        self._ai = clientstyle._ai
        self._style = clientstyle._style
        try:
            pickle.dump(self, f)
            voice.info([105])
        except:
            exception("save game failed")
            voice.alert([1029]) # hostile sound
        self.world.restore_links_for_savegame()

    def run_on(self):
        sounds.enter_map(self.map.mapfile)
        self.world.restore_links_for_savegame()
        clientstyle._rules = self._rules
        clientstyle._ai = self._ai
        clientstyle._style = self._style
        clientworld.update_orders_list() # when style has changed
        self.interface.set_self_as_listener()
        self.interface.loop()
        self._record_stats(self.world)
        self.post_run()
        self.world.clean()

class TrainingGame(_MultiplayerGame, _Savable):

    game_type_name = "training"

    def __init__(self, map, computers):
        self.map = map
        self.me = DirectClient(config.login, self)
        self.players = [self.me] + [DummyClient(x) for x in computers]

class MissionGame(_Game, _Savable):

    game_type_name = "mission"
    _has_victory = False

    def __init__(self, map):
        self.map = map
        self.me = DirectClient(config.login, self)
        self.players = [self.me]

    def pre_run(self):
        if self.world.intro: # only in missions (streaming sound)
            sounds.play_sequence(self.world.intro)

    def post_run(self):
        _Game.post_run(self)
        self._has_victory = self.me.has_victory()

    def has_victory(self):
        return self._has_victory

    def run_on(self):
        sounds.enter_campaign(self.map.campaign.path)
        _Savable.run_on(self)
        self.map.run_next_step(self)
        sounds.exit_campaign()

# maps and campaigns

class MissionChapter(Map):

    def __init__(self, p, campaign, id):
        Map.__init__(self, p)
        self.campaign = campaign
        self.id = id

    def _get_next(self):
        return self.campaign.get_next(self)

    def _victory(self):
        menu = clientmenu.Menu([], [])
        menu.append([4011], self._get_next()) # continue
        menu.append([4009], None) # cancel
        menu.run()

    def _defeat(self):
        menu = clientmenu.Menu([], [])
        menu.append([4266], self) # restart
        menu.append([4009], None) # cancel
        menu.run()

    def run(self):
        voice.important(self.title)
        game = MissionGame(self)
        game.run()
        self.run_next_step(game)

    def run_next_step(self, game):
        if game.has_victory():
            self.campaign.unlock_next(self)
            if self._get_next():
                self._victory()
        else:
            self._defeat()

class CutSceneChapter:

    def __init__(self, path, campaign=None, id=None):
        self.path = path
        self.campaign = campaign
        self.id = id
        self._load()

    def _load(self):
        s = open(self.path, "U").read() # "universal newlines"
        # header
        m = re.search("(?m)^title[ \t]+([0-9 ]+)$", s)
        if m:
            l = m.group(1).split(" ")
            l = [int(x) for x in l]
        else:
            l = number(self.id)
        self.title = l
        # content
        m = re.search("(?m)^sequence[ \t]+([0-9 ]+)$", s)
        if m:
            l = m.group(1).split(" ")
        else:
            l = []
        self.sequence = l

    def _get_next(self):
        return self.campaign.get_next(self)

    def run(self):
        voice.important(self.title)
        sounds.play_sequence(self.sequence)
        self.campaign.unlock_next(self)
        if self._get_next():
            self._get_next().run()

class Campaign(object):

    def __init__(self, path, title=None):
        self.path = path
        if title:
            self.title = title
        else:
            self.title = [os.path.split(path)[1]]
        self.chapters = []
        i = 0
        while True:
            cp = os.path.join(self.path, "%s.txt" % i)
            if not os.path.isfile(cp):
                cp = os.path.join(self.path, "%s" % i)
                if not os.path.isdir(cp):
                    break
            if os.path.isfile(cp) and \
               open(cp, "U").readline() == "cut_scene_chapter\n":
                c = CutSceneChapter(cp, campaign=self, id=i)
            else:
                c = MissionChapter(cp, campaign=self, id=i)
            self.chapters.append(c)
            i += 1

    def _get(self, id):
        if id < len(self.chapters):
            return self.chapters[id]
        else:
            return None

    def get_next(self, chapter):
        return self._get(chapter.id + 1)

    def _get_id(self):
        return re.sub("[^a-zA-Z0-9]", "_", self.path)

    def _get_bookmark(self):
        c = ConfigParser.SafeConfigParser()
        if os.path.isfile(CAMPAIGNS_CONFIG_PATH):
            c.readfp(open(CAMPAIGNS_CONFIG_PATH))
        try:
            return c.getint(self._get_id(), "chapter")
        except:
            return 0

    def _available_chapters(self):
        return self.chapters[:self._get_bookmark() + 1]

    def _set_bookmark(self, number):
        c = ConfigParser.SafeConfigParser()
        if os.path.isfile(CAMPAIGNS_CONFIG_PATH):
            c.readfp(open(CAMPAIGNS_CONFIG_PATH))
        if self._get_id() not in c.sections():
            c.add_section(self._get_id())
        c.set(self._get_id(), "chapter", repr(number))
        c.write(open(CAMPAIGNS_CONFIG_PATH, "w"))

    def unlock_next(self, chapter):
        if self._get_bookmark() == chapter.id:
            next_chapter = self.get_next(chapter)
            if next_chapter:
                self._set_bookmark(next_chapter.id)

    def run(self):
        sounds.enter_campaign(self.path)
        menu = clientmenu.Menu(self.title, [],
                default_choice_index=len(self._available_chapters()) - 1)
        for ch in self._available_chapters():
            menu.append(ch.title, ch)
#        menu.append([4113], "restore")
        menu.append([4118], None) # "cancel"
        menu.run()
        sounds.exit_campaign()
<commit_msg>bug fixed: impossible to restore a game<commit_after>import ConfigParser
import pickle
import re
import sys
import threading
import time

from clientmedia import *
import clientgame
import clientmenu
import clientstyle
import clientworld
import config
from constants import *
from mapfile import *
from paths import *
import res
import tts
from world import *
from worldclient import *

class _Game(object):

    default_triggers = () # empty tuple; a tuple is immutable
    seed = None
    game_type_name = None
    alliances = ()
    races = ()

    def _game_type(self):
        return "%s/%s/%s" % (VERSION,
                             self.game_type_name + "-" + self.map.get_name(),
                             self.nb_human_players)

    def _record_stats(self, world):
        s = stats.Stats(STATS_PATH, METASERVER_URL)
        s.add(self._game_type(), int(world.time / 1000))

    def run(self, speed=config.speed):
        self.world = World(self.default_triggers, self.seed)
        if self.world.load_and_build_map(self.map):
            load_style(res.get_text("ui/style", append=True, locale=True),
                       self.map.campaign_style,
                       self.map.additional_style)
            sounds.enter_map(self.map.mapfile)
            clientworld.update_orders_list() # when style has changed
            self.pre_run()
            self.interface = clientgame.GameInterface(self.me, speed=speed)
            self.interface.load_bindings(
                res.get_text("ui/bindings", append=True, locale=True) + "\n" +
                self.map.get_campaign("ui/bindings.txt") + "\n" +
                self.map.get_additional("ui/bindings.txt"))
            self.world.populate_map(self.players, self.alliances, self.races)
            self.nb_human_players = self.world.current_nb_human_players()
            threading.Thread(target=self.world.loop).start()
            self.interface.loop()
            self._record_stats(self.world)
            self.post_run()
            sounds.exit_map()
            self.world.clean()
        else:
            voice.alert([1029]) # hostile sound
            if tts.is_available:
                voice.alert([self.world.map_error])

    def pre_run(self):
        pass

    def post_run(self):
        self.say_score()

    def say_score(self):
        for msg in self.interface.get_score_msgs():
            voice.info(msg)
        voice.flush()

class _MultiplayerGame(_Game):

    default_triggers = (
        ["players", ["no_enemy_player_left"], ["victory"]],
        ["players", ["no_building_left"], ["defeat"]],
        ["computers", ["no_unit_left"], ["defeat"]],
        ) # a tuple is immutable

class MultiplayerGame(_MultiplayerGame):

    game_type_name = "multiplayer"

    def __init__(self, map, players, my_login, main_server, seed, speed):
        self.map = map
        computers, humans = self._computers_and_humans(players, my_login)
        self.me = Coordinator(my_login, main_server, humans)
        humans[humans.index(None)] = self.me
        self.players = humans + computers # humans first because the first in the list is the game admin
        self.seed = seed
        self.speed = speed
        self.main_server = main_server

    def run(self):
        _MultiplayerGame.run(self, speed=self.speed)

    def _countdown(self):
        voice.important([4062]) # "the game starts in 5 seconds"
        for n in [5, 4, 3, 2, 1, 0]:
            voice.item(nombre(n))
            time.sleep(1)
        pygame.event.clear(KEYDOWN)

    def pre_run(self):
        nb_human_players = len([p for p in self.players if p.login != "ai"])
        if nb_human_players > 1:
            if compatibility_version() != COMPATIBILITY_VERSION:
                warning("rules.txt or ai.txt has been modified"
                        " after the program started: exit...")
                sys.exit()
            send_platform_version_to_metaserver(self.map.get_name(), nb_human_players)
            self._countdown()

    def post_run(self):
        # alert the server of the exit from the game interface
        if self.interface.forced_quit:
            self.main_server.write_line("abort_game")
        else:
            self.main_server.write_line("quit_game")
        self.say_score() # say score only after quit_game to avoid blocking the main server
        voice.menu([4010, 4030]) # "menu" "please make a selection" (long enough to allow history navigation)

    def _computers_and_humans(self, players, my_login):
        computers = []
        humans = []
        for p in players:
            if p in ["ai_aggressive", "ai_easy"]:
                computers.append(DummyClient(p[3:]))
            else:
                if p != my_login:
                    humans.append(HalfDummyClient(p))
                else:
                    humans.append(None) # marked for further replacement, because the order must be the same (the worlds must be the same)
        return computers, humans

class _Savable(object):

    def save(self):
        f = open(SAVE_PATH, "w")
        i = stats.Stats(None, None)._get_weak_user_id()
        f.write("%s\n" % i)
        self.world.remove_links_for_savegame()
        self._rules = clientstyle._rules
        self._ai = clientstyle._ai
        self._style = clientstyle._style
        try:
            pickle.dump(self, f)
            voice.info([105])
        except:
            exception("save game failed")
            voice.alert([1029]) # hostile sound
        self.world.restore_links_for_savegame()

    def run_on(self):
        sounds.enter_map(self.map.mapfile)
        self.world.restore_links_for_savegame()
        clientstyle._rules = self._rules
        clientstyle._ai = self._ai
        clientstyle._style = self._style
        clientworld.update_orders_list() # when style has changed
        self.interface.set_self_as_listener()
        threading.Thread(target=self.world.loop).start()
        self.interface.loop()
        self._record_stats(self.world)
        self.post_run()
        self.world.clean()

class TrainingGame(_MultiplayerGame, _Savable):

    game_type_name = "training"

    def __init__(self, map, computers):
        self.map = map
        self.me = DirectClient(config.login, self)
        self.players = [self.me] + [DummyClient(x) for x in computers]

class MissionGame(_Game, _Savable):

    game_type_name = "mission"
    _has_victory = False

    def __init__(self, map):
        self.map = map
        self.me = DirectClient(config.login, self)
        self.players = [self.me]

    def pre_run(self):
        if self.world.intro: # only in missions (streaming sound)
            sounds.play_sequence(self.world.intro)

    def post_run(self):
        _Game.post_run(self)
        self._has_victory = self.me.has_victory()

    def has_victory(self):
        return self._has_victory

    def run_on(self):
        sounds.enter_campaign(self.map.campaign.path)
        _Savable.run_on(self)
        self.map.run_next_step(self)
        sounds.exit_campaign()

# maps and campaigns

class MissionChapter(Map):

    def __init__(self, p, campaign, id):
        Map.__init__(self, p)
        self.campaign = campaign
        self.id = id

    def _get_next(self):
        return self.campaign.get_next(self)

    def _victory(self):
        menu = clientmenu.Menu([], [])
        menu.append([4011], self._get_next()) # continue
        menu.append([4009], None) # cancel
        menu.run()

    def _defeat(self):
        menu = clientmenu.Menu([], [])
        menu.append([4266], self) # restart
        menu.append([4009], None) # cancel
        menu.run()

    def run(self):
        voice.important(self.title)
        game = MissionGame(self)
        game.run()
        self.run_next_step(game)

    def run_next_step(self, game):
        if game.has_victory():
            self.campaign.unlock_next(self)
            if self._get_next():
                self._victory()
        else:
            self._defeat()

class CutSceneChapter:

    def __init__(self, path, campaign=None, id=None):
        self.path = path
        self.campaign = campaign
        self.id = id
        self._load()

    def _load(self):
        s = open(self.path, "U").read() # "universal newlines"
        # header
        m = re.search("(?m)^title[ \t]+([0-9 ]+)$", s)
        if m:
            l = m.group(1).split(" ")
            l = [int(x) for x in l]
        else:
            l = number(self.id)
        self.title = l
        # content
        m = re.search("(?m)^sequence[ \t]+([0-9 ]+)$", s)
        if m:
            l = m.group(1).split(" ")
        else:
            l = []
        self.sequence = l

    def _get_next(self):
        return self.campaign.get_next(self)

    def run(self):
        voice.important(self.title)
        sounds.play_sequence(self.sequence)
        self.campaign.unlock_next(self)
        if self._get_next():
            self._get_next().run()

class Campaign(object):

    def __init__(self, path, title=None):
        self.path = path
        if title:
            self.title = title
        else:
            self.title = [os.path.split(path)[1]]
        self.chapters = []
        i = 0
        while True:
            cp = os.path.join(self.path, "%s.txt" % i)
            if not os.path.isfile(cp):
                cp = os.path.join(self.path, "%s" % i)
                if not os.path.isdir(cp):
                    break
            if os.path.isfile(cp) and \
               open(cp, "U").readline() == "cut_scene_chapter\n":
                c = CutSceneChapter(cp, campaign=self, id=i)
            else:
                c = MissionChapter(cp, campaign=self, id=i)
            self.chapters.append(c)
            i += 1

    def _get(self, id):
        if id < len(self.chapters):
            return self.chapters[id]
        else:
            return None

    def get_next(self, chapter):
        return self._get(chapter.id + 1)

    def _get_id(self):
        return re.sub("[^a-zA-Z0-9]", "_", self.path)

    def _get_bookmark(self):
        c = ConfigParser.SafeConfigParser()
        if os.path.isfile(CAMPAIGNS_CONFIG_PATH):
            c.readfp(open(CAMPAIGNS_CONFIG_PATH))
        try:
            return c.getint(self._get_id(), "chapter")
        except:
            return 0

    def _available_chapters(self):
        return self.chapters[:self._get_bookmark() + 1]

    def _set_bookmark(self, number):
        c = ConfigParser.SafeConfigParser()
        if os.path.isfile(CAMPAIGNS_CONFIG_PATH):
            c.readfp(open(CAMPAIGNS_CONFIG_PATH))
        if self._get_id() not in c.sections():
            c.add_section(self._get_id())
        c.set(self._get_id(), "chapter", repr(number))
        c.write(open(CAMPAIGNS_CONFIG_PATH, "w"))

    def unlock_next(self, chapter):
        if self._get_bookmark() == chapter.id:
            next_chapter = self.get_next(chapter)
            if next_chapter:
                self._set_bookmark(next_chapter.id)

    def run(self):
        sounds.enter_campaign(self.path)
        menu = clientmenu.Menu(self.title, [],
                default_choice_index=len(self._available_chapters()) - 1)
        for ch in self._available_chapters():
            menu.append(ch.title, ch)
#        menu.append([4113], "restore")
        menu.append([4118], None) # "cancel"
        menu.run()
        sounds.exit_campaign()
<|endoftext|>


<commit_before>import numpy as np
import cv2
import os

#function that creates 3D array
#function that walks through the os, varies with user

def img_array():
	path = 'c:/Users/Charles/Desktop/landsat8_images'
	my_array = []
	for (path, dirs, files) in os.walk(path):
		#print(path)
		#print(files)
		for file in files:
			if file.endswith('.TIF'):
				thefile = os.path.join(path, file)
				#print(thefile)
				my_array.append(thefile)
	print(my_array)
	return (my_array)

def print_img(my_array):
	for elt in my_array:
		location = elt
		img = cv2.imread(location)
		resize_img = cv2.resize(img, (184,184))
		cv2.imshow('image_1', resize_img)
		'''
		b, g, r = cv2.split(img)
		cv2.imshow('image_2',b)
		cv2.imshow('image_3',g)
		cv2.imshow('image_4',r)
		'''
		k = cv2.waitKey()
		#press ESC key to close window
		while (k != 27):
			k = cv2.waitKey()
		cv2.destroyAllWindows()
		return

def main():
	my_array = img_array()
	print_img(my_array)
main()
<commit_msg>move return<commit_after>import numpy as np
import cv2
import os

#function that creates 3D array
#function that walks through the os, varies with user

def img_array():
	path = 'c:/Users/Charles/Desktop/landsat8_images'
	my_array = []
	for (path, dirs, files) in os.walk(path):
		#print(path)
		#print(files)
		for file in files:
			if file.endswith('.TIF'):
				thefile = os.path.join(path, file)
				#print(thefile)
				my_array.append(thefile)
	print(my_array)
	return (my_array)

def print_img(my_array):
	for elt in my_array:
		location = elt
		img = cv2.imread(location)
		resize_img = cv2.resize(img, (184,184))
		cv2.imshow('image_1', resize_img)
		'''
		b, g, r = cv2.split(img)
		cv2.imshow('image_2',b)
		cv2.imshow('image_3',g)
		cv2.imshow('image_4',r)
		'''
		k = cv2.waitKey()
		#press ESC key to close window
		while (k != 27):
			k = cv2.waitKey()
		cv2.destroyAllWindows()
	return

def main():
	my_array = img_array()
	print_img(my_array)
main()
<|endoftext|>