<commit_before>#!/usr/bin/env python3

"""
A machine learning classifier for identifying minimal semantic units. See the README.md for more information.
"""

from enum import Enum
from typing import Dict, List, Tuple

FeatureID = int

class Label(Enum):
    O = 0
    B = 1
    I = 2

class Frequency:
    def __init__(self):
        self._frequencies: Dict[Label, int] = {label: 0 for label in Label}

    def __getitem__(self, label: Label):
        return self._frequencies[label]

    def __setitem__(self, key: Label, value: int):
        self._frequencies[key] = value

class Prediction:
    def __init__(self, line_number: int, predicted_label: Label, actual_label: Label, features: List[FeatureID]):
        self.line_number = line_number
        self.predicted_label = predicted_label
        self.actual_label = actual_label
        self.features = features

class MWE:
    def __init__(self):
        self.frequencies: Dict[FeatureID, Frequency] = {}

    def train(self, training_datafile: str):
        with open(training_datafile) as df:
            for line in df:
                label, features = self._read_line(line)
                for feature in features:
                    frequency = self.frequencies.get(feature)
                    if frequency is None:
                        frequency = Frequency()
                        self.frequencies[feature] = frequency
                    frequency[label] += 1

    def test(self, testing_datafile: str):
        pass

    @staticmethod
    def _read_line(line: str) -> Tuple[Label, List[FeatureID]]:
        # Lines are assumed to be written in the form:
        #   <label> <feature_1> <feature_2> ...
        parts = [int(part) for part in line.strip().split()]
        label = Label(parts[0])
        features = parts[1:]
        return label, features

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('training_datafile')
    parser.add_argument('testing_datafile')
    args = parser.parse_args()

    mwe = MWE()

    # Train the MWE recognizer.
    mwe.train(args.training_datafile)

    # Now test it!
    mwe.test(args.testing_datafile)
<commit_msg>Add total occurrences counter to Frequency<commit_after>#!/usr/bin/env python3

"""
A machine learning classifier for identifying minimal semantic units. See the README.md for more information.
"""

from enum import Enum
from typing import Dict, List, Tuple

FeatureID = int

class Label(Enum):
    O = 0
    B = 1
    I = 2

class Frequency:
    def __init__(self):
        self._frequencies: Dict[Label, int] = {label: 0 for label in Label}

    def __getitem__(self, label: Label) -> int:
        return self._frequencies[label]

    def __setitem__(self, key: Label, value: int):
        self._frequencies[key] = value

    @property
    def total_occurrences(self) -> int:
        return sum(self._frequencies.values())

class Prediction:
    def __init__(self, line_number: int, predicted_label: Label, actual_label: Label, features: List[FeatureID]):
        self.line_number = line_number
        self.predicted_label = predicted_label
        self.actual_label = actual_label
        self.features = features

class MWE:
    def __init__(self):
        self.frequencies: Dict[FeatureID, Frequency] = {}

    def train(self, training_datafile: str):
        with open(training_datafile) as df:
            for line in df:
                label, features = self._read_line(line)
                for feature in features:
                    frequency = self.frequencies.get(feature)
                    if frequency is None:
                        frequency = Frequency()
                        self.frequencies[feature] = frequency
                    frequency[label] += 1

    def test(self, testing_datafile: str):
        pass

    @staticmethod
    def _read_line(line: str) -> Tuple[Label, List[FeatureID]]:
        # Lines are assumed to be written in the form:
        #   <label> <feature_1> <feature_2> ...
        parts = [int(part) for part in line.strip().split()]
        label = Label(parts[0])
        features = parts[1:]
        return label, features

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('training_datafile')
    parser.add_argument('testing_datafile')
    args = parser.parse_args()

    mwe = MWE()

    # Train the MWE recognizer.
    mwe.train(args.training_datafile)

    # Now test it!
    mwe.test(args.testing_datafile)
<|endoftext|>


<commit_before># Copyright 2016 Jon Wayne Parrott
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import nox

@nox.session
def default(session):
    session.install('-r', 'requirements-test.txt')
    session.install('-e', '.[tox_to_nox]')
    tests = session.posargs or ['tests/']
    session.run(
        'py.test', '--cov=nox', '--cov-config', '.coveragerc',
        '--cov-report=', *tests)
    session.notify('cover')

@nox.session
@nox.parametrize('version', ['2.7', '3.4', '3.5', '3.6'])
def interpreters(session, version):
    default(session)
    session.interpreter = 'python' + version

@nox.session
def cover(session):
    session.install('coverage')
    session.run('coverage', 'report', '--fail-under=100', '--show-missing')
    session.run('coverage', 'erase')

@nox.session
def lint(session):
    session.install('flake8', 'flake8-import-order')
    session.run(
        'flake8',
        '--import-order-style=google',
        '--application-import-names=nox,tests',
        'nox', 'tests')

@nox.session
def docs(session):
    session.interpreter = 'python3.6'
    session.run('rm', '-rf', 'docs/_build')
    session.install('-r', 'requirements-test.txt')
    # Readthedocs uses setuptools to install the package, so we do as well.
    session.run('python', 'setup.py', 'install')
    session.cd('docs')
    session.run(
        'sphinx-build',
        '-b', 'html',
        '-d', '_build/doctrees',
        '.', '_build/html')
<commit_msg>Adding coverage exemption for AppVeyor.<commit_after># Copyright 2016 Jon Wayne Parrott
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

import nox

ON_APPVEYOR = os.environ.get('APPVEYOR') == 'True'

@nox.session
def default(session):
    session.install('-r', 'requirements-test.txt')
    session.install('-e', '.[tox_to_nox]')
    tests = session.posargs or ['tests/']
    session.run(
        'py.test', '--cov=nox', '--cov-config', '.coveragerc',
        '--cov-report=', *tests)
    session.notify('cover')

@nox.session
@nox.parametrize('version', ['2.7', '3.4', '3.5', '3.6'])
def interpreters(session, version):
    default(session)
    session.interpreter = 'python' + version

@nox.session
def cover(session):
    session.install('coverage')
    if ON_APPVEYOR:
        fail_under = '--fail-under=99'
    else:
        fail_under = '--fail-under=100'
    session.run(
        'coverage',
        'report',
        fail_under,
        '--show-missing',
    )
    session.run('coverage', 'erase')

@nox.session
def lint(session):
    session.install('flake8', 'flake8-import-order')
    session.run(
        'flake8',
        '--import-order-style=google',
        '--application-import-names=nox,tests',
        'nox', 'tests')

@nox.session
def docs(session):
    session.interpreter = 'python3.6'
    session.run('rm', '-rf', 'docs/_build')
    session.install('-r', 'requirements-test.txt')
    # Readthedocs uses setuptools to install the package, so we do as well.
    session.run('python', 'setup.py', 'install')
    session.cd('docs')
    session.run(
        'sphinx-build',
        '-b', 'html',
        '-d', '_build/doctrees',
        '.', '_build/html')
<|endoftext|>


<commit_before># Copyright 2017 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import os

import nox
import nox.command

LOCAL_DEPS = (
    os.path.join('..', 'api_core'),
    os.path.join('..', 'core'),
)

@nox.session
def default(session):
    """Default unit test session.

    This is intended to be run **without** an interpreter set, so
    that the current ``python`` (on the ``PATH``) or the version of
    Python corresponding to the ``nox`` binary the ``PATH`` can
    run the tests.
    """
    # Install all test dependencies, then install this package in-place.
    session.install('mock', 'pytest', 'pytest-cov', *LOCAL_DEPS)
    session.install('-e', '.')

    # Run py.test against the unit tests.
    session.run(
        'py.test',
        '--quiet',
        '--cov=google.cloud.firestore',
        '--cov=google.cloud.firestore_v1beta1',
        '--cov=tests.unit',
        '--cov-append',
        '--cov-config=.coveragerc',
        '--cov-report=',
        '--cov-fail-under=97',
        os.path.join('tests', 'unit'),
        *session.posargs
    )

@nox.session
@nox.parametrize('py', ['2.7', '3.4', '3.5', '3.6'])
def unit(session, py):
    """Run the unit test suite."""

    # Run unit tests against all supported versions of Python.
    session.interpreter = 'python{}'.format(py)

    # Set the virtualenv dirname.
    session.virtualenv_dirname = 'unit-' + py

    default(session)

@nox.session
@nox.parametrize('py', ['2.7', '3.6'])
def system(session, py):
    """Run the system test suite."""
    # Sanity check: Only run system tests if the environment variable is set.
    if not os.environ.get('FIRESTORE_APPLICATION_CREDENTIALS'):
        session.skip('Credentials must be set via environment variable.')

    # Run the system tests against latest Python 2 and Python 3 only.
    session.interpreter = 'python{}'.format(py)

    # Set the virtualenv dirname.
    session.virtualenv_dirname = 'sys-' + py

    # Install all test dependencies, then install this package into the
    # virtualenv's dist-packages.
    session.install('mock', 'pytest', *LOCAL_DEPS)
    session.install(os.path.join('..', 'test_utils'))
    session.install('.')

    # Run py.test against the system tests.
    session.run(
        'py.test',
        os.path.join('tests', 'system.py'),
        *session.posargs
    )

@nox.session
def lint(session):
    """Run linters.

    Returns a failure if the linters find linting errors or sufficiently
    serious code quality issues.
    """
    session.interpreter = 'python3.6'
    session.install('flake8', *LOCAL_DEPS)
    session.install('.')
    session.run('flake8', 'google', 'tests')

@nox.session
def lint_setup_py(session):
    """Verify that setup.py is valid (including RST check)."""
    session.interpreter = 'python3.6'

    # Set the virtualenv dirname.
    session.virtualenv_dirname = 'setup'

    session.install('docutils', 'Pygments')
    session.run(
        'python', 'setup.py', 'check', '--restructuredtext', '--strict')

@nox.session
def cover(session):
    """Run the final coverage report.

    This outputs the coverage report aggregating coverage from the unit
    test runs (not system test runs), and then erases coverage data.
    """
    session.interpreter = 'python3.6'
    session.chdir(os.path.dirname(__file__))
    session.install('coverage', 'pytest-cov')
    session.run('coverage', 'report', '--show-missing', '--fail-under=100')
    session.run('coverage', 'erase')
<commit_msg>Add Test runs for Python 3.7 and remove 3.4 (#5295)<commit_after># Copyright 2017 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import os

import nox
import nox.command

LOCAL_DEPS = (
    os.path.join('..', 'api_core'),
    os.path.join('..', 'core'),
)

@nox.session
def default(session):
    """Default unit test session.

    This is intended to be run **without** an interpreter set, so
    that the current ``python`` (on the ``PATH``) or the version of
    Python corresponding to the ``nox`` binary the ``PATH`` can
    run the tests.
    """
    # Install all test dependencies, then install this package in-place.
    session.install('mock', 'pytest', 'pytest-cov', *LOCAL_DEPS)
    session.install('-e', '.')

    # Run py.test against the unit tests.
    session.run(
        'py.test',
        '--quiet',
        '--cov=google.cloud.firestore',
        '--cov=google.cloud.firestore_v1beta1',
        '--cov=tests.unit',
        '--cov-append',
        '--cov-config=.coveragerc',
        '--cov-report=',
        '--cov-fail-under=97',
        os.path.join('tests', 'unit'),
        *session.posargs
    )

@nox.session
@nox.parametrize('py', ['2.7', '3.5', '3.6', '3.7'])
def unit(session, py):
    """Run the unit test suite."""

    # Run unit tests against all supported versions of Python.
    session.interpreter = 'python{}'.format(py)

    # Set the virtualenv dirname.
    session.virtualenv_dirname = 'unit-' + py

    default(session)

@nox.session
@nox.parametrize('py', ['2.7', '3.6'])
def system(session, py):
    """Run the system test suite."""
    # Sanity check: Only run system tests if the environment variable is set.
    if not os.environ.get('FIRESTORE_APPLICATION_CREDENTIALS'):
        session.skip('Credentials must be set via environment variable.')

    # Run the system tests against latest Python 2 and Python 3 only.
    session.interpreter = 'python{}'.format(py)

    # Set the virtualenv dirname.
    session.virtualenv_dirname = 'sys-' + py

    # Install all test dependencies, then install this package into the
    # virtualenv's dist-packages.
    session.install('mock', 'pytest', *LOCAL_DEPS)
    session.install(os.path.join('..', 'test_utils'))
    session.install('.')

    # Run py.test against the system tests.
    session.run(
        'py.test',
        os.path.join('tests', 'system.py'),
        *session.posargs
    )

@nox.session
def lint(session):
    """Run linters.

    Returns a failure if the linters find linting errors or sufficiently
    serious code quality issues.
    """
    session.interpreter = 'python3.6'
    session.install('flake8', *LOCAL_DEPS)
    session.install('.')
    session.run('flake8', 'google', 'tests')

@nox.session
def lint_setup_py(session):
    """Verify that setup.py is valid (including RST check)."""
    session.interpreter = 'python3.6'

    # Set the virtualenv dirname.
    session.virtualenv_dirname = 'setup'

    session.install('docutils', 'Pygments')
    session.run(
        'python', 'setup.py', 'check', '--restructuredtext', '--strict')

@nox.session
def cover(session):
    """Run the final coverage report.

    This outputs the coverage report aggregating coverage from the unit
    test runs (not system test runs), and then erases coverage data.
    """
    session.interpreter = 'python3.6'
    session.chdir(os.path.dirname(__file__))
    session.install('coverage', 'pytest-cov')
    session.run('coverage', 'report', '--show-missing', '--fail-under=100')
    session.run('coverage', 'erase')
<|endoftext|>


<commit_before># Copyright 2016 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import os
import sys

import nox

LOCAL_DEPS = (
    os.path.join('..', 'api_core'),
    os.path.join('..', 'core'),
)
UNIT_TEST_DEPS = (
    'mock',
    'pytest',
    'pytest-cov',
    'flask',
    'webapp2',
    'webob',
)

@nox.session
def default(session):
    """Default unit test session.

    This is intended to be run **without** an interpreter set, so
    that the current ``python`` (on the ``PATH``) or the version of
    Python corresponding to the ``nox`` binary the ``PATH`` can
    run the tests.
    """
    # Install all test dependencies, then install this package in-place.
    deps = UNIT_TEST_DEPS
    if session.interpreter == 'python2.7':
        deps += ('django >= 1.11.0, < 2.0.0dev',)
    elif session.interpreter is None and sys.version_info[:2] == (2, 7):
        deps += ('django >= 1.11.0, < 2.0.0dev',)
    else:
        deps += ('django',)

    deps += LOCAL_DEPS
    session.install(*deps)
    session.install('-e', '.')

    # Run py.test against the unit tests.
    session.run(
        'py.test',
        '--quiet',
        '--cov=google.cloud.logging',
        '--cov=tests.unit',
        '--cov-append',
        '--cov-config=.coveragerc',
        '--cov-report=',
        '--cov-fail-under=97',
        'tests/unit',
        *session.posargs
    )

@nox.session
@nox.parametrize('py', ['2.7', '3.4', '3.5', '3.6'])
def unit(session, py):
    """Run the unit test suite."""

    # Run unit tests against all supported versions of Python.
    session.interpreter = 'python{}'.format(py)

    # Set the virtualenv dirname.
    session.virtualenv_dirname = 'unit-' + py

    default(session)

@nox.session
@nox.parametrize('py', ['2.7', '3.6'])
def system(session, py):
    """Run the system test suite."""

    # Sanity check: Only run system tests if the environment variable is set.
    if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
        session.skip('Credentials must be set via environment variable.')

    # Run the system tests against latest Python 2 and Python 3 only.
    session.interpreter = 'python{}'.format(py)

    # Set the virtualenv dirname.
    session.virtualenv_dirname = 'sys-' + py

    # Install all test dependencies, then install this package into the
    # virtualenv's dist-packages.
    session.install('mock', 'pytest', *LOCAL_DEPS)
    session.install('../test_utils/', '../bigquery/', '../pubsub/',
                    '../storage/')
    session.install('.')

    # Run py.test against the system tests.
    session.run(
        'py.test',
        '-vvv',
        '-s',
        'tests/system',
        *session.posargs)

@nox.session
def lint(session):
    """Run linters.

    Returns a failure if the linters find linting errors or sufficiently
    serious code quality issues.
    """
    session.interpreter = 'python3.6'
    session.install('flake8')
    session.install('.')
    session.run('flake8', 'google', 'tests')

@nox.session
def lint_setup_py(session):
    """Verify that setup.py is valid (including RST check)."""
    session.interpreter = 'python3.6'

    # Set the virtualenv dirname.
    session.virtualenv_dirname = 'setup'

    session.install('docutils', 'Pygments')
    session.run(
        'python', 'setup.py', 'check', '--restructuredtext', '--strict')

@nox.session
def cover(session):
    """Run the final coverage report.

    This outputs the coverage report aggregating coverage from the unit
    test runs (not system test runs), and then erases coverage data.
    """
    session.interpreter = 'python3.6'
    session.install('coverage', 'pytest-cov')
    session.run('coverage', 'report', '--show-missing', '--fail-under=100')
    session.run('coverage', 'erase')
<commit_msg>Install local dependencies when running lint (#4936)<commit_after># Copyright 2016 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import os
import sys

import nox

LOCAL_DEPS = (
    os.path.join('..', 'api_core'),
    os.path.join('..', 'core'),
)
UNIT_TEST_DEPS = (
    'mock',
    'pytest',
    'pytest-cov',
    'flask',
    'webapp2',
    'webob',
)

@nox.session
def default(session):
    """Default unit test session.

    This is intended to be run **without** an interpreter set, so
    that the current ``python`` (on the ``PATH``) or the version of
    Python corresponding to the ``nox`` binary the ``PATH`` can
    run the tests.
    """
    # Install all test dependencies, then install this package in-place.
    deps = UNIT_TEST_DEPS
    if session.interpreter == 'python2.7':
        deps += ('django >= 1.11.0, < 2.0.0dev',)
    elif session.interpreter is None and sys.version_info[:2] == (2, 7):
        deps += ('django >= 1.11.0, < 2.0.0dev',)
    else:
        deps += ('django',)

    deps += LOCAL_DEPS
    session.install(*deps)
    session.install('-e', '.')

    # Run py.test against the unit tests.
    session.run(
        'py.test',
        '--quiet',
        '--cov=google.cloud.logging',
        '--cov=tests.unit',
        '--cov-append',
        '--cov-config=.coveragerc',
        '--cov-report=',
        '--cov-fail-under=97',
        'tests/unit',
        *session.posargs
    )

@nox.session
@nox.parametrize('py', ['2.7', '3.4', '3.5', '3.6'])
def unit(session, py):
    """Run the unit test suite."""

    # Run unit tests against all supported versions of Python.
    session.interpreter = 'python{}'.format(py)

    # Set the virtualenv dirname.
    session.virtualenv_dirname = 'unit-' + py

    default(session)

@nox.session
@nox.parametrize('py', ['2.7', '3.6'])
def system(session, py):
    """Run the system test suite."""

    # Sanity check: Only run system tests if the environment variable is set.
    if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
        session.skip('Credentials must be set via environment variable.')

    # Run the system tests against latest Python 2 and Python 3 only.
    session.interpreter = 'python{}'.format(py)

    # Set the virtualenv dirname.
    session.virtualenv_dirname = 'sys-' + py

    # Install all test dependencies, then install this package into the
    # virtualenv's dist-packages.
    session.install('mock', 'pytest', *LOCAL_DEPS)
    session.install('../test_utils/', '../bigquery/', '../pubsub/',
                    '../storage/')
    session.install('.')

    # Run py.test against the system tests.
    session.run(
        'py.test',
        '-vvv',
        '-s',
        'tests/system',
        *session.posargs)

@nox.session
def lint(session):
    """Run linters.

    Returns a failure if the linters find linting errors or sufficiently
    serious code quality issues.
    """
    session.interpreter = 'python3.6'
    session.install('flake8', *LOCAL_DEPS)
    session.install('.')
    session.run('flake8', 'google', 'tests')

@nox.session
def lint_setup_py(session):
    """Verify that setup.py is valid (including RST check)."""
    session.interpreter = 'python3.6'

    # Set the virtualenv dirname.
    session.virtualenv_dirname = 'setup'

    session.install('docutils', 'Pygments')
    session.run(
        'python', 'setup.py', 'check', '--restructuredtext', '--strict')

@nox.session
def cover(session):
    """Run the final coverage report.

    This outputs the coverage report aggregating coverage from the unit
    test runs (not system test runs), and then erases coverage data.
    """
    session.interpreter = 'python3.6'
    session.install('coverage', 'pytest-cov')
    session.run('coverage', 'report', '--show-missing', '--fail-under=100')
    session.run('coverage', 'erase')
<|endoftext|>


<commit_before>import zipfile
import os
import xml.etree.ElementTree as etree

class ODTPage:
    def getPage(self, name="test.odt"):
        odt = ODT(name)
        #res = odt.read()
        return self.getHeader()+self.getBody(odt)+self.getFooter()

    def getHeader(self):
        return """

            <title>ODT</title>
            <link rel="stylesheet" type="text/css" title="styles" href="odt.css"/>

        """
        #

    def getBody(self, odt):
        res = odt.read()
        if not res:
            cntx = "
Invalid file
"
        else:
            data = odt.parseStyles()
            cntx = "
Ok: %s
"%(data)
        return """

ODT-HTML5

            %s

"""%(cntx)

    def getFooter(self):
        return """"""

class ODT:
    def __init__(self, name):
        self._name = name
        self._zip = None

    def open(self):
        if not os.path.isfile(self._name):
            self._zip = None
            return False
        try:
            self._zip = zipfile.ZipFile(self._name, 'r')
        except zipfile.BadZipfile:
            self._zip = None
            return False
        return True

    def close(self):
        self._zip.close()

    def extract(self, file):
        if self._zip == None:
            return None
        return self._zip.read(file)

    def parseStyles(self):
        if self._styles == None:
            return None
        root = etree.fromstring(self._styles)
        res = " %s "%root.tag
        for child in root:
            res += " %s "%(child.tag)
        return res
        #parser = etree.XMLParser()
        #parser.feed(self._styles)
        #parser.close()

    def read(self):
        if not self.open():
            return False

        self._styles = self.extract("styles.xml")
        self._content = self.extract("content.xml")

        self.close()
        return True
<commit_msg>continue new parser work<commit_after>import zipfile
import os
import xml.etree.ElementTree as etree
import re

class ODTPage:
    def getPage(self, name="test.odt"):
        odt = ODT(name)
        #res = odt.read()
        return self.getHeader()+self.getBody(odt)+self.getFooter()

    def getHeader(self):
        return """

            <title>ODT</title>
            <link rel="stylesheet" type="text/css" title="styles" href="odt.css"/>

        """
        #

    def getBody(self, odt):
        res = odt.read()
        if not res:
            cntx = "
Invalid file
"
        else:
            data = odt.parseStyles()
            odt.parseTextProperties()
            cntx = "
Ok: %s
" % (data)
        return """

ODT-HTML5

            %s

"""%(cntx)

    def getFooter(self):
        return """"""

class ODT:
    def __init__(self, name):
        self._name = name
        self._zip = None
        self._styles = {}
        self._styles_xml = None
        self._content_xml = None
        self._stylename = None

    def open(self):
        if not os.path.isfile(self._name):
            self._zip = None
            return False
        try:
            self._zip = zipfile.ZipFile(self._name, 'r')
        except zipfile.BadZipfile:
            self._zip = None
            return False
        return True

    def close(self):
        self._zip.close()

    def extract(self, file):
        if self._zip == None:
            return None
        return self._zip.read(file)

    def cleanTag(self, tag):
        return re.sub("{[^}]+}","",tag).strip()

    def findElement(self, root, name):
        res = []
        #if self.cleanTag(root.tag) == name:
        if root.tag == name:
            res.append(root)
        for child in root:
            if child.tag == name:
                res.append(child)
            res += self.findElement(child, name)
        return res

    def parseStyleTag(self, styles):
        res = {}
        for style in styles:
            tmp = self.getAttrib(style, "name")
            if tmp is not None:
                res[tmp] = {}
                self._stylename = tmp

            pstyle = self.getAttrib(style, "parent-style-name")
            if pstyle is not None and res is not None:
                res[self._stylename]["parent"] = pstyle
        return res

    def filterAttributes(self, datas, keep):
        res = []
        for prop in props:
            style = {}
            for val in prop.attrib:
                if val in valid_text_attrs:
                    style[val] = prop.attrib[val]

            if style:
                res.append(style)

        return res

    def parseTextPropertyTag(self, props):
        valid_text_attrs = ["font-size", "color", "background-color", "font-weight",
            "font-style", "text-underline-style", "text-underline-color",
            "text-overline-style", "text-line-through-style" ]

        return filterAttributes(props, valid_text_attrs)

    def parseParagraphPropertyTag(self, props):
        valid_para_attrs = [ "break-before", "text-align", "color", "background-color",
            "text-indent", "margin-left", "margin-right", "margin-top", "margin-bottom" ]

        return filterAttributes(props, valid_para_attrs)

    def getAttrib(self, tag, name):
        for attrib in tag.attrib:
            #if self.cleanTag(attrib)==name:
            if attrib == name:
                return tag.attrib[attrib]
        return None

    def stripNamespace(self, root):
        for el in root.getiterator():
            if el.tag[0] == "{":
                el.tag = el.tag.split('}', 1)[1]
            tmp = {}
            for attr in el.attrib:
                if attr[0] == "{":
                    tmp[attr.split('}', 1)[1]] = el.attrib[attr]
                else:
                    tmp[attr] = el.attrib[attr]
            el.attrib = tmp

    def parseXML(self):
        if self._styles_xml == None:
            return None
        self.__root = etree.fromstring(self._styles_xml)
        self.stripNamespace(self.__root)

    def parseParagraphProperties(self):
        tags = self.findElement(self.__root, "paragraph-properties")
        self.parseParagraphPropertyTag(tags)

    def parseTextProperties(self):
        tags = self.findElement(self.__root, "text-properties")
        return self.parseTextPropertyTag(tags)

    def parseStyles(self):
        styles = self.findElement(self.__root, "style")
        return self.parseStyleTag(styles)

        #res += "
 %s
\n"%self.findElement(root, "style")
        #res += "
 %s
\n"%self.findElement(root, "text-properties")
        #res += "
 %s
\n"%self.findElement(root, "paragraph-properties")
        #parser = etree.XMLParser()
        #parser.feed(self._styles)
        #parser.close()

    def read(self):
        if not self.open():
            return False

        self._styles_xml = self.extract("styles.xml")
        self._content_xml = self.extract("content.xml")
        self.parseXML()

        self.close()
        return True
<|endoftext|>


<commit_before>import tensorflow as tf
import numpy as np
import os
import pickle
import time
from PIL import Image

# some global params
# the directory to which teh results will be saved
# imgSize = [96,128]
imgSize = [64,48]
spaceSize = [64,48]
batch_size = 20
# folder to save the results in
resDir = 'results/'
# folder to log the training progress in
log_dir  = 'train_log/1/'

learning_rate = 1e-5
# below is the coefficient for l2 loss
alpha = 0.001

model_save_path = ['savedModels/model_1.ckpt',
                    'savedModels/model_2.ckpt']

# in every traning example, these will be the names of the 5 spaces
nameList = ['red',
            'green',
            'blue',
            'yellow',
            'white']
# And this dictionary provides the color as RGB tuple for a space with a certian name
colors = {
    'red':(255,0,0),
    'green':(0,255,0),
    'blue':(0,0,255),
    'yellow':(255,255,0),
    'white':(255,255,255)
}

# this method saves the model
def saveModel(sess, savePath):
    print('\n...saving the models, please wait...')
    saver = tf.train.Saver()
    saver.save(sess, savePath)
    print('Saved the model to %s'%savePath)

# this method loads the saved model
def loadModel(sess, savedPath):
    print('\n...loading the models, please wait...')
    saver = tf.train.Saver()
    saver.restore(sess, savedPath)
    print('Loaded the model from %s'%savedPath)

# weight variable
def weightVariable(shape, name):
    initializer = tf.truncated_normal_initializer(mean=0.0, stddev=0.05)
    weight = tf.get_variable(name=name, shape=shape, initializer=initializer)
    return weight

# bias variable
def biasVariable(shape, name):
    initializer = tf.constant_initializer(0.01)
    bias = tf.get_variable(name=name, shape=shape, initializer=initializer)
    return bias

# 2d convolutional layer
def conv2d(x, W, strides = [1,2,2,1]):
    return tf.nn.conv2d(x, W, strides=strides, padding='SAME')

# 3d convolutional layer
def conv3d(x, W, strides = [1,2,2,1,1]):
    return tf.nn.conv3d(x,W,strides = strides, padding='SAME')

# deconv layer
def deConv3d(y, w, outShape, strides=[1,2,2,2,1]):
    return tf.nn.conv3d_transpose(y, w, output_shape = outShape, strides=strides, padding='SAME')

# this is max-pooling for 3d convolutional layers
def max_pool2x2x1(x):
    return tf.nn.max_pool3d(x,ksize=[1,2,2,1,1],strides=[1,2,2,1,1],padding='SAME')

# this is a 2x2 max-pooling layer for 2d convolutional layers
# this is max-pooling for 3d convolutional layers
def max_pool2x2(x):
    return tf.nn.max_pool(x,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')

# converts data to image
def toImage(data):
    data = np.reshape(data, [48,64])
    newData = 255*data
    # converting new data into integer format to make it possible to export it as a bitmap
    # in this case converting it into 8 bit integer
    newData = newData.astype(np.uint8)
    return Image.fromarray(newData)

# this method converts a list of images into a feed ready batch
def prepareImages(fileList):
    batch = []
    for fileName in fileList:
        img = Image.open(fileName).convert("L")
        arr = prepareImage(img)
        batch.append(arr)
    return np.array(batch)

# this one prepares a single image and returns it as an array
def prepareImage(img):
    arr = np.array(img)
    arr = arr.astype(np.float32)
    arr /= 255

    return arr

# this method returns the time estimate as a string
def estimate_time(startTime, totalCycles, finishedCycles):
    timePast = time.time() - startTime
    if timePast < 10:
        return '...calculating...'
    cps = finishedCycles/timePast
    if cps == 0:
        return '...calculating...'

    secsLeft = (totalCycles - finishedCycles)/cps

    # double slash is an integer division, returns the quotient without the decimal
    hrs, secs = secsLeft//3600, secsLeft % 3600
    mins, secs = secs//60, secs % 60

    timeStr = '%.0fh%.0fm%.0fs remaining'%(hrs, mins, secs) + ' '*10
    return timeStr

# this is the dataset object which is responsible with supplying data for training as well as
# testing purposes
class dataset:
    # the dirPath is the path of the directory
    # testFile is the name of the file with the test examples
    def __init__(self, dirPath = './', testFile='test'):
        self.dirPath = dirPath
        self.testFileName = testFile + '.pkl'
        # getting the list of files in the dirPath
        self.trainFileList = os.listdir(self.dirPath)
        # removing the test data file from the list of training data files
        if len(self.trainFileList) > 1:
            self.trainFileList.remove(self.testFileName)

        #removing any other file than pickled data files from the list
        i = 0
        while i < len(self.trainFileList):
            if not self.trainFileList[i].endswith('.pkl'):
                # print('removing %s'%self.trainFileList[i])
                del self.trainFileList[i]
                i -= 1

            i += 1

        # print(self.trainFileList)

        self.curFile = None
        if len(self.trainFileList) > 0:
            self.curFile = self.trainFileList[0]
        self.data = None
        self.test_data = None
        self.load_data()

        # this is the number of data samples currently loaded
        self.data_num = self.data[0].shape[0]
        self.test_data_num = self.test_data[0].shape[0]

        # this is the counter for where we are reading the data in the currently loaded set
        self.c = 0
        self.tc = 0

    # returns the name of the next file to be used once the current file is exhausted
    def next_file(self):
        fileNum = self.trainFileList.index(self.curFile)
        fileNum = (fileNum + 1)%len(self.trainFileList)
        nextFile = self.trainFileList[fileNum]
        return nextFile

    # this loads the data from the current file into the self.data
    def load_data(self, silent = False):
        with open(self.dirPath + self.curFile,'rb') as inp:
            if not silent: print('\nLoading data from %s...'%self.curFile)
            dSet = pickle.load(inp)

        self.data = [np.expand_dims(np.array(dSet[0]),3), np.array(dSet[1])]
        # self.data = [np.array(dSet[0]), np.array(dSet[1])]
        # self.data = dSet

        if self.test_data is None:
            with open(self.dirPath + self.testFileName,'rb') as inp:
                if not silent: print('\nLoading test data from %s...'%self.testFileName)
                dSet = pickle.load(inp)

            self.test_data = [np.expand_dims(np.array(dSet[0]),3), np.array(dSet[1])]
            # self.test_data = [np.array(dSet[0]), np.array(dSet[1])]
            # self.test_data = dSet
        if not silent: print('\nDataset in %s is successfully loaded\n'%self.dirPath)

    # this returns the next batch of the size - size
    def next_batch(self, size):
        if self.c + size >= self.data_num:
            end = self.data_num
            dataNeeded = size-(end-self.c)

            batch1 = [self.data[0][self.c: end], self.data[1][self.c: end]]

            # now getting the remaining data from the next file
            end = (self.c + size)%self.data_num
            self.c = 0
            self.curFile = self.next_file()
            self.load_data(silent = True)

            batch2 = [self.data[0][self.c: end], self.data[1][self.c: end]]
            # now joining batch 2 to the original batch
            batch = [np.concatenate((batch1[0], batch2[0]), axis=0), np.concatenate((batch1[1], batch2[1]), axis=0)]

            self.c = end

        elif self.c + size < self.data_num:
            end = self.c + size

            batch = [self.data[0][self.c: end], self.data[1][self.c: end]]
            self.c = end

        return batch

    # this fetches a test batch
    def test_batch(self, size):
        if self.tc + size >= self.test_data_num:
            end = self.test_data_num
            dataNeeded = size-(end-self.tc)

            batch1 = [self.test_data[0][self.tc: end], self.test_data[1][self.tc: end]]

            end = (self.tc + size)%self.test_data_num
            self.tc = 0

            batch2 = [self.test_data[0][self.tc: end], self.test_data[1][self.tc: end]]
            # now joining batch 2 to the original batch
            batch = [np.concatenate((batch1[0], batch2[0]), axis=0), np.concatenate((batch1[1], batch2[1]), axis=0)]

            self.tc = end

        elif self.tc + size < self.test_data_num:
            end = self.tc + size

            batch = [self.test_data[0][self.tc: end], self.test_data[1][self.tc: end]]
            self.tc = end

        return batch

# this just saves any given data to any given path
def writeToFile(data, path):
    with open(path, 'wb') as output:
        pickle.dump(data, output, pickle.HIGHEST_PROTOCOL)

# this creates summaries for variables to be sued by tensorboard
def summarize(varT):
    varName = varT.name[:-2]
    with tf.name_scope(varName):
        var_mean = tf.reduce_mean(varT)
        var_sum = tf.reduce_sum(varT)
        tf.summary.scalar('mean', var_mean)
        tf.summary.scalar('sum', var_sum)

        with tf.name_scope('stddev'):
            stddev = tf.sqrt(tf.reduce_mean(tf.square(varT - var_mean)))

        tf.summary.scalar('stddev', stddev)
        tf.summary.scalar('max', tf.reduce_max(varT))
        tf.summary.scalar('min', tf.reduce_min(varT))
        tf.summary.histogram('histogram', varT)

# this returns the writer objects for training and testing
def getSummaryWriters(sess):
    train_writer = tf.summary.FileWriter(log_dir + 'train', sess.graph)
    test_writer = tf.summary.FileWriter(log_dir + 'test')

    return [train_writer, test_writer]
# from here down is the sandbox place to check and verify the code above before using it in
# the other files<commit_msg>changed the dataset folder<commit_after>import tensorflow as tf
import numpy as np
import os
import pickle
import time
from PIL import Image

# some global params
# the directory to which teh results will be saved
# imgSize = [96,128]
imgSize = [64,48]
spaceSize = [64,48]
batch_size = 20
# folder to save the results in
resDir = 'results/'
# folder to log the training progress in
log_dir  = 'train_log/2/'

learning_rate = 1e-5
# below is the coefficient for l2 loss
alpha = 0.001

model_save_path = ['savedModels/model_1.ckpt',
                    'savedModels/model_2.ckpt']

# in every traning example, these will be the names of the 5 spaces
nameList = ['red',
            'green',
            'blue',
            'yellow',
            'white']
# And this dictionary provides the color as RGB tuple for a space with a certian name
colors = {
    'red':(255,0,0),
    'green':(0,255,0),
    'blue':(0,0,255),
    'yellow':(255,255,0),
    'white':(255,255,255)
}

# this method saves the model
def saveModel(sess, savePath):
    print('\n...saving the models, please wait...')
    saver = tf.train.Saver()
    saver.save(sess, savePath)
    print('Saved the model to %s'%savePath)

# this method loads the saved model
def loadModel(sess, savedPath):
    print('\n...loading the models, please wait...')
    saver = tf.train.Saver()
    saver.restore(sess, savedPath)
    print('Loaded the model from %s'%savedPath)

# weight variable
def weightVariable(shape, name):
    initializer = tf.truncated_normal_initializer(mean=0.0, stddev=0.05)
    weight = tf.get_variable(name=name, shape=shape, initializer=initializer)
    return weight

# bias variable
def biasVariable(shape, name):
    initializer = tf.constant_initializer(0.01)
    bias = tf.get_variable(name=name, shape=shape, initializer=initializer)
    return bias

# 2d convolutional layer
def conv2d(x, W, strides = [1,2,2,1]):
    return tf.nn.conv2d(x, W, strides=strides, padding='SAME')

# 3d convolutional layer
def conv3d(x, W, strides = [1,2,2,1,1]):
    return tf.nn.conv3d(x,W,strides = strides, padding='SAME')

# deconv layer
def deConv3d(y, w, outShape, strides=[1,2,2,2,1]):
    return tf.nn.conv3d_transpose(y, w, output_shape = outShape, strides=strides, padding='SAME')

# this is max-pooling for 3d convolutional layers
def max_pool2x2x1(x):
    return tf.nn.max_pool3d(x,ksize=[1,2,2,1,1],strides=[1,2,2,1,1],padding='SAME')

# this is a 2x2 max-pooling layer for 2d convolutional layers
# this is max-pooling for 3d convolutional layers
def max_pool2x2(x):
    return tf.nn.max_pool(x,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')

# converts data to image
def toImage(data):
    data = np.reshape(data, [48,64])
    newData = 255*data
    # converting new data into integer format to make it possible to export it as a bitmap
    # in this case converting it into 8 bit integer
    newData = newData.astype(np.uint8)
    return Image.fromarray(newData)

# this method converts a list of images into a feed ready batch
def prepareImages(fileList):
    batch = []
    for fileName in fileList:
        img = Image.open(fileName).convert("L")
        arr = prepareImage(img)
        batch.append(arr)
    return np.array(batch)

# this one prepares a single image and returns it as an array
def prepareImage(img):
    arr = np.array(img)
    arr = arr.astype(np.float32)
    arr /= 255

    return arr

# this method returns the time estimate as a string
def estimate_time(startTime, totalCycles, finishedCycles):
    timePast = time.time() - startTime
    if timePast < 10:
        return '...calculating...'
    cps = finishedCycles/timePast
    if cps == 0:
        return '...calculating...'

    secsLeft = (totalCycles - finishedCycles)/cps

    # double slash is an integer division, returns the quotient without the decimal
    hrs, secs = secsLeft//3600, secsLeft % 3600
    mins, secs = secs//60, secs % 60

    timeStr = '%.0fh%.0fm%.0fs remaining'%(hrs, mins, secs) + ' '*10
    return timeStr

# this is the dataset object which is responsible with supplying data for training as well as
# testing purposes
class dataset:
    # the dirPath is the path of the directory
    # testFile is the name of the file with the test examples
    def __init__(self, dirPath = './', testFile='test'):
        self.dirPath = dirPath
        self.testFileName = testFile + '.pkl'
        # getting the list of files in the dirPath
        self.trainFileList = os.listdir(self.dirPath)
        # removing the test data file from the list of training data files
        if len(self.trainFileList) > 1:
            self.trainFileList.remove(self.testFileName)

        #removing any other file than pickled data files from the list
        i = 0
        while i < len(self.trainFileList):
            if not self.trainFileList[i].endswith('.pkl'):
                # print('removing %s'%self.trainFileList[i])
                del self.trainFileList[i]
                i -= 1

            i += 1

        # print(self.trainFileList)

        self.curFile = None
        if len(self.trainFileList) > 0:
            self.curFile = self.trainFileList[0]
        self.data = None
        self.test_data = None
        self.load_data()

        # this is the number of data samples currently loaded
        self.data_num = self.data[0].shape[0]
        self.test_data_num = self.test_data[0].shape[0]

        # this is the counter for where we are reading the data in the currently loaded set
        self.c = 0
        self.tc = 0

    # returns the name of the next file to be used once the current file is exhausted
    def next_file(self):
        fileNum = self.trainFileList.index(self.curFile)
        fileNum = (fileNum + 1)%len(self.trainFileList)
        nextFile = self.trainFileList[fileNum]
        return nextFile

    # this loads the data from the current file into the self.data
    def load_data(self, silent = False):
        with open(self.dirPath + self.curFile,'rb') as inp:
            if not silent: print('\nLoading data from %s...'%self.curFile)
            dSet = pickle.load(inp)

        self.data = [np.expand_dims(np.array(dSet[0]),3), np.array(dSet[1])]
        # self.data = [np.array(dSet[0]), np.array(dSet[1])]
        # self.data = dSet

        if self.test_data is None:
            with open(self.dirPath + self.testFileName,'rb') as inp:
                if not silent: print('\nLoading test data from %s...'%self.testFileName)
                dSet = pickle.load(inp)

            self.test_data = [np.expand_dims(np.array(dSet[0]),3), np.array(dSet[1])]
            # self.test_data = [np.array(dSet[0]), np.array(dSet[1])]
            # self.test_data = dSet
        if not silent: print('\nDataset in %s is successfully loaded\n'%self.dirPath)

    # this returns the next batch of the size - size
    def next_batch(self, size):
        if self.c + size >= self.data_num:
            end = self.data_num
            dataNeeded = size-(end-self.c)

            batch1 = [self.data[0][self.c: end], self.data[1][self.c: end]]

            # now getting the remaining data from the next file
            end = (self.c + size)%self.data_num
            self.c = 0
            self.curFile = self.next_file()
            self.load_data(silent = True)

            batch2 = [self.data[0][self.c: end], self.data[1][self.c: end]]
            # now joining batch 2 to the original batch
            batch = [np.concatenate((batch1[0], batch2[0]), axis=0), np.concatenate((batch1[1], batch2[1]), axis=0)]

            self.c = end

        elif self.c + size < self.data_num:
            end = self.c + size

            batch = [self.data[0][self.c: end], self.data[1][self.c: end]]
            self.c = end

        return batch

    # this fetches a test batch
    def test_batch(self, size):
        if self.tc + size >= self.test_data_num:
            end = self.test_data_num
            dataNeeded = size-(end-self.tc)

            batch1 = [self.test_data[0][self.tc: end], self.test_data[1][self.tc: end]]

            end = (self.tc + size)%self.test_data_num
            self.tc = 0

            batch2 = [self.test_data[0][self.tc: end], self.test_data[1][self.tc: end]]
            # now joining batch 2 to the original batch
            batch = [np.concatenate((batch1[0], batch2[0]), axis=0), np.concatenate((batch1[1], batch2[1]), axis=0)]

            self.tc = end

        elif self.tc + size < self.test_data_num:
            end = self.tc + size

            batch = [self.test_data[0][self.tc: end], self.test_data[1][self.tc: end]]
            self.tc = end

        return batch

# this just saves any given data to any given path
def writeToFile(data, path):
    with open(path, 'wb') as output:
        pickle.dump(data, output, pickle.HIGHEST_PROTOCOL)

# this creates summaries for variables to be sued by tensorboard
def summarize(varT):
    varName = varT.name[:-2]
    with tf.name_scope(varName):
        var_mean = tf.reduce_mean(varT)
        var_sum = tf.reduce_sum(varT)
        tf.summary.scalar('mean', var_mean)
        tf.summary.scalar('sum', var_sum)

        with tf.name_scope('stddev'):
            stddev = tf.sqrt(tf.reduce_mean(tf.square(varT - var_mean)))

        tf.summary.scalar('stddev', stddev)
        tf.summary.scalar('max', tf.reduce_max(varT))
        tf.summary.scalar('min', tf.reduce_min(varT))
        tf.summary.histogram('histogram', varT)

# this returns the writer objects for training and testing
def getSummaryWriters(sess):
    train_writer = tf.summary.FileWriter(log_dir + 'train', sess.graph)
    test_writer = tf.summary.FileWriter(log_dir + 'test')

    return [train_writer, test_writer]
# from here down is the sandbox place to check and verify the code above before using it in
# the other files<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

# Requires boost pre-installed.
# This is slightly special if you want to use Venture. The GNU c++
# compilers use the standard library libstdc++, while Mac's c++
# compiler on Mavericks uses libc++. In order for Venture to build,
# you must build Boost using libstdc++, and then build Venture using
# the same. This can be accomplished by building both Boost and
# Venture using GNU gcc (installed via Homebrew) instead of Mac's
# compiler. The correct version of gcc is set for Venture installation
# in the setup.py file. To install Boost with the correct library,
# call:
#
#    brew install boost --cc=gcc-4.9
#    brew install boost-python --cc=gcc-4.9

# Requires locate to be working, so we can find a pre-installed boost.
# Requires virtualenv pre-installed.
# Requires read access to the listed git repos.
GIT_REPOS = ['crosscat', 'bayeslite', 'bdbcontrib']
PEG = {  # None means head.
  'crosscat': None,
  'bdbcontrib': None,
  'bayeslite': None
  }
PAUSE_TO_MODIFY = False

import distutils.spawn
import errno
import os
import os.path
root = os.path.dirname(os.path.abspath(__file__))
here = os.getcwd()
import subprocess
import time
import tempfile

START_TIME = time.time()

try:
  from setuptools import setup
except ImportError:
  from distutils.core import setup

BUILD_DIR = tempfile.mkdtemp(prefix='BayesLite-app-')
os.chdir(BUILD_DIR)
print "Building in", BUILD_DIR

def run(cmd):
  print cmd
  assert not os.system(cmd)

def outputof(cmd, **kwargs):
  print cmd
  output = subprocess.check_output(cmd, **kwargs)
  print "OUTPUT:", output
  return output

def shellquote(s):
  """Return `s`, quoted appropriately for use in a shell script."""
  return "'" + s.replace("'", "'\\''") + "'"

# Do not specify --python to virtualenv, bc eg python27 is a 32-bit version that
# will produce a .app that osx will not open.
# You get errors like:
#  You can’t open the application “...” because PowerPC applications are no longer supported.
# Or less helpfully:
#  The application “...” can’t be opened.
# and when investigating:
#  lipo: can't figure out the architecture type of: ...
# Instead, use the built-in python by not specifying anything, and get fine results.
# Rather than merely praying that the built-in python is language-compatible, let's check.
PYVER = outputof('python --version 2>&1', shell=True)
assert "Python 2.7" == PYVER[:10]

def get_version(project_dir):
  here = os.getcwd()
  with open(os.path.join(project_dir, 'VERSION'), 'rU') as f:
    version = f.readline().strip()

  # Append the Git commit id if this is a development version.
  if version.endswith('+'):
    tag = 'v' + version[:-1]
    try:
      os.chdir(project_dir)
      desc = outputof(['git', 'describe', '--dirty', '--match', tag])
      os.chdir(here)
    except Exception:
      version += 'unknown'
    else:
      assert desc.startswith(tag)
      version = desc[1:].strip()
  return version

VERSION = ''
for project in GIT_REPOS:
  print "Checking out", project
  run("git clone git@github.com:probcomp/%s.git %s"
      % (project, os.path.join(BUILD_DIR, project)))
  if PEG[project]:
    repodir = os.path.join(BUILD_DIR, project)
    branch = PEG[project]
    run('cd %s && git checkout %s' % (shellquote(repodir), shellquote(branch)))
  if os.path.exists(os.path.join(project, 'VERSION')):
    project_version = get_version(project)
    if project_version:
      VERSION += "-" + project[:5] + project_version
      print "Project", project, "version is", project_version

VENV_DIR=os.path.join(BUILD_DIR, "venv")

# Do not specify --python version here. See explanation where we fail fast above.
run('virtualenv %s' % (shellquote(VENV_DIR),))

os.chdir(VENV_DIR)
def venv_run(cmd):
  print cmd
  assert not os.system('source bin/activate; %s' % cmd)

# Preprocessing
# =============
print "Deps for BdbContrib"
LIBBOOST_DIR = os.path.dirname(outputof("locate -l 1 libboost_atomic-mt.dylib", shell=True))
assert os.path.exists(LIBBOOST_DIR), ("We need libboost-dev already installed: %s" %
                                      LIBBOOST_DIR)

venv_run("pip install cython")  # If we don't, crosscat's setup tries and fails.
venv_run("pip install numpy")
print "Deps for BayesLite"
# Assume that osx has sqlite3 already.
# http://computechtips.com/619/upgrade-sqlite-os-x-mavericks-yosemite
# which suggests that OS 10.10 and above have a sufficiently new sqlite.
venv_run("pip install pytest sphinx cov-core dill ")

BUILD_EXAMPLES = os.path.join(BUILD_DIR, "examples")
run("mkdir -p %s" % (shellquote(BUILD_EXAMPLES),))

# Main installs
# =============
for project in GIT_REPOS:
  reqfile = os.path.join(BUILD_DIR, project, "requirements.txt")
  if os.path.exists(reqfile):
    print "Installing dependencies for", project
    venv_run("pip install -r %s" % (shellquote(reqfile),))
  setupfile = os.path.join(BUILD_DIR, project, "setup.py")
  if os.path.exists(setupfile):
    print "Installing", project, "into", BUILD_DIR
    repodir = os.path.join(BUILD_DIR, project)
    venv_run("cd %s && pip install ." % (shellquote(repodir),))
  examplesdir = os.path.join(BUILD_DIR, project, "examples")
  if os.path.exists(examplesdir):
    print "Copying examples from", examplesdir
    run("/bin/cp -r %s/* %s/" %
        (shellquote(examplesdir), shellquote(BUILD_EXAMPLES)))

# Postprocessing
# ==============
run("curl http://probcomp.csail.mit.edu/bayesdb/analyses/satellites.bdb > %s/satellites.bdb"
    % (shellquote(BUILD_EXAMPLES),))

# This app's only other dependency:
venv_run("pip install 'ipython[notebook]' runipy")

print "Ready to start packaging the app!"
venv_run('virtualenv --relocatable %s' % (shellquote(VENV_DIR),))
# Sadly, that doesn't actually fix the most critical file, the activate script.
relocable = '''VIRTUAL_ENV=$(dirname "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )")\n'''
new_activate = tempfile.NamedTemporaryFile(delete=False)
old_activate_path = os.path.join(VENV_DIR, "bin", "activate")
with open(old_activate_path, "r") as old_activate:
  for line in old_activate:
    if line[:len("VIRTUAL_ENV=")] == "VIRTUAL_ENV=":
      new_activate.write(relocable)
    else:
      new_activate.write(line)
new_activate.close()
run("mv %s %s" %
    (shellquote(new_activate.name), shellquote(old_activate_path)))
# Also, we are hard linking to the python that we were built with, which may be different
# than the python that we want the client to execute. Let's just assume that on the client,
# we want to use the generic /System/Library py2.7, rather than a special one.
PYTHON_DYLIB_LINK = os.path.join(VENV_DIR, ".Python")
run("ln -fs /System/Library/Frameworks/Python.framework/Versions/2.7/Python %s" %
    (shellquote(PYTHON_DYLIB_LINK),))
# And we still have a copy of python that my otherwise reference its
# own dependencies, rather than relying on the built-in python. So
# remove that.
run("rm -f %s" % (shellquote(os.path.join(VENV_DIR, "bin", "python")),))
run("ln -s /usr/bin/python %s" %
    (shellquote(os.path.join(VENV_DIR, "bin", "python")),))

NAME="BayesDB%s" % VERSION
DIST_DIR = os.path.join(BUILD_DIR, "BayesDB")
MACOS_PATH = os.path.join(DIST_DIR, NAME + ".app", "Contents", "MacOS")
os.makedirs(MACOS_PATH)
run("/bin/cp -r %s %s/" % (shellquote(BUILD_EXAMPLES), shellquote(MACOS_PATH)))
run("/bin/ln -s /Applications %s" % (shellquote(DIST_DIR),))

STARTER = '''#!/bin/bash

set -e
wd=`dirname -- "$0"`
cd -- "$wd"
wd=`pwd -P`
NAME=`basename -- "$(dirname "$(dirname -- "$wd")")" .app`

activate="$wd/venv/bin/activate"
ldpath="$wd/lib"

source "$activate"
export DYLD_LIBRARY_PATH="$ldpath"

# Copy the examples to someplace writeable:
rsync -r --ignore-existing -- "$wd/examples"/* "$HOME/Documents/$NAME"
ipython notebook "$HOME/Documents/$NAME"
'''

startsh_path = os.path.join(MACOS_PATH, "start.sh")
with open(startsh_path, "w") as startsh:
  startsh.write(STARTER)
run("chmod +x '%s'" % startsh_path)

LAUNCHER = '''#!/bin/bash

wd=`dirname -- "$0"`
cd -- "$wd"
wd=`pwd -P`

osacmd="tell application \\"Terminal\\" to do script"
script="/bin/bash -- '$wd/start.sh'"
osascript -e "$osacmd \\"$script\\""
'''

launchsh_path = os.path.join(MACOS_PATH, NAME)  # Must be the same as NAME in MACOS_PATH
with open(launchsh_path, "w") as launchsh:
  launchsh.write(LAUNCHER)
run("chmod +x %s" % (shellquote(launchsh_path),))
run("mv -f %s %s" % (shellquote(VENV_DIR), shellquote(MACOS_PATH)))

# Basic sanity check.
ipynb = os.path.join(MACOS_PATH, "examples", "Satellites.ipynb")
venv_run("runipy %s" % (shellquote(ipynb),))

if PAUSE_TO_MODIFY:
  print "Pausing to let you modify %s before packaging it up." % (MACOS_PATH,)
  os.system('read -s -n 1 -p "Press any key to continue..."')

DMG_PATH = os.path.join(os.environ['HOME'], 'Desktop', '%s.dmg' % (NAME,))
naming_attempt = 0
while os.path.exists(DMG_PATH):
  naming_attempt += 1
  DMG_PATH = os.path.join(os.environ['HOME'], 'Desktop',
                          "%s (%d).dmg" % (NAME, naming_attempt))
run("hdiutil create -format UDBZ -size 1g -srcfolder %s %s" %
    (shellquote(DIST_DIR), shellquote(DMG_PATH)))
run("/bin/rm -fr %s" % (shellquote(BUILD_DIR),))

print "Done. %d seconds elapsed" % (time.time() - START_TIME)
<commit_msg>relative dirs might start with hyphen, protect.<commit_after># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

# Requires boost pre-installed.
# This is slightly special if you want to use Venture. The GNU c++
# compilers use the standard library libstdc++, while Mac's c++
# compiler on Mavericks uses libc++. In order for Venture to build,
# you must build Boost using libstdc++, and then build Venture using
# the same. This can be accomplished by building both Boost and
# Venture using GNU gcc (installed via Homebrew) instead of Mac's
# compiler. The correct version of gcc is set for Venture installation
# in the setup.py file. To install Boost with the correct library,
# call:
#
#    brew install boost --cc=gcc-4.9
#    brew install boost-python --cc=gcc-4.9

# Requires locate to be working, so we can find a pre-installed boost.
# Requires virtualenv pre-installed.
# Requires read access to the listed git repos.
GIT_REPOS = ['crosscat', 'bayeslite', 'bdbcontrib']
PEG = {  # None means head.
  'crosscat': None,
  'bdbcontrib': None,
  'bayeslite': None
  }
PAUSE_TO_MODIFY = False

import distutils.spawn
import errno
import os
import os.path
root = os.path.dirname(os.path.abspath(__file__))
here = os.getcwd()
import subprocess
import time
import tempfile

START_TIME = time.time()

try:
  from setuptools import setup
except ImportError:
  from distutils.core import setup

BUILD_DIR = tempfile.mkdtemp(prefix='BayesLite-app-')
os.chdir(BUILD_DIR)
print "Building in", BUILD_DIR

def run(cmd):
  print cmd
  assert not os.system(cmd)

def outputof(cmd, **kwargs):
  print cmd
  output = subprocess.check_output(cmd, **kwargs)
  print "OUTPUT:", output
  return output

def shellquote(s):
  """Return `s`, quoted appropriately for use in a shell script."""
  return "'" + s.replace("'", "'\\''") + "'"

# Do not specify --python to virtualenv, bc eg python27 is a 32-bit version that
# will produce a .app that osx will not open.
# You get errors like:
#  You can’t open the application “...” because PowerPC applications are no longer supported.
# Or less helpfully:
#  The application “...” can’t be opened.
# and when investigating:
#  lipo: can't figure out the architecture type of: ...
# Instead, use the built-in python by not specifying anything, and get fine results.
# Rather than merely praying that the built-in python is language-compatible, let's check.
PYVER = outputof('python --version 2>&1', shell=True)
assert "Python 2.7" == PYVER[:10]

def get_version(project_dir):
  here = os.getcwd()
  with open(os.path.join(project_dir, 'VERSION'), 'rU') as f:
    version = f.readline().strip()

  # Append the Git commit id if this is a development version.
  if version.endswith('+'):
    tag = 'v' + version[:-1]
    try:
      os.chdir(project_dir)
      desc = outputof(['git', 'describe', '--dirty', '--match', tag])
      os.chdir(here)
    except Exception:
      version += 'unknown'
    else:
      assert desc.startswith(tag)
      version = desc[1:].strip()
  return version

VERSION = ''
for project in GIT_REPOS:
  print "Checking out", project
  run("git clone git@github.com:probcomp/%s.git %s"
      % (project, os.path.join(BUILD_DIR, project)))
  if PEG[project]:
    repodir = os.path.join(BUILD_DIR, project)
    branch = PEG[project]
    run('cd %s && git checkout %s' % (shellquote(repodir), shellquote(branch)))
  if os.path.exists(os.path.join(project, 'VERSION')):
    project_version = get_version(project)
    if project_version:
      VERSION += "-" + project[:5] + project_version
      print "Project", project, "version is", project_version

VENV_DIR=os.path.join(BUILD_DIR, "venv")

# Do not specify --python version here. See explanation where we fail fast above.
run('virtualenv %s' % (shellquote(VENV_DIR),))

os.chdir(VENV_DIR)
def venv_run(cmd):
  print cmd
  assert not os.system('source bin/activate; %s' % cmd)

# Preprocessing
# =============
print "Deps for BdbContrib"
LIBBOOST_DIR = os.path.dirname(outputof("locate -l 1 libboost_atomic-mt.dylib", shell=True))
assert os.path.exists(LIBBOOST_DIR), ("We need libboost-dev already installed: %s" %
                                      LIBBOOST_DIR)

venv_run("pip install cython")  # If we don't, crosscat's setup tries and fails.
venv_run("pip install numpy")
print "Deps for BayesLite"
# Assume that osx has sqlite3 already.
# http://computechtips.com/619/upgrade-sqlite-os-x-mavericks-yosemite
# which suggests that OS 10.10 and above have a sufficiently new sqlite.
venv_run("pip install pytest sphinx cov-core dill ")

BUILD_EXAMPLES = os.path.join(BUILD_DIR, "examples")
run("mkdir -p %s" % (shellquote(BUILD_EXAMPLES),))

# Main installs
# =============
for project in GIT_REPOS:
  reqfile = os.path.join(BUILD_DIR, project, "requirements.txt")
  if os.path.exists(reqfile):
    print "Installing dependencies for", project
    venv_run("pip install -r %s" % (shellquote(reqfile),))
  setupfile = os.path.join(BUILD_DIR, project, "setup.py")
  if os.path.exists(setupfile):
    print "Installing", project, "into", BUILD_DIR
    repodir = os.path.join(BUILD_DIR, project)
    venv_run("cd %s && pip install ." % (shellquote(repodir),))
  examplesdir = os.path.join(BUILD_DIR, project, "examples")
  if os.path.exists(examplesdir):
    print "Copying examples from", examplesdir
    run("/bin/cp -r %s/* %s/" %
        (shellquote(examplesdir), shellquote(BUILD_EXAMPLES)))

# Postprocessing
# ==============
run("curl http://probcomp.csail.mit.edu/bayesdb/analyses/satellites.bdb > %s/satellites.bdb"
    % (shellquote(BUILD_EXAMPLES),))

# This app's only other dependency:
venv_run("pip install 'ipython[notebook]' runipy")

print "Ready to start packaging the app!"
venv_run('virtualenv --relocatable %s' % (shellquote(VENV_DIR),))
# Sadly, that doesn't actually fix the most critical file, the activate script.
relocable = '''VIRTUAL_ENV=$(dirname "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )")\n'''
new_activate = tempfile.NamedTemporaryFile(delete=False)
old_activate_path = os.path.join(VENV_DIR, "bin", "activate")
with open(old_activate_path, "r") as old_activate:
  for line in old_activate:
    if line[:len("VIRTUAL_ENV=")] == "VIRTUAL_ENV=":
      new_activate.write(relocable)
    else:
      new_activate.write(line)
new_activate.close()
run("mv %s %s" %
    (shellquote(new_activate.name), shellquote(old_activate_path)))
# Also, we are hard linking to the python that we were built with, which may be different
# than the python that we want the client to execute. Let's just assume that on the client,
# we want to use the generic /System/Library py2.7, rather than a special one.
PYTHON_DYLIB_LINK = os.path.join(VENV_DIR, ".Python")
run("ln -fs /System/Library/Frameworks/Python.framework/Versions/2.7/Python %s" %
    (shellquote(PYTHON_DYLIB_LINK),))
# And we still have a copy of python that my otherwise reference its
# own dependencies, rather than relying on the built-in python. So
# remove that.
run("rm -f %s" % (shellquote(os.path.join(VENV_DIR, "bin", "python")),))
run("ln -s /usr/bin/python %s" %
    (shellquote(os.path.join(VENV_DIR, "bin", "python")),))

NAME="BayesDB%s" % VERSION
DIST_DIR = os.path.join(BUILD_DIR, "BayesDB")
MACOS_PATH = os.path.join(DIST_DIR, NAME + ".app", "Contents", "MacOS")
os.makedirs(MACOS_PATH)
run("/bin/cp -r %s %s/" % (shellquote(BUILD_EXAMPLES), shellquote(MACOS_PATH)))
run("/bin/ln -s /Applications %s" % (shellquote(DIST_DIR),))

STARTER = '''#!/bin/bash

set -e
wd=`dirname -- "$0"`
cd -- "$wd"
wd=`pwd -P`
NAME=`basename -- "$(dirname -- "$(dirname -- "$wd")")" .app`

activate="$wd/venv/bin/activate"
ldpath="$wd/lib"

source "$activate"
export DYLD_LIBRARY_PATH="$ldpath"

# Copy the examples to someplace writeable:
rsync -r --ignore-existing -- "$wd/examples"/* "$HOME/Documents/$NAME"
ipython notebook "$HOME/Documents/$NAME"
'''

startsh_path = os.path.join(MACOS_PATH, "start.sh")
with open(startsh_path, "w") as startsh:
  startsh.write(STARTER)
run("chmod +x '%s'" % startsh_path)

LAUNCHER = '''#!/bin/bash

wd=`dirname -- "$0"`
cd -- "$wd"
wd=`pwd -P`

osacmd="tell application \\"Terminal\\" to do script"
script="/bin/bash -- '$wd/start.sh'"
osascript -e "$osacmd \\"$script\\""
'''

launchsh_path = os.path.join(MACOS_PATH, NAME)  # Must be the same as NAME in MACOS_PATH
with open(launchsh_path, "w") as launchsh:
  launchsh.write(LAUNCHER)
run("chmod +x %s" % (shellquote(launchsh_path),))
run("mv -f %s %s" % (shellquote(VENV_DIR), shellquote(MACOS_PATH)))

# Basic sanity check.
ipynb = os.path.join(MACOS_PATH, "examples", "Satellites.ipynb")
venv_run("runipy %s" % (shellquote(ipynb),))

if PAUSE_TO_MODIFY:
  print "Pausing to let you modify %s before packaging it up." % (MACOS_PATH,)
  os.system('read -s -n 1 -p "Press any key to continue..."')

DMG_PATH = os.path.join(os.environ['HOME'], 'Desktop', '%s.dmg' % (NAME,))
naming_attempt = 0
while os.path.exists(DMG_PATH):
  naming_attempt += 1
  DMG_PATH = os.path.join(os.environ['HOME'], 'Desktop',
                          "%s (%d).dmg" % (NAME, naming_attempt))
run("hdiutil create -format UDBZ -size 1g -srcfolder %s %s" %
    (shellquote(DIST_DIR), shellquote(DMG_PATH)))
run("/bin/rm -fr %s" % (shellquote(BUILD_DIR),))

print "Done. %d seconds elapsed" % (time.time() - START_TIME)
<|endoftext|>


<commit_before>import collections
import json
import os, os.path
import re
import time
import urllib.request

import psxml

def f_clean():
    """f_clean() -> function(string)

    This function returns a function that cleans disc names.  Cleaning is done
    by stripping all non-alphanumeric characters.  The remaining characters are
    then converted to lowercase.

    """
    regex = re.compile(r"[^a-zA-Z0-9]")
    return lambda x: regex.sub(r"", x).lower()

def images():
    """images() -> None

    Download all disc images from the DJMAX site.  Files are saved in a new
    directory named "images" under "./DJRivals/".  An image will be skipped if
    it is determined that the file already exists.  Existence is checked using a
    simple filename lookup.

    """
    # todo: make multithreaded
    url = "http://djmaxcrew.com/ranking/GetRankPopMixing.asp?p={}"
    image_url = "http://img3.djmaxcrew.com/icon/disc/110/{}"
    image_dir = "./DJRivals/images/"
    clean = f_clean()
    if not os.path.exists(image_dir):
        os.makedirs(image_dir)
    for page in range(1, 9):
        reply = json.loads(urllib.request.urlopen(url.format(page)).read().decode())
        for record in reply["DATA"]["RECORD"]:
            name, extension = os.path.splitext(record["DISCIMG"])
            for chart in range(1, 5):
                theirname = "{}{}{}".format(name[:-1], chart, extension)
                myname = "{}_{}{}".format(clean(record["DISCNAME"]), chart, extension)
                if os.path.exists(image_dir + myname):
                    continue
                with open(image_dir + myname, "wb") as f:
                    f.write(urllib.request.urlopen(image_url.format(theirname)).read())
                print('Wrote: "{}{}"'.format(image_dir, myname))

def index(refresh=False):
    """index([boolean]) -> dictionary

    An auto-generated dictionary with manually maintained elements.  The
    dictionary is saved as a plain text file in JSON format with the name
    "pop_index.json" under "./DJRivals/".  An optional boolean value (default:
    False) controls whether or not it should refresh its contents by checking
    the DJMAX site.  Refer to data_structures.txt for the format and contents of
    this file.

    Note: Because the DJMAX site does not list the difficulty level of charts
    anywhere, these entries are manually maintained.  The dictionary can be
    edited by hand in any text editor, and subsequent executions of this
    function will not clobber the manual entries.  However, if this function
    should encounter any errors while attempting to read the file on disk, a new
    one will be generated as a replacement.  It is therefore recommended to have
    a backup of the current file before using this function.

    """
    url = "http://djmaxcrew.com/ranking/GetRankPopMixing.asp?p={}"
    index_dir = "./DJRivals/"
    index_file = index_dir + "pop_index.json"
    clean = f_clean()
    if not os.path.exists(index_dir):
        os.mkdir(index_dir)
    try:
        with open(index_file, "rb") as f:
            index = json.loads(f.read().decode())
    except:
        index = {}
    if refresh or not index:
        for page in range(1, 9):
            reply = json.loads(urllib.request.urlopen(url.format(page)).read().decode())
            for record in reply["DATA"]["RECORD"]:
                name = record["DISCNAME"]
                if clean(name) not in index:
                    index[clean(name)] = [name, 0, 0, 0, 0, page]
                else:  # do this because the name and page could have changed
                    index[clean(name)][0] = name
                    index[clean(name)][5] = page
        output = json.dumps(index, indent=4)
        output = re.sub(r'\[\n +(".+",) +\n +(\d+,) +\n +(\d+,) +\n +(\d+,) +\n +(\d+,) +\n +(\d+)\n +\](,?) *', r"[\1 \2 \3 \4 \5 \6]\7", output)
        with open(index_file, "wb") as f:
            f.write(output.encode())
        print('Wrote: "{}"'.format(index_file))
    return index

def f_identifier():
    """f_identifier() -> function(string)

    This function returns a function that retrieves a disc identifier from the
    DJMAX site by disc name.

    """
    url = "http://djmaxcrew.com/ranking/GetRankPopMixing.asp?p={}"
    clean = f_clean()
    page = index()
    cache = {}

    # since DJMAX returns up to 20 identifiers at a time (each page lists 20
    # discs except the last page which has less than 20), the cache dictionary
    # saves these identifers to expedite future lookups.  however, since it is
    # unknown how long these identifiers stay valid, the cache size has been
    # limited to about 25% of the total number of available discs.  when the
    # cache exceeds this limit, it will clear itself and rebuild.

    def identifier(name):
        name = clean(name)
        if name not in cache:
            if len(cache) > 20:  # value of 20 means at most 40
                cache.clear()
            reply = json.loads(urllib.request.urlopen(url.format(page[name][5])).read().decode())
            cache.update({clean(record["DISCNAME"]): record["DISCID"] for record in reply["DATA"]["RECORD"]})
        return cache[name]

    return identifier

def ranking(disc, chart, pages=1):
    """ranking(string, string[, integer]) -> list of tuples

    Retrieve the ranking of a specified disc name and chart.  The chart is a
    string that can be either NM, HD, MX, or EX; any other value will be treated
    as EX.  The optional integer specifies how many pages worth of rankings to
    retrieve (default: 1).  Specify 0 for a full listing.

    """
    # todo: make multithreaded
    url = "http://djmaxcrew.com/ranking/GetRankPopMixingMusic.asp?c={}&pt={}&p={}"
    identifier = f_identifier()
    disc_id = identifier(disc)
    chart = (lambda x: 1 if x == "nm" else 2 if x == "hd" else 3 if x == "mx" else 4)(chart.lower())
    results = []
    for page in range(1, 1 + (100 if pages == 0 else pages)):
        reply = json.loads(urllib.request.urlopen(url.format(disc_id, chart, page)).read().decode())
        results.extend([(record["RANK"], record["DJICON"], record["DJNAME"], record["SCORE"]) for record in reply["DATA"]["RECORD"]])
        if len(reply["DATA"]["RECORD"]) < 20:
            break
    return results

def database(disc_list=[]):
    """database([list]) -> None

    Create a local database of scores with information obtained from the DJMAX
    site.  The database is implemented as a collection of JSON files.  One JSON
    file is created for each disc.  In addition, one JSON file will be created
    for each DJ based on the information just acquired.  The optional argument
    is a list of strings (default: []) of cleaned disc names.  By default, it
    will create the complete database.  When given a list it will create a
    database of only those discs.  Files are saved in
    "./DJRivals/rankings/pop/disc/" and "./DJRivals/rankings/pop/dj/".  Refer to
    data_structures.txt for the format and contents of these files.

    """
    # todo: write the code to generate the DJ JSON files
    # todo: include EX charts
    start_time = time.time()
    disc_dir = "./DJRivals/rankings/pop/disc/"
    dj_dir = "./DJRivals/rankings/pop/dj/"
    disc_info = index()
    for directory in [disc_dir, dj_dir]:
        if not os.path.exists(directory):
            os.makedirs(directory)
    if not disc_list:
        disc_list = [title[0] for title in sorted(disc_info.items(), key=lambda i: i[1])]  # sort by page to maximize identifier() cache hits
    while len(disc_list):
        print("{} discs remaining.".format(len(disc_list)))
        disc = disc_list.pop()
        charts = ["nm", "hd", "mx"]
        output = collections.OrderedDict()
        output["updated"] = time.strftime("%Y-%m-%d %H:%M:%S")
        output["name"] = collections.OrderedDict(zip(["clean", "full"], [disc, disc_info[disc][0]]))
        output["image"] = collections.OrderedDict([(i, disc + j) for i, j in zip(["eyecatch"] + charts, [".png", "_1.png", "_2.png", "_3.png", "_4.png"])])
        output["difficulty"] = collections.OrderedDict([(i, disc_info[disc][j]) for i, j in zip(charts, [1, 2, 3, 4])])
        output["ranking"] = collections.OrderedDict()
        output["ranking"]["records"] = collections.OrderedDict()
        for chart in charts:
            try:
                results = ranking(disc, chart, 0)
                output["ranking"][chart] = results
                output["ranking"]["records"][chart] = len(results)
                print("{} {} complete.  Sleeping...".format(disc, chart))
                time.sleep(15)
            except:
                print("{} {} error.  Sleeping for 5 minutes before retrying.".format(disc, chart))
                charts.insert(0, chart)
                time.sleep(300)
        output = json.dumps(output, indent=4)
        output = re.sub(r'\[\n +(\d+,) +\n +(".*",) +\n +(".*",) +\n +(\d{6})\n +\](,?) *', r"[\1 \2 \3 \4]\5", output)
        with open(disc_dir + disc + ".json", "wb") as f:
            f.write(output.encode())
        print('Wrote: "{}{}.json"\n'.format(disc_dir, disc))
    print("Operation complete!")
    elapsed_time = round((time.time() - start_time) / 60)
    print("Database creation took {} minutes.".format(elapsed_time))

def html():
    """html() -> None

    Crawl through the database and automatically generate the DJRivals HTML user
    interface.  The HTML file is saved as "index.html" under "./DJRivals/".

    """
    # todo: include EX charts
    pop_disc_dir = "./DJRivals/rankings/pop/disc/"
    html_file = "./DJRivals/index.html"
    charts = ["nm", "hd", "mx"]
    disc_info = []
    ps = psxml.PrettySimpleXML()
    for directory in [pop_disc_dir]:
        if not os.path.exists(directory):
            os.makedirs(directory)
    for disc in sorted(os.listdir(pop_disc_dir)):
        with open(pop_disc_dir + disc, "rb") as f:
            data = json.loads(f.read().decode())
            extracted = {}
            extracted["name"] = data["name"]
            extracted["image"] = data["image"]
            extracted["level"] = data["difficulty"]
            extracted["records"] = data["ranking"]["records"]
            disc_info.append(extracted)
    ps.raw("<!DOCTYPE html>")
    ps.start("html")
    ps.start("head")
    ps.empty("meta", ['charset="UTF-8"'])
    ps.start("title", value="DJRivals", newline=False).end()
    ps.empty("link", ['rel="stylesheet"', 'type="text/css"', 'href="./css/ui-lightness/jquery-ui-1.8.18.custom.css"'])
    ps.empty("link", ['rel="stylesheet"', 'type="text/css"', 'href="./djrivals.css"'])
    ps.start("script", ['type="text/javascript"', 'src="./js/jquery-1.7.1.min.js"'], newline=False).end()
    ps.start("script", ['type="text/javascript"', 'src="./js/jquery-ui-1.8.18.custom.min.js"'], newline=False).end()
    ps.start("script", ['type="text/javascript"', 'src="./djrivals.js"'], newline=False).end()
    ps.end()  # head
    ps.start("body")
    ps.start("div", attr=['class="accordion"'])
    ps.start("h3", newline=False).start("a", ['href="#"'], "Pop", newline=False).end(False).end()
    ps.start("div")
    ps.start("div", attr=['class="accordion"'])
    for chart in charts:
        ps.start("h3", newline=False).start("a", ['href="#"'], chart.upper(), newline=False).end(False).end()
        ps.start("div")
        ps.start("div", attr=['class="pop accordion"'])
        for disc in [disc for disc in disc_info if disc["records"][chart] > 0]:
            ps.start("h3", ['title="{}"'.format(chart)], newline=False)
            ps.start("a", ['href="#"'], newline=False)
            ps.empty("img", ['src="./images/{}"'.format(disc["image"][chart])], newline=False)
            ps.raw("&nbsp " + disc["name"]["full"], newline=False)
            ps.end(False)  # a
            ps.end()  # h3
            ps.start("div", newline=False).start("p", value="Loading...", newline=False).end(False).end()
        ps.end()  # div
        ps.end()  # div
    for alpha in ["ABCD", "EFGH", "IJKL", "MNOP", "QRSTU", "VWXYZ"]:
        ps.start("h3", newline=False).start("a", ['href="#"'], "{} - {}".format(alpha[0], alpha[-1]), newline=False).end(False).end()
        ps.start("div")
        ps.start("div", attr=['class="pop accordion"'])
        for dc in [(disc, chart) for disc in disc_info for chart in charts if disc["name"]["clean"][0] in alpha.lower() and disc["records"][chart] > 0]:
            ps.start("h3", ['title="{}"'.format(dc[1])], newline=False)
            ps.start("a", ['href="#"'], newline=False)
            ps.empty("img", ['src="./images/{}"'.format(dc[0]["image"][dc[1]])], newline=False)
            ps.raw("&nbsp " + dc[0]["name"]["full"], newline=False)
            ps.end(False)  # a
            ps.end()  # h3
            ps.start("div", newline=False).start("p", value="Loading...", newline=False).end(False).end()
        ps.end()  # div
        ps.end()  # div
    for level in range(1, 13):
        ps.start("h3", newline=False).start("a", ['href="#"'], "Level {}".format(level), newline=False).end(False).end()
        ps.start("div")
        ps.start("div", attr=['class="pop accordion"'])
        for dc in [(disc, chart) for disc in disc_info for chart in charts if disc["level"][chart] == level]:
            ps.start("h3", ['title="{}"'.format(dc[1])], newline=False)
            ps.start("a", ['href="#"'], newline=False)
            ps.empty("img", ['src="./images/{}"'.format(dc[0]["image"][dc[1]])], newline=False)
            ps.raw("&nbsp " + dc[0]["name"]["full"], newline=False)
            ps.end(False)  # a
            ps.end()  # h3
            ps.start("div", newline=False).start("p", value="Loading...", newline=False).end(False).end()
        ps.end()  # div
        ps.end()  # div
    ps.end_all()  # div, div, div, body, html
    with open(html_file, "wb") as f:
        f.write(ps.get().encode())
    print('Wrote: "{}"'.format(html_file))
<commit_msg>New function to download DJ icons.  Updated output directories and documentation.  Removed the letter and level folders.<commit_after>import collections
import json
import os, os.path
import re
import time
import urllib.request

import psxml

def f_clean():
    """f_clean() -> function(string)

    This function returns a function that cleans disc names.  Cleaning is done
    by stripping all non-alphanumeric characters.  The remaining characters are
    then converted to lowercase.

    """
    regex = re.compile(r"[^a-zA-Z0-9]")
    return lambda x: regex.sub(r"", x).lower()

def discs():
    """discs() -> None

    Download all disc images from the DJMAX site.  Files are saved under
    "./DJRivals/images/disc/".  An image will be skipped if it is determined
    that the file already exists.  Existence is checked using a simple filename
    lookup.

    """
    # todo: make multithreaded
    url = "http://djmaxcrew.com/ranking/GetRankPopMixing.asp?p={}"
    image_url = "http://img3.djmaxcrew.com/icon/disc/110/"
    image_dir = "./DJRivals/images/disc/"
    clean = f_clean()
    if not os.path.exists(image_dir):
        os.makedirs(image_dir)
    for page in range(1, 9):
        reply = json.loads(urllib.request.urlopen(url.format(page)).read().decode())
        for record in reply["DATA"]["RECORD"]:
            name, extension = os.path.splitext(record["DISCIMG"])
            for chart in range(1, 5):
                theirname = "{}{}{}".format(name[:-1], chart, extension)
                myname = "{}_{}{}".format(clean(record["DISCNAME"]), chart, extension)
                if os.path.exists(image_dir + myname):
                    continue
                with open(image_dir + myname, "wb") as f:
                    f.write(urllib.request.urlopen(image_url + theirname).read())
                print('Wrote: "{}{}"'.format(image_dir, myname))

def icons():
    """icons() -> None

    Crawl through the local database and download all necessary DJ icons from
    the DJMAX site.  Files are saved under "./DJRivals/images/icon/".  An image
    will be skipped if it is determined that the file already exists.  Existence
    is checked using a simple filename lookup.

    """
    # todo: make multithreaded
    url = "http://img3.djmaxcrew.com/icon/djicon/104/"
    pop_db_dir = "./DJRivals/rankings/pop/disc/"
    image_dir = "./DJRivals/images/icon/"
    icons = []
    for directory in [pop_db_dir, image_dir]:
        if not os.path.exists(directory):
            os.makedirs(directory)
    for disc in os.listdir(pop_db_dir):
        with open(pop_db_dir + disc, "rb") as f:
            data = json.loads(f.read().decode())
            icons.extend([dj[1] for chart in ['nm', 'hd', 'mx', 'ex'] for dj in data["ranking"][chart]])
    for icon in set(icons):
        if os.path.exists(image_dir + icon):
            continue
        with open(image_dir + icon, "wb") as f:
            f.write(urllib.request.urlopen(url + icon).read())
        print('Wrote: "{}{}"'.format(image_dir, icon))

def index(refresh=False):
    """index([boolean]) -> dictionary

    An auto-generated dictionary with manually maintained elements.  The
    dictionary is saved in JSON format as "./DJRivals/pop_index.json".  An
    optional boolean value (default: False) controls whether or not it should
    refresh its contents by checking the DJMAX site.  Refer to
    data_structures.txt for the format and contents of this file.

    Note: Because the DJMAX site does not list the difficulty level of charts
    anywhere, these entries are manually maintained.  The dictionary can be
    edited by hand in any text editor, and subsequent executions of this
    function will not clobber the manual entries.  However, if this function
    should encounter any errors while attempting to read the file, a new one
    will be generated as a replacement.  It is therefore recommended to have a
    backup of the current file before using this function.

    """
    url = "http://djmaxcrew.com/ranking/GetRankPopMixing.asp?p={}"
    index_dir = "./DJRivals/"
    index_file = index_dir + "pop_index.json"
    clean = f_clean()
    if not os.path.exists(index_dir):
        os.mkdir(index_dir)
    try:
        with open(index_file, "rb") as f:
            index = json.loads(f.read().decode())
    except:
        index = {}
    if refresh or not index:
        for page in range(1, 9):
            reply = json.loads(urllib.request.urlopen(url.format(page)).read().decode())
            for record in reply["DATA"]["RECORD"]:
                name = record["DISCNAME"]
                if clean(name) not in index:
                    index[clean(name)] = [name, 0, 0, 0, 0, page]
                else:  # do this because the name and page could have changed
                    index[clean(name)][0] = name
                    index[clean(name)][5] = page
        output = json.dumps(index, indent=4)
        output = re.sub(r'\[\n +(".+",) +\n +(\d+,) +\n +(\d+,) +\n +(\d+,) +\n +(\d+,) +\n +(\d+)\n +\](,?) *', r"[\1 \2 \3 \4 \5 \6]\7", output)
        with open(index_file, "wb") as f:
            f.write(output.encode())
        print('Wrote: "{}"'.format(index_file))
    return index

def f_identifier():
    """f_identifier() -> function(string)

    This function returns a function that retrieves a disc identifier from the
    DJMAX site by disc name.

    """
    url = "http://djmaxcrew.com/ranking/GetRankPopMixing.asp?p={}"
    clean = f_clean()
    page = index()
    cache = {}

    # since DJMAX returns up to 20 identifiers at a time (each page lists 20
    # discs except the last page which has less than 20), the cache dictionary
    # saves these identifers to expedite future lookups.  however, since it is
    # unknown how long these identifiers stay valid, the cache size has been
    # limited to about 25% of the total number of available discs.  when the
    # cache exceeds this limit, it will clear itself and rebuild.

    def identifier(name):
        name = clean(name)
        if name not in cache:
            if len(cache) > 20:  # value of 20 means at most 40
                cache.clear()
            reply = json.loads(urllib.request.urlopen(url.format(page[name][5])).read().decode())
            cache.update({clean(record["DISCNAME"]): record["DISCID"] for record in reply["DATA"]["RECORD"]})
        return cache[name]

    return identifier

def ranking(disc, chart, pages=1):
    """ranking(string, string[, integer]) -> list of tuples

    Retrieve the ranking of a specified disc name and chart.  The chart is a
    string that can be either NM, HD, MX, or EX; any other value will be treated
    as EX.  The optional integer specifies how many pages worth of rankings to
    retrieve (default: 1).  Specify 0 for a full listing.

    """
    # todo: make multithreaded
    url = "http://djmaxcrew.com/ranking/GetRankPopMixingMusic.asp?c={}&pt={}&p={}"
    identifier = f_identifier()
    disc_id = identifier(disc)
    chart = (lambda x: 1 if x == "nm" else 2 if x == "hd" else 3 if x == "mx" else 4)(chart.lower())
    results = []
    for page in range(1, 1 + (100 if pages == 0 else pages)):
        reply = json.loads(urllib.request.urlopen(url.format(disc_id, chart, page)).read().decode())
        results.extend([(record["RANK"], record["DJICON"], record["DJNAME"], record["SCORE"]) for record in reply["DATA"]["RECORD"]])
        if len(reply["DATA"]["RECORD"]) < 20:
            break
    return results

def database(disc_list=[]):
    """database([list]) -> None

    Create a local database of scores with information obtained from the DJMAX
    site.  The database is implemented as a collection of JSON files.  One JSON
    file is created for each disc.  In addition, one JSON file will be created
    for each DJ based on the information just acquired.  The optional argument
    is a list of strings (default: []) of cleaned disc names.  By default, it
    will create the entire database.  Given a list, it will create a database of
    only those discs.  Files are saved under "./DJRivals/rankings/pop/disc/" and
    "./DJRivals/rankings/pop/dj/".  Refer to data_structures.txt for the format
    and contents of these files.

    """
    # todo: write the code to generate the DJ JSON files
    start_time = time.time()
    disc_dir = "./DJRivals/rankings/pop/disc/"
    dj_dir = "./DJRivals/rankings/pop/dj/"
    disc_info = index()
    for directory in [disc_dir, dj_dir]:
        if not os.path.exists(directory):
            os.makedirs(directory)
    if not disc_list:
        disc_list = [title[0] for title in sorted(disc_info.items(), key=lambda i: i[1])]  # sort by page to maximize identifier() cache hits
    while len(disc_list):
        print("{} discs remaining.".format(len(disc_list)))
        disc = disc_list.pop()
        charts = ["nm", "hd", "mx", "ex"]
        output = collections.OrderedDict()
        output["name"] = collections.OrderedDict(zip(["clean", "full"], [disc, disc_info[disc][0]]))
        output["image"] = collections.OrderedDict([(i, disc + j) for i, j in zip(["eyecatch"] + charts, [".png", "_1.png", "_2.png", "_3.png", "_4.png"])])
        output["difficulty"] = collections.OrderedDict([(i, disc_info[disc][j]) for i, j in zip(charts, [1, 2, 3, 4])])
        output["ranking"] = collections.OrderedDict()
        output["ranking"]["records"] = collections.OrderedDict()
        for chart in charts:
            try:
                results = ranking(disc, chart, 0)
                output["ranking"][chart] = results
                output["ranking"]["records"][chart] = len(results)
                print("{} {} complete.  Sleeping...".format(disc, chart))
                time.sleep(10)
            except:
                print("{} {} error.  Sleeping for 5 minutes before retrying.".format(disc, chart))
                charts.insert(0, chart)
                time.sleep(300)
        output = json.dumps(output, indent=4)
        output = re.sub(r'\[\n +(\d+,) +\n +(".*",) +\n +(".*",) +\n +(\d{6})\n +\](,?) *', r"[\1 \2 \3 \4]\5", output)
        with open(disc_dir + disc + ".json", "wb") as f:
            f.write(output.encode())
        print('Wrote: "{}{}.json"\n'.format(disc_dir, disc))
    print("Operation complete!")
    elapsed_time = round((time.time() - start_time) / 60)
    print("Database creation took {} minutes.".format(elapsed_time))

def html():
    """html() -> None

    Crawl through the local database and automatically generate the DJRivals
    HTML user interface.  The HTML file is saved as "./DJRivals/index.html".

    """
    pop_disc_dir = "./DJRivals/rankings/pop/disc/"
    html_file = "./DJRivals/index.html"
    charts = ["nm", "hd", "mx", "ex"]
    disc_info = []
    ps = psxml.PrettySimpleXML()
    for directory in [pop_disc_dir]:
        if not os.path.exists(directory):
            os.makedirs(directory)
    for disc in sorted(os.listdir(pop_disc_dir)):
        with open(pop_disc_dir + disc, "rb") as f:
            data = json.loads(f.read().decode())
            extracted = {}
            extracted["name"] = data["name"]
            extracted["image"] = data["image"]
            extracted["level"] = data["difficulty"]
            extracted["records"] = data["ranking"]["records"]
            disc_info.append(extracted)
    ps.raw("<!DOCTYPE html>")
    ps.start("html")
    ps.start("head")
    ps.empty("meta", ['charset="UTF-8"'])
    ps.start("title", value="DJRivals", newline=False).end()
    ps.empty("link", ['rel="stylesheet"', 'type="text/css"', 'href="./css/ui-lightness/jquery-ui-1.8.18.custom.css"'])
    ps.empty("link", ['rel="stylesheet"', 'type="text/css"', 'href="./djrivals.css"'])
    ps.start("script", ['type="text/javascript"', 'src="./js/jquery-1.7.1.min.js"'], newline=False).end()
    ps.start("script", ['type="text/javascript"', 'src="./js/jquery-ui-1.8.18.custom.min.js"'], newline=False).end()
    ps.start("script", ['type="text/javascript"', 'src="./djrivals.js"'], newline=False).end()
    ps.end()  # head
    ps.start("body")
    ps.start("div", attr=['class="accordion"'])
    ps.start("h3", newline=False).start("a", ['href="#"'], "Pop", newline=False).end(False).end()
    ps.start("div")
    ps.start("div", attr=['class="accordion"'])
    for chart in charts:
        ps.start("h3", newline=False).start("a", ['href="#"'], chart.upper(), newline=False).end(False).end()
        ps.start("div")
        ps.start("div", attr=['class="pop accordion"'])
        for disc in [disc for disc in disc_info if disc["records"][chart] > 0]:
            ps.start("h3", newline=False)
            ps.start("a", ['href="#"'], newline=False)
            ps.empty("img", ['src="./images/{}"'.format(disc["image"][chart])], newline=False)
            ps.raw("&nbsp " + disc["name"]["full"], newline=False)
            ps.end(False)  # a
            ps.end()  # h3
            ps.start("div", newline=False).start("p", value="Loading...", newline=False).end(False).end()
        ps.end()  # div
        ps.end()  # div
    ps.end_all()  # div, div, div, body, html
    with open(html_file, "wb") as f:
        f.write(ps.get().encode())
    print('Wrote: "{}"'.format(html_file))
<|endoftext|>