<commit_before>import csv

import logging
log = logging.getLogger(__name__)

def deinterlace(path):
    with open(path) as csvfile:
        raw = [row for row in csv.reader(
            csvfile,
        )]
        l = len(raw)
        data = []
        i = 0
        #  Deinterlace and build list of dictionaries.
        while i < l:
            if (i % 2 == 0):  # zero-indexed rows
                row = raw[i]
                row.extend(raw[i + 1])
                data.append(row)
            else:  # Skip interlaced row; added supra
                pass
            i += 1
        return data

def parse_chorus(path, round):
    data = deinterlace(path)
    output = []
    # Split first cell and chapter cell
    for row in data:
        row.extend([row[0].split(' ', 1)[0]])
        row.extend([row[8].split('[', 1)[1].split(']', 1)[0]])
        row[0] = row[0].split(' ', 1)[1]
        row[8] = row[8].split('[', 1)[0]
        # Add round
        row.extend([round])
        output.append(row)
    # Strip space
    for row in output:
        i = 0
        l = len(row)
        while i < l:
            row[i] = row[i].strip()
            i += 1
    # Reorder in list -- SUPER Kludge!
    for row in output:
        row.insert(0, row.pop(-1))
        row.insert(1, row.pop(-2))
        row.insert(2, row.pop(10))
        row.insert(4, row.pop(-1))
        row.pop(10)
        row.pop(9)
        row.insert(-1, row.pop(9))
        row.pop(13)
        row.pop(13)
        row.pop(-1)

    #  Write output
    with open('output.csv', 'wb') as csvfile:
        writer = csv.writer(csvfile)
        for row in output:
            writer.writerow(row)
    return output

def parse_quarters(path):
    data = deinterlace(path)
    output = []
    # Split first cell and chapter cell
    for row in data:
        row.extend([row[0].split(' ', 1)[0]])
        row.extend([row[7].split('[', 1)[1].split(']', 1)[0]])
        row[0] = row[0].split(' ', 1)[1]
        row[7] = row[7].split('[', 1)[0]
        # Add round
        row.extend(['3'])
        output.append(row)
    # Strip space
    for row in output:
        i = 0
        l = len(row)
        while i < l:
            row[i] = row[i].strip()
            i += 1
    # Reorder in list -- SUPER Kludge!
    for row in output:
        row.insert(0, row.pop(-1))
        row.insert(1, row.pop(-2))
        row.insert(3, row.pop(-2))
        row.insert(4, row.pop(-1))
        row.pop(9)
        row.pop(9)
        row.pop(9)

    # Write output
    with open('output.csv', 'wb') as csvfile:
        writer = csv.writer(csvfile)
        for row in output:
            writer.writerow(row)
    return output

def parse_semis(path):
    data = deinterlace(path)
    output = []
    # Split first cell and chapter cell
    for row in data:
        row.extend([row[0].split(' ', 1)[0]])
        row.extend([row[9].split('[', 1)[1].split(']', 1)[0]])
        row[0] = row[0].split(' ', 1)[1]
        row[9] = row[9].split('[', 1)[0]
        # Add round
        row.extend(['2'])
        output.append(row)
    # Strip space
    for row in output:
        i = 0
        l = len(row)
        while i < l:
            row[i] = row[i].strip()
            i += 1
    # Reorder in list -- SUPER Kludge!
    for row in output:
        row.insert(0, row.pop(-1))
        row.insert(1, row.pop(-2))
        row.insert(3, row.pop(-2))
        row.insert(4, row.pop(-1))
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(-1)
        row.pop(-1)

    # Write output
    with open('output.csv', 'wb') as csvfile:
        writer = csv.writer(csvfile)
        for row in output:
            writer.writerow(row)
    return output

def parse_finals(path):
    data = deinterlace(path)
    output = []
    # Split first cell and chapter cell
    for row in data:
        row.extend([row[0].split(' ', 1)[0]])
        row.extend([row[9].split('[', 1)[1].split(']', 1)[0]])
        row[0] = row[0].split(' ', 1)[1]
        row[9] = row[9].split('[', 1)[0]
        # Add round
        row.extend(['1'])
        output.append(row)
    # Strip space
    for row in output:
        i = 0
        l = len(row)
        while i < l:
            row[i] = row[i].strip()
            i += 1
    # Reorder in list -- SUPER Kludge!
    for row in output:
        row.insert(0, row.pop(-1))
        row.insert(1, row.pop(-2))
        row.insert(3, row.pop(-2))
        row.insert(4, row.pop(-1))
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(-1)
        row.pop(-1)

    # Write output
    with open('output.csv', 'wb') as csvfile:
        writer = csv.writer(csvfile)
        for row in output:
            writer.writerow(row)
    return output
<commit_msg>District chorus importer<commit_after>import os
import csv

import logging
log = logging.getLogger(__name__)

def deinterlace(path):
    with open(path) as csvfile:
        raw = [row for row in csv.reader(
            csvfile,
        )]
        l = len(raw)
        data = []
        i = 0
        #  Deinterlace and build list of dictionaries.
        while i < l:
            if (i % 2 == 0):  # zero-indexed rows
                row = raw[i]
                row.extend(raw[i + 1])
                data.append(row)
            else:  # Skip interlaced row; added supra
                pass
            i += 1
        return data

def parse_chorus(path, round):
    data = deinterlace(path)
    output = []
    # Split first cell and chapter cell
    for row in data:
        row.extend([row[0].split(' ', 1)[0]])
        row.extend([row[8].split('[', 1)[1].split(']', 1)[0]])
        row[0] = row[0].split(' ', 1)[1]
        row[8] = row[8].split('[', 1)[0]
        # Add round
        row.extend([round])
        output.append(row)
    # Strip space
    for row in output:
        i = 0
        l = len(row)
        while i < l:
            row[i] = row[i].strip()
            i += 1
    # Reorder in list -- SUPER Kludge!
    for row in output:
        row.insert(0, row.pop(-1))
        row.insert(1, row.pop(-2))
        row.insert(2, row.pop(10))
        row.insert(4, row.pop(-1))
        row.pop(10)
        row.pop(9)
        row.insert(-1, row.pop(9))
        row.pop(13)
        row.pop(13)
        row.pop(-1)

    # Write output
    rename = os.path.splitext(os.path.basename)[0]
    with open('{0}.csv'.format(rename), 'wb') as csvfile:
        writer = csv.writer(csvfile)
        for row in output:
            writer.writerow(row)
    return output

def parse_district_chorus(path, district):
    data = deinterlace(path)
    output = []
    # Split first cell and chapter cell
    for row in data:
        row.extend([row[0].split(' ', 1)[0]])
        row.extend([row[9].split('(', 1)[0]])
        row[0] = row[0].split(' ', 1)[1]
        row[9] = row[9].split('(', 1)[0]
        # Add round
        row.extend(['1'])
        # Overwrite district
        row[19] = district
        output.append(row)
    # Strip space
    for row in output:
        i = 0
        l = len(row)
        while i < l:
            row[i] = row[i].strip()
            i += 1
    # Reorder in list -- SUPER Kludge!
    for row in output:
        row.insert(0, row.pop(-1))
        row.insert(1, row.pop(-2))
        row.insert(2, row.pop(11))
        row.insert(4, row.pop(-1))
        row.insert(-1, row.pop(12))
        row.pop(5)
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(13)
        row.pop(13)
        row.pop(14)

    # # Write output
    # rename = os.path.splitext(os.path.basename)[0]
    # with open('{0}.csv'.format(rename), 'wb') as csvfile:
    #     writer = csv.writer(csvfile)
    #     for row in output:
    #         writer.writerow(row)
    return output

def parse_quarters(path):
    data = deinterlace(path)
    output = []
    # Split first cell and chapter cell
    for row in data:
        row.extend([row[0].split(' ', 1)[0]])
        row.extend([row[7].split('[', 1)[1].split(']', 1)[0]])
        row[0] = row[0].split(' ', 1)[1]
        row[7] = row[7].split('[', 1)[0]
        # Add round
        row.extend(['3'])
        output.append(row)
    # Strip space
    for row in output:
        i = 0
        l = len(row)
        while i < l:
            row[i] = row[i].strip()
            i += 1
    # Reorder in list -- SUPER Kludge!
    for row in output:
        row.insert(0, row.pop(-1))
        row.insert(1, row.pop(-2))
        row.insert(3, row.pop(-2))
        row.insert(4, row.pop(-1))
        row.pop(9)
        row.pop(9)
        row.pop(9)

    # Write output
    rename = os.path.splitext(os.path.basename)[0]
    with open('{0}.csv'.format(rename), 'wb') as csvfile:
        writer = csv.writer(csvfile)
        for row in output:
            writer.writerow(row)
    return output

def parse_semis(path):
    data = deinterlace(path)
    output = []
    # Split first cell and chapter cell
    for row in data:
        row.extend([row[0].split(' ', 1)[0]])
        row.extend([row[9].split('[', 1)[1].split(']', 1)[0]])
        row[0] = row[0].split(' ', 1)[1]
        row[9] = row[9].split('[', 1)[0]
        # Add round
        row.extend(['2'])
        output.append(row)
    # Strip space
    for row in output:
        i = 0
        l = len(row)
        while i < l:
            row[i] = row[i].strip()
            i += 1
    # Reorder in list -- SUPER Kludge!
    for row in output:
        row.insert(0, row.pop(-1))
        row.insert(1, row.pop(-2))
        row.insert(3, row.pop(-2))
        row.insert(4, row.pop(-1))
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(-1)
        row.pop(-1)

    # Write output
    rename = os.path.splitext(os.path.basename)[0]
    with open('{0}.csv'.format(rename), 'wb') as csvfile:
        writer = csv.writer(csvfile)
        for row in output:
            writer.writerow(row)
    return output

def parse_finals(path):
    data = deinterlace(path)
    output = []
    # Split first cell and chapter cell
    for row in data:
        row.extend([row[0].split(' ', 1)[0]])
        row.extend([row[9].split('[', 1)[1].split(']', 1)[0]])
        row[0] = row[0].split(' ', 1)[1]
        row[9] = row[9].split('[', 1)[0]
        # Add round
        row.extend(['1'])
        output.append(row)
    # Strip space
    for row in output:
        i = 0
        l = len(row)
        while i < l:
            row[i] = row[i].strip()
            i += 1
    # Reorder in list -- SUPER Kludge!
    for row in output:
        row.insert(0, row.pop(-1))
        row.insert(1, row.pop(-2))
        row.insert(3, row.pop(-2))
        row.insert(4, row.pop(-1))
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(9)
        row.pop(-1)
        row.pop(-1)

    # Write output
    rename = os.path.splitext(os.path.basename)[0]
    with open('{0}.csv'.format(rename), 'wb') as csvfile:
        writer = csv.writer(csvfile)
        for row in output:
            writer.writerow(row)
    return output
<|endoftext|>


<commit_before>import os
import re
import sys
from os.path import abspath, isfile, join, pardir

"""
required packages
numpy
(numba)

these packages have to be installed in virtual environment in use:

right python version! (will influence the tox environments!)
for testing:
conda install pytest
conda install isort
conda install twine

pip install rstcheck pip-tools

rstcheck>=3.3.1
twine for uploading securely

documentation generation:
conda install sphinx
https://docs.readthedocs.io/en/stable/intro/getting-started-with-sphinx.html

Use the Makefile to build the docs, like so:
make html

--cov-config=tox.ini

pip-tools package:
TODO write bash script for this
its important to pin requirements to get reproducible errors!
compile a new requirements file (with the latest versions)

source activate tzEnv
pip-compile --upgrade
same as?!:
pip-compile --output-file requirements_tests.txt requirements_tests.in
pip-compile --output-file requirements_numba.txt requirements_numba.in
only update the flask package:
pip-compile --upgrade-package flask
compile a new requirements file (with versions currently used in the virtual env )
pip-compile --generate-hashes requirements_numba.in

do NOT sync. will install ONLY the packages specified! (tox etc. would not be installed any more!)
pip-sync

commands
tox -r to rebuild your tox virtualenvs when you've made changes to requirements setup
# rstcheck will complain about non referenced hyperlinks in doc .rst files! (cannot detect cross file references!)
rstcheck *.rst
tox -r -e codestyle
tox -r -e py37
tox -r -e py37-numba

automatically update imports: isort -rc .
dont use for example.py

Use the Makefile to build the docs, like so:
cd ./docs
make html
# for online build of docs, release tag must be created!

use bandit to check for vulnerabilities:

conda install bandit
bandit ./timezonefinder/*.py

"""

PACKAGE = 'timezonefinder'
VERSION_FILE = 'VERSION'

# print('Enter virtual env name:')
VIRT_ENV_NAME = 'tzEnv'
VIRT_ENV_ACT_CMD = f'. ~/miniconda3/etc/profile.d/conda.sh; conda activate {VIRT_ENV_NAME}; '

def get_version():
    return open(VERSION_FILE, 'r').read().strip()

def parse_version(new_version_input='', old_version_str='1.0.0'):
    new_version_input = re.search(r'\d\.\d\.\d+', new_version_input)

    if new_version_input is None:
        raise ValueError  # will cause new input request
    else:
        new_version_input = new_version_input.group()

    # print(new_version_input)

    split_new_version = [int(x) for x in new_version_input.split('.')]
    # print(split_new_version)
    split_old_version = [int(x) for x in old_version_str.split('.')]
    # print(split_old_version)

    for i in range(3):
        if split_new_version[i] > split_old_version[i]:
            break
        if split_new_version[i] < split_old_version[i]:
            raise ValueError  # will cause new input request

    return new_version_input

def set_version(new_version_str):
    with open(VERSION_FILE, 'w') as version_file:
        version_file.write(new_version_str)

def routine(cmd=None, message='', option1='next', option2='exit'):
    while 1:
        print(message)

        if cmd:
            print('running command:', cmd)
            os.system(cmd)

        print('__________\nDone. Options:')
        print('1)', option1)
        print('2)', option2)
        print('anything else to repeat this step.')
        try:
            inp = int(input())

            if inp == 1:
                print('==============')
                break
            if inp == 2:
                sys.exit()

        except ValueError:
            pass
        print('================')

if __name__ == "__main__":

    print('Do you want to switch to the "dev" branch? Commit before switching branch!')
    print('1) yes, change now.')
    print('2) no, exit')
    print('anything else skip.')
    try:
        inp = int(input())
        if inp == 1:
            os.system('git checkout dev')
            print('==============')
        if inp == 2:
            sys.exit()
    except ValueError:
        pass

    old_version = get_version()

    print('The actual version number is:', old_version)
    print('Enter new version number:')
    version_input = None
    while 1:
        try:
            version_input = input()
            version_str = parse_version(version_input, old_version)
            set_version(version_str)
            break
        except ValueError:
            print(
                f'Invalid version input. Should be of format "x.x.xxx" and higher than the old version {old_version}.')
            pass  # try again

    version = get_version()
    print('the version number has been set to:', version)
    print('=====================')

    # TODO data could contain errors, test before upload
    routine(None, 'have you uploaded new data and pinned the version in setup.py (under extras)?!', 'OK. Continue',
            'Exit')
    routine(None, 'Are all data files listed in global_settings.py?!', 'OK. Continue', 'Exit')
    routine(None, 'Remember to list all relevant importable objects in __all__ variable in __init__.py!',
            'OK. Continue', 'Exit')
    routine(None, 'Remember to keep helpers.py and helpers_numba.py consistent!', 'OK. Continue', 'Exit')
    routine(None,
            'Maybe re-pin the test dependencies (requirements.txt) with pip-compile!'
            ' Commands are written in the beginning of this script',
            'Done. Run tests', 'Exit')
    routine(None,
            'Are all pinned dependencies written in setup.py and the Documentation?',
            'OK. Continue',
            'Exit')
    routine(None, 'Are all (new) features documented?', 'OK. Continue', 'Exit')
    routine(None, 'Remember to write a changelog now for version %s' % version, 'Done. Continue', 'Exit')

    print('___________')
    print('Running TESTS:')

    # routine(VIRT_ENV_ACT_CMD + "pip-compile requirements_numba.in;pip-sync",
    #      'pinning the requirements.txt and bringing virtualEnv to exactly the specified state:', 'next: build check')

    routine(VIRT_ENV_ACT_CMD + "rstcheck *.rst", 'checking syntax of all .rst files:', 'next: build check')

    print('generating documentation now...')
    os.system('(cd ./docs && exec make html)')
    print('done.')
    # TODO test

    # IMPORTANT: -r flag to rebuild tox virtual env
    # only when dependencies have changed!
    rebuild_flag = ''
    print('when the dependencies (in requirements.txt) have changed enter 1 (-> rebuild tox)')
    try:
        inp = int(input())
        if inp == 1:
            rebuild_flag = ' -r'
    except ValueError:
        pass

    # routine(VIRT_ENV_ACT_CMD + "tox" + rebuild_flag, 'checking syntax, codestyle and imports', 'continue')
    routine(VIRT_ENV_ACT_CMD + "tox" + rebuild_flag + " -e codestyle",
            'checking syntax, codestyle and imports', 'continue')
    routine(VIRT_ENV_ACT_CMD + "tox" + rebuild_flag + " -e py37", 'build tests py3')
    routine(VIRT_ENV_ACT_CMD + "tox" + rebuild_flag + " -e py37-numba", 'build tests with numba installed')

    print('Tests finished.')

    routine(None,
            'Please commit your changes, push and wait if Travis tests build successfully. '
            'Only then merge them into the master.',
            'Build successful. try packaging.')

    # TODO do this automatically, problem are the commit messages (often the same as changelog)
    # git commit --message
    # git push dev

    # TODO wait for Travis to finish positively

    # if not in master

    # TODO ask to push in master
    # git merge ...

    # TODO switching to master

    print('=================')
    print('PUBLISHING:')

    # routine("python3 setup.py sdist bdist_wheel upload", 'Uploading the package now.') # deprecated
    # new twine publishing routine:
    # https://packaging.python.org/tutorials/packaging-projects/
    routine("python3 setup.py sdist bdist_wheel", 'building the package now.', 'packaging done. try publishing.')

    path = abspath(join(__file__, pardir, 'dist'))
    all_archives_this_version = [f for f in os.listdir(path) if isfile(join(path, f)) and version_str in f]
    paths2archives = [abspath(join(path, f)) for f in all_archives_this_version]
    command = "twine upload --repository-url https://test.pypi.org/legacy/ " + ' '.join(paths2archives)

    # upload all archives of this version
    routine(VIRT_ENV_ACT_CMD + command, 'testing if upload works.', 'publishing test done. start real publishing.')

    command = "twine upload " + ' '.join(paths2archives)
    routine(VIRT_ENV_ACT_CMD + command, 'real upload to PyPI.')

    # tag erstellen
    routine(None, 'Do you want to create a git release tag?', 'Yes', 'No')

    routine(f"git tag -a v{version} -m 'Version {version}'", 'Creating tag', 'Continue')

    routine(None, 'Do you want to push the git release tag?', 'Yes', 'No')
    # in den master pushen
    os.system("git push --tags")

    print('______________')
    print('Publishing Done.')
    print('now run:')
    print("only when the upload didn't work: python3 setup.py bdist_wheel upload")
    print(f'sudo -H pip install {PACKAGE} --upgrade')
<commit_msg>explicit py versions for wheel<commit_after>import os
import re
import sys
from os.path import abspath, isfile, join, pardir

"""
required packages
numpy
(numba)

these packages have to be installed in virtual environment in use:

right python version! (will influence the tox environments!)
for testing:
conda install pytest
conda install isort
conda install twine

pip install rstcheck pip-tools

rstcheck>=3.3.1
twine for uploading securely

documentation generation:
conda install sphinx
https://docs.readthedocs.io/en/stable/intro/getting-started-with-sphinx.html

Use the Makefile to build the docs, like so:
make html

--cov-config=tox.ini

pip-tools package:
TODO write bash script for this
its important to pin requirements to get reproducible errors!
compile a new requirements file (with the latest versions)

source activate tzEnv
pip-compile --upgrade
same as?!:
pip-compile --output-file requirements_tests.txt requirements_tests.in
pip-compile --output-file requirements_numba.txt requirements_numba.in
only update the flask package:
pip-compile --upgrade-package flask
compile a new requirements file (with versions currently used in the virtual env )
pip-compile --generate-hashes requirements_numba.in

do NOT sync. will install ONLY the packages specified! (tox etc. would not be installed any more!)
pip-sync

commands
tox -r to rebuild your tox virtualenvs when you've made changes to requirements setup
# rstcheck will complain about non referenced hyperlinks in doc .rst files! (cannot detect cross file references!)
rstcheck *.rst
tox -r -e codestyle
tox -r -e py37
tox -r -e py37-numba

automatically update imports: isort -rc .
dont use for example.py

Use the Makefile to build the docs, like so:
cd ./docs
make html
# for online build of docs, release tag must be created!

use bandit to check for vulnerabilities:

conda install bandit
bandit ./timezonefinder/*.py

"""

PACKAGE = 'timezonefinder'
VERSION_FILE = 'VERSION'

# print('Enter virtual env name:')
VIRT_ENV_NAME = 'tzEnv'
VIRT_ENV_ACT_CMD = f'. ~/miniconda3/etc/profile.d/conda.sh; conda activate {VIRT_ENV_NAME}; '
PY_VERSION_IDS = ['36', '37', '38']  # the supported python versions to create wheels for
PYTHON_TAG = '.'.join([f'py{v}'for v in PY_VERSION_IDS])

def get_version():
    return open(VERSION_FILE, 'r').read().strip()

def parse_version(new_version_input='', old_version_str='1.0.0'):
    new_version_input = re.search(r'\d\.\d\.\d+', new_version_input)

    if new_version_input is None:
        raise ValueError  # will cause new input request
    else:
        new_version_input = new_version_input.group()

    # print(new_version_input)

    split_new_version = [int(x) for x in new_version_input.split('.')]
    # print(split_new_version)
    split_old_version = [int(x) for x in old_version_str.split('.')]
    # print(split_old_version)

    for i in range(3):
        if split_new_version[i] > split_old_version[i]:
            break
        if split_new_version[i] < split_old_version[i]:
            raise ValueError  # will cause new input request

    return new_version_input

def set_version(new_version_str):
    with open(VERSION_FILE, 'w') as version_file:
        version_file.write(new_version_str)

def routine(cmd=None, message='', option1='next', option2='exit'):
    while 1:
        print(message)

        if cmd:
            print('running command:', cmd)
            os.system(cmd)

        print('__________\nDone. Options:')
        print('1)', option1)
        print('2)', option2)
        print('anything else to repeat this step.')
        try:
            inp = int(input())

            if inp == 1:
                print('==============')
                break
            if inp == 2:
                sys.exit()

        except ValueError:
            pass
        print('================')

if __name__ == "__main__":

    print('Do you want to switch to the "dev" branch? Commit before switching branch!')
    print('1) yes, change now.')
    print('2) no, exit')
    print('anything else skip.')
    try:
        inp = int(input())
        if inp == 1:
            os.system('git checkout dev')
            print('==============')
        if inp == 2:
            sys.exit()
    except ValueError:
        pass

    old_version = get_version()

    print('The actual version number is:', old_version)
    print('Enter new version number:')
    version_input = None
    while 1:
        try:
            version_input = input()
            version_str = parse_version(version_input, old_version)
            set_version(version_str)
            break
        except ValueError:
            print(
                f'Invalid version input. Should be of format "x.x.xxx" and higher than the old version {old_version}.')
            pass  # try again

    version = get_version()
    print('the version number has been set to:', version)
    print('=====================')

    # TODO data could contain errors, test before upload
    routine(None, 'Remember to properly specify all supported python versions in publish.py and setup.py')
    routine(None, 'Remember to uploaded new EXTERNAL data and pin the version in setup.py (under extras)')
    routine(None, 'Remember to list all LOCAL data files in global_settings.py')
    routine(None, 'Remember to list all relevant importable objects in __all__ variable in __init__.py')
    routine(None, 'Remember to keep helpers.py and helpers_numba.py consistent')
    routine(None, 'Maybe re-pin the test dependencies (requirements.txt) with pip-compile!'
            ' Commands are written in the beginning of this script')
    routine(None, 'Have all pinned dependencies been listed in setup.py and the Documentation?',)
    routine(None, 'Have all (new) features been documented?')
    routine(None, f'Remember to write a changelog now for version {version}')

    print('___________')
    print('Running TESTS:')

    # routine(VIRT_ENV_ACT_CMD + "pip-compile requirements_numba.in;pip-sync",
    #      'pinning the requirements.txt and bringing virtualEnv to exactly the specified state:', 'next: build check')

    routine(VIRT_ENV_ACT_CMD + "rstcheck *.rst", 'checking syntax of all .rst files:', 'next: build check')

    print('generating documentation now...')
    os.system('(cd ./docs && exec make html)')
    print('done.')
    # TODO test

    # IMPORTANT: -r flag to rebuild tox virtual env
    # only when dependencies have changed!
    rebuild_flag = ''
    print('when the dependencies (in requirements.txt) have changed enter 1 (-> rebuild tox)')
    try:
        inp = int(input())
        if inp == 1:
            rebuild_flag = ' -r'
    except ValueError:
        pass

    # routine(VIRT_ENV_ACT_CMD + "tox" + rebuild_flag, 'checking syntax, codestyle and imports', 'continue')
    routine(VIRT_ENV_ACT_CMD + "tox" + rebuild_flag + " -e codestyle",
            'checking syntax, codestyle and imports', 'continue')
    routine(VIRT_ENV_ACT_CMD + "tox" + rebuild_flag + " -e py37", 'build tests py3')
    routine(VIRT_ENV_ACT_CMD + "tox" + rebuild_flag + " -e py37-numba", 'build tests with numba installed')

    print('Tests finished.')

    routine(None,
            'Please commit your changes, push and wait if Travis tests build successfully. '
            'Only then merge them into the master.',
            'Build successful. try packaging.')

    # TODO do this automatically, problem are the commit messages (often the same as changelog)
    # git commit --message
    # git push dev

    # TODO wait for Travis to finish positively

    # if not in master

    # TODO ask to push in master
    # git merge ...

    # TODO switching to master

    print('=================')
    print('PUBLISHING:')

    # routine("python3 setup.py sdist bdist_wheel upload", 'Uploading the package now.') # deprecated
    # new twine publishing routine:
    # https://packaging.python.org/tutorials/packaging-projects/
    routine(f"python setup.py sdist bdist_wheel --python-tag {PYTHON_TAG}", 'building the package now.')

    path = abspath(join(__file__, pardir, 'dist'))
    all_archives_this_version = [f for f in os.listdir(path) if isfile(join(path, f)) and version_str in f]
    paths2archives = [abspath(join(path, f)) for f in all_archives_this_version]
    command = "twine upload --repository-url https://test.pypi.org/legacy/ " + ' '.join(paths2archives)

    # upload all archives of this version
    routine(VIRT_ENV_ACT_CMD + command, 'testing if upload works.', 'publishing test done. start real publishing.')

    command = "twine upload " + ' '.join(paths2archives)
    routine(VIRT_ENV_ACT_CMD + command, 'real upload to PyPI.')

    # tag erstellen
    routine(None, 'Do you want to create a git release tag?', 'Yes', 'No')

    routine(f"git tag -a v{version} -m 'Version {version}'", 'Creating tag', 'Continue')

    routine(None, 'Do you want to push the git release tag?', 'Yes', 'No')
    # in den master pushen
    os.system("git push --tags")

    print('______________')
    print('Publishing Done.')
    print('now run:')
    print("only when the upload didn't work: python3 setup.py bdist_wheel upload")
    print(f'sudo -H pip install {PACKAGE} --upgrade')
<|endoftext|>


<commit_before>import logging
log = logging.getLogger(__name__)

from rest_framework import (
    viewsets,
)

from .models import (
    Convention,
    Chorus,
    Quartet,
)

from .serializers import (
    ConventionSerializer,
    ChorusSerializer,
    QuartetSerializer,
)

class ConventionViewSet(viewsets.ModelViewSet):
    queryset = Convention.objects.filter(
        slug__startswith='bhs-summer',
    )
    serializer_class = ConventionSerializer
    lookup_field = 'slug'

class ChorusViewSet(viewsets.ModelViewSet):
    queryset = Chorus.objects.all()
    serializer_class = ChorusSerializer
    lookup_field = 'slug'

class QuartetViewSet(viewsets.ModelViewSet):
    queryset = Quartet.objects.all()
    serializer_class = QuartetSerializer
    lookup_field = 'slug'
<commit_msg>Filter and prefetch_related to improve performance<commit_after>import logging
log = logging.getLogger(__name__)

from rest_framework import (
    viewsets,
)

from .models import (
    Convention,
    Chorus,
    Quartet,
)

from .serializers import (
    ConventionSerializer,
    ChorusSerializer,
    QuartetSerializer,
)

class ConventionViewSet(viewsets.ModelViewSet):
    queryset = Convention.objects.filter(
        name__in=[
            'Philadelphia 2010',
            'Kansas City 2011',
            'Portland 2012',
            'Toronto 2013',
            'Las Vegas 2014',
            'Pittsburgh 2015',
        ]
    ).prefetch_related('contests__contestants__performances')
    serializer_class = ConventionSerializer
    lookup_field = 'slug'

class ChorusViewSet(viewsets.ModelViewSet):
    queryset = Chorus.objects.all().prefetch_related('contestants__performances')
    serializer_class = ChorusSerializer
    lookup_field = 'slug'

class QuartetViewSet(viewsets.ModelViewSet):
    queryset = Quartet.objects.all()
    serializer_class = QuartetSerializer
    lookup_field = 'slug'
<|endoftext|>


<commit_before><commit_msg>Remove girder_plugin_dir symlink to fix build<commit_after><|endoftext|>


<commit_before>'''
Dummy data for use in web development

This data is meant to be used for example purposes only. While the intention
is that this data be representative of the variables presented, it is not final
and should not be used in production.
'''

from __future__ import absolute_import
import os
import itertools
import logging
from functools import reduce
import xarray as xr
import pandas as pd

from climate.toolbox import (
    load_climate_data,
    weighted_aggregate_grid_to_regions)

__author__ = 'Michael Delgado'
__contact__ = 'mdelgado@rhg.com'
__version__ = '0.0.1a1'

BCSD_orig_files = os.path.join(
    '/shares/gcp/sources/BCSD-original/{rcp}/day/atmos/{variable}/r1i1p1/v1.0',
    '{variable}_day_BCSD_rcp85_r1i1p1_{model}_{year}.nc')

WRITE_PATH = os.path.join(
    '/shares/gcp/outputs/diagnostics/web/gcp/climate',
    '{variable}/{variable}_{model}_{period}.nc')

ADDITIONAL_METADATA = dict(
    description=__file__.__doc__,
    author=__author__,
    contact=__contact__,
    version=__version__,
    project='gcp',
    team='climate',
    geography='hierid',
    weighting='areawt',
    frequency='year_sample')

def tasmin_under_32F(ds):
    '''
    Count of days with tasmin under 32F/0C
    '''
    return ds.tasmin.where((ds.tasmin- 273.15) < 0).count(dim='time')

def tasmax_over_95F(ds):
    '''
    Count of days with tasmax over 95F/35C
    '''
    return ds.tasmax.where((ds.tasmax- 273.15) > 35).count(dim='time')

def average_seasonal_temp(ds):
    '''
    Average seasonal tas
    '''
    return ds.tas.groupby('time.season').mean(dim='time')

JOBS = [
    dict(variable='tasmax', transformation=tasmax_over_95F),
    dict(variable='tasmin', transformation=tasmin_under_32F),
    dict(variable='tas', transformation=average_seasonal_temp)]

PERIODS = [
    dict(rcp='hist', pername='1986', years=list(range(1996, 1997)))
    # dict(rcp='rcp85', pername='2020', years=list(range(2020, 2040))),
    # dict(rcp='rcp85', pername='2040', years=list(range(2040, 2060))),
    # dict(rcp='rcp85', pername='2080', years=list(range(2080, 2100)))
    ]

MODELS = [
    dict(model='ACCESS1-0'),
    dict(model='CESM1-BGC'),
    dict(model='GFDL-ESM2M')]

ITERATION_COMPONENTS = (JOBS, PERIODS, MODELS)

def run_job(variable, transformation, rcp, pername, years, model):

    # Build job metadata
    metadata = {k: v for k, v in ADDITIONAL_METADATA.items()}
    metadata.update(dict(
        variable=variable, model=model, period=pername, rcp=rcp))

    metadata['transformation'] = transformation.__doc__
    metadata['time_horizon'] = '{}-{}'.format(years[0], years[-1])

    # Get transformed data
    transformed = xr.concat([
        (load_climate_data(
            BCSD_orig_files.format(year=y, model=model, variable=variable),
                variable)
            .pipe(transformation))
        for y in years],
        dim=pd.Index(years, name='year')).mean(dim='year')

    # Reshape to regions
    wtd = weighted_aggregate_grid_to_regions(
            transformed, variable, 'areawt', 'hierid')

    # Update netCDF metadata
    wtd.attrs.update(**metadata)

    # Write output
    fp = WRITE_PATH.format(**metadata)
    if not os.path.isdir(os.path.dirname(fp)):
        os.makedirs(os.path.dirname(fp))
    wtd.to_netcdf(fp)

FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(format=FORMAT)
logger = logging.getLogger('uploader')
logger.setLevel('INFO')

def main():

    njobs = reduce(lambda x, y: x*y, map(len, ITERATION_COMPONENTS))

    for i, job_components in enumerate(
            itertools.product(*ITERATION_COMPONENTS)):

        job = {}
        for job_component in job_components:
            job.update(job_component)

        logger.info('beginning job {} of {}'.format(i, njobs))

        try:
            run_job(**job)
        except Exception, e:
            logger.error(
                'Error encountered in job {} of {}:\n\nJob spec:\n{}\n\n'
                    .format(i, njobs, job),
                exc_info=e)

if __name__ == '__main__':
    main()
<commit_msg>bug fix<commit_after>'''
Dummy data for use in web development

This data is meant to be used for example purposes only. While the intention
is that this data be representative of the variables presented, it is not final
and should not be used in production.
'''

from __future__ import absolute_import
import os
import itertools
import logging
from functools import reduce
import xarray as xr
import pandas as pd

from climate.toolbox import (
    load_climate_data,
    weighted_aggregate_grid_to_regions)

__author__ = 'Michael Delgado'
__contact__ = 'mdelgado@rhg.com'
__version__ = '0.0.1a1'

BCSD_orig_files = os.path.join(
    '/shares/gcp/sources/BCSD-original/{rcp}/day/atmos/{variable}/r1i1p1/v1.0',
    '{variable}_day_BCSD_rcp85_r1i1p1_{model}_{year}.nc')

WRITE_PATH = os.path.join(
    '/shares/gcp/outputs/diagnostics/web/gcp/climate',
    '{variable}/{variable}_{model}_{period}.nc')

ADDITIONAL_METADATA = dict(
    description=__file__.__doc__,
    author=__author__,
    contact=__contact__,
    version=__version__,
    project='gcp',
    team='climate',
    geography='hierid',
    weighting='areawt',
    frequency='year_sample')

def tasmin_under_32F(ds):
    '''
    Count of days with tasmin under 32F/0C
    '''
    return ds.tasmin.where((ds.tasmin- 273.15) < 0).count(dim='time')

def tasmax_over_95F(ds):
    '''
    Count of days with tasmax over 95F/35C
    '''
    return ds.tasmax.where((ds.tasmax- 273.15) > 35).count(dim='time')

def average_seasonal_temp(ds):
    '''
    Average seasonal tas
    '''
    return ds.tas.groupby('time.season').mean(dim='time')

JOBS = [
    dict(variable='tasmax', transformation=tasmax_over_95F),
    dict(variable='tasmin', transformation=tasmin_under_32F),
    dict(variable='tas', transformation=average_seasonal_temp)]

PERIODS = [
    dict(rcp='hist', pername='1986', years=list(range(1996, 1997)))
    # dict(rcp='rcp85', pername='2020', years=list(range(2020, 2040))),
    # dict(rcp='rcp85', pername='2040', years=list(range(2040, 2060))),
    # dict(rcp='rcp85', pername='2080', years=list(range(2080, 2100)))
    ]

MODELS = [
    dict(model='ACCESS1-0'),
    dict(model='CESM1-BGC'),
    dict(model='GFDL-ESM2M')]

ITERATION_COMPONENTS = (JOBS, PERIODS, MODELS)

def run_job(variable, transformation, rcp, pername, years, model):

    # Build job metadata
    metadata = {k: v for k, v in ADDITIONAL_METADATA.items()}
    metadata.update(dict(
        variable=variable, model=model, period=pername, rcp=rcp))

    metadata['transformation'] = transformation.__doc__
    metadata['time_horizon'] = '{}-{}'.format(years[0], years[-1])

    # Get transformed data
    transformed = xr.concat([
        (load_climate_data(BCSD_orig_files.format(**metadata))
            .pipe(transformation))
        for y in years],
        dim=pd.Index(years, name='year')).mean(dim='year')

    # Reshape to regions
    wtd = weighted_aggregate_grid_to_regions(
            transformed, variable, 'areawt', 'hierid')

    # Update netCDF metadata
    wtd.attrs.update(**metadata)

    # Write output
    fp = WRITE_PATH.format(**metadata)
    if not os.path.isdir(os.path.dirname(fp)):
        os.makedirs(os.path.dirname(fp))
    wtd.to_netcdf(fp)

FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(format=FORMAT)
logger = logging.getLogger('uploader')
logger.setLevel('INFO')

def main():

    njobs = reduce(lambda x, y: x*y, map(len, ITERATION_COMPONENTS))

    for i, job_components in enumerate(
            itertools.product(*ITERATION_COMPONENTS)):

        job = {}
        for job_component in job_components:
            job.update(job_component)

        logger.info('beginning job {} of {}'.format(i, njobs))

        try:
            run_job(**job)
        except Exception, e:
            logger.error(
                'Error encountered in job {} of {}:\n\nJob spec:\n{}\n\n'
                    .format(i, njobs, job),
                exc_info=e)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import logging, json
from Queue import Queue
from threading import Thread
from UserDict import DictMixin

from mobiletrans.mtdistmap.utils import shapefile

from django.contrib.gis.geos import Point
from pyproj import Geod

logger = logging.getLogger('simple_example')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

g = Geod(ellps='WGS84')

class GridPoint(object):
    """
    This represents a geographic coordinate and
    where it falls on the Grid object.  The value
    of x and y refer to the x or y distance from the
    origin.  The xint and yint are the spacings between
    points.
    """
    def __init__(self, x, y, point,):
        self.x = x
        self.y = y
        self.point = point

    @property
    def grid_coords(self):
        return (self.x, self.y)

    @property
    def geo_coords(self):
        return self.point.x, self.point.y

    @property
    def geo_coords_r(self):
        return self.point.y, self.point.x

    def __repr__(self):
        return "GridPoint(lng=%s (%s), lat=%s (%s))" % (
                self.point.x, self.x, self.point.y, self.y)

"""
from mobiletrans.mtdistmap.area_grid import generate_grid, GridPoint
region = CityBorder.objects.all()[0]
gp = GridPoint(0, 0, region.area.centroid)
g = generate_grid(region, gp)
"""

class Grid(DictMixin):
    """
    This represents a grid of GridPoints, the center of which
    is at (0,0).  The key to this dict-like object is an (x,y)
    tuple representing the distance from the origin (0,0).
    """
    def __init__(self, xint, yint, *args, **kwargs):
        self.grid = {}
        self.xint = xint
        self.yint = yint
    def __getitem__(self, key):
        return self.grid[key]
    def __setitem__(self, key, item):
        self.grid[key] = item
    def __delitem__(self, key):
        del self.grid[key]
    def keys(self):
        return self.grid.keys()
    def json(self):
        features = []
        for k, point in self.items():
            d = { "type": "Feature",
                   "geometry": {"type": "Point", "coordinates": point.geo_coords},
            }
            features.append(d)

        json_dict = { "type": "FeatureCollection",
          "features": features
         }
        return json.dumps(json_dict)
    def generate_shapefile(self, shape_file_name='shapefiles/test'):
        """
        Exports the data grid into a set of shapefiles.

        Input:
        shape_file_name="/home/test/shapefiles/myshape"

        Output:
        In directory /home/test/shapefiles/
           myshape.shp
           myshape.shx
           myshape.dbf

        """
        w = shapefile.Writer(shapeType=shapefile.POINT)
        w.autoBalance = 1
        w.field('ID', 'N')  # id field of type Number
        w.field('x' , "N")
        w.field('y', "N")

        count = 1
        for i in self.items():
           graphpoint = i[1]
           w.point(graphpoint.point.x, graphpoint.point.y)
           w.record(count, graphpoint.x, graphpoint.y, )
           count += 1

        w.save(shape_file_name)

class GridGenerator(object):
    grid = None
    directions = ['north', 'south', 'east', 'west']
    num_worker_threads=4

    def __init__(self, region, grid, *args, **kwargs):
        self.q = Queue()
        self.region = region
        self.grid = grid

    @property
    def grid_point_class(self):
        return GridPoint

    def transpose(self, grid_point, azimuth, distance):
        return g.fwd(grid_point.point.x, grid_point.point.y, azimuth, distance)

    def north(self, grid_point):
        lng, lat, az =  self.transpose(grid_point, 0, self.grid.yint)
        return self.grid_point_class(grid_point.x, grid_point.y + self.grid.yint, Point(lng, lat))

    def south(self, grid_point):
        lng, lat, az =  self.transpose(grid_point, 180, self.grid.yint)
        return self.grid_point_class(grid_point.x, grid_point.y - self.grid.yint, Point(lng, lat), )

    def east(self, grid_point):
        lng, lat, az = self.transpose(grid_point, 90, self.grid.xint)
        return self.grid_point_class(grid_point.x + self.grid.xint, grid_point.y, Point(lng, lat), )

    def west(self, grid_point):
        lng, lat, az = self.transpose(grid_point, 270, self.grid.xint)
        return self.grid_point_class(grid_point.x - self.grid.xint, grid_point.y, Point(lng, lat),)

    def create_grid(self, grid_point):
        for direction in self.directions:
            new_point = getattr(self, direction)(grid_point)
            if not self.grid.has_key((new_point.x, new_point.y)):
                self.grid[(new_point.x, new_point.y)] = new_point
                if self.region.area.contains(new_point.point):
                    self.work(new_point)
                    self.q.put(new_point)

    def work(self, new_point):
        pass

    def worker(self):
        while True:
            item = self.q.get()
            self.create_grid(item)
            self.q.task_done()

    def run(self, start_grid_point):
        for i in range(self.num_worker_threads):
             t = Thread(target=self.worker)
             t.daemon = True
             t.start()
        self.create_grid(start_grid_point)
        self.q.join()
        return self.grid

"""
#from mobiletrans.mtdistmap.area_grid import GridGenerator, GridPoint, Grid
region = CityBorder.objects.all()[0]
center = region.area.centroid
grid = Grid(xint=300, yint=300)
gridgen = GridGenerator(region, grid)
g = gridgen.run(GridPoint(0,0,center))
"""

from mobiletrans.mtdistmap.cta_conn import load_transitnetwork
from mobiletrans.mtdistmap.route_planner import RoutePlanner
from django.contrib.gis.geos import Point, fromstr

class RouteGridGenerator(GridGenerator):

    def __init__(self, to_point, *args, **kwargs):
        self.to_point = to_point
        tn = load_transitnetwork()
        self.t = RoutePlanner(tn)
        super(RouteGridGenerator, self).__init__(*args, **kwargs)

    @property
    def grid_point_class(self):
        return RouteGridPoint

    def work(self, new_point):
        from_point = new_point.point
        p = self.t.fastest_route_from_point(from_point, self.to_point)
        new_point.routes += p

def RouteGrid(Grid):

    def __init__(self, *args, **kwargs):
        super(RouteGrid, self).__init__(*args, **kwargs)

    def generate_shapefile(self, shape_file_name='shapefiles/test'):
        """
        Exports the data grid into a set of shapefiles.

        Input:
        shape_file_name="/home/test/shapefiles/myshape"

        Output:
        In directory /home/test/shapefiles/
           myshape.shp
           myshape.shx
           myshape.dbf

        """
        w = shapefile.Writer(shapeType=shapefile.POINT)
        w.autoBalance = 1
        w.field('ID', 'N')  # id field of type Number
        w.field('NUM_ROUTES', 'N')  # route field of type Number
        w.field('x' , "N")
        w.field('y', "N")
        w.field('total_time', 'N', decimal=4) # long

        count = 1
        for i in self.items():
           graphpoint = i[1]
           w.point(graphpoint.point.x, graphpoint.point.y)
           w.record(count, len(graphpoint.routes), graphpoint.x, graphpoint.y, "%s" % graphpoint.shortest_time() )
           count += 1

        w.save(shape_file_name)

class RouteGridPoint(GridPoint):

    def __init__(self, *args, **kwargs):
        super(RouteGridPoint, self).__init__(*args, **kwargs)
        self.routes = []

    def __repr__(self):
        return "Route" + super(RouteGridPoint, self).__repr__()

    def shortest_route(self):
        return_value = None
        if self.routes:
            return_value = sorted( self.routes, key=lambda x: x.total_time,reverse=True)[0]
        return return_value

    def shortest_time(self, default=9999999):
        return_value = default
        route = self.shortest_route()
        if route:
            return_value = route.total_time
        return return_value

"""

# generate the grid with routes
from mobiletrans.mtdistmap.area_grid import RouteGridGenerator, RouteGridPoint, RouteGrid
region = CityBorder.objects.all()[0]
center = region.area.centroid
grid = RouteGrid(xint=300, yint=300)
gridgen = RouteGridGenerator('41320', region, grid)
g = gridgen.run(RouteGridPoint(0,0,center))

# print the grid
for k, v in g.items():
    print v.geo_coords_r, v.routes

# display grid as a shapefile
from mobiletrans.mtdistmap.utils import shapefile
w = shapefile.Writer(shapeType=shapefile.POINT)
w.autoBalance = 1
w.field('ID', 'N')  # id field of type Number
w.field('NUM_ROUTES', 'N')  # route field of type Number
w.field('x' , "N")
w.field('y', "N")
w.field('total_time', 'N', decimal=4) # long

count = 1
for i in g.items():
   graphpoint = i[1]
   w.point(graphpoint.point.x, graphpoint.point.y)
   w.record(count, len(graphpoint.routes), graphpoint.x, graphpoint.y, "%s" % graphpoint.shortest_time() )
   count += 1

w.save('shapefiles/chicago_pts4')

"""
<commit_msg>fixes<commit_after>import logging, json
from Queue import Queue
from threading import Thread
from UserDict import DictMixin

from mobiletrans.mtdistmap.utils import shapefile

from django.contrib.gis.geos import Point
from pyproj import Geod

logger = logging.getLogger('simple_example')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

g = Geod(ellps='WGS84')

class GridPoint(object):
    """
    This represents a geographic coordinate and
    where it falls on the Grid object.  The value
    of x and y refer to the x or y distance from the
    origin.  The xint and yint are the spacings between
    points.
    """
    def __init__(self, x, y, point,):
        self.x = x
        self.y = y
        self.point = point

    @property
    def grid_coords(self):
        return (self.x, self.y)

    @property
    def geo_coords(self):
        return self.point.x, self.point.y

    @property
    def geo_coords_r(self):
        return self.point.y, self.point.x

    def __repr__(self):
        return "GridPoint(lng=%s (%s), lat=%s (%s))" % (
                self.point.x, self.x, self.point.y, self.y)

"""
from mobiletrans.mtdistmap.area_grid import generate_grid, GridPoint
region = CityBorder.objects.all()[0]
gp = GridPoint(0, 0, region.area.centroid)
g = generate_grid(region, gp)
"""

class Grid(DictMixin):
    """
    This represents a grid of GridPoints, the center of which
    is at (0,0).  The key to this dict-like object is an (x,y)
    tuple representing the distance from the origin (0,0).
    """
    def __init__(self, xint, yint, *args, **kwargs):
        self.grid = {}
        self.xint = xint
        self.yint = yint
    def __getitem__(self, key):
        return self.grid[key]
    def __setitem__(self, key, item):
        self.grid[key] = item
    def __delitem__(self, key):
        del self.grid[key]
    def keys(self):
        return self.grid.keys()
    def json(self):
        features = []
        for k, point in self.items():
            d = { "type": "Feature",
                   "geometry": {"type": "Point", "coordinates": point.geo_coords},
            }
            features.append(d)

        json_dict = { "type": "FeatureCollection",
          "features": features
         }
        return json.dumps(json_dict)
    def generate_shapefile(self, shape_file_name='shapefiles/test'):
        """
        Exports the data grid into a set of shapefiles.

        Input:
        shape_file_name="/home/test/shapefiles/myshape"

        Output:
        In directory /home/test/shapefiles/
           myshape.shp
           myshape.shx
           myshape.dbf

        """
        w = shapefile.Writer(shapeType=shapefile.POINT)
        w.autoBalance = 1
        w.field('ID', 'N')  # id field of type Number
        w.field('x' , "N")
        w.field('y', "N")

        count = 1
        for i in self.items():
           graphpoint = i[1]
           w.point(graphpoint.point.x, graphpoint.point.y)
           w.record(count, graphpoint.x, graphpoint.y, )
           count += 1

        w.save(shape_file_name)

class GridGenerator(object):
    grid = None
    directions = ['north', 'south', 'east', 'west']
    num_worker_threads=4

    def __init__(self, region, grid, *args, **kwargs):
        self.q = Queue()
        self.region = region
        self.grid = grid

    @property
    def grid_point_class(self):
        return GridPoint

    def transpose(self, grid_point, azimuth, distance):
        return g.fwd(grid_point.point.x, grid_point.point.y, azimuth, distance)

    def north(self, grid_point):
        lng, lat, az =  self.transpose(grid_point, 0, self.grid.yint)
        return self.grid_point_class(grid_point.x, grid_point.y + self.grid.yint, Point(lng, lat))

    def south(self, grid_point):
        lng, lat, az =  self.transpose(grid_point, 180, self.grid.yint)
        return self.grid_point_class(grid_point.x, grid_point.y - self.grid.yint, Point(lng, lat), )

    def east(self, grid_point):
        lng, lat, az = self.transpose(grid_point, 90, self.grid.xint)
        return self.grid_point_class(grid_point.x + self.grid.xint, grid_point.y, Point(lng, lat), )

    def west(self, grid_point):
        lng, lat, az = self.transpose(grid_point, 270, self.grid.xint)
        return self.grid_point_class(grid_point.x - self.grid.xint, grid_point.y, Point(lng, lat),)

    def create_grid(self, grid_point):
        for direction in self.directions:
            new_point = getattr(self, direction)(grid_point)
            if not self.grid.has_key((new_point.x, new_point.y)):
                self.grid[(new_point.x, new_point.y)] = new_point
                if self.region.area.contains(new_point.point):
                    self.work(new_point)
                    self.q.put(new_point)

    def work(self, new_point):
        pass

    def worker(self):
        while True:
            item = self.q.get()
            self.create_grid(item)
            self.q.task_done()

    def run(self, start_grid_point):
        for i in range(self.num_worker_threads):
             t = Thread(target=self.worker)
             t.daemon = True
             t.start()
        self.create_grid(start_grid_point)
        self.q.join()
        return self.grid

"""
#from mobiletrans.mtdistmap.area_grid import GridGenerator, GridPoint, Grid
region = CityBorder.objects.all()[0]
center = region.area.centroid
grid = Grid(xint=300, yint=300)
gridgen = GridGenerator(region, grid)
g = gridgen.run(GridPoint(0,0,center))
"""

from mobiletrans.mtdistmap.cta_conn import load_transitnetwork
from mobiletrans.mtdistmap.route_planner import RoutePlanner
from django.contrib.gis.geos import Point, fromstr

class RouteGridGenerator(GridGenerator):

    def __init__(self, to_point, *args, **kwargs):
        self.to_point = to_point
        tn = load_transitnetwork()
        self.t = RoutePlanner(tn)
        super(RouteGridGenerator, self).__init__(*args, **kwargs)

    @property
    def grid_point_class(self):
        return RouteGridPoint

    def work(self, new_point):
        from_point = new_point.point
        p = self.t.fastest_route_from_point(from_point, self.to_point)
        new_point.routes += p

class RouteGrid(Grid):

    def generate_shapefile(self, shape_file_name='shapefiles/test'):
        """
        Exports the data grid into a set of shapefiles.

        Input:
        shape_file_name="/home/test/shapefiles/myshape"

        Output:
        In directory /home/test/shapefiles/
           myshape.shp
           myshape.shx
           myshape.dbf

        """
        w = shapefile.Writer(shapeType=shapefile.POINT)
        w.autoBalance = 1
        w.field('ID', 'N')  # id field of type Number
        w.field('NUM_ROUTES', 'N')  # route field of type Number
        w.field('x' , "N")
        w.field('y', "N")
        w.field('total_time', 'N', decimal=4) # long

        count = 1
        for i in self.items():
           graphpoint = i[1]
           w.point(graphpoint.point.x, graphpoint.point.y)
           w.record(count, len(graphpoint.routes), graphpoint.x, graphpoint.y, "%s" % graphpoint.shortest_time() )
           count += 1

        w.save(shape_file_name)

class RouteGridPoint(GridPoint):

    def __init__(self, *args, **kwargs):
        super(RouteGridPoint, self).__init__(*args, **kwargs)
        self.routes = []

    def __repr__(self):
        return "Route" + super(RouteGridPoint, self).__repr__()

    def shortest_route(self):
        return_value = None
        if self.routes:
            return_value = sorted( self.routes, key=lambda x: x.total_time,reverse=True)[0]
        return return_value

    def shortest_time(self, default=9999999):
        return_value = default
        route = self.shortest_route()
        if route:
            return_value = route.total_time
        return return_value

"""

# generate the grid with routes
from mobiletrans.mtdistmap.area_grid import RouteGridGenerator, RouteGridPoint, RouteGrid
region = CityBorder.objects.all()[0]
center = region.area.centroid
grid = RouteGrid(xint=300, yint=300)
gridgen = RouteGridGenerator('41320', region, grid)
g = gridgen.run(RouteGridPoint(0,0,center))

# print the grid
for k, v in g.items():
    print v.geo_coords_r, v.routes

# display grid as a shapefile
from mobiletrans.mtdistmap.utils import shapefile
w = shapefile.Writer(shapeType=shapefile.POINT)
w.autoBalance = 1
w.field('ID', 'N')  # id field of type Number
w.field('NUM_ROUTES', 'N')  # route field of type Number
w.field('x' , "N")
w.field('y', "N")
w.field('total_time', 'N', decimal=4) # long

count = 1
for i in g.items():
   graphpoint = i[1]
   w.point(graphpoint.point.x, graphpoint.point.y)
   w.record(count, len(graphpoint.routes), graphpoint.x, graphpoint.y, "%s" % graphpoint.shortest_time() )
   count += 1

w.save('shapefiles/chicago_pts4')

"""
<|endoftext|>


<commit_before>import os, json, unittest, time, shutil, sys
# not needed, but in case you move it down to subdir
sys.path.extend(['.','..'])
import h2o_cmd
import h2o
import h2o_browse as h2b

class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        h2o.build_cloud(node_count=3)

    @classmethod
    def tearDownClass(cls):
        h2o.tear_down_cloud()

    def test_A_Basic(self):
        h2o.verify_cloud_size()

    def test_B_RF_iris2(self):
        h2o_cmd.runRF(trees=6, timeoutSecs=10,
                csvPathname = h2o.find_file('smalldata/iris/iris2.csv'))

    def test_C_RF_poker100(self):
        h2o_cmd.runRF(trees=6, timeoutSecs=10,
                csvPathname = h2o.find_file('smalldata/poker/poker100'))

    def test_D_GenParity1(self):
        trees = 50
        h2o_cmd.runRF(trees=50, timeoutSecs=15,
                csvPathname = h2o.find_file('smalldata/parity_128_4_100_quad.data'))

    def test_E_ParseManyCols(self):
        csvPathname=h2o.find_file('smalldata/fail1_100x11000.csv.gz')
        parseKey = h2o_cmd.parseFile(None, csvPathname, timeoutSecs=10)
        inspect = h2o_cmd.runInspect(None, parseKey['destination_key'], offset=-1, view=5)

    def test_F_StoreView(self):
        storeView = h2o.nodes[0].store_view()

    def test_G_Slower_JUNIT(self):
        h2o.tear_down_cloud()
        h2o.build_cloud(node_count=2)
        # we don't have the port or ip configuration here
        # that util/h2o.py does? Keep this in synch with spawn_h2o there.
        # also don't have --nosigar here?
        (ps, stdout, stderr) = h2o.spawn_cmd('junit', [
                'java',
                '-Dh2o.arg.ice_root='+h2o.tmp_dir('ice.'),
                '-Dh2o.arg.name='+h2o.cloud_name(),
                '-Dh2o.arg.ip='+h2o.get_ip_address(),
                '-ea', '-jar', h2o.find_file('target/h2o.jar'),
                '-mainClass', 'org.junit.runner.JUnitCore',
                # The tests
                'water.ConcurrentKeyTest',
                'hex.MinorityClassTest',
                'water.exec.RBigDataTest',
                'water.parser.ParseFolderTestBig'
                ])
        rc = ps.wait(None)
        out = file(stdout).read()
        err = file(stderr).read()
        if rc is None:
            ps.terminate()
            raise Exception("junit timed out.\nstdout:\n%s\n\nstderr:\n%s" % (out, err))
        elif rc != 0:
            raise Exception("junit failed.\nstdout:\n%s\n\nstderr:\n%s" % (out, err))

if __name__ == '__main__':
    h2o.unit_main()
<commit_msg>removed testParseFolderBig test from the test.py (throws file not found) .<commit_after>import os, json, unittest, time, shutil, sys
# not needed, but in case you move it down to subdir
sys.path.extend(['.','..'])
import h2o_cmd
import h2o
import h2o_browse as h2b

class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        h2o.build_cloud(node_count=3)

    @classmethod
    def tearDownClass(cls):
        h2o.tear_down_cloud()

    def test_A_Basic(self):
        h2o.verify_cloud_size()

    def test_B_RF_iris2(self):
        h2o_cmd.runRF(trees=6, timeoutSecs=10,
                csvPathname = h2o.find_file('smalldata/iris/iris2.csv'))

    def test_C_RF_poker100(self):
        h2o_cmd.runRF(trees=6, timeoutSecs=10,
                csvPathname = h2o.find_file('smalldata/poker/poker100'))

    def test_D_GenParity1(self):
        trees = 50
        h2o_cmd.runRF(trees=50, timeoutSecs=15,
                csvPathname = h2o.find_file('smalldata/parity_128_4_100_quad.data'))

    def test_E_ParseManyCols(self):
        csvPathname=h2o.find_file('smalldata/fail1_100x11000.csv.gz')
        parseKey = h2o_cmd.parseFile(None, csvPathname, timeoutSecs=10)
        inspect = h2o_cmd.runInspect(None, parseKey['destination_key'], offset=-1, view=5)

    def test_F_StoreView(self):
        storeView = h2o.nodes[0].store_view()

    def test_G_Slower_JUNIT(self):
        h2o.tear_down_cloud()
        h2o.build_cloud(node_count=2)
        # we don't have the port or ip configuration here
        # that util/h2o.py does? Keep this in synch with spawn_h2o there.
        # also don't have --nosigar here?
        (ps, stdout, stderr) = h2o.spawn_cmd('junit', [
                'java',
                '-Dh2o.arg.ice_root='+h2o.tmp_dir('ice.'),
                '-Dh2o.arg.name='+h2o.cloud_name(),
                '-Dh2o.arg.ip='+h2o.get_ip_address(),
                '-ea', '-jar', h2o.find_file('target/h2o.jar'),
                '-mainClass', 'org.junit.runner.JUnitCore',
                # The tests
                'water.ConcurrentKeyTest',
                'hex.MinorityClassTest',
                'water.exec.RBigDataTest'
                ])
        rc = ps.wait(None)
        out = file(stdout).read()
        err = file(stderr).read()
        if rc is None:
            ps.terminate()
            raise Exception("junit timed out.\nstdout:\n%s\n\nstderr:\n%s" % (out, err))
        elif rc != 0:
            raise Exception("junit failed.\nstdout:\n%s\n\nstderr:\n%s" % (out, err))

if __name__ == '__main__':
    h2o.unit_main()
<|endoftext|>


<commit_before>import platform
import getpass
import subprocess
import cpuinfo
import psutil
from screeninfo import get_monitors
from multiprocessing.pool import ThreadPool
from uptime import uptime

w = '\033[1;37;40m'
y = '\033[1;33;40m'
r = '\033[0;31;40m'
lr = '\033[0;1;31m'
g = '\033[0;32;40m'
b = '\033[0;34;40m'
xx = '\033[0m'

def humanbytes(B):
   """Return the given bytes as a human friendly KB, MB, GB, or TB string"""
   B = float(B)
   KB = float(1024)
   MB = float(KB ** 2) # 1,048,576
   GB = float(KB ** 3) # 1,073,741,824
   TB = float(KB ** 4) # 1,099,511,627,776

   if B < KB:
      return '{0} {1}'.format(B,'Bytes' if 0 == B > 1 else 'Byte')
   elif KB <= B < MB:
      return '{0:.2f} KB'.format(B/KB)
   elif MB <= B < GB:
      return '{0:.2f} MB'.format(B/MB)
   elif GB <= B < TB:
      return '{0:.2f} GB'.format(B/GB)
   elif TB <= B:
      return '{0:.2f} TB'.format(B/TB)

def ddhhmmss(seconds):
    day = seconds // (24 * 3600)
    seconds = seconds % (24 * 3600)
    hour = seconds // 3600
    seconds %= 3600
    minute = seconds // 60
    return "%d days %d hours %d minutes" % (day, hour, minute)

def linux():
    import distro
    distro_info = distro.linux_distribution()
    dist_name = distro_info[2].split(' ')[0]
    return '{} {} {}'.format(distro_info[0], distro_info[1], dist_name)

def get_uname():
    return platform.uname()

def get_user():
    return getpass.getuser()

def get_uptime():
    return ddhhmmss(int(uptime()))

def get_cpu():
    cpu = cpuinfo.get_cpu_info()
    brand = cpu['brand'].rstrip()
    hz = cpu['hz_actual']
    return brand, hz

def get_mem():
    mem = psutil.virtual_memory()
    return humanbytes(mem.total), humanbytes(mem.used)

def get_packages():
    return subprocess.run("dpkg -l | grep -c '^ii'", shell=True, stdout=subprocess.PIPE, universal_newlines=True).stdout.strip('\n')

def get_shell(os):
    if 'linux' in os:
        return subprocess.run("$SHELL --version", shell=True, stdout=subprocess.PIPE, universal_newlines=True).stdout.strip('\n').split('(')[0]
    else:
        return subprocess.run("bash --version", shell=True, stdout=subprocess.PIPE, universal_newlines=True).stdout.strip('\n').split('(')[0]

def get_disk():
    disk = psutil.disk_usage(".")
    return humanbytes(disk.free), humanbytes(disk.total)

def get_screen():
    try:
        screen = str(get_monitors()[0]).strip('monitor()').split('+')[0]
        return screen
    except:
        return None

def get_motherboard(os):
    if 'linux' in os:
        manufacturer_and_name = subprocess.run("grep '' /sys/class/dmi/id/board_vendor && grep '' /sys/class/dmi/id/board_name", shell=True, stdout=subprocess.PIPE, universal_newlines=True).stdout.split('\n')
        manufacturer = manufacturer_and_name[0]
        name = manufacturer_and_name[1]
    else:
        manufacturer = subprocess.run("wmic baseboard get manufacturer", shell=True, stdout=subprocess.PIPE, universal_newlines=True).stdout.split('\n')[2].rstrip()
        name = subprocess.run("wmic baseboard get product", shell=True, stdout=subprocess.PIPE, universal_newlines=True).stdout.strip('\n\t').split('\n')[2].rstrip()
    return manufacturer, name

class LogoMaker:
    def __init__(self, username, os, kernel, uptime, packages, shell, hdd, cpu, ram, motherboard, screen=False):
        self.username = username
        self.os = os
        self.kernel = kernel
        self.uptime = uptime
        self.packages = packages
        self.shell = shell
        self.hdd = hdd
        self.cpu = cpu
        self.ram = ram
        self.motherboard = motherboard
        self.screen = screen

    def ubuntu(self):
        print(f"{w}              .-.                                                              {self.username}\n"
              f"{y}        .-'``{w}(|||)                                                             {self.os}\n"
              f"{y}     ,`\ \    {w}`-`{y}.                 88                         88               {self.kernel}\n"
              f"{y}    /   \ '``-.   `                88                         88               {self.uptime}\n"
              f"{y}  .-.  ,       `___:      88   88  88,888,  88   88  ,88888, 88888  88   88    {self.packages}\n"
              f"{y} (:::) :       {w} ___{y}       88   88  88   88  88   88  88   88  88    88   88    {self.shell}\n"
              f"{y}  `-`  `       {w},   :{y}      88   88  88   88  88   88  88   88  88    88   88    {self.hdd}\n"
              f"{y}    \   /{w} ,..-`   ,{y}       88   88  88   88  88   88  88   88  88    88   88    {self.cpu}\n"
              f"{y}     `./ {w}/    {y}.-.{w}`{y}        '88888'  '88888'  '88888'  88   88  '8888 '88888'    {self.ram}")
        if self.screen:
          print(f"{w}        `-..-{y}(   )                                                                          {self.screen}")
          print(f"{y}              `-`                                                                                {self.motherboard}")
        else:
          print(f"{w}        `-..-{y}(   )                                                             {self.motherboard}")
        print(f"{y}              `-`{xx}")

    def win810(self):
        print(f"{b}                                  ..,   {self.username}\n"
              f"{b}                      ....,,:;+ccllll   {self.os}\n"
              f"{b}        ...,,+:;  cllllllllllllllllll   {self.kernel}\n"
              f"{b}  ,cclllllllllll  lllllllllllllllllll   {self.uptime}\n"
              f"{b}  llllllllllllll  lllllllllllllllllll   {self.shell}\n"
              f"{b}  llllllllllllll  lllllllllllllllllll   {self.hdd}\n"
              f"{b}  llllllllllllll  lllllllllllllllllll   {self.cpu}\n"
              f"{b}  llllllllllllll  lllllllllllllllllll   {self.ram}")
        if self.screen:
              print(f"{b}  llllllllllllll  lllllllllllllllllll   {self.screen}\n"
                    f"                                        {self.motherboard}")
        else:
            print(f"{b}  llllllllllllll  lllllllllllllllllll   {self.motherboard}\n")

        print(f"{b}  llllllllllllll  lllllllllllllllllll\n"
              f"{b}  llllllllllllll  lllllllllllllllllll\n"
              f"{b}  llllllllllllll  lllllllllllllllllll\n"
              f"{b}  llllllllllllll  lllllllllllllllllll\n"
              f"{b}  llllllllllllll  lllllllllllllllllll\n"
              f"{b}  llllllllllllll  lllllllllllllllllll\n"
              f"{b}  `'ccllllllllll  lllllllllllllllllll\n"
              f"{b}           `'""*::  :ccllllllllllllllll\n"
              f"{b}                        ````''\"*::cll\n"
              f"{b}                                   ``")

    def not_win10(self):
        print(f"{lr}         ,.=:^!^!t3Z3z.,\n"
              f"{lr}        :tt:::tt333EE3                  {self.username}\n"
              f"{lr}        Et:::ztt33EEE  {g}@Ee.,      ..,\n"
              f"{lr}       ;tt:::tt333EE7 {g};EEEEEEttttt33#   {self.os}\n"
              f"{lr}      :Et:::zt333EEQ. {g}SEEEEEttttt33QL   {self.kernel}\n"
              f"{lr}      it::::tt333EEF {g}@EEEEEEttttt33F    {self.uptime}\n"
              f"{lr}     ;3=*^```'*4EEV {g}:EEEEEEttttt33@.    {self.shell}\n"
              f"{b}     ,.=::::it=., {lr}` {g}@EEEEEEtttz33QF     {self.hdd}\n"
              f"{b}    ;::::::::zt33)   {g}'4EEEtttji3P*      {self.cpu}\n"
              f"{b}   :t::::::::tt33.{y}:Z3z..  {g}`` {y},..g.      {self.ram}")
        if self.screen:
            print(f"{b}   i::::::::zt33F {y}AEEEtttt::::ztF       {self.screen}\n"
                  f"{b}  ;:::::::::t33V {y};EEEttttt::::t3        {self.motherboard}")
        else:
            print(f"{b}   i::::::::zt33F {y}AEEEtttt::::ztF       \n"
                  f"{b}  ;:::::::::t33V {y};EEEttttt::::t3")
        print(f"{b}  E::::::::zt33L {y}@EEEtttt::::z3F        \n"
              f"{b} (3=*^```'*4E3) {y};EEEtttt:::::tZ`        \n"
              f"{b}             ` {y}:EEEEtttt::::z7          \n"
              f"{y}                 'VEzjt:;;z>*`        ")

    def nothing(self):
        print(f"{self.username}\n{self.os}\n{self.kernel}\n{self.uptime}\n{self.shell}\n{self.hdd}\n{self.cpu}\n{self.ram}")
        if self.screen:
            print(self.screen)
        print(f"{self.motherboard}\nThis OS does not have a logo added yet. Post an issue on the GitHub repository to request support")

def get_specs(username, uname, uptime, mem_total, mem_used, cpu_brand, cpu_hz, disk_free, disk_total, screen, motherboard_vendor, motherboard_name, packages=None, shell_version=None):
    username = f'{lr}{username}{w}@{lr}{uname.node}{xx}'
    kernel = f'{lr}Kernel: {xx}{uname.machine}'

    if 'linux' in uname.system.lower():
        os = f'{lr}OS: {xx}{system}'
        kernel = f'{lr}Kernel: {xx}{uname.machine} Linux {uname.release}'
    else:
        os = f'{lr}OS: {xx}{uname.system} {uname.release}'
    uptime = f'{lr}Uptime: {xx}{uptime}'

    packages = None

    if 'linux' in uname.system.lower():
        packages = f'{lr}Packages: {xx}{packages}'

    shell = f'{lr}Shell: {xx}{shell_version}'
    hdd = f'{lr}HDD: {xx}{disk_free} / {disk_total} (Free/Total)'
    cpu = f'{lr}CPU: {xx}{cpu_brand} @ {cpu_hz}'
    ram = f'{lr}RAM: {xx}{mem_used} / {mem_total} (Used/Total)'

    if screen:
        screen = f'{lr}Resolution: {xx}{screen}'

    motherboard = f'{lr}Motherboard: {xx}{motherboard_vendor} {motherboard_name}'
    make_logo = LogoMaker(username,os,kernel,uptime,packages,shell,hdd,cpu,ram,motherboard,screen)

    if 'ubuntu' in uname.version.lower():
        make_logo.ubuntu()
    elif 'windows' in uname.system.lower():
        if '10' or '8' in uname.release:
            #Add Win10 Logo!!!
            make_logo.win810()
        else:
            make_logo.not_win10()
    else:
        make_logo.nothing()

if __name__ == '__main__':
    pool = ThreadPool(processes=5)

    mem_future = pool.apply_async(get_mem)
    mem_total, mem_used = mem_future.get()

    uname_future = pool.apply_async(get_uname)
    uname = uname_future.get()

    username_future = pool.apply_async(get_user)
    username = username_future.get()

    uptime_future = pool.apply_async(get_uptime)
    uptime = uptime_future.get()

    cpu_future = pool.apply_async(get_cpu)
    cpu_brand, cpu_hz = cpu_future.get()

    disk_future = pool.apply_async(get_disk)
    disk_free, disk_total = disk_future.get()

    screen_future = pool.apply_async(get_screen)
    screen = screen_future.get()

    system = uname.system
    node = uname.node
    release = uname.release
    version = uname.version
    machine = uname.machine

    shell_future = pool.apply_async(get_shell, (system,))
    shell = shell_future.get()

    if 'linux' in uname.system.lower():
        linux_future = pool.apply_async(linux)
        system = linux_future.get()

        packages_future = pool.apply_async(get_packages)
        packages = packages_future.get()

        motherboard_future = pool.apply_async(get_motherboard, ('linux',))
        motherboard_vendor, motherboard_name = motherboard_future.get()

        get_specs(username, uname, uptime, mem_total, mem_used, cpu_brand, cpu_hz, disk_free, disk_total, screen, motherboard_vendor, motherboard_name, packages=packages, shell_version=shell)
    else:
        motherboard_future = pool.apply_async(get_motherboard, ('windows',))
        motherboard_vendor, motherboard_name = motherboard_future.get()
        get_specs(username, uname, uptime, mem_total, mem_used, cpu_brand, cpu_hz, disk_free, disk_total, screen, motherboard_vendor, motherboard_name, packages=None, shell_version=shell)
<commit_msg>Major improvements<commit_after>import platform
import getpass
import subprocess
import cpuinfo
import psutil
from screeninfo import get_monitors
from uptime import uptime

def humanbytes(B):
   """Return the given bytes as a human friendly KB, MB, GB, or TB string"""
   B = float(B)
   KB = float(1024)
   MB = float(KB ** 2) # 1,048,576
   GB = float(KB ** 3) # 1,073,741,824
   TB = float(KB ** 4) # 1,099,511,627,776

   if B < KB:
      return '{0} {1}'.format(B,'Bytes' if 0 == B > 1 else 'Byte')
   elif KB <= B < MB:
      return '{0:.2f} KB'.format(B/KB)
   elif MB <= B < GB:
      return '{0:.2f} MB'.format(B/MB)
   elif GB <= B < TB:
      return '{0:.2f} GB'.format(B/GB)
   elif TB <= B:
      return '{0:.2f} TB'.format(B/TB)

def ddhhmmss(seconds):
    day = seconds // (24 * 3600)
    seconds = seconds % (24 * 3600)
    hour = seconds // 3600
    seconds %= 3600
    minute = seconds // 60
    return "%d days %d hours %d minutes" % (day, hour, minute)

class LogoMaker:
    def __init__(self):
        computer = Computer()
        self.w = '\033[1;37;40m'
        self.y = '\033[1;33;40m'
        self.r = '\033[0;31;40m'
        self.lr = '\033[0;1;31m'
        self.g = '\033[0;32;40m'
        self.b = '\033[0;34;40m'
        self.xx = '\033[0m'

        self.username = f'{self.lr}{computer.username}{self.w}@{self.lr}{computer.uname.node}{self.xx}'
        self.kernel = f'{self.lr}Kernel: {self.xx}{computer.uname.machine}'

        if 'linux' in computer.uname.system.lower():
            self.system = f'{self.lr}OS: {self.xx}{computer.uname.system}'
            self.kernel = f'{self.lr}Kernel: {self.xx}{computer.uname.machine} Linux {computer.uname.release}'
        else:
            self.system = f'{self.lr}OS: {self.xx}{computer.uname.system} {computer.uname.release}'
        self.uptime = f'{self.lr}Uptime: {self.xx}{computer.uptime}'

        self.packages = None

        if 'linux' in computer.uname.system.lower():
            self.packages = f'{self.lr}Packages: {self.xx}{computer.packages}'

        self.shell = f'{self.lr}Shell: {self.xx}{computer.shell}'
        self.hdd = f'{self.lr}HDD: {self.xx}{computer.disk_free} / {computer.disk_total} (Free/Total)'
        self.cpu = f'{self.lr}CPU: {self.xx}{computer.brand} @ {computer.hz}'
        self.ram = f'{self.lr}RAM: {self.xx}{computer.mem_used} / {computer.mem_total} (Used/Total)'

        self.screen = None

        if computer.screen:
            self.screen = f'{self.lr}Resolution: {self.xx}{computer.screen}'

        self.motherboard = f'{self.lr}Motherboard: {self.xx}{computer.motherboard_vendor} {computer.motherboard_name}'

        if computer.os == 'ubuntu':
            self.ubuntu()
        elif computer.os == '10/8':
            self.win10_8()
        elif computer.os == 'win':
            self.old_win()
        else:
            self.n_a()

    def ubuntu(self):
        print(f"{self.w}              .-.                                                              {self.username}\n"
              f"{self.y}        .-'``{self.w}(|||)                                                             {self.system}\n"
              f"{self.y}     ,`\ \    {self.w}`-`{self.y}.                 88                         88               {self.kernel}\n"
              f"{self.y}    /   \ '``-.   `                88                         88               {self.uptime}\n"
              f"{self.y}  .-.  ,       `___:      88   88  88,888,  88   88  ,88888, 88888  88   88    {self.packages}\n"
              f"{self.y} (:::) :       {self.w} ___{self.y}       88   88  88   88  88   88  88   88  88    88   88    {self.shell}\n"
              f"{self.y}  `-`  `       {self.w},   :{self.y}      88   88  88   88  88   88  88   88  88    88   88    {self.hdd}\n"
              f"{self.y}    \   /{self.w} ,..-`   ,{self.y}       88   88  88   88  88   88  88   88  88    88   88    {self.cpu}\n"
              f"{self.y}     `./ {self.w}/    {self.y}.-.{self.w}`{self.y}        '88888'  '88888'  '88888'  88   88  '8888 '88888'    {self.ram}")
        if self.screen:
            print(
                f"{self.w}        `-..-{self.y}(   )                                                                          {self.screen}")
            print(
                f"{self.y}              `-`                                                                                {self.motherboard}")
        else:
            print(
                f"{self.w}        `-..-{self.y}(   )                                                             {self.motherboard}")
        print(f"{self.y}              `-`{self.xx}")

    def win10_8(self):
        print(f"{self.b}                                  ..,   {self.username}\n"
              f"{self.b}                      ....,,:;+ccllll   {self.system}\n"
              f"{self.b}        ...,,+:;  cllllllllllllllllll   {self.kernel}\n"
              f"{self.b}  ,cclllllllllll  lllllllllllllllllll   {self.uptime}\n"
              f"{self.b}  llllllllllllll  lllllllllllllllllll   {self.shell}\n"
              f"{self.b}  llllllllllllll  lllllllllllllllllll   {self.hdd}\n"
              f"{self.b}  llllllllllllll  lllllllllllllllllll   {self.cpu}\n"
              f"{self.b}  llllllllllllll  lllllllllllllllllll   {self.ram}")
        if self.screen:
            print(f"{self.b}  llllllllllllll  lllllllllllllllllll   {self.screen}\n"
                  f"                                        {self.motherboard}")
        else:
            print(f"{self.b}  llllllllllllll  lllllllllllllllllll   {self.motherboard}\n")

        print(f"{self.b}  llllllllllllll  lllllllllllllllllll\n"
              f"{self.b}  llllllllllllll  lllllllllllllllllll\n"
              f"{self.b}  llllllllllllll  lllllllllllllllllll\n"
              f"{self.b}  llllllllllllll  lllllllllllllllllll\n"
              f"{self.b}  llllllllllllll  lllllllllllllllllll\n"
              f"{self.b}  llllllllllllll  lllllllllllllllllll\n"
              f"{self.b}  `'ccllllllllll  lllllllllllllllllll\n"
              f"{self.b}           `'""*::  :ccllllllllllllllll\n"
              f"{self.b}                        ````''\"*::cll\n"
              f"{self.b}                                   ``")

    def old_win(self):
        print(f"{self.lr}         ,.=:^!^!t3Z3z.,\n"
              f"{self.lr}        :tt:::tt333EE3                  {self.username}\n"
              f"{self.lr}        Et:::ztt33EEE  {self.g}@Ee.,      ..,\n"
              f"{self.lr}       ;tt:::tt333EE7 {self.g};EEEEEEttttt33#   {self.system}\n"
              f"{self.lr}      :Et:::zt333EEQ. {self.g}SEEEEEttttt33QL   {self.kernel}\n"
              f"{self.lr}      it::::tt333EEF {self.g}@EEEEEEttttt33F    {self.uptime}\n"
              f"{self.lr}     ;3=*^```'*4EEV {self.g}:EEEEEEttttt33@.    {self.shell}\n"
              f"{self.b}     ,.=::::it=., {self.lr}` {self.g}@EEEEEEtttz33QF     {self.hdd}\n"
              f"{self.b}    ;::::::::zt33)   {self.g}'4EEEtttji3P*      {self.cpu}\n"
              f"{self.b}   :t::::::::tt33.{self.y}:Z3z..  {self.g}`` {self.y},..g.      {self.ram}")
        if self.screen:
            print(f"{self.b}   i::::::::zt33F {self.y}AEEEtttt::::ztF       {self.screen}\n"
                  f"{self.b}  ;:::::::::t33V {self.y};EEEttttt::::t3        {self.motherboard}")
        else:
            print(f"{self.b}   i::::::::zt33F {self.y}AEEEtttt::::ztF       \n"
                  f"{self.b}  ;:::::::::t33V {self.y};EEEttttt::::t3")
        print(f"{self.b}  E::::::::zt33L {self.y}@EEEtttt::::z3F        \n"
              f"{self.b} (3=*^```'*4E3) {self.y};EEEtttt:::::tZ`        \n"
              f"{self.b}             ` {self.y}:EEEEtttt::::z7          \n"
              f"{self.y}                 'VEzjt:;;z>*`        ")

    def n_a(self):
        print(
            f"{self.username}\n{self.system}\n{self.kernel}\n{self.uptime}\n{self.shell}\n{self.hdd}\n{self.cpu}\n{self.ram}")
        if self.screen:
            print(self.screen)
        print(
            f"{self.motherboard}\nThis OS does not have a logo added yet. Post an issue on the GitHub repository to request support")

class Computer(object):
    def __init__(self):
        linux_distro = ['ubuntu','fedora','mint']
        self.uname = platform.uname()
        self.username = getpass.getuser()
        self.uptime = ddhhmmss(int(uptime()))
        self.cpu = cpuinfo.get_cpu_info()
        self.brand = self.cpu['brand'].rstrip()
        self.hz = self.cpu['hz_actual']
        mem = psutil.virtual_memory()
        self.mem_total = humanbytes(mem.total)
        self.mem_used = humanbytes(mem.used)
        disk = psutil.disk_usage(".")
        self.disk_free = humanbytes(disk.free)
        self.disk_total = humanbytes(disk.total)
        self.screen = self.get_screen()

        if 'ubuntu' in self.uname.version.lower():
            self.os = 'ubuntu'
        elif 'windows' in self.uname.system.lower():
            if '10' or '8' in self.uname.release:
                self.os = '10/8'
            else:
                self.os = 'win'
        else:
            self.os = 'nothing'

        self.packages = None

        if self.os in linux_distro:
            self.packages = self.get_packages()
            manufacturer_and_name = subprocess.run("grep '' /sys/class/dmi/id/board_vendor && grep '' /sys/class/dmi/id/board_name", shell=True, stdout=subprocess.PIPE, universal_newlines=True).stdout.split('\n')
            self.motherboard_vendor, self.motherboard_name = manufacturer_and_name[0], manufacturer_and_name[1]
        else:
            self.motherboard_vendor = subprocess.run("wmic baseboard get manufacturer", shell=True, stdout=subprocess.PIPE, universal_newlines=True).stdout.split('\n')[2].rstrip()
            self.motherboard_name = subprocess.run("wmic baseboard get product", shell=True, stdout=subprocess.PIPE, universal_newlines=True).stdout.strip('\n\t').split('\n')[2].rstrip()

        try:
            self.shell = subprocess.run("bash --version", shell=True, stdout=subprocess.PIPE, universal_newlines=True).stdout.strip('\n').split('(')[0]
        except:
            self.shell = 'N/A'

    def get_screen(self):
        try:
            return str(get_monitors()[0]).strip('monitor()').split('+')[0]
        except:
            return None

    def get_packages(self):
        try:
            return subprocess.run("dpkg -l | grep -c '^ii'", shell=True, stdout=subprocess.PIPE, universal_newlines=True).stdout.strip('\n')
        except:
            return None

if __name__ == '__main__':
    make_logo = LogoMaker()
<|endoftext|>