<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import re
import pwd
import sys
sys.dont_write_bytecode = True

from copy import deepcopy
from ruamel import yaml
from pathlib import Path

from argparse import ArgumentParser, Action
from contextlib import contextmanager
from fnmatch import fnmatch

SCRIPT_FILE = os.path.abspath(__file__)
SCRIPT_NAME = os.path.basename(SCRIPT_FILE)
SCRIPT_PATH = os.path.dirname(SCRIPT_FILE)
if os.path.islink(__file__):
    REAL_FILE = os.path.abspath(os.readlink(__file__))
    REAL_NAME = os.path.basename(REAL_FILE)
    REAL_PATH = os.path.dirname(REAL_FILE)

SECTIONS = [
    'link',
    'ppa',
    'apt',
    'dnf',
    'npm',
    'pip3',
    'github',
    'script',
]

UID = os.getuid()
GID = pwd.getpwuid(UID).pw_gid
USER = pwd.getpwuid(UID).pw_name

LINKER = '''
linker() {
    file=$(realpath "$1")
    link="${2/#\~/$HOME}"
    echo "$link -> $file"
    if [ -f "$link" ] && [ "$file" != "$(readlink $link)" ]; then
        orig="$link.orig"
        $VERBOSE && echo "backing up $orig"
        mv $link $orig
    elif [ ! -f "$link" ] && [ -L "$link" ]; then
        $VERBOSE && echo "removing broken link $link"
        unlink $link
    fi
    if [ -f "$link" ]; then
        echo "[exists] $link"
    else
        echo "[create] $link -> $file"
        mkdir -p $(dirname $link); ln -s $file $link
    fi
}
'''.lstrip('\n').rstrip()

class UnknownPkgmgrError(Exception):
    def __init__(self):
        super(UnknownPkgmgrError, self).__init__('unknown pkgmgr!')

@contextmanager
def cd(*args, **kwargs):
    mkdir = kwargs.pop('mkdir', True)
    verbose = kwargs.pop('verbose', False)
    path = os.path.sep.join(args)
    path = os.path.normpath(path)
    path = os.path.expanduser(path)
    prev = os.getcwd()
    if path != prev:
        if mkdir:
            os.system(f'mkdir -p {path}')
        os.chdir(path)
        curr = os.getcwd()
        sys.path.append(curr)
        if verbose:
            print(f'cd {curr}')
    try:
        yield
    finally:
        if path != prev:
            sys.path.remove(curr)
            os.chdir(prev)
            if verbose:
                print('cd {prev}')

def check_hash(program):
    from subprocess import check_call, CalledProcessError, PIPE
    try:
        check_call(f'hash {program}', shell=True, stdout=PIPE, stderr=PIPE)
        return True
    except CalledProcessError:
        return False

def get_pkgmgr():
    if check_hash('dpkg'):
        return 'deb'
    elif check_hash('rpm'):
        return 'rpm'
    elif check_hash('brew'):
        return 'brew'
    raise UnknownPkgmgrError

def render_item(item):
    if isinstance(item, tuple):
        return ' '.join(item)
    return str(item)

class ManifestType():
    def __repr__(self):
        return f'{type(self).__name__}(items = {self.items})'

    __str__ = __repr__

    def functions(self):
        return ''

    def render(self):
        raise NotImplementedError

class PackageType(ManifestType):
    def __init__(self, spec, patterns, **kwargs):
        self.items = sift(spec.get('items', None), patterns)

    def __repr__(self):
        return f'{type(self).__name__}(items={self.items})'

    __str__ = __repr__

    def render_header(self):
        return f'''
echo "{type(self).__name__.lower()}s:"
        '''.lstrip('\n').rstrip()

    def render_block(self):
        raise NotImplementedError

class HeredocPackageType(PackageType):
    def render_items(self):
        return '\n'.join([render_item(item) for item in self.items])

    def render(self):
        return f'''
{self.render_header()}

while read pkg; do
{self.render_block()}
done<<EOM
{self.render_items()}
EOM
        '''.lstrip('\n').rstrip()

class ContinuePackageType(PackageType):
    def render_items(self):
        return ' \\\n    '.join([render_item(item) for item in self.items])

    def render(self):
        return f'''
{self.render_header()}

{self.render_block()} {self.render_items()}
        '''.lstrip('\n').rstrip()

class Link(HeredocPackageType):
    def __init__(self, spec, patterns, cwd=None, user=None, **kwargs):
        self.cwd = cwd
        self.user = user
        self.recursive = spec.pop('recursive', False)
        def interpolate_rootpath(filepath, rootpath):
            return os.path.realpath(re.sub(f'^{srcpath}', rootpath, filepath))
        def interpolate_user(filepath, user):
            return re.sub(f'USER', user, filepath)
        if self.recursive:
            self.items = []
            for srcpath, dstpath in spec.items():
                for item in [item for item in Path(srcpath).rglob('*') if not item.is_dir()]:
                    src = os.path.join(cwd, item.as_posix())
                    dst = interpolate_rootpath(src, dstpath)
                    dst = interpolate_user(dst, user)
                    self.items += [(src, dst)]
        else:
            self.items = [(os.path.join(cwd, src), interpolate_user(dst, user)) for src, dst in spec.items()]

    def __repr__(self):
        return f'{type(self).__name__}(recursive={self.recursive}, items={self.items})'

    __str__ = __repr__

    def functions(self):
        return LINKER

    def render(self):
        return f'''
echo "links:"
while read -r file link; do
    linker $file $link
done<<EOM
{self.render_items()}
EOM
        '''.lstrip('\n').rstrip()

def sift(items, includes=None):
    if None in (items, includes) or '*' in includes:
        return items
    def match(item, includes):
        return any([fnmatch(item, include) for include in includes])
    return [item for item in items if match(item, includes)]

class APT(ContinuePackageType):
    def render_header(self):
        return f'''
{PackageType.render_header(self)}

sudo apt update && sudo apt upgrade -y && sudo apt install -y software-properties-common
        '''.lstrip('\n').rstrip()

    def render_block(self):
        return '''
sudo apt install -y
        '''.lstrip('\n').rstrip()

class DNF(ContinuePackageType):
    def render_block(self):
        return '''
    sudo dnf install -y $pkg
        '''.lstrip('\n').rstrip()

class PPA(HeredocPackageType):
    def render_block(self):
        if not self.items:
            return ''
        return f'''
    ppas=$(find /etc/apt/ -name *.list | xargs cat | grep ^[[:space:]]*deb | grep -v deb-src)
    if [[ $ppas != *"$ppa"* ]]; then
        sudo add-apt-repository -y "ppa:$ppa"
    fi
'''.lstrip('\n').rstrip()

class NPM(ContinuePackageType):
    def render_block(self):
        return f'''
sudo npm install -g
'''.lstrip('\n').rstrip()

class PIP3(ContinuePackageType):
    def render_header(self):
        return f'''
{PackageType.render_header(self)}

sudo apt-get install -y python3-dev
sudo -H pip3 install --upgrade pip setuptools
'''.lstrip('\n').rstrip()

    def render_block(self):
        return f'''
sudo -H pip3 install --upgrade
'''.lstrip('\n').rstrip()

class Repo():
    def __init__(self, baseurl, reponame, spec, repopath, **kwargs):
        self.baseurl = baseurl
        self.reponame = reponame
        self.repopath = repopath
        self.link = Link(spec.get('link'), None, cwd=os.path.join(kwargs.pop('cwd'), repopath, reponame), **kwargs) if 'link' in spec else None
        self.script = Script(dict(reponame=spec.get('script')), None, **kwargs) if 'script' in spec else None

    def __repr__(self):
        return f'{type(self).__name__}(baseurl={self.baseurl}, reponame={self.reponame}, repopath={self.repopath}, link={self.link}, script={self.script})'

    __str__ = __repr__

    def render(self):
        link = self.link.render() + '\n' if self.link else ''
        script = self.script.render() + '\n' if self.script else ''
        return f'''
echo "{self.reponame}:"
git clone --recursive {self.baseurl}/{self.reponame} {self.repopath}/{self.reponame}
(cd {self.repopath}/{self.reponame} && pwd && git pull && git checkout HEAD)
{link}
{script}
'''.lstrip('\n').rstrip()

class Github(ManifestType):
    def __init__(self, spec, patterns, **kwargs):
        repopath = spec.pop('repopath', 'repos')
        self.repos = [Repo('https://github.com', reponame, spec[reponame], repopath, **kwargs) for reponame in sift(spec.keys(), patterns)]

    def __repr__(self):
        return f'{type(self).__name__}(repos={self.repos})'

    __str__ = __repr__

    def functions(self):
        return LINKER

    def render(self):
        if not self.repos:
            return ''
        return 'echo "github repos:"\n\n' + '\n\n'.join([repo.render() for repo in self.repos]).strip()

class Script(ManifestType):
    def __init__(self, spec, patterns, **kwargs):
        self.items = {name: spec[name].strip() for name in sift(spec.keys(), patterns)}

    def __repr__(self):
        return f'{type(self).__name__}(items={self.items})'

    __str__ = __repr__

    def render(self):
        if not self.items:
            return ''
        return 'echo "scripts:"\n\n' +  '\n\n'.join([f"echo \"{name}:\"\nbash << 'EOM'\n{script}\nEOM" for name, script in self.items.items()])

class Manifest():
    def __init__(
            self,
            spec=None,
            complete=True,
            pkgmgr=None,
            link=None,
            ppa=None,
            apt=None,
            dnf=None,
            npm=None,
            pip3=None,
            github=None,
            script=None,
            **kwargs):
        self.verbose = spec.pop('verbose', False)
        self.errors = spec.pop('errors', False)
        self.sections = []
        if complete or link != None:
            self.sections += [Link(spec['link'], link, **kwargs)]
        if complete or ppa != None:
            self.sections += [PPA(spec['ppa'], ppa, **kwargs)]
        pkgs = spec.get('pkg', {}).get('items', [])
        apts = pkgs + spec.get('apt', {}).get('items', []) if complete or apt != None else []
        dnfs = pkgs + spec.get('dnf', {}).get('items', []) if complete or dnf != None else []
        if pkgmgr == 'deb' and apts:
            self.sections += [APT(dict(items=apts), apt, **kwargs)]
        elif pkgmgr == 'rpm' and dnfs:
            self.sections += [DNF(dict(items=dnfs), dnf, **kwargs)]
        if complete or npm != None:
            self.sections += [NPM(spec['npm'], npm, **kwargs)]
        if complete or pip3 != None:
            self.sections += [PIP3(spec['pip3'], pip3, **kwargs)]
        if complete or github != None:
            self.sections += [Github(spec['github'], github, **kwargs)]
        if complete or script != None:
            self.sections += [Script(spec['script'], script, **kwargs)]

    def __repr__(self):
        return f'{type(self).__name__}(verbose={self.verbose}, errors={self.errors}, sections={self.sections})'

    def render_header(self):
        return '''
#!/bin/bash
# generated file by manifest.py
# src: https://github.com/scottidler/.../blob/master/manifest.py

'''.lstrip('\n')

    def render_functions(self, sep='\n\n', prefix=None, suffix='\n\n'):
        if not self.sections:
            return ''
        result = sep.join(set([section.functions() for section in self.sections])).lstrip('\n').rstrip()
        if result:
            if prefix:
                result = prefix + result
            if suffix:
                result += suffix
        return result

    def render_body(self, sep='\n\n', prefix=None, suffix='\n\n'):
        if not self.sections:
            return ''
        result = sep.join([section.render() for section in self.sections]).lstrip('\n').rstrip()
        if result:
            if prefix:
                result = prefix + result
            if suffix:
                result += suffix
        return result

    def render_footer(self):
        return ''

    def render(self):
        return ''.join([
            self.render_header(),
            self.render_functions(),
            self.render_body(),
            self.render_footer(),
        ])

    __str__ = __repr__

def load_manifest(complete=True, config=None, **kwargs):
    spec = yaml.safe_load(open(config))
    manifest = Manifest(spec=spec, complete=complete, **kwargs)
    return manifest

def complete(ns):
    return not any([getattr(ns, sec) for sec in SECTIONS])

class ManifestAction(Action):
    def __call__(self, parser, namespace, values, option_strings=None):
        setattr(namespace, self.dest, values if values else ['*'])

def main(args):
    parser = ArgumentParser()
    parser.add_argument(
        '-C', '--config',
        default=f'{SCRIPT_PATH}/manifest.yml',
        help='default="%(default)s"; specify the config path')
    parser.add_argument(
        '-D', '--cwd',
        default=os.getcwd(),
        help='default="%(default)s"; set the cwd')
    parser.add_argument(
        '-U', '--user',
        default=USER,
        help='default="%(default)s"; specify user if not current')
    parser.add_argument(
        '-M', '--pkgmgr',
        default=get_pkgmgr(),
        help=f'default="%(default)s"; override pkgmgr')
    parser.add_argument(
        '-l', '--link',
        metavar='LINK',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match links')
    parser.add_argument(
        '-p', '--ppa',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match ppa items')
    parser.add_argument(
        '-a', '--apt',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match apt items')
    parser.add_argument(
        '-d', '--dnf',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match dnf items')
    parser.add_argument(
        '-n', '--npm',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match npm items')
    parser.add_argument(
        '-P', '--pip3',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match pip3 items')
    parser.add_argument(
        '-g', '--github',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match github repos')
    parser.add_argument(
        '-s', '--script',
        metavar='SCRIPT',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match script names')
    ns = parser.parse_args()
    manifest = load_manifest(complete=complete(ns), **ns.__dict__)
    try:
        print(manifest.render())
        sys.stdout.flush()
    except IOError:
        sys.stderr.write("on running: " + str(sys.exc_info()))
    try:
        sys.stdout.close()
    except IOError:
        sys.stderr.write("on closing: " + str(sys.exc_info()))

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>added debug capability to every manifest gen'd bash; fixed ppas code<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import re
import pwd
import sys
sys.dont_write_bytecode = True

from copy import deepcopy
from ruamel import yaml
from pathlib import Path

from argparse import ArgumentParser, Action
from contextlib import contextmanager
from fnmatch import fnmatch

SCRIPT_FILE = os.path.abspath(__file__)
SCRIPT_NAME = os.path.basename(SCRIPT_FILE)
SCRIPT_PATH = os.path.dirname(SCRIPT_FILE)
if os.path.islink(__file__):
    REAL_FILE = os.path.abspath(os.readlink(__file__))
    REAL_NAME = os.path.basename(REAL_FILE)
    REAL_PATH = os.path.dirname(REAL_FILE)

SECTIONS = [
    'link',
    'ppa',
    'apt',
    'dnf',
    'npm',
    'pip3',
    'github',
    'script',
]

UID = os.getuid()
GID = pwd.getpwuid(UID).pw_gid
USER = pwd.getpwuid(UID).pw_name

DEBUG = '''
if [ -n "$DEBUG" ]; then
    PS4=':${LINENO}+'
    set -x
fi
'''.lstrip('\n').rstrip()

LINKER = '''
linker() {
    file=$(realpath "$1")
    link="${2/#\~/$HOME}"
    echo "$link -> $file"
    if [ -f "$link" ] && [ "$file" != "$(readlink $link)" ]; then
        orig="$link.orig"
        $VERBOSE && echo "backing up $orig"
        mv $link $orig
    elif [ ! -f "$link" ] && [ -L "$link" ]; then
        $VERBOSE && echo "removing broken link $link"
        unlink $link
    fi
    if [ -f "$link" ]; then
        echo "[exists] $link"
    else
        echo "[create] $link -> $file"
        mkdir -p $(dirname $link); ln -s $file $link
    fi
}
'''.lstrip('\n').rstrip()

class UnknownPkgmgrError(Exception):
    def __init__(self):
        super(UnknownPkgmgrError, self).__init__('unknown pkgmgr!')

@contextmanager
def cd(*args, **kwargs):
    mkdir = kwargs.pop('mkdir', True)
    verbose = kwargs.pop('verbose', False)
    path = os.path.sep.join(args)
    path = os.path.normpath(path)
    path = os.path.expanduser(path)
    prev = os.getcwd()
    if path != prev:
        if mkdir:
            os.system(f'mkdir -p {path}')
        os.chdir(path)
        curr = os.getcwd()
        sys.path.append(curr)
        if verbose:
            print(f'cd {curr}')
    try:
        yield
    finally:
        if path != prev:
            sys.path.remove(curr)
            os.chdir(prev)
            if verbose:
                print('cd {prev}')

def check_hash(program):
    from subprocess import check_call, CalledProcessError, PIPE
    try:
        check_call(f'hash {program}', shell=True, stdout=PIPE, stderr=PIPE)
        return True
    except CalledProcessError:
        return False

def get_pkgmgr():
    if check_hash('dpkg'):
        return 'deb'
    elif check_hash('rpm'):
        return 'rpm'
    elif check_hash('brew'):
        return 'brew'
    raise UnknownPkgmgrError

def render_item(item):
    if isinstance(item, tuple):
        return ' '.join(item)
    return str(item)

class ManifestType():
    def __repr__(self):
        return f'{type(self).__name__}(items = {self.items})'

    __str__ = __repr__

    def functions(self):
        return ''

    def render(self):
        raise NotImplementedError

class PackageType(ManifestType):
    def __init__(self, spec, patterns, **kwargs):
        self.items = sift(spec.get('items', None), patterns)

    def __repr__(self):
        return f'{type(self).__name__}(items={self.items})'

    __str__ = __repr__

    def render_header(self):
        return f'''
echo "{type(self).__name__.lower()}s:"
        '''.lstrip('\n').rstrip()

    def render_block(self):
        raise NotImplementedError

class HeredocPackageType(PackageType):
    def render_items(self):
        return '\n'.join([render_item(item) for item in self.items])

    def render(self):
        return f'''
{self.render_header()}

while read pkg; do
{self.render_block()}
done<<EOM
{self.render_items()}
EOM
        '''.lstrip('\n').rstrip()

class ContinuePackageType(PackageType):
    def render_items(self):
        return ' \\\n    '.join([render_item(item) for item in self.items])

    def render(self):
        return f'''
{self.render_header()}

{self.render_block()} {self.render_items()}
        '''.lstrip('\n').rstrip()

class Link(HeredocPackageType):
    def __init__(self, spec, patterns, cwd=None, user=None, **kwargs):
        self.cwd = cwd
        self.user = user
        self.recursive = spec.pop('recursive', False)
        def interpolate_rootpath(filepath, rootpath):
            return os.path.realpath(re.sub(f'^{srcpath}', rootpath, filepath))
        def interpolate_user(filepath, user):
            return re.sub(f'USER', user, filepath)
        if self.recursive:
            self.items = []
            for srcpath, dstpath in spec.items():
                for item in [item for item in Path(srcpath).rglob('*') if not item.is_dir()]:
                    src = os.path.join(cwd, item.as_posix())
                    dst = interpolate_rootpath(src, dstpath)
                    dst = interpolate_user(dst, user)
                    self.items += [(src, dst)]
        else:
            self.items = [(os.path.join(cwd, src), interpolate_user(dst, user)) for src, dst in spec.items()]

    def __repr__(self):
        return f'{type(self).__name__}(recursive={self.recursive}, items={self.items})'

    __str__ = __repr__

    def functions(self):
        return LINKER

    def render(self):
        return f'''
echo "links:"
while read -r file link; do
    linker $file $link
done<<EOM
{self.render_items()}
EOM
        '''.lstrip('\n').rstrip()

def sift(items, includes=None):
    if None in (items, includes) or '*' in includes:
        return items
    def match(item, includes):
        return any([fnmatch(item, include) for include in includes])
    return [item for item in items if match(item, includes)]

class APT(ContinuePackageType):
    def render_header(self):
        return f'''
{PackageType.render_header(self)}

sudo apt update && sudo apt upgrade -y && sudo apt install -y software-properties-common
        '''.lstrip('\n').rstrip()

    def render_block(self):
        return '''
sudo apt install -y
        '''.lstrip('\n').rstrip()

class DNF(ContinuePackageType):
    def render_block(self):
        return '''
    sudo dnf install -y $pkg
        '''.lstrip('\n').rstrip()

class PPA(HeredocPackageType):
    def render_block(self):
        if not self.items:
            return ''
        return f'''
    ppas=$(find /etc/apt/ -name *.list | xargs cat | grep ^[[:space:]]*deb | grep -v deb-src)
    if [[ $ppas != *"$pkg"* ]]; then
        sudo add-apt-repository -y "ppa:$pkg"
    fi
'''.lstrip('\n').rstrip()

class NPM(ContinuePackageType):
    def render_block(self):
        return f'''
sudo npm install -g
'''.lstrip('\n').rstrip()

class PIP3(ContinuePackageType):
    def render_header(self):
        return f'''
{PackageType.render_header(self)}

sudo apt-get install -y python3-dev
sudo -H pip3 install --upgrade pip setuptools
'''.lstrip('\n').rstrip()

    def render_block(self):
        return f'''
sudo -H pip3 install --upgrade
'''.lstrip('\n').rstrip()

class Repo():
    def __init__(self, baseurl, reponame, spec, repopath, **kwargs):
        self.baseurl = baseurl
        self.reponame = reponame
        self.repopath = repopath
        self.link = Link(spec.get('link'), None, cwd=os.path.join(kwargs.pop('cwd'), repopath, reponame), **kwargs) if 'link' in spec else None
        self.script = Script(dict(reponame=spec.get('script')), None, **kwargs) if 'script' in spec else None

    def __repr__(self):
        return f'{type(self).__name__}(baseurl={self.baseurl}, reponame={self.reponame}, repopath={self.repopath}, link={self.link}, script={self.script})'

    __str__ = __repr__

    def render(self):
        link = self.link.render() + '\n' if self.link else ''
        script = self.script.render() + '\n' if self.script else ''
        return f'''
echo "{self.reponame}:"
git clone --recursive {self.baseurl}/{self.reponame} {self.repopath}/{self.reponame}
(cd {self.repopath}/{self.reponame} && pwd && git pull && git checkout HEAD)
{link}
{script}
'''.lstrip('\n').rstrip()

class Github(ManifestType):
    def __init__(self, spec, patterns, **kwargs):
        repopath = spec.pop('repopath', 'repos')
        self.repos = [Repo('https://github.com', reponame, spec[reponame], repopath, **kwargs) for reponame in sift(spec.keys(), patterns)]

    def __repr__(self):
        return f'{type(self).__name__}(repos={self.repos})'

    __str__ = __repr__

    def functions(self):
        return LINKER

    def render(self):
        if not self.repos:
            return ''
        return 'echo "github repos:"\n\n' + '\n\n'.join([repo.render() for repo in self.repos]).strip()

class Script(ManifestType):
    def __init__(self, spec, patterns, **kwargs):
        self.items = {name: spec[name].strip() for name in sift(spec.keys(), patterns)}

    def __repr__(self):
        return f'{type(self).__name__}(items={self.items})'

    __str__ = __repr__

    def render(self):
        if not self.items:
            return ''
        return 'echo "scripts:"\n\n' +  '\n\n'.join([f"echo \"{name}:\"\nbash << 'EOM'\n{script}\nEOM" for name, script in self.items.items()])

class Manifest():
    def __init__(
            self,
            spec=None,
            complete=True,
            pkgmgr=None,
            link=None,
            ppa=None,
            apt=None,
            dnf=None,
            npm=None,
            pip3=None,
            github=None,
            script=None,
            **kwargs):
        self.verbose = spec.pop('verbose', False)
        self.errors = spec.pop('errors', False)
        self.sections = []
        if complete or link != None:
            self.sections += [Link(spec['link'], link, **kwargs)]
        if complete or ppa != None:
            self.sections += [PPA(spec['ppa'], ppa, **kwargs)]
        pkgs = spec.get('pkg', {}).get('items', [])
        apts = pkgs + spec.get('apt', {}).get('items', []) if complete or apt != None else []
        dnfs = pkgs + spec.get('dnf', {}).get('items', []) if complete or dnf != None else []
        if pkgmgr == 'deb' and apts:
            self.sections += [APT(dict(items=apts), apt, **kwargs)]
        elif pkgmgr == 'rpm' and dnfs:
            self.sections += [DNF(dict(items=dnfs), dnf, **kwargs)]
        if complete or npm != None:
            self.sections += [NPM(spec['npm'], npm, **kwargs)]
        if complete or pip3 != None:
            self.sections += [PIP3(spec['pip3'], pip3, **kwargs)]
        if complete or github != None:
            self.sections += [Github(spec['github'], github, **kwargs)]
        if complete or script != None:
            self.sections += [Script(spec['script'], script, **kwargs)]

    def __repr__(self):
        return f'{type(self).__name__}(verbose={self.verbose}, errors={self.errors}, sections={self.sections})'

    def render_header(self):
        return f'''
#!/bin/bash
# generated file by manifest.py
# src: https://github.com/scottidler/.../blob/master/manifest.py

{DEBUG}

'''.lstrip('\n')

    def render_functions(self, sep='\n\n', prefix=None, suffix='\n\n'):
        if not self.sections:
            return ''
        result = sep.join(set([section.functions() for section in self.sections])).lstrip('\n').rstrip()
        if result:
            if prefix:
                result = prefix + result
            if suffix:
                result += suffix
        return result

    def render_body(self, sep='\n\n', prefix=None, suffix='\n\n'):
        if not self.sections:
            return ''
        result = sep.join([section.render() for section in self.sections]).lstrip('\n').rstrip()
        if result:
            if prefix:
                result = prefix + result
            if suffix:
                result += suffix
        return result

    def render_footer(self):
        return ''

    def render(self):
        return ''.join([
            self.render_header(),
            self.render_functions(),
            self.render_body(),
            self.render_footer(),
        ])

    __str__ = __repr__

def load_manifest(complete=True, config=None, **kwargs):
    spec = yaml.safe_load(open(config))
    manifest = Manifest(spec=spec, complete=complete, **kwargs)
    return manifest

def complete(ns):
    return not any([getattr(ns, sec) for sec in SECTIONS])

class ManifestAction(Action):
    def __call__(self, parser, namespace, values, option_strings=None):
        setattr(namespace, self.dest, values if values else ['*'])

def main(args):
    parser = ArgumentParser()
    parser.add_argument(
        '-C', '--config',
        default=f'{SCRIPT_PATH}/manifest.yml',
        help='default="%(default)s"; specify the config path')
    parser.add_argument(
        '-D', '--cwd',
        default=os.getcwd(),
        help='default="%(default)s"; set the cwd')
    parser.add_argument(
        '-U', '--user',
        default=USER,
        help='default="%(default)s"; specify user if not current')
    parser.add_argument(
        '-M', '--pkgmgr',
        default=get_pkgmgr(),
        help=f'default="%(default)s"; override pkgmgr')
    parser.add_argument(
        '-l', '--link',
        metavar='LINK',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match links')
    parser.add_argument(
        '-p', '--ppa',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match ppa items')
    parser.add_argument(
        '-a', '--apt',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match apt items')
    parser.add_argument(
        '-d', '--dnf',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match dnf items')
    parser.add_argument(
        '-n', '--npm',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match npm items')
    parser.add_argument(
        '-P', '--pip3',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match pip3 items')
    parser.add_argument(
        '-g', '--github',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match github repos')
    parser.add_argument(
        '-s', '--script',
        metavar='SCRIPT',
	action=ManifestAction,
        nargs='*',
        help='specify list of glob patterns to match script names')
    ns = parser.parse_args()
    manifest = load_manifest(complete=complete(ns), **ns.__dict__)
    try:
        print(manifest.render())
        sys.stdout.flush()
    except IOError:
        sys.stderr.write("on running: " + str(sys.exc_info()))
    try:
        sys.stdout.close()
    except IOError:
        sys.stderr.write("on closing: " + str(sys.exc_info()))

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>import math
import numpy as np
import sys
import sklearn
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LinearRegression, Lasso
from sklearn.preprocessing import StandardScaler, MaxAbsScaler
from sklearn.decomposition import PCA
from sklearn.feature_selection import VarianceThreshold
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVR
import pandas as pd
from os.path import join
from src.dump_results import dump
from sklearn.tree import DecisionTreeRegressor
from src.pre_process import process_level2, load_binary
from sklearn.linear_model import ElasticNet

def run():
    data = load_binary()

    # (Opt: user clustering)

    # Extract features
    user_feat_matrix = process_level2(data)  # X

    del user_feat_matrix['X']['user_id']
    X = user_feat_matrix['X'].values
    X[np.isnan(X)] = 0
    Y = user_feat_matrix['Y']
    Y.fillna(0, inplace=True)
    del user_feat_matrix['X_all']['user_id']
    X_all = user_feat_matrix['X_all'].values
    X_all[np.isnan(X_all)] = 0

    cols = list(Y.columns.values)
    symptoms = ['happy', 'pms', 'sad', 'sensitive_emotion', 'energized', 'exhausted',
                'high_energy', 'low_energy', 'cramps', 'headache', 'ovulation_pain',
                'tender_breasts', 'acne_skin', 'good_skin', 'oily_skin', 'dry_skin']
    with open("result.txt", 'w') as f:
        f.write("user_id,day_in_cycle,symptom,probability\n")

    for symptom in symptoms:
        print(symptom)
        s_Y = Y[[x for x in cols if x[1] == symptom]]
        pipeline = Pipeline([
            ('remove_low_variance_features', VarianceThreshold(threshold=0.0)),
            ('standard_scale', StandardScaler()),
            ('estimator', ElasticNet()),
        ])

        model = pipeline
        model.fit(X, s_Y.values)

        print("dumping...")
        data_dir = 'data'
        cycles0 = pd.read_csv(join(data_dir, 'cycles0.csv'))
        c_length = {k:v for k,v in zip(cycles0.user_id.values, cycles0.expected_cycle_length)}
        dump(symptom, model, X_all, c_length, data['users'].user_id)

if __name__ == '__main__':
    run()

    """
    symptoms=['happy']
    for symptom in symptoms:
        print(symptom)
        s_Y = Y[[x for x in cols if x[1] == symptom]]
        print("Lasso")
        pipeline = Pipeline([
            ('remove_low_variance_features', VarianceThreshold(threshold=0.0)),
            #('pca', PCA()),
            ('estimator', SVR(kernel='rbf')),
        ])

        param_grid = {'estimator__gamma': [math.pow(2, x) for x in range(-5, 2)],
                      'estimator__C': [math.pow(2, x) for x in range(-5, 2)]}
                      #'pca__n_components': [5]}
        model = GridSearchCV(pipeline,
                             param_grid,
                             verbose=10,
                             cv=5,
                             n_jobs=1
                 )

        model.fit(X, s_Y.values)

        model.best_score_

    DTAD:
    for symptom in symptoms:
        print(symptom)
        s_Y = Y[[x for x in cols if x[1] == symptom]]
        pipeline = Pipeline([
            ('remove_low_variance_features', VarianceThreshold(threshold=0.0)),
            ('standard_scale', StandardScaler()),
            ('estimator', DecisionTreeRegressor(max_depth=5)),
        ])

        param_grid = {'estimator__max_depth': [3, 5, 7],
                      'estimator__max_features': ['auto', 'sqrt', 'log2']}
        model = GridSearchCV(pipeline, param_grid = param_grid, n_jobs = 1,
                             verbose=2)
        model.fit(X, s_Y.values)

    ENGS:
    for symptom in symptoms:
        print(symptom)
        s_Y = Y[[x for x in cols if x[1] == symptom]]
        pipeline = Pipeline([
            ('remove_low_variance_features', VarianceThreshold(threshold=0.0)),
            ('standard_scale', StandardScaler()),
            ('estimator', ElasticNet()),
        ])

        param_grid = {'estimator__alpha': [.1, .3, .5, .7, .8],
                      'estimator__l1_ratio': [.2, .5, .8]}
        model = GridSearchCV(pipeline, param_grid = param_grid, n_jobs = 4,
                             verbose=2)
        model.fit(X, s_Y.values)

    ENLD
    for symptom in symptoms:
        print(symptom)
        s_Y = Y[[x for x in cols if x[1] == symptom]]
        pipeline = Pipeline([
            ('remove_low_variance_features', VarianceThreshold(threshold=0.0)),
            ('standard_scale', StandardScaler()),
            ('estimator', ElasticNet()),
        ])

        model = pipeline
        model.fit(X, s_Y.values)

        for symptom in symptoms:
        print(symptom)
        s_Y = Y[[x for x in cols if x[1] == symptom]]
        pipeline = Pipeline([
            ('remove_low_variance_features', VarianceThreshold(threshold=0.0)),
            ('standard_scale', MaxAbsScaler()),
            ('estimator', DecisionTreeRegressor(max_depth=5)),
        ])

        param_grid = {'estimator__max_depth': [3, 5, 7],
                      'estimator__max_features': ['auto', 'sqrt', 'log2']}
        model = GridSearchCV(pipeline, param_grid = param_grid, n_jobs = 3,
                             verbose=2)
        model.fit(X, s_Y.values)

    """
<commit_msg>cleanup<commit_after><|endoftext|>


<commit_before># Copyright 2014 LinkedIn Corp. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

"""
Classes that handles browsers to view creatives and analyze the network logs.
"""

import os
import os.path
import re
import json
import time
import socket
import requests
import threading
import subprocess

from adscan.issue import IssueType

class BrowserHost(threading.Thread):
  """
  Class that launches an external PhantomJS process to view multiple urls one by one. The PhantomJS browser is launched
  with an option that accepts any invalid SSL certificates to capture all the requests made by each creative. The PhantomJS
  creates a log file containing the requested urls. By sending a request to each url (again but without ignoring invalid SSL
  certificates), we can check if the urls have any problems or not.
  """

  @classmethod
  def verify_http_url(cls, session, url):
    """
    Check if the url is avaiable over https or not.

    :param session: the session of requests library.
    :param url: the URL found on the log file. Its protocol should be either of http or https.
    :return: an issue id defined in :class:`adscan.issue.IssueType`.
    """
    issue_id = IssueType.NO_ISSUE if re.match(r'^https', url) else IssueType.HTTPS_AVAIL
    try:
      req_url = re.sub(r'^http\:', 'https:', url)
      res = session.get(req_url, verify=True, allow_redirects=True)
      if res.status_code == 403:
        issue_id = IssueType.INVALID_CERT
      if res.status_code >= 500:
        issue_id = IssueType.SERVER_ERROR
      elif res.status_code >= 400:
        issue_id = IssueType.CLIENT_ERROR
    except requests.exceptions.SSLError:
      issue_id = IssueType.INVALID_CERT
    except Exception:
      issue_id = IssueType.NO_SSL_SERVER
    return issue_id

  def __init__(self, creative_urls, protocol, phantomjs, browserjs, display_id, log_dir, cookie_dir, callback=None, debug=False):
    """
    Initialize the instance.

    :param creative_urls: a dictionary of a creative key and its url to be scanned.
    :param protocol: the server protocol, 'https' or 'http'.
    :param phantomjs: the path to the phantomjs command.
    :param browserjs: the path to the browser.js file.
    :param display_id: the number that represents the xvfb display id.
    :param log_dir: the directory into which the browser process saves the scan log.
    :param cookie_dir: the directory that contains cookies used by browsers while scanning.
    :param callback: the function called for passing the urls and issue ids found during this scanning process.
    :param debug: Turn on the debug mode, which temporarily to allow access to private network that host test creatives.
    """
    threading.Thread.__init__(self)
    self.creative_urls = creative_urls
    self.protocol = protocol
    self.phantomjs = phantomjs
    self.browserjs = browserjs
    self.display_id = display_id
    self.log_dir = log_dir
    self.cookie_dir = cookie_dir
    self.callback = callback
    self.debug = debug
    self.abort = False

  def _run_command(self, command):
    """
    Create a new process and execute the `command`.

    :param command: a list of strings that represents the command and command arguments.
    """
    print command
    process = subprocess.Popen(
      command, shell=False, env={'DISPLAY': ':%d' % self.display_id}, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    process.wait()
    print 'return: %d' % (process.wait(),)
    print 'stdout: %s' % (process.stdout.readlines(),)
    print 'stderr: %s' % (process.stderr.readlines(),)

  def _verify_each_url(self, creative_id, logfile):
    """
    Analyzes the browser's network log to check if https protocols are avaiable or not.

    :param creative_id: a list of creative ids.
    :param logfile: the path to the file in which the network log is saved.
    """
    if not os.path.exists(logfile):
      return

    data = None
    with open(logfile, 'r') as fp:
      data = json.load(fp)

    found = False

    session = requests.Session()
    session.max_redirects = 100
    if data:
      for key, value in data.iteritems():
        try:
          int(key)
        except ValueError:
          continue

        # When the request is aborted, PhantomJS will set 1 (Connection Refused Error) for the error code.
        # When IP address is used, PhantomJS will set 301 (Protocol Unknown Error) for the error code.
        if data[key]['error'] and data[key]['error']['errorCode'] and data[key]['error']['errorCode'] in [1, 301]:
          issue_id = IssueType.PRIVATE_NETWORK
        else:
          url = data[key]['request']['url']
          issue_id = IssueType.NO_ISSUE
          if re.match(r'^https?:\/\/%s' % socket.gethostname(), url):
            continue
          elif re.match(r'^http', url):
            found = True
            issue_id = self.verify_http_url(session, url)
        if self.callback:
          self.callback(creative_id, issue_id, self.protocol, url=url)

    if not found and self.callback:
      self.callback(creative_id, IssueType.NO_EXTERNAL, self.protocol)

  def _create_command(self, url_obj, log_file):
    """
    Create a new process for a browser. The `url` will be viewed and the network log will be saved into `log_file`.

    :param url_obj: a dictionary of a url and a boolean flag indicating if the url is hosted locally or not.
    :param log_file: the path to the file to which network log is saved.
    :return: a string list of a command and its arguments.
    """
    command = []
    if self.display_id < 0:
      command.extend(['/usr/bin/xvfb-run', '--auto-servernum', '--server-args', '-screen 0 1024x768x24'])
    command.append(self.phantomjs)
    command.extend(['--web-security', 'false'])
    command.extend(['--load-plugins', 'true'])
    command.extend(['--ignore-ssl-errors', 'true'])
    command.append(self.browserjs)
    command.extend(['--use-cookie', 'true'])
    command.extend(['--enable-javascript', 'true'])
    command.extend(['--hosted-locally', url_obj['hosted_locally']])
    command.extend(['--url', url_obj['url']])
    command.extend(['--log-file', log_file])
    command.extend(['--debug', 'true' if self.debug else 'false'])
    if self.cookie_dir:
      command.extend(['--cookie-dir', self.cookie_dir])
    return command

  def run(self):
    """
    Start browsing the urls one by one.
    """
    try:
      for creative_id, url_obj in self.creative_urls.iteritems():
        if self.abort:
          break
        elif url_obj:
          log_file = '%s/%s.json' % (self.log_dir, creative_id)
          command = self._create_command(url_obj, log_file)
          self._run_command(command)
          self._verify_each_url(creative_id, log_file)
        elif self.callback:
          self.callback(creative_id, IssueType.NO_EXTERNAL, self.protocol)
    except:
      raise

  def shutdown(self):
    """
    Prevent the next scan to shutdown this browser.
    """
    self.abort = True

class BrowserController(object):
  """
  Class that allots creatives to BrowserHost instances.
  """

  @classmethod
  def create_html(cls, html, dest_file):
    """
    Create an html file and save it in `dest_file`.

    :param html: a string that contains html tags.
    :param dest_file: the path to file into which the html will be written.
    """
    with open(dest_file, 'w') as fp:
      fp.write(html.encode('utf-8'))

  def __init__(self, creatives, protocol, ports, browser_count, phantomjs, browserjs, cookie_dir, workspace, log_func, modify_func, debug=False):
    """
    Initiate an instance.

    :param creatives: a list of creatives.
    :param protocol: a protocol, `https` or `http`.
    :param ports: a list of port numbers.
    :param browser_count: a number of browsers to be launched.
    :param phantomjs: the path to the phantomjs command.
    :param browserjs: the path to the browser.js file.
    :param cookie_dir: the directory that contains cookies, which will be used by browsers during scanning.
    :param workspace: the directory where the html files and scan logs are saved.
    :param log_func: the function called for passing the urls and issue ids found during this scanning process.
    :param modify_func: the function called for modifying the creatives.
    :param debug: Turn on the debug mode, which temporarily to allow access to private network that host test creatives.
    """
    self.creatives = creatives
    self.protocol = protocol
    self.ports = ports
    self.browser_count = browser_count
    self.phantomjs = phantomjs
    self.browserjs = browserjs
    self.cookie_dir = cookie_dir
    self.workspace = workspace
    self.log_func = log_func
    self.modify_func = modify_func
    self.threads = []
    self.hostname = socket.gethostname()
    self.debug = debug

  def _create_urls_to_scan(self, creatives, port):
    """
    Create urls to scan these creatives.

    :param creatives: a list of creatives.
    :param port: the port number used to scan these creatives.
    :return: a dictionary of a creative id and its url to be scanned.
    """
    url_dict = {}
    for creative in creatives:
      self.modify_func(creative)
      snippet = creative.modified_scan_snippet if self.protocol == 'https' else creative.scan_snippet

      if snippet:
        url_obj = None
        if re.match(r'^http', snippet, re.IGNORECASE):
          url_obj = {
            'url': snippet,
            'hosted_locally': 'false'
          }
        else:
          save_file = '%s/%s.html' % (self.workspace, creative.creative_id)
          self.create_html(snippet, save_file)
          url_obj = {
            'url': '%s://%s:%d/%s' % (self.protocol, self.hostname, port, save_file),
            'hosted_locally': 'true'
          }
        url_dict[str(creative.creative_id)] = url_obj
    return url_dict

  def start(self):
    """
    Start browsers.
    """
    # Caluculate how many creatives are allotted to each browser process.
    allotment = len(self.creatives) / self.browser_count
    if allotment == 0:
      allotment = 1
    if len(self.creatives) % allotment != 0:
      allotment += 1

    for i in xrange(0, self.browser_count):
      display_id = i + 1
      log_dir = '%s/%d' % (self.workspace, i)

      allotted_creatives = self.creatives[(i * allotment):((i + 1) * allotment)]
      allotted_port = self.ports[i % len(self.ports)]

      url_dict = self._create_urls_to_scan(allotted_creatives, allotted_port)
      if len(url_dict) > 0:
        thread = BrowserHost(url_dict, self.protocol, self.phantomjs, self.browserjs, display_id, log_dir, self.cookie_dir, self.log_func, self.debug)
        thread.start()
        self.threads.append(thread)
        time.sleep(1)

  def wait(self):
    """
    Wait until all the threads done.
    """
    for thread in self.threads:
      if thread and thread.isAlive():
        thread.join()

  def shutdown(self):
    """
    Cloase all the browser threads.
    """
    for thread in self.threads:
      if thread and thread.isAlive():
        thread.shutdown()
<commit_msg>Support any type of ssl certificates<commit_after># Copyright 2014 LinkedIn Corp. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

"""
Classes that handles browsers to view creatives and analyze the network logs.
"""

import os
import os.path
import re
import json
import time
import socket
import requests
import threading
import subprocess

from adscan.issue import IssueType

class BrowserHost(threading.Thread):
  """
  Class that launches an external PhantomJS process to view multiple urls one by one. The PhantomJS browser is launched
  with an option that accepts any invalid SSL certificates to capture all the requests made by each creative. The PhantomJS
  creates a log file containing the requested urls. By sending a request to each url (again but without ignoring invalid SSL
  certificates), we can check if the urls have any problems or not.
  """

  @classmethod
  def verify_http_url(cls, session, url):
    """
    Check if the url is avaiable over https or not.

    :param session: the session of requests library.
    :param url: the URL found on the log file. Its protocol should be either of http or https.
    :return: an issue id defined in :class:`adscan.issue.IssueType`.
    """
    issue_id = IssueType.NO_ISSUE if re.match(r'^https', url) else IssueType.HTTPS_AVAIL
    try:
      req_url = re.sub(r'^http\:', 'https:', url)
      res = session.get(req_url, verify=True, allow_redirects=True)
      if res.status_code == 403:
        issue_id = IssueType.INVALID_CERT
      if res.status_code >= 500:
        issue_id = IssueType.SERVER_ERROR
      elif res.status_code >= 400:
        issue_id = IssueType.CLIENT_ERROR
    except requests.exceptions.SSLError:
      issue_id = IssueType.INVALID_CERT
    except Exception:
      issue_id = IssueType.NO_SSL_SERVER
    return issue_id

  def __init__(self, creative_urls, protocol, phantomjs, browserjs, display_id, log_dir, cookie_dir, callback=None, debug=False):
    """
    Initialize the instance.

    :param creative_urls: a dictionary of a creative key and its url to be scanned.
    :param protocol: the server protocol, 'https' or 'http'.
    :param phantomjs: the path to the phantomjs command.
    :param browserjs: the path to the browser.js file.
    :param display_id: the number that represents the xvfb display id.
    :param log_dir: the directory into which the browser process saves the scan log.
    :param cookie_dir: the directory that contains cookies used by browsers while scanning.
    :param callback: the function called for passing the urls and issue ids found during this scanning process.
    :param debug: Turn on the debug mode, which temporarily to allow access to private network that host test creatives.
    """
    threading.Thread.__init__(self)
    self.creative_urls = creative_urls
    self.protocol = protocol
    self.phantomjs = phantomjs
    self.browserjs = browserjs
    self.display_id = display_id
    self.log_dir = log_dir
    self.cookie_dir = cookie_dir
    self.callback = callback
    self.debug = debug
    self.abort = False

  def _run_command(self, command):
    """
    Create a new process and execute the `command`.

    :param command: a list of strings that represents the command and command arguments.
    """
    print command
    process = subprocess.Popen(
      command, shell=False, env={'DISPLAY': ':%d' % self.display_id}, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    process.wait()
    print 'return: %d' % (process.wait(),)
    print 'stdout: %s' % (process.stdout.readlines(),)
    print 'stderr: %s' % (process.stderr.readlines(),)

  def _verify_each_url(self, creative_id, logfile):
    """
    Analyzes the browser's network log to check if https protocols are avaiable or not.

    :param creative_id: a list of creative ids.
    :param logfile: the path to the file in which the network log is saved.
    """
    if not os.path.exists(logfile):
      return

    data = None
    with open(logfile, 'r') as fp:
      data = json.load(fp)

    found = False

    session = requests.Session()
    session.max_redirects = 100
    if data:
      for key, value in data.iteritems():
        try:
          int(key)
        except ValueError:
          continue

        # When the request is aborted, PhantomJS will set 1 (Connection Refused Error) for the error code.
        # When IP address is used, PhantomJS will set 301 (Protocol Unknown Error) for the error code.
        if data[key]['error'] and data[key]['error']['errorCode'] and data[key]['error']['errorCode'] in [1, 301]:
          issue_id = IssueType.PRIVATE_NETWORK
        else:
          url = data[key]['request']['url']
          issue_id = IssueType.NO_ISSUE
          if re.match(r'^https?:\/\/%s' % socket.gethostname(), url):
            continue
          elif re.match(r'^http', url):
            found = True
            issue_id = self.verify_http_url(session, url)
        if self.callback:
          self.callback(creative_id, issue_id, self.protocol, url=url)

    if not found and self.callback:
      self.callback(creative_id, IssueType.NO_EXTERNAL, self.protocol)

  def _create_command(self, url_obj, log_file):
    """
    Create a new process for a browser. The `url` will be viewed and the network log will be saved into `log_file`.

    :param url_obj: a dictionary of a url and a boolean flag indicating if the url is hosted locally or not.
    :param log_file: the path to the file to which network log is saved.
    :return: a string list of a command and its arguments.
    """
    command = []
    if self.display_id < 0:
      command.extend(['/usr/bin/xvfb-run', '--auto-servernum', '--server-args', '-screen 0 1024x768x24'])
    command.append(self.phantomjs)
    command.extend(['--web-security', 'false'])
    command.extend(['--load-plugins', 'true'])
    command.extend(['--ignore-ssl-errors', 'true'])
    command.extend(['--ssl-protocol', 'any'])  # Accept any ssl protocol, default only accepts SSLv3.
    command.append(self.browserjs)
    command.extend(['--use-cookie', 'true'])
    command.extend(['--enable-javascript', 'true'])
    command.extend(['--hosted-locally', url_obj['hosted_locally']])
    command.extend(['--url', url_obj['url']])
    command.extend(['--log-file', log_file])
    command.extend(['--debug', 'true' if self.debug else 'false'])
    if self.cookie_dir:
      command.extend(['--cookie-dir', self.cookie_dir])
    return command

  def run(self):
    """
    Start browsing the urls one by one.
    """
    try:
      for creative_id, url_obj in self.creative_urls.iteritems():
        if self.abort:
          break
        elif url_obj:
          log_file = '%s/%s.json' % (self.log_dir, creative_id)
          command = self._create_command(url_obj, log_file)
          self._run_command(command)
          self._verify_each_url(creative_id, log_file)
        elif self.callback:
          self.callback(creative_id, IssueType.NO_EXTERNAL, self.protocol)
    except:
      raise

  def shutdown(self):
    """
    Prevent the next scan to shutdown this browser.
    """
    self.abort = True

class BrowserController(object):
  """
  Class that allots creatives to BrowserHost instances.
  """

  @classmethod
  def create_html(cls, html, dest_file):
    """
    Create an html file and save it in `dest_file`.

    :param html: a string that contains html tags.
    :param dest_file: the path to file into which the html will be written.
    """
    with open(dest_file, 'w') as fp:
      fp.write(html.encode('utf-8'))

  def __init__(self, creatives, protocol, ports, browser_count, phantomjs, browserjs, cookie_dir, workspace, log_func, modify_func, debug=False):
    """
    Initiate an instance.

    :param creatives: a list of creatives.
    :param protocol: a protocol, `https` or `http`.
    :param ports: a list of port numbers.
    :param browser_count: a number of browsers to be launched.
    :param phantomjs: the path to the phantomjs command.
    :param browserjs: the path to the browser.js file.
    :param cookie_dir: the directory that contains cookies, which will be used by browsers during scanning.
    :param workspace: the directory where the html files and scan logs are saved.
    :param log_func: the function called for passing the urls and issue ids found during this scanning process.
    :param modify_func: the function called for modifying the creatives.
    :param debug: Turn on the debug mode, which temporarily to allow access to private network that host test creatives.
    """
    self.creatives = creatives
    self.protocol = protocol
    self.ports = ports
    self.browser_count = browser_count
    self.phantomjs = phantomjs
    self.browserjs = browserjs
    self.cookie_dir = cookie_dir
    self.workspace = workspace
    self.log_func = log_func
    self.modify_func = modify_func
    self.threads = []
    self.hostname = socket.gethostname()
    self.debug = debug

  def _create_urls_to_scan(self, creatives, port):
    """
    Create urls to scan these creatives.

    :param creatives: a list of creatives.
    :param port: the port number used to scan these creatives.
    :return: a dictionary of a creative id and its url to be scanned.
    """
    url_dict = {}
    for creative in creatives:
      self.modify_func(creative)
      snippet = creative.modified_scan_snippet if self.protocol == 'https' else creative.scan_snippet

      if snippet:
        url_obj = None
        if re.match(r'^http', snippet, re.IGNORECASE):
          url_obj = {
            'url': snippet,
            'hosted_locally': 'false'
          }
        else:
          save_file = '%s/%s.html' % (self.workspace, creative.creative_id)
          self.create_html(snippet, save_file)
          url_obj = {
            'url': '%s://%s:%d/%s' % (self.protocol, self.hostname, port, save_file),
            'hosted_locally': 'true'
          }
        url_dict[str(creative.creative_id)] = url_obj
    return url_dict

  def start(self):
    """
    Start browsers.
    """
    # Caluculate how many creatives are allotted to each browser process.
    allotment = len(self.creatives) / self.browser_count
    if allotment == 0:
      allotment = 1
    if len(self.creatives) % allotment != 0:
      allotment += 1

    for i in xrange(0, self.browser_count):
      display_id = i + 1
      log_dir = '%s/%d' % (self.workspace, i)

      allotted_creatives = self.creatives[(i * allotment):((i + 1) * allotment)]
      allotted_port = self.ports[i % len(self.ports)]

      url_dict = self._create_urls_to_scan(allotted_creatives, allotted_port)
      if len(url_dict) > 0:
        thread = BrowserHost(url_dict, self.protocol, self.phantomjs, self.browserjs, display_id, log_dir, self.cookie_dir, self.log_func, self.debug)
        thread.start()
        self.threads.append(thread)
        time.sleep(1)

  def wait(self):
    """
    Wait until all the threads done.
    """
    for thread in self.threads:
      if thread and thread.isAlive():
        thread.join()

  def shutdown(self):
    """
    Cloase all the browser threads.
    """
    for thread in self.threads:
      if thread and thread.isAlive():
        thread.shutdown()
<|endoftext|>


<commit_before># __BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
# __END_LICENSE__

from datetime import datetime

from django.db import models
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType

from django.template.loader import get_template
from django.template import Context

from geocamUtil.models import AbstractEnumModel, ServerIdModel
from geocamUtil.models.UuidField import UuidField, makeUuid
from geocamUtil.modelJson import modelToDict
from geocamUtil.defaultSettings import HOSTNAME

from treebeard.mp_tree import MP_Node
from taggit.models import TagBase, ItemBase
from taggit.managers import TaggableManager

class HierarchichalTag(TagBase, MP_Node):
    node_order_by = ['name']
    abbreviation = models.CharField(max_length=8, blank=True)
    description = models.TextField(blank=True, null=True)

    def preSave(self):
        self.name = self.name.lower()

    def getTreeJson(self):
        """ get the JSON block for fancytree """
        title = self.name
        if self.abbreviation:
            title = title + '(' + self.abbreviation + ')'
        result = {"title": title,
                  "key": self.pk,
                  "tooltip": self.description,
                  "extraClasses": 'tag',
                  "lazy": True,
                  "data": {"parentId": None,
                           "name": self.name,
                           "abbreviation": self.abbreviation,
                           "description": self.description
                           }
                  }
        parent = self.get_parent()
        if parent:
            result['data']['parentId'] = parent.pk
        return result

    @classmethod
    def getFormFields(cls):
        return ["abbreviation", "name", "description"]

    class Meta(TagBase.Meta):
        db_table = 'taggit_tag'

class TaggedNote(ItemBase):
    uuid = UuidField(default=makeUuid(), primary_key=True)
    content_object = models.ForeignKey(settings.XGDS_NOTES_NOTE_MODEL)
    tag = models.ForeignKey('HierarchichalTag', related_name='tags')

class MissionDay(models.Model):
    name = models.TextField()
    start_time = models.DateTimeField()
    end_time = models.DateTimeField()

    def __unicode__(self):
        return unicode(self.name)

    @property
    def range(self):
        return (self.start_time, self.end_time)

    def contains(self, date_time):
        return date_time >= self.start_time and date_time <= self.end_time

    @classmethod
    def for_datetime(cls, date_time):
        qs = cls.objects.filter(start_time__lte=date_time, end_time__gte=date_time)
        if qs.count() == 1:
            return qs[0]
        elif qs.count() == 0:
            return None
        else:
            raise Exception("Found more than one MissionDay for datetime %s" % str(date_time))

class Role(AbstractEnumModel):
    pass

class Location(AbstractEnumModel):
    pass

class AbstractUserSession(models.Model):
    role = models.ForeignKey(Role)

    @classmethod
    def getFormFields(cls):
        return ['role']

    class Meta:
        abstract = True

class UserSession(AbstractUserSession):
    location = models.ForeignKey(Location)

    @classmethod
    def getFormFields(cls):
        return ['role',
                'location']

class AbstractNote(models.Model):
    ''' Abstract base class for notes
    '''
#     # custom id field for uniqueness
#     id = models.CharField(max_length=128,
#                           unique=True, blank=False,
#                           editable=False, primary_key=True)

    # Override this to specify a list of related fields
    # to be join-query loaded when notes are listed, as an optimization
    prefetch_related_fields = []

    # select related for forward releationships.  prefetch for reverse or for many to many.
    select_related_fields = ['author', 'role']

    uuid = UuidField(default=makeUuid())

    show_on_map = models.BooleanField(default=False) # broadcast this note on the map by default

    event_time = models.DateTimeField(null=True, blank=False)
    creation_time = models.DateTimeField(blank=True, default=datetime.utcnow(), editable=False)
    modification_time = models.DateTimeField(blank=True, default=datetime.utcnow(), editable=False)

    author = models.ForeignKey(User)
    role = models.ForeignKey(Role, null=True)
    location = models.ForeignKey(Location, null=True)

    content = models.TextField(blank=True, null=True)

    tags = TaggableManager(through=TaggedNote, blank=True)

    content_type = models.ForeignKey(ContentType, null=True, blank=True)
    object_id = models.CharField(max_length=128, null=True, blank=True)
    content_object = GenericForeignKey('content_type', 'object_id')

    class Meta:
        abstract = True
        ordering = ['event_time']
        permissions = (
            ('edit_all_notes', 'Can edit notes authored by others.'),
        )

    def __unicode__(self):
        return "%s: %s" % (self.event_time, self.content)

    @classmethod
    def getFormFields(cls):
        return ['event_time',
                'content',
                'tags',
                'show_on_map',
                ]

# For sphinx to work right - these next 2 lines *MUST* be in the inherited concrete
# class. Will fail miserably if it tries to search on this abstract class.
#    if "SphinxSearch" in globals():
#        search = SphinxSearch()

    def adjustedEventTime(self):
        """
        This is so derived classes can adapt event time for display
        """
        return self.event_time

    def calculateDelayedEventTime(self):
        """
        This is so derived classes can adapt event time in the case of being in "delay" mode
        """
        return self.event_time

    def to_kml(self, animated=False, timestampLabels=False, ignoreLabels=None):
        "Define me, or override in your subclass"
        raise NotImplementedError

    def getLabel(self):
        """
        Get the KML note label, use tag if exists or content otherwise
        """
        result = ""
#         if self.tags:
#             tags_list = self.tags.all()
#             result = "".join(str(x) + " " for x in tags_list)
        if result == "":
            result = self.content[:12]
        return result

    def getAuthorName(self):
        authorname = self.author.username
        if self.author.first_name:
            authorname = self.author.first_name
            if self.author.last_name:
                authorname = authorname + " " + self.author.last_name
        return authorname

    def toMapDict(self):
        """
        Return a reduced dictionary that will be turned to JSON for rendering in a map
        """
        result = modelToDict(self)

        result['type'] = self.__class__.__name__
        result['author'] = self.getAuthorName()
        result['role'] = self.role.display_name
        result['event_time'] = self.adjustedEventTime()

        if self.label:
            if 'tags' in result:
                result['tags'].append(self.label.display_name)
            else:
                result['tags'] = [self.label.display_name]
        else:
            result['tags'] = ''

        return result

    @staticmethod
    def getMapBoundedQuery(minLon, minLat, maxLon, maxLat, today=False):
        """
        Implement this in your note that has a position to return a query that will filter notes by position
        """
        return None

    def getChannels(self):
        """ for sse, return a list of channels """
        return [settings.XGDS_NOTES2_NOTES_CHANNEL]

class Note(AbstractNote):
    '''
    Non-Abstract note class (with no position)
    '''
    pass

class LocatedNote(AbstractNote):
    """ This is a basic note with a location, pulled from the current settings for geocam track past position model.
    """
    position = models.ForeignKey(settings.GEOCAM_TRACK_PAST_POSITION_MODEL, null=True, blank=True)

    def to_kml(self, animated=False, timestampLabels=False, ignoreLabels=None):
        if ignoreLabels is None:
            ignoreLabels = []
        # if no label is associated with the note, we'll fall back to a timetamp-based label anyway.
        notelabel = None
        if timestampLabels or (self.label and self.label.value in ignoreLabels):
            atime = self.adjustedEventTime()
            notelabel = "%02d:%02d" % (atime.hour, atime.minute)
        else:
            notelabel = self.getLabel()

        if not notelabel:
            atime = self.adjustedEventTime()
            try:
                altitude = self.position.altitude
                notelabel = "%s %02d:%02d" % (altitude, atime.hour, atime.minute)
            except:
                notelabel = "%02d:%02d" % (atime.hour, atime.minute)

        return get_template('note.kml').render(Context({
            'note': self,
            'animated': animated,
            'notelabel': notelabel,
        }))

    def toMapDict(self):
        """
        Return a reduced dictionary that will be turned to JSON for rendering in a map
        """
        result = AbstractNote.toMapDict(self)
        if result['position']:
            result['lon'] = self.position.longitude
            result['lat'] = self.position.latitude
            try:
                result['heading'] = self.position.heading
            except:
                result['heading'] = ''
            try:
                result['altitude'] = self.position.altitude
            except:
                result['altitude'] = ''

        else:
            result['lat'] = ''
            result['lon'] = ''
            result['heading'] = ''
            result['altitude'] = ''

        del(result['position'])
        return result

    @staticmethod
    def getMapBoundedQuery(minLon, minLat, maxLon, maxLat, today=False):
        bounds = {'minLon': minLon,
                  'minLat': minLat,
                  'maxLon': maxLon,
                  'maxLat': maxLat}
        qstring = 'select n.* from %s n, %s a where (n.position_id = a.id) and '
        qstring += '(a.longitude<=%(maxLon).9f and a.longitude>=%(minLon).9f) and '
        qstring += '(a.latitude>=%(minLat).9f and a.latitude<=%(maxLat).9f) union '
        qstring += 'select n.* from plrpExplorer_note n, plrpExplorer_newassetposition b where (n.new_asset_position_id = b.id) and '
        qstring += '(b.longitude<=%(maxLon).9f and b.longitude>=%(minLon).9f) and '
        qstring += '(b.latitude>=%(minLat).9f and b.latitude<=%(maxLat).9f);'

        result = qstring % (settings.XGDS_NOTES_NOTE_MODEL, settings.GEOCAM_TRACK_PAST_POSITION_MODEL, bounds)
        return result

<commit_msg>notes now use server id model<commit_after># __BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
# __END_LICENSE__

from datetime import datetime

from django.db import models
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType

from django.template.loader import get_template
from django.template import Context

from geocamUtil.models import AbstractEnumModel
from geocamUtil.models.ServerIdModel import ServerIdModel
from geocamUtil.models.UuidField import UuidField, makeUuid
from geocamUtil.modelJson import modelToDict
from geocamUtil.defaultSettings import HOSTNAME

from treebeard.mp_tree import MP_Node
from taggit.models import TagBase, ItemBase
from taggit.managers import TaggableManager

class HierarchichalTag(TagBase, MP_Node):
    node_order_by = ['name']
    abbreviation = models.CharField(max_length=8, blank=True)
    description = models.TextField(blank=True, null=True)

    def preSave(self):
        self.name = self.name.lower()

    def getTreeJson(self):
        """ get the JSON block for fancytree """
        title = self.name
        if self.abbreviation:
            title = title + '(' + self.abbreviation + ')'
        result = {"title": title,
                  "key": self.pk,
                  "tooltip": self.description,
                  "extraClasses": 'tag',
                  "lazy": True,
                  "data": {"parentId": None,
                           "name": self.name,
                           "abbreviation": self.abbreviation,
                           "description": self.description
                           }
                  }
        parent = self.get_parent()
        if parent:
            result['data']['parentId'] = parent.pk
        return result

    @classmethod
    def getFormFields(cls):
        return ["abbreviation", "name", "description"]

    class Meta(TagBase.Meta):
        db_table = 'taggit_tag'

class TaggedNote(ItemBase):
    uuid = UuidField(default=makeUuid(), primary_key=True)
    content_object = models.ForeignKey(settings.XGDS_NOTES_NOTE_MODEL)
    tag = models.ForeignKey('HierarchichalTag', related_name='tags')

class MissionDay(models.Model):
    name = models.TextField()
    start_time = models.DateTimeField()
    end_time = models.DateTimeField()

    def __unicode__(self):
        return unicode(self.name)

    @property
    def range(self):
        return (self.start_time, self.end_time)

    def contains(self, date_time):
        return date_time >= self.start_time and date_time <= self.end_time

    @classmethod
    def for_datetime(cls, date_time):
        qs = cls.objects.filter(start_time__lte=date_time, end_time__gte=date_time)
        if qs.count() == 1:
            return qs[0]
        elif qs.count() == 0:
            return None
        else:
            raise Exception("Found more than one MissionDay for datetime %s" % str(date_time))

class Role(AbstractEnumModel):
    pass

class Location(AbstractEnumModel):
    pass

class AbstractUserSession(models.Model):
    role = models.ForeignKey(Role)

    @classmethod
    def getFormFields(cls):
        return ['role']

    class Meta:
        abstract = True

class UserSession(AbstractUserSession):
    location = models.ForeignKey(Location)

    @classmethod
    def getFormFields(cls):
        return ['role',
                'location']

class AbstractNote(ServerIdModel):
    ''' Abstract base class for notes
    '''
#     # custom id field for uniqueness
#     id = models.CharField(max_length=128,
#                           unique=True, blank=False,
#                           editable=False, primary_key=True)

    # Override this to specify a list of related fields
    # to be join-query loaded when notes are listed, as an optimization
    prefetch_related_fields = []

    # select related for forward releationships.  prefetch for reverse or for many to many.
    select_related_fields = ['author', 'role']

    uuid = UuidField(default=makeUuid())

    show_on_map = models.BooleanField(default=False) # broadcast this note on the map by default

    event_time = models.DateTimeField(null=True, blank=False)
    creation_time = models.DateTimeField(blank=True, default=datetime.utcnow(), editable=False)
    modification_time = models.DateTimeField(blank=True, default=datetime.utcnow(), editable=False)

    author = models.ForeignKey(User)
    role = models.ForeignKey(Role, null=True)
    location = models.ForeignKey(Location, null=True)

    content = models.TextField(blank=True, null=True)

    tags = TaggableManager(through=TaggedNote, blank=True)

    content_type = models.ForeignKey(ContentType, null=True, blank=True)
    object_id = models.CharField(max_length=128, null=True, blank=True)
    content_object = GenericForeignKey('content_type', 'object_id')

    class Meta:
        abstract = True
        ordering = ['event_time']
        permissions = (
            ('edit_all_notes', 'Can edit notes authored by others.'),
        )

    def __unicode__(self):
        return "%s: %s" % (self.event_time, self.content)

    @classmethod
    def getFormFields(cls):
        return ['event_time',
                'content',
                'tags',
                'show_on_map',
                ]

# For sphinx to work right - these next 2 lines *MUST* be in the inherited concrete
# class. Will fail miserably if it tries to search on this abstract class.
#    if "SphinxSearch" in globals():
#        search = SphinxSearch()

    def adjustedEventTime(self):
        """
        This is so derived classes can adapt event time for display
        """
        return self.event_time

    def calculateDelayedEventTime(self):
        """
        This is so derived classes can adapt event time in the case of being in "delay" mode
        """
        return self.event_time

    def to_kml(self, animated=False, timestampLabels=False, ignoreLabels=None):
        "Define me, or override in your subclass"
        raise NotImplementedError

    def getLabel(self):
        """
        Get the KML note label, use tag if exists or content otherwise
        """
        result = ""
#         if self.tags:
#             tags_list = self.tags.all()
#             result = "".join(str(x) + " " for x in tags_list)
        if result == "":
            result = self.content[:12]
        return result

    def getAuthorName(self):
        authorname = self.author.username
        if self.author.first_name:
            authorname = self.author.first_name
            if self.author.last_name:
                authorname = authorname + " " + self.author.last_name
        return authorname

    def toMapDict(self):
        """
        Return a reduced dictionary that will be turned to JSON for rendering in a map
        """
        result = modelToDict(self)

        result['type'] = self.__class__.__name__
        result['author'] = self.getAuthorName()
        result['role'] = self.role.display_name
        result['event_time'] = self.adjustedEventTime()

        if self.label:
            if 'tags' in result:
                result['tags'].append(self.label.display_name)
            else:
                result['tags'] = [self.label.display_name]
        else:
            result['tags'] = ''

        return result

    @staticmethod
    def getMapBoundedQuery(minLon, minLat, maxLon, maxLat, today=False):
        """
        Implement this in your note that has a position to return a query that will filter notes by position
        """
        return None

    def getChannels(self):
        """ for sse, return a list of channels """
        return [settings.XGDS_NOTES2_NOTES_CHANNEL]

class Note(AbstractNote):
    '''
    Non-Abstract note class (with no position)
    '''
    pass

class LocatedNote(AbstractNote):
    """ This is a basic note with a location, pulled from the current settings for geocam track past position model.
    """
    position = models.ForeignKey(settings.GEOCAM_TRACK_PAST_POSITION_MODEL, null=True, blank=True)

    def to_kml(self, animated=False, timestampLabels=False, ignoreLabels=None):
        if ignoreLabels is None:
            ignoreLabels = []
        # if no label is associated with the note, we'll fall back to a timetamp-based label anyway.
        notelabel = None
        if timestampLabels or (self.label and self.label.value in ignoreLabels):
            atime = self.adjustedEventTime()
            notelabel = "%02d:%02d" % (atime.hour, atime.minute)
        else:
            notelabel = self.getLabel()

        if not notelabel:
            atime = self.adjustedEventTime()
            try:
                altitude = self.position.altitude
                notelabel = "%s %02d:%02d" % (altitude, atime.hour, atime.minute)
            except:
                notelabel = "%02d:%02d" % (atime.hour, atime.minute)

        return get_template('note.kml').render(Context({
            'note': self,
            'animated': animated,
            'notelabel': notelabel,
        }))

    def toMapDict(self):
        """
        Return a reduced dictionary that will be turned to JSON for rendering in a map
        """
        result = AbstractNote.toMapDict(self)
        if result['position']:
            result['lon'] = self.position.longitude
            result['lat'] = self.position.latitude
            try:
                result['heading'] = self.position.heading
            except:
                result['heading'] = ''
            try:
                result['altitude'] = self.position.altitude
            except:
                result['altitude'] = ''

        else:
            result['lat'] = ''
            result['lon'] = ''
            result['heading'] = ''
            result['altitude'] = ''

        del(result['position'])
        return result

    @staticmethod
    def getMapBoundedQuery(minLon, minLat, maxLon, maxLat, today=False):
        bounds = {'minLon': minLon,
                  'minLat': minLat,
                  'maxLon': maxLon,
                  'maxLat': maxLat}
        qstring = 'select n.* from %s n, %s a where (n.position_id = a.id) and '
        qstring += '(a.longitude<=%(maxLon).9f and a.longitude>=%(minLon).9f) and '
        qstring += '(a.latitude>=%(minLat).9f and a.latitude<=%(maxLat).9f) union '
        qstring += 'select n.* from plrpExplorer_note n, plrpExplorer_newassetposition b where (n.new_asset_position_id = b.id) and '
        qstring += '(b.longitude<=%(maxLon).9f and b.longitude>=%(minLon).9f) and '
        qstring += '(b.latitude>=%(minLat).9f and b.latitude<=%(maxLat).9f);'

        result = qstring % (settings.XGDS_NOTES_NOTE_MODEL, settings.GEOCAM_TRACK_PAST_POSITION_MODEL, bounds)
        return result

<|endoftext|>


<commit_before>from lexos.helpers.general_functions import get_encoding, make_preview_from, \
    generate_d3_object, merge_list, load_stastic, matrix_to_dict, \
    dict_to_matrix, html_escape, apply_function_exclude_tags, decode_bytes
import unittest

class TestGeneralFunctions(unittest.TestCase):
    def test_get_encoding(self):
        assert get_encoding(b"asdf") == "ascii"

    def test_make_preview_from(self):
        newline = '\n'
        one_char = "x"
        less_than_500_char = "modgaecq"
        str_250 = "gjzeqagitanbwnuwjkfbtpixhkcxltlcmvrbunoxovjzhyoiptckkxmd" \
                  "brcnshyefsrqexbdeczdbqjvprgiyjwwsacutlahuwhmscyuwkqxfnxq" \
                  "zxyozedtwmrztwzzvoxrjnaypzbrkxfytpqeqmemxylvrvgtsthbalai" \
                  "byzxnoxxbtofhnpdepatvbihjoungenjidckhepgdlsmnrbqdgaalidw" \
                  "gccbardglcnedcqqxduuaauzyv"
        str_500 = str_250 + str_250
        more_than_500_char_even = \
            str_250 + less_than_500_char + less_than_500_char + str_250
        more_than_500_char_odd = \
            str_250 + less_than_500_char + one_char + less_than_500_char + \
            str_250
        middle = '\u2026 ' + newline + newline + '\u2026'
        assert make_preview_from(less_than_500_char) == less_than_500_char
        assert make_preview_from(str_500) == str_500

        assert make_preview_from(
            more_than_500_char_odd) == str_250 + middle + str_250
        assert make_preview_from(
            more_than_500_char_even) == str_250 + middle + str_250

    def test_generate_d3_object(self):
        assert generate_d3_object(
            {'a': 1, 'b': 2, 'c': 3, 'd': 4},
            "object", "word", "count") == {'name': 'object', 'children': [{
            'word': 'a', 'count': 1}, {'word': 'b', 'count': 2}, {
            'word': 'c', 'count': 3}, {'word': 'd', 'count': 4}]}

    def test_merge_list(self):
        assert merge_list(
            [{"a": 1, "b": 2}, {"c": 3, "d": 4}]) == {
                   'a': 1, 'b': 2, 'c': 3, 'd': 4}

    def test_load_stastic(self):
        assert load_stastic(
            "this is a string string") == {
            "this": 1, "is": 1, "a": 1, "string": 2}

    def test_matrix_to_dict(self):
        assert matrix_to_dict(
            [['', 'a', 'b', 'c', 'd'], [0, 1, 2, 3, 4]]) == \
               [{'a': 1, 'b': 2, 'c': 3, 'd': 4}]

    def test_dict_to_matrix(self):
        assert dict_to_matrix(
            [{'a': 1, 'b': 2, 'c': 3, 'd': 4}]) == (
            [['', 'a', 'b', 'c', 'd'], [0, 1, 2, 3, 4]],
            ['a', 'b', 'c', 'd'])

    def test_html_escape(self):
        assert html_escape("&") == "&"
        assert html_escape('"') == """
        assert html_escape("'") == "'"
        assert html_escape(">") == ">"
        assert html_escape("<") == "<"

    def test_apply_function_exclude_tags(self):
        input_str = "<tag>asdf</tag>"

        def dummy_function(input_string):
            return input_string + input_string

        assert apply_function_exclude_tags(
            input_str, [dummy_function]) == "<tag>asdfasdf</tag>"

    def test_decode_bytes(self):
        assert decode_bytes(u'哈哈'.encode('gb2312')) == "¹þ¹þ"
<commit_msg>fix indentation errors<commit_after>from lexos.helpers.general_functions import get_encoding, make_preview_from, \
    generate_d3_object, merge_list, load_stastic, matrix_to_dict, \
    dict_to_matrix, html_escape, apply_function_exclude_tags, decode_bytes
import unittest

class TestGeneralFunctions(unittest.TestCase):
    def test_get_encoding(self):
        assert get_encoding(b"asdf") == "ascii"

    def test_make_preview_from(self):
        newline = '\n'
        one_char = "x"
        less_than_500_char = "modgaecq"
        str_250 = "gjzeqagitanbwnuwjkfbtpixhkcxltlcmvrbunoxovjzhyoiptckkxmd" \
                  "brcnshyefsrqexbdeczdbqjvprgiyjwwsacutlahuwhmscyuwkqxfnxq" \
                  "zxyozedtwmrztwzzvoxrjnaypzbrkxfytpqeqmemxylvrvgtsthbalai" \
                  "byzxnoxxbtofhnpdepatvbihjoungenjidckhepgdlsmnrbqdgaalidw" \
                  "gccbardglcnedcqqxduuaauzyv"
        str_500 = str_250 + str_250
        more_than_500_char_even = \
            str_250 + less_than_500_char + less_than_500_char + str_250
        more_than_500_char_odd = \
            str_250 + less_than_500_char + one_char + less_than_500_char + \
            str_250
        middle = '\u2026 ' + newline + newline + '\u2026'
        assert make_preview_from(less_than_500_char) == less_than_500_char
        assert make_preview_from(str_500) == str_500

        assert make_preview_from(
            more_than_500_char_odd) == str_250 + middle + str_250
        assert make_preview_from(
            more_than_500_char_even) == str_250 + middle + str_250

    def test_generate_d3_object(self):
        assert generate_d3_object(
            {'a': 1, 'b': 2, 'c': 3, 'd': 4},
            "object", "word", "count") == {'name': 'object', 'children': [{
                'word': 'a', 'count': 1}, {'word': 'b', 'count': 2}, {
                'word': 'c', 'count': 3}, {'word': 'd', 'count': 4}]}

    def test_merge_list(self):
        assert merge_list([{"a": 1, "b": 2}, {"c": 3, "d": 4}]) == {
            'a': 1, 'b': 2, 'c': 3, 'd': 4}

    def test_load_stastic(self):
        assert load_stastic(
            "this is a string string") == {
            "this": 1, "is": 1, "a": 1, "string": 2}

    def test_matrix_to_dict(self):
        assert matrix_to_dict(
            [['', 'a', 'b', 'c', 'd'], [0, 1, 2, 3, 4]]) == \
               [{'a': 1, 'b': 2, 'c': 3, 'd': 4}]

    def test_dict_to_matrix(self):
        assert dict_to_matrix(
            [{'a': 1, 'b': 2, 'c': 3, 'd': 4}]) == (
            [['', 'a', 'b', 'c', 'd'], [0, 1, 2, 3, 4]],
            ['a', 'b', 'c', 'd'])

    def test_html_escape(self):
        assert html_escape("&") == "&"
        assert html_escape('"') == """
        assert html_escape("'") == "'"
        assert html_escape(">") == ">"
        assert html_escape("<") == "<"

    def test_apply_function_exclude_tags(self):
        input_str = "<tag>asdf</tag>"

        def dummy_function(input_string):
            return input_string + input_string

        assert apply_function_exclude_tags(
            input_str, [dummy_function]) == "<tag>asdfasdf</tag>"

    def test_decode_bytes(self):
        assert decode_bytes(u'哈哈'.encode('gb2312')) == "¹þ¹þ"
<|endoftext|>