<commit_before>from __future__ import print_function

import sys
import os
import traceback
import pprint
from getopt import getopt
from glob import glob
from threading import Thread, Event
if sys.version_info[0] < 3: #major
    from Queue import Queue, Empty
else:
    from queue import Queue, Empty
import json
import pickle
import marshal

import yara

"""
A command line YARA rules scanning utility...

[mjdorma@gmail.com]
"""

DEFAULT_THREAD_POOL = 4

class Scanner:
    """Scan process IDs or file paths."""
    def __init__(self, pids=[], paths=[],
                       rules_rootpath=yara.YARA_RULES_ROOT,
                       whitelist=[],
                       blacklist=[],
                       thread_pool=DEFAULT_THREAD_POOL,
                       fast_match=False,
                       externals={}):
        """Scanner yields scan results in a tuple of (path|pid, result)

        kwargs:
            pids - list of process ids to scan
            paths - globbed out list of paths to scan
            rules_rootpath - path to the root of the rules directory
            whitelist - whitelist of rules to use in scanner
            blacklist - blacklist of rules to not use in scanner
            thread_pool - number of threads to use in scanner
            fast_match - scan fast True / False
            externals - externally defined variables
        """
        self._rules = yara.load_rules(rules_rootpath=rules_rootpath,
                                      blacklist=blacklist,
                                      whitelist=whitelist,
                                      includes=True,
                                      externals=externals,
                                      fast_match=fast_match)
        print(self._rules, file=sys.stderr)
        self._jq = Queue()
        self._rq = Queue()
        self._empty = Event()
        self.scanned = 0
        self.quit = Event()

        for i in range(thread_pool):
            t = Thread(target=self._run)
            t.start()

        if pids:
            for pid in pids:
                self._jq.put((self._rules.match_proc, pid))
        else:
            for path in paths:
                for p in glob(path):
                    if os.path.isdir(p):
                        for dirpath, dirnames, filenames in os.walk(p):
                            for filename in filenames:
                                a = os.path.join(dirpath, filename)
                                self._jq.put((self._rules.match_path, a))
                    else:
                        self._jq.put((self._rules.match_path, p))
        self._jq.put(None)

    @property
    def sq_size(self):
        """contains the current scan queue size"""
        return self._jq.unfinished_tasks

    @property
    def rq_size(self):
        """contains the current result queue size"""
        return self._rq.unfinished_tasks

    def _run(self):
        while not self._empty.is_set() and not self.quit.is_set():
            try:
                job = self._jq.get(timeout=0.1)
            except Empty:
                continue
            if job is None:
                self._jq.task_done()
                self._jq.join()
                self._rq.put(None)
                self._empty.set()
                break
            try:
                self.scanned += 1
                f, a = job
                r = f(a)
            except Exception:
                r = traceback.format_exc()
            finally:
                self._rq.put((a, r))
                self._jq.task_done()

    def __iter__(self):
        return self

    def __next__(self):
        r = self._rq.get()
        self._rq.task_done()
        if r is None:
            raise StopIteration()
        return r

    def next(self):
        return self.__next__()

__help__ = """
NAME scan - scan files or processes against yara signatures

SYNOPSIS
    python scan.py [OPTIONS]... [FILE(s)|PID(s)]...

DESCRIPTION

Scan control:
    --proc
        scan PIDs. This indicate that trailing args are PIDS

    --ext=
        file extension inclusion filter (comma separate list)

    --thread_pool=%s
        size of the thread pool used for scanning

    --fast
        fast matching mode

Rule control:
    -d <identifier>=<value>
        define external variable.

    -w, --whitelist=
        whitelist of comma separated YARA namespaces to include in scan

    -b, --blacklist=
        blacklist of comma separated YARA namespaces to exclude from scan

    --list
        list available YARA namespaces

    --root=(env[YARA_RULES] or <pkg>/yara/rules/)
        set the YARA_RULES path (path to the root of the rules directory)

Output control:
    --fmt=dict
        output format [dict|pprint|json|pickle|marshal]
    -o
        outfile path -> redirect stdout results to outfile

    -t  [tag1,tag2,tag3, ...]
        print matches that contain specific tags and filter out the rest

    -i  [ident1,ident2,ident3, ...]
        print matches that contain specific identifiers and filter out the rest

""" % DEFAULT_THREAD_POOL

def match_filter(tags_filter, idents_filter, res):
    if tags_filter is not None:
        new_res = {}
        for ns, matches in res.iteritems():
            for match in matches:
                if tags_filter.intersection(match['tags']):
                    mlist = new_res.get(ns, [])
                    mlist.append(match)
                    new_res[ns] = mlist
        res = new_res
    if idents_filter is not None:
        new_res = {}
        for ns, matches in res.iteritems():
            for match in matches:
                idents = [s['identifier'] for s in match['strings']]
                if idents_filter.intersection(idents):
                    mlist = new_res.get(ns, [])
                    mlist.append(match)
                    new_res[ns] = mlist
        res = new_res
    return res

def main(args):

    try:
        opts, args = getopt(args, 'hw:b:t:o:i:d:', ['proc',
                                              'whitelist=',
                                              'blacklist=',
                                              'thread_pool=',
                                              'root=',
                                              'list',
                                              'fmt=',
                                              'fast',
                                              'help'])
    except Exception as exc:
        print("Getopt error: %s" % (exc), file=sys.stderr)
        return -1

    whitelist = []
    blacklist = []
    thread_pool = 4
    externals = {}
    fast_match = False
    pids = []
    paths = args
    rules_rootpath = yara.YARA_RULES_ROOT
    list_rules = False
    stream = sys.stdout
    stream_fmt = str
    tags_filter = None
    idents_filter = None

    for opt, arg in opts:
        if opt in ['-h', '--help']:
            print(__help__)
            return 0
        elif opt in ['--root']:
            if os.path.exists(rules_rootpath):
                print("root path '%s' does not exist" % arg, file=sys.stderr)
                return -1
            rules_rootpath = arg
        elif opt in ['-d']:
            try:
                externals.update(eval("dict(%s)" % arg))
            except SyntaxError:
                print("external '%s' syntax error" % arg, file=sys.stderr)
                return -1
        elif opt in ['--fast']:
            fast_match = True
        elif opt in ['--list']:
            list_rules = True
        elif opt in ['-o']:
            stream = open(arg, 'wb')
        elif opt in ['-t']:
            tags_filter = set(arg.split(','))
        elif opt in ['-i']:
            idents_filter = arg
        elif opt in ['-w', '--whitelist']:
            whitelist = arg.split(',')
        elif opt in ['b', '--blacklist']:
            blacklist = arg.split(',')
        elif opt in ['--fmt']:
            if arg == 'pickle':
                stream_fmt = pickle.dumps
            elif arg == 'json':
                stream_fmt = lambda a: json.dumps(a, ensure_ascii=False,
                                                check_circular=False, indent=4)
            elif arg == 'pprint':
                stream_fmt = pprint.pformat
            elif arg == 'marshal':
                stream_fmt = marshal.dumps
            elif arg == 'dict':
                stream_fmt = str
            else:
                print("unknown output format %s" % arg, file=sys.stderr)
                return -1
        elif opt in ['t', '--thread_pool']:
            try:
                thread_pool = int(arg)
            except ValueError:
                print("-t param %s was not an int" % (arg), file=sys.stderr)
                return -1
        elif opt in ['--proc']:
            paths = []
            if not args:
                print("no PIDs specified")
                return -1
            for pid in args:
                try:
                    pids.append(int(pid))
                except ValueError:
                    print("PID %s was not an int" % (pid), file=sys.stderr)

    if list_rules is True:
        rules = yara.load_rules(rules_rootpath=rules_rootpath,
                                blacklist=blacklist,
                                whitelist=whitelist)
        print(rules)
        return 0

    scanner = Scanner(paths=paths, pids=pids,
                      rules_rootpath=rules_rootpath,
                      whitelist=whitelist,
                      blacklist=blacklist,
                      thread_pool=thread_pool,
                      fast_match=fast_match,
                      externals=externals)

    try:
        status_template = "scan queue: %-7s result queue: %-7s"
        i = 0
        for arg, res in scanner:
            i += 1
            if i % 20 == 0:
                status = status_template % (scanner.sq_size, scanner.rq_size)
                sys.stderr.write("\b" * len(status) + status)

            res = match_filter(tags_filter, idents_filter)
            if res:
                print("<scan arg='%s'>" % arg, file=self.stream)
                print(stream_fmt(res), file=self.stream)
                print("</scan>", file=self.stream)
    finally:
        scanner.quit.set()
        status = status_template % (scanner.sq_size, scanner.rq_size)
        sys.stderr.write("\b" * len(status) + status)
        print("\nscanned %s items... done." % scanner.scanned, file=sys.stderr)

if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))
<commit_msg>can now specify a specific rule file using the tag -r or --rule=<commit_after>from __future__ import print_function

import sys
import os
import traceback
import pprint
from getopt import getopt
from glob import glob
from threading import Thread, Event
if sys.version_info[0] < 3: #major
    from Queue import Queue, Empty
else:
    from queue import Queue, Empty
import json
import pickle
import marshal

import yara

"""
A command line YARA rules scanning utility...

[mjdorma@gmail.com]
"""

DEFAULT_THREAD_POOL = 4

class Scanner:
    """Scan process IDs or file paths."""
    def __init__(self, pids=[], paths=[],
                       rules_rootpath=yara.YARA_RULES_ROOT,
                       whitelist=[],
                       blacklist=[],
                       rule_filepath=None,
                       thread_pool=DEFAULT_THREAD_POOL,
                       fast_match=False,
                       externals={}):
        """Scanner yields scan results in a tuple of (path|pid, result)

        kwargs:
            pids - list of process ids to scan
            paths - globbed out list of paths to scan
            rules_rootpath - path to the root of the rules directory
            whitelist - whitelist of rules to use in scanner
            blacklist - blacklist of rules to not use in scanner
            thread_pool - number of threads to use in scanner
            fast_match - scan fast True / False
            externals - externally defined variables
        """
        if rule_filepath is None:
            self._rules = yara.load_rules(rules_rootpath=rules_rootpath,
                                      blacklist=blacklist,
                                      whitelist=whitelist,
                                      includes=True,
                                      externals=externals,
                                      fast_match=fast_match)
        else:
            self._rules = yara.compile(filepath=rules_filepath)

        print(self._rules, file=sys.stderr)
        self._jq = Queue()
        self._rq = Queue()
        self._empty = Event()
        self.scanned = 0
        self.quit = Event()

        for i in range(thread_pool):
            t = Thread(target=self._run)
            t.start()

        if pids:
            for pid in pids:
                self._jq.put((self._rules.match_proc, pid))
        else:
            for path in paths:
                for p in glob(path):
                    if os.path.isdir(p):
                        for dirpath, dirnames, filenames in os.walk(p):
                            for filename in filenames:
                                a = os.path.join(dirpath, filename)
                                self._jq.put((self._rules.match_path, a))
                    else:
                        self._jq.put((self._rules.match_path, p))
        self._jq.put(None)

    @property
    def sq_size(self):
        """contains the current scan queue size"""
        return self._jq.unfinished_tasks

    @property
    def rq_size(self):
        """contains the current result queue size"""
        return self._rq.unfinished_tasks

    def _run(self):
        while not self._empty.is_set() and not self.quit.is_set():
            try:
                job = self._jq.get(timeout=0.1)
            except Empty:
                continue
            if job is None:
                self._jq.task_done()
                self._jq.join()
                self._rq.put(None)
                self._empty.set()
                break
            try:
                self.scanned += 1
                f, a = job
                r = f(a)
            except Exception:
                r = traceback.format_exc()
            finally:
                self._rq.put((a, r))
                self._jq.task_done()

    def __iter__(self):
        return self

    def __next__(self):
        r = self._rq.get()
        self._rq.task_done()
        if r is None:
            raise StopIteration()
        return r

    def next(self):
        return self.__next__()

__help__ = """
NAME scan - scan files or processes against yara signatures

SYNOPSIS
    python scan.py [OPTIONS]... [FILE(s)|PID(s)]...

DESCRIPTION

Scan control:
    --proc
        scan PIDs. This indicate that trailing args are PIDS

    --ext=
        file extension inclusion filter (comma separate list)

    --thread_pool=%s
        size of the thread pool used for scanning

    --fast
        fast matching mode

    -d <identifier>=<value>
        define external variable.

Load rules control:
  namespace:
    --list
        list available YARA namespaces

    -w, --whitelist=
        whitelist of comma separated YARA namespaces to include in scan

    -b, --blacklist=
        blacklist of comma separated YARA namespaces to exclude from scan

    --root=(env[YARA_RULES] or <pkg>/yara/rules/)
        set the YARA_RULES path (path to the root of the rules directory)

  filepath:
    -r, --rule=
        Use the rule file specified by this input argument and ignore the
        YARA namespaces

Output control:
    --fmt=dict
        output format [dict|pprint|json|pickle|marshal]
    -o
        outfile path -> redirect stdout results to outfile

    -t  [tag1,tag2,tag3, ...]
        print matches that contain specific tags and filter out the rest

    -i  [ident1,ident2,ident3, ...]
        print matches that contain specific identifiers and filter out the rest

""" % DEFAULT_THREAD_POOL

def match_filter(tags_filter, idents_filter, res):
    if tags_filter is not None:
        new_res = {}
        for ns, matches in res.iteritems():
            for match in matches:
                if tags_filter.intersection(match['tags']):
                    mlist = new_res.get(ns, [])
                    mlist.append(match)
                    new_res[ns] = mlist
        res = new_res
    if idents_filter is not None:
        new_res = {}
        for ns, matches in res.iteritems():
            for match in matches:
                idents = [s['identifier'] for s in match['strings']]
                if idents_filter.intersection(idents):
                    mlist = new_res.get(ns, [])
                    mlist.append(match)
                    new_res[ns] = mlist
        res = new_res
    return res

def main(args):

    try:
        opts, args = getopt(args, 'hw:b:t:o:i:d:r:', ['proc',
                                              'whitelist=',
                                              'blacklist=',
                                              'thread_pool=',
                                              'root=',
                                              'list',
                                              'fmt=',
                                              'rule=',
                                              'fast',
                                              'help'])
    except Exception as exc:
        print("Getopt error: %s" % (exc), file=sys.stderr)
        return -1

    whitelist = []
    blacklist = []
    rule_filepath = None
    thread_pool = 4
    externals = {}
    fast_match = False
    pids = []
    paths = args
    rules_rootpath = yara.YARA_RULES_ROOT
    list_rules = False
    stream = sys.stdout
    stream_fmt = str
    tags_filter = None
    idents_filter = None

    for opt, arg in opts:
        if opt in ['-h', '--help']:
            print(__help__)
            return 0
        elif opt in ['--root']:
            if os.path.exists(rules_rootpath):
                print("root path '%s' does not exist" % arg, file=sys.stderr)
                return -1
            rules_rootpath = arg
        elif opt in ['-d']:
            try:
                externals.update(eval("dict(%s)" % arg))
            except SyntaxError:
                print("external '%s' syntax error" % arg, file=sys.stderr)
                return -1
        elif opt in ['--fast']:
            fast_match = True
        elif opt in ['--list']:
            list_rules = True
        elif opt in ['-o']:
            stream = open(arg, 'wb')
        elif opt in ['-t']:
            tags_filter = set(arg.split(','))
        elif opt in ['-i']:
            idents_filter = arg
        elif opt in ['-r', '--rule']:
            if os.path.exists(rules_rootpath):
                print("rule path '%s' does not exist" % arg, file=sys.stderr)
                return -1
            rule_filepath = arg
        elif opt in ['-w', '--whitelist']:
            whitelist = arg.split(',')
        elif opt in ['b', '--blacklist']:
            blacklist = arg.split(',')
        elif opt in ['--fmt']:
            if arg == 'pickle':
                stream_fmt = pickle.dumps
            elif arg == 'json':
                stream_fmt = lambda a: json.dumps(a, ensure_ascii=False,
                                                check_circular=False, indent=4)
            elif arg == 'pprint':
                stream_fmt = pprint.pformat
            elif arg == 'marshal':
                stream_fmt = marshal.dumps
            elif arg == 'dict':
                stream_fmt = str
            else:
                print("unknown output format %s" % arg, file=sys.stderr)
                return -1
        elif opt in ['t', '--thread_pool']:
            try:
                thread_pool = int(arg)
            except ValueError:
                print("-t param %s was not an int" % (arg), file=sys.stderr)
                return -1
        elif opt in ['--proc']:
            paths = []
            if not args:
                print("no PIDs specified")
                return -1
            for pid in args:
                try:
                    pids.append(int(pid))
                except ValueError:
                    print("PID %s was not an int" % (pid), file=sys.stderr)

    if list_rules is True:
        rules = yara.load_rules(rules_rootpath=rules_rootpath,
                                blacklist=blacklist,
                                whitelist=whitelist)
        print(rules)
        return 0

    scanner = Scanner(paths=paths, pids=pids,
                      rules_rootpath=rules_rootpath,
                      whitelist=whitelist,
                      blacklist=blacklist,
                      rules_filepath=rules_filepath,
                      thread_pool=thread_pool,
                      fast_match=fast_match,
                      externals=externals)

    try:
        status_template = "scan queue: %-7s result queue: %-7s"
        i = 0
        for arg, res in scanner:
            i += 1
            if i % 20 == 0:
                status = status_template % (scanner.sq_size, scanner.rq_size)
                sys.stderr.write("\b" * len(status) + status)

            res = match_filter(tags_filter, idents_filter, res)
            if res:
                print("<scan arg='%s'>" % arg, file=stream)
                print(stream_fmt(res), file=stream)
                print("</scan>", file=stream)
    finally:
        scanner.quit.set()
        status = status_template % (scanner.sq_size, scanner.rq_size)
        sys.stderr.write("\b" * len(status) + status)
        print("\nscanned %s items... done." % scanner.scanned, file=sys.stderr)

if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))
<|endoftext|>


<commit_before>#!/usr/bin/python
"""
/*
This file is part of Bohrium and copyright (c) 2012 the Bohrium
team <http://www.bh107.org>.

Bohrium is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as
published by the Free Software Foundation, either version 3
of the License, or (at your option) any later version.

Bohrium is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the
GNU Lesser General Public License along with Bohrium.

If not, see <http://www.gnu.org/licenses/>.
*/
"""

import sys
import os
from os.path import join, expanduser, exists
import shutil
import getopt
import subprocess

OK   = '\033[92m'
FAIL = '\033[91m'
ENDC = '\033[0m'

makecommand = "make"
makefilename = "Makefile"

def build(components,interpreter):
    for (name, dir, fatal) in components:
        print "***Building %s***"%name
        mkfile = makefilename

        if not exists(join(install_dir, dir, makefilename)) and exists(join(install_dir, dir, "Makefile")):
            mkfile = "Makefile"

        try:
            p = subprocess.Popen([makecommand, "-f", mkfile,"BH_PYTHON=%s"%interpreter], cwd=join(install_dir, dir))
            err = p.wait()
        except KeyboardInterrupt:
            p.terminate()

        if err == 0:
            print OK, "Build %s successfully."%name, ENDC
        else:
            if fatal:
                print FAIL, "A build error in %s is fatal. Exiting."%name, ENDC
                sys.exit(-1)
            else:
                print FAIL, "A build error in %s is not fatal. Continuing."%name, ENDC

def clean(components):
    for (name, dir, fatal) in components:
        print "***Cleaning %s***"%name
        mkfile = makefilename

        if not exists(join(install_dir, dir, makefilename)) and exists(join(install_dir, dir, "Makefile")):
            mkfile = "Makefile"

        try:
            p = subprocess.Popen([makecommand, "-f", mkfile, "clean"], cwd=join(install_dir, dir))
            err = p.wait()
        except KeyboardInterrupt:
            p.terminate()

def install(components,prefix,interpreter):
    if not exists(join(prefix,"lib")):
        os.mkdir(join(prefix,"lib"))
    for (name, dir, fatal) in components:
        print "***Installing %s***"%name
        mkfile = makefilename

        if not exists(join(install_dir, dir, makefilename)) and exists(join(install_dir, dir, "Makefile")):
            mkfile = "Makefile"

        try:
            p = subprocess.Popen([makecommand, "-f", mkfile,"install","BH_PYTHON=%s"%interpreter,"INSTALLDIR=%s"%prefix], cwd=join(install_dir, dir))
            err = p.wait()
        except KeyboardInterrupt:
            p.terminate()

        if err == 0:
            print OK, "Build %s successfully."%name, ENDC
        else:
            if fatal:
                print FAIL, "A build error in %s is fatal. Exiting."%name, ENDC
                sys.exit(-1)
            else:
                print FAIL, "A build error in %s is not fatal. Continuing."%name, ENDC

def install_config(prefix):
    if os.geteuid() == 0:#Root user
        HOME_CONFIG = "/etc/bohrium"
    else:
        HOME_CONFIG = join(join(expanduser("~"),".bohrium"))
    if not exists(HOME_CONFIG):
        os.mkdir(HOME_CONFIG)
    dst = join(HOME_CONFIG, "config.ini")
    src = join(install_dir,"config.ini.example")
    if not exists(dst):
        src_file = open(src, "r")
        src_str = src_file.read()
        src_file.close()
        dst_str = src_str.replace("/opt/bohrium",prefix)
        if sys.platform.startswith('darwin'):
            dst_str = dst_str.replace(".so",".dylib")
        dst_file = open(dst,"w")
        dst_file.write(dst_str)
        dst_file.close()
        print "Write default config file to %s"%(dst)

if __name__ == "__main__":
    debug = False
    interactive = False
    if not sys.platform.startswith('win32') and os.geteuid() == 0:#Root user
        prefix = "/opt/bohrium"
    else:
        prefix = join(join(expanduser("~"),".local"))
    interpreter = sys.executable
    try:
        install_dir = os.path.abspath(os.path.dirname(__file__))
    except NameError:
        print "The build script cannot run interactively."
        sys.exit(-1)

    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:],"d",["debug","prefix=","interactive","interpreter="])
    except getopt.GetoptError, err:
        print str(err)
        sys.exit(2)
    for o, a in opts:
        if o in ("-d","--debug"):
            debug = True
        elif o in ("--prefix"):
            prefix = a
        elif o in ("--interactive"):
            interactive = True
        elif o in ("--interpreter"):
            interpreter = a
        else:
            assert False, "unhandled option"

    if sys.platform.startswith('win32'):
        makecommand="nmake"
        makefilename="Makefile.win"
    elif sys.platform.startswith('darwin'):
        makefilename="Makefile.osx"

    if interactive:
        import readline, glob
        def complete(text, state):#For autocomplete
            return (glob.glob(text+'*')+[None])[state]
        readline.set_completer_delims(' \t\n;')
        readline.parse_and_bind("tab: complete")
        readline.set_completer(complete)

        print "Please specify the installation directory:"
        answer = raw_input("[%s] "%prefix)
        if answer != "":
            prefix = expanduser(answer)
    try:
        cmd = args[0]
    except IndexError:
        print "No command given"
        print ""
        print "Known commands: build, clean, install, rebuild"
        sys.exit(-1)

    components = [\
                  ("OPCODES","core/codegen",True),\
                  ("INIPARSER","iniparser",True),\
                  ("CORE-BHIR", "core/bhir", True),\
                  ("CORE", "core", True),\
                  ("VE-GPU", "ve/gpu", False),\
                  ("VE-CPU", "ve/cpu", True),\
                  ("VEM-NODE", "vem/node", True),\
                  ("VEM-CLUSTER", "vem/cluster", False),\
                  ("FILTER-PPRINT", "filter/pprint", True),\
                  ("FILTER-TRANSITIVE-REDUCTION", "filter/transitive_reduction", True),\
                  ("EXT-METHOD-MATMUL", "extmethods/matmul", True),\
                  ("BRIDGE-C++", "bridge/cpp", True),\
                  ("BRIDGE-NUMPY", "bridge/numpy", True),\
                  ("BRIDGE-C", "bridge/c", True),\
                  ("BHNUMPY", "bohrium", True),\
#                  ("FILTER-POWER", "filter/power", False),\
#                  ("FILTER-FUSION", "filter/fusion", False),\
#                  ("FILTER-STREAMING", "filter/streaming", False),\
#                  ("NumCIL", "bridge/NumCIL", False)\
                  ]

    if cmd == "rebuild":
        clean(components)
    if cmd == "build" or cmd == "rebuild":
        build(components,interpreter)
    elif cmd == "clean":
        clean(components)
    elif cmd == "install":
        prefix = os.path.abspath(prefix)
        if exists(prefix):
            assert os.path.isdir(prefix),"The prefix points to an existing file"
        else:
            os.makedirs(prefix)
        install(components,prefix,interpreter)
        install_config(prefix);
    else:
        print "Unknown command: '%s'."%cmd
        print ""
        print "Known commands: build, clean, install"
<commit_msg>Added some color to the build script<commit_after>#!/usr/bin/python
"""
/*
This file is part of Bohrium and copyright (c) 2012 the Bohrium
team <http://www.bh107.org>.

Bohrium is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as
published by the Free Software Foundation, either version 3
of the License, or (at your option) any later version.

Bohrium is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the
GNU Lesser General Public License along with Bohrium.

If not, see <http://www.gnu.org/licenses/>.
*/
"""

import sys
import os
from os.path import join, expanduser, exists
import shutil
import getopt
import subprocess

OK   = '\033[92m'
FAIL = '\033[91m'
ENDC = '\033[0m'
HEAD = '\033[93m'

makecommand = "make"
makefilename = "Makefile"

def build(components,interpreter):
    for (name, dir, fatal) in components:
        print HEAD,"***Building %s***"%name, ENDC
        mkfile = makefilename

        if not exists(join(install_dir, dir, makefilename)) and exists(join(install_dir, dir, "Makefile")):
            mkfile = "Makefile"

        try:
            p = subprocess.Popen([makecommand, "-f", mkfile,"BH_PYTHON=%s"%interpreter], cwd=join(install_dir, dir))
            err = p.wait()
        except KeyboardInterrupt:
            p.terminate()

        if err == 0:
            print OK, "Build %s successfully."%name, ENDC
        else:
            if fatal:
                print FAIL, "A build error in %s is fatal. Exiting."%name, ENDC
                sys.exit(-1)
            else:
                print FAIL, "A build error in %s is not fatal. Continuing."%name, ENDC

def clean(components):
    for (name, dir, fatal) in components:
        print HEAD,"***Cleaning %s***"%name, ENDC
        mkfile = makefilename

        if not exists(join(install_dir, dir, makefilename)) and exists(join(install_dir, dir, "Makefile")):
            mkfile = "Makefile"

        try:
            p = subprocess.Popen([makecommand, "-f", mkfile, "clean"], cwd=join(install_dir, dir))
            err = p.wait()
        except KeyboardInterrupt:
            p.terminate()

def install(components,prefix,interpreter):
    if not exists(join(prefix,"lib")):
        os.mkdir(join(prefix,"lib"))
    for (name, dir, fatal) in components:
        print HEAD,"***Installing %s***"%name, ENDC
        mkfile = makefilename

        if not exists(join(install_dir, dir, makefilename)) and exists(join(install_dir, dir, "Makefile")):
            mkfile = "Makefile"

        try:
            p = subprocess.Popen([makecommand, "-f", mkfile,"install","BH_PYTHON=%s"%interpreter,"INSTALLDIR=%s"%prefix], cwd=join(install_dir, dir))
            err = p.wait()
        except KeyboardInterrupt:
            p.terminate()

        if err == 0:
            print OK, "Build %s successfully."%name, ENDC
        else:
            if fatal:
                print FAIL, "A build error in %s is fatal. Exiting."%name, ENDC
                sys.exit(-1)
            else:
                print FAIL, "A build error in %s is not fatal. Continuing."%name, ENDC

def install_config(prefix):
    if os.geteuid() == 0:#Root user
        HOME_CONFIG = "/etc/bohrium"
    else:
        HOME_CONFIG = join(join(expanduser("~"),".bohrium"))
    if not exists(HOME_CONFIG):
        os.mkdir(HOME_CONFIG)
    dst = join(HOME_CONFIG, "config.ini")
    src = join(install_dir,"config.ini.example")
    if not exists(dst):
        src_file = open(src, "r")
        src_str = src_file.read()
        src_file.close()
        dst_str = src_str.replace("/opt/bohrium",prefix)
        if sys.platform.startswith('darwin'):
            dst_str = dst_str.replace(".so",".dylib")
        dst_file = open(dst,"w")
        dst_file.write(dst_str)
        dst_file.close()
        print "Write default config file to %s"%(dst)

if __name__ == "__main__":
    debug = False
    interactive = False
    if not sys.platform.startswith('win32') and os.geteuid() == 0:#Root user
        prefix = "/opt/bohrium"
    else:
        prefix = join(join(expanduser("~"),".local"))
    interpreter = sys.executable
    try:
        install_dir = os.path.abspath(os.path.dirname(__file__))
    except NameError:
        print "The build script cannot run interactively."
        sys.exit(-1)

    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:],"d",["debug","prefix=","interactive","interpreter="])
    except getopt.GetoptError, err:
        print str(err)
        sys.exit(2)
    for o, a in opts:
        if o in ("-d","--debug"):
            debug = True
        elif o in ("--prefix"):
            prefix = a
        elif o in ("--interactive"):
            interactive = True
        elif o in ("--interpreter"):
            interpreter = a
        else:
            assert False, "unhandled option"

    if sys.platform.startswith('win32'):
        makecommand="nmake"
        makefilename="Makefile.win"
    elif sys.platform.startswith('darwin'):
        makefilename="Makefile.osx"

    if interactive:
        import readline, glob
        def complete(text, state):#For autocomplete
            return (glob.glob(text+'*')+[None])[state]
        readline.set_completer_delims(' \t\n;')
        readline.parse_and_bind("tab: complete")
        readline.set_completer(complete)

        print "Please specify the installation directory:"
        answer = raw_input("[%s] "%prefix)
        if answer != "":
            prefix = expanduser(answer)
    try:
        cmd = args[0]
    except IndexError:
        print "No command given"
        print ""
        print "Known commands: build, clean, install, rebuild"
        sys.exit(-1)

    components = [\
                  ("OPCODES","core/codegen",True),\
                  ("INIPARSER","iniparser",True),\
                  ("CORE-BHIR", "core/bhir", True),\
                  ("CORE", "core", True),\
                  ("VE-GPU", "ve/gpu", False),\
                  ("VE-CPU", "ve/cpu", True),\
                  ("VEM-NODE", "vem/node", True),\
                  ("VEM-CLUSTER", "vem/cluster", False),\
                  ("FILTER-PPRINT", "filter/pprint", True),\
                  ("FILTER-TRANSITIVE-REDUCTION", "filter/transitive_reduction", True),\
                  ("EXT-METHOD-MATMUL", "extmethods/matmul", True),\
                  ("BRIDGE-C++", "bridge/cpp", True),\
                  ("BRIDGE-NUMPY", "bridge/numpy", True),\
                  ("BRIDGE-C", "bridge/c", True),\
                  ("BHNUMPY", "bohrium", True),\
#                  ("FILTER-POWER", "filter/power", False),\
#                  ("FILTER-FUSION", "filter/fusion", False),\
#                  ("FILTER-STREAMING", "filter/streaming", False),\
#                  ("NumCIL", "bridge/NumCIL", False)\
                  ]

    if cmd == "rebuild":
        clean(components)
    if cmd == "build" or cmd == "rebuild":
        build(components,interpreter)
    elif cmd == "clean":
        clean(components)
    elif cmd == "install":
        prefix = os.path.abspath(prefix)
        if exists(prefix):
            assert os.path.isdir(prefix),"The prefix points to an existing file"
        else:
            os.makedirs(prefix)
        install(components,prefix,interpreter)
        install_config(prefix);
    else:
        print "Unknown command: '%s'."%cmd
        print ""
        print "Known commands: build, clean, install"
<|endoftext|>


<commit_before>"""
Filename:     calc_vwind_rotation.py
Author:       Damien Irving, d.irving@student.unimelb.edu.au
Description:  Calculate meridional wind according to a new coordinate axis

"""

import sys
import os

import argparse
import numpy
import re

import cdms2

module_dir = os.path.join(os.environ['HOME'], 'modules')
sys.path.insert(0, module_dir)
import netcdf_io as nio
import coordinate_rotation as rot

import pdb

def rotate_vwind(dataU, dataV, new_np, pm_point, res=1.0, anomaly=None):
    """Define the new meridional wind field, according to the
    position of the new north pole.

    FIX: Need a more general method of dealing with input axes
    (e.g. what if there is multiple levels)

    """

    assert isinstance(dataU, cdms2.tvariable.TransientVariable)
    assert isinstance(dataV, cdms2.tvariable.TransientVariable)

    assert 'yx' in dataU.getOrder(), \
    'Input data must have a latitude and longitude axis'

    lat_axis = dataU.getLatitude()[:]
    lon_axis = dataU.getLongitude()[:]
    lats, lons = nio.coordinate_pairs(lat_axis, lon_axis)

    # Calculate new vwind on the native grid

    vwind_rot = calc_vwind(dataU, dataV, lat_axis, lon_axis, new_np)

    if anomaly:
        date_pattern = '([0-9]{4})-([0-9]{1,2})-([0-9]{1,2})'
        if re.search(date_pattern, anomaly[0]) and re.search(date_pattern, anomaly[1]):
            period = anomaly
        else:
            print """Input anomaly base period either invalid format or 'all' - base climatology is entire period"""
            period = None

	vwind_rot = cdms2.createVariable(vwind_rot, grid=dataU.getGrid(), axes=dataU.getAxisList())
	vwind_rot = nio.temporal_aggregation(vwind_rot, 'ANNUALCYCLE', 'anomaly', time_period=period)  #hard wired to annual cycle

    # Rotate the coordinate axis to desired grid

    nlats = int((180.0 / res) + 1)
    nlons = int(360.0 / res)
    grid = cdms2.createUniformGrid(-90.0, nlats, res, 0.0, nlons, res)
    lat_axis_rot = grid.getLatitude()
    lon_axis_rot = grid.getLongitude()

    vwind_rot_switch = rot.switch_regular_axes(vwind_rot, lats, lons, lat_axis_rot[:], lon_axis_rot[:], new_np, pm_point=pm_point, invert=True)

    if 't' in dataU.getOrder():
        axis_list = [dataU.getTime(), lat_axis_rot, lon_axis_rot]
    else:
        axis_list = [lat_axis_rot, lon_axis_rot]

    vwind_rot_swtich = cdms2.createVariable(vwind_rot_switch, grid=grid, axes=axis_list)

    return vwind_rot, vwind_rot_swtich

def calc_vwind(dataU, dataV, lat_axis, lon_axis, new_np, old_np=(90.0, 0.0)):
    """Calculate the new meridional wind field, according to the
    new position of the north pole"""

    lats, lons = nio.coordinate_pairs(lat_axis, lon_axis)
    theta = rot.rotation_angle(old_np[0], old_np[1], new_np[0], new_np[1],
                               lats, lons, reshape=[len(lat_axis), len(lon_axis)])
    theta = numpy.resize(theta, numpy.shape(dataU))

    dataV_rot = vwind_trig(dataU, dataV, theta)

    return dataV_rot

def vwind_trig(u, v, theta):
    """Do the trigonometry required to calculate new meridional
    wind from the old one.

    u      -  zonal wind data
    v      -  meridional wind data
    theta  -  angle through which the x/y coordinate axes must be rotated
              (measured anticlockwise starting on the original positive x-axis
	      (i.e. at 90 deg in a 360 circle))
    phi    -  angle that the wind vector makes with original positive x-axis
              (measured anticlockwise starting on the original positive x-axis)
    alpha  -  angle that the wind vector makes with the new positive x-axis
              (measured anticlockwise starting on the new positive x-axis)

    """

    wsp = numpy.sqrt(numpy.square(numpy.array(u)) + numpy.square(numpy.array(v)))
    phi = numpy.arctan2(v, u)
    alpha = phi - theta

    return wsp * numpy.sin(alpha)

def main(inargs):
    """Run the program."""

    # Prepate input data #

    indataU = nio.InputData(inargs.infileU, inargs.variableU,
                           **nio.dict_filter(vars(inargs), ['time', 'region', 'latitude', 'longitude', 'grid']))
    indataV = nio.InputData(inargs.infileV, inargs.variableV,
                           **nio.dict_filter(vars(inargs), ['time', 'region', 'latitude', 'longitude', 'grid']))

    # Calulate the new vwind #

    vwind_rot, vwind_rot_switch = rotate_vwind(indataU.data, indataV.data, inargs.north_pole, inargs.pm, anomaly=inargs.anomaly)

    # Write the output file #

    if inargs.anomaly:
        name_insert = ' anomaly'
        clim = 'Base period: %s - %s' %(inargs.anomaly[0], inargs.anomaly[1])
    else:
        name_insert = ''
	clim = ''

    history = 'Location of north pole: %s N, %s E. Prime meridian point = %s N, %s E. %s' %(str(inargs.north_pole[0]), str(inargs.north_pole[1]),
											    str(inargs.pm[0]), str(inargs.pm[1]), clim)
    indata_list = [indataU, indataV,]
    if inargs.noswitch:
        vrot_atts = {'id': 'vrot',
                     'name': 'Rotated meridional wind'+name_insert,
                     'long_name': 'Meridional wind'+name_insert+' on a rotated coordinate grid (but the poles have not yet been shifted)',
                     'units': indataU.data.units,
                     'history': history}
        outdata_list = [vwind_rot,]
        outvar_atts_list = [vrot_atts,]
        outvar_axes_list = [vwind_rot.getAxisList(),]
    else:
        vrot_atts = {'id': 'vrot',
                     'name': 'Rotated meridional wind'+name_insert,
                     'long_name': 'Meridional wind'+name_insert+' on a rotated coordinate grid (the poles have been shifted)',
                     'units': indataU.data.units,
                     'history': history}
        outdata_list = [vwind_rot_swtich,]
        outvar_atts_list = [vrot_atts,]
        outvar_axes_list = [vwind_rot_switch.getAxisList(),]

    nio.write_netcdf(inargs.outfile, 'Rotated meridional wind'+name_insert,
                     indata_list,
                     outdata_list,
                     outvar_atts_list,
                     outvar_axes_list)

if __name__ == '__main__':

    extra_info ="""
example (abyss.earthsci.unimelb.edu.au):
  /usr/local/uvcdat/1.2.0rc1/bin/cdat calc_vwind_rotation.py
  /work/dbirving/datasets/Merra/data/ua_Merra_250hPa_monthly_native.nc ua
  /work/dbirving/datasets/Merra/data/va_Merra_250hPa_monthly_native.nc va
  /work/dbirving/datasets/Merra/data/processed/vrot_Merra_250hPa_monthly-anom-wrt-all_y181x360_np30-270.nc
  --north_pole 30 270
  --anomaly all all
  --grid -90.0 181 1.0 0.0 360 1.0

required improvements:
  1. There might be some funny point in the grid switch that need to be looked at. This
     might be able to be fixed by using a higher resolution grid.

author:
  Damien Irving, d.irving@student.unimelb.edu.au

"""

    description='Rotate grid and calculate new meridional wind'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info,
                                     argument_default=argparse.SUPPRESS,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("infileU", type=str, help="Input file name, containing the zonal wind")
    parser.add_argument("variableU", type=str, help="Input file variable, for the zonal wind")
    parser.add_argument("infileV", type=str, help="Input file name, containing the meridional wind")
    parser.add_argument("variableV", type=str, help="Input file variable, for the meridional wind")
    parser.add_argument("outfile", type=str, help="Output file name")

    parser.add_argument("--region", type=str, choices=nio.regions.keys(),
                        help="Region [default = entire]")
    parser.add_argument("--latitude", type=float, nargs=2, metavar=('START', 'END'),
                        help="Latitude range [default = entire]")
    parser.add_argument("--longitude", type=float, nargs=2, metavar=('START', 'END'),
                        help="Longitude range [default = entire]")
    parser.add_argument("--time", type=str, nargs=3, metavar=('START_DATE', 'END_DATE', 'MONTHS'),
                        help="Time period [default = entire]")
    parser.add_argument("--grid", type=float, nargs=6, metavar=('START_LAT', 'NLAT', 'DELTALAT', 'START_LON', 'NLON', 'DELTALON'),
                        #default=(-90.0, 73, 2.5, 0.0, 144, 2.5),
                        help="Uniform regular grid to regrid data to [default = None]")
    parser.add_argument("--north_pole", type=float, nargs=2, metavar=('LAT', 'LON'), default=[90.0, 0.0],
                        help="Location of north pole [default = (90, 0)] - (30, 270) for PSA pattern")
    parser.add_argument("--pm", type=float, nargs=2, metavar=('LAT', 'LON'), default=(0.0, 0.0),
                        help="Location of the prime meridian point")
    parser.add_argument("--anomaly", type=str, nargs=2, metavar=('START_DATE', 'END_DATE'), default=None,
                        help="""Output the anomaly timeseries (calculated from annual cycle monthly climatology). Each date can be 'all' or 'YYYY-MM-DD' [default=False]""")
    parser.add_argument("--noswitch", action="store_true", default=False,
                        help="Switch for outputing the vwind on the original grid, as opposed to the switched (or rotated) grid [default: False]")

    args = parser.parse_args()

    print 'Input files: ', args.infileU, args.infileV
    print 'Output file: ', args.outfile

    main(args)
<commit_msg>Fixed typo in calc_vwind_rotation.py<commit_after>"""
Filename:     calc_vwind_rotation.py
Author:       Damien Irving, d.irving@student.unimelb.edu.au
Description:  Calculate meridional wind according to a new coordinate axis

"""

import sys
import os

import argparse
import numpy
import re

import cdms2

module_dir = os.path.join(os.environ['HOME'], 'modules')
sys.path.insert(0, module_dir)
import netcdf_io as nio
import coordinate_rotation as rot

import pdb

def rotate_vwind(dataU, dataV, new_np, pm_point, res=1.0, anomaly=None):
    """Define the new meridional wind field, according to the
    position of the new north pole.

    FIX: Need a more general method of dealing with input axes
    (e.g. what if there is multiple levels)

    """

    assert isinstance(dataU, cdms2.tvariable.TransientVariable)
    assert isinstance(dataV, cdms2.tvariable.TransientVariable)

    assert 'yx' in dataU.getOrder(), \
    'Input data must have a latitude and longitude axis'

    lat_axis = dataU.getLatitude()[:]
    lon_axis = dataU.getLongitude()[:]
    lats, lons = nio.coordinate_pairs(lat_axis, lon_axis)

    # Calculate new vwind on the native grid

    vwind_rot = calc_vwind(dataU, dataV, lat_axis, lon_axis, new_np)

    if anomaly:
        date_pattern = '([0-9]{4})-([0-9]{1,2})-([0-9]{1,2})'
        if re.search(date_pattern, anomaly[0]) and re.search(date_pattern, anomaly[1]):
            period = anomaly
        else:
            print """Input anomaly base period either invalid format or 'all' - base climatology is entire period"""
            period = None

	vwind_rot = cdms2.createVariable(vwind_rot, grid=dataU.getGrid(), axes=dataU.getAxisList())
	vwind_rot = nio.temporal_aggregation(vwind_rot, 'ANNUALCYCLE', 'anomaly', time_period=period)  #hard wired to annual cycle

    # Rotate the coordinate axis to desired grid

    nlats = int((180.0 / res) + 1)
    nlons = int(360.0 / res)
    grid = cdms2.createUniformGrid(-90.0, nlats, res, 0.0, nlons, res)
    lat_axis_rot = grid.getLatitude()
    lon_axis_rot = grid.getLongitude()

    vwind_rot_switch = rot.switch_regular_axes(vwind_rot, lats, lons, lat_axis_rot[:], lon_axis_rot[:], new_np, pm_point=pm_point, invert=True)

    if 't' in dataU.getOrder():
        axis_list = [dataU.getTime(), lat_axis_rot, lon_axis_rot]
    else:
        axis_list = [lat_axis_rot, lon_axis_rot]

    vwind_rot_swtich = cdms2.createVariable(vwind_rot_switch, grid=grid, axes=axis_list)

    return vwind_rot, vwind_rot_swtich

def calc_vwind(dataU, dataV, lat_axis, lon_axis, new_np, old_np=(90.0, 0.0)):
    """Calculate the new meridional wind field, according to the
    new position of the north pole"""

    lats, lons = nio.coordinate_pairs(lat_axis, lon_axis)
    theta = rot.rotation_angle(old_np[0], old_np[1], new_np[0], new_np[1],
                               lats, lons, reshape=[len(lat_axis), len(lon_axis)])
    theta = numpy.resize(theta, numpy.shape(dataU))

    dataV_rot = vwind_trig(dataU, dataV, theta)

    return dataV_rot

def vwind_trig(u, v, theta):
    """Do the trigonometry required to calculate new meridional
    wind from the old one.

    u      -  zonal wind data
    v      -  meridional wind data
    theta  -  angle through which the x/y coordinate axes must be rotated
              (measured anticlockwise starting on the original positive x-axis
	      (i.e. at 90 deg in a 360 circle))
    phi    -  angle that the wind vector makes with original positive x-axis
              (measured anticlockwise starting on the original positive x-axis)
    alpha  -  angle that the wind vector makes with the new positive x-axis
              (measured anticlockwise starting on the new positive x-axis)

    """

    wsp = numpy.sqrt(numpy.square(numpy.array(u)) + numpy.square(numpy.array(v)))
    phi = numpy.arctan2(v, u)
    alpha = phi - theta

    return wsp * numpy.sin(alpha)

def main(inargs):
    """Run the program."""

    # Prepate input data #

    indataU = nio.InputData(inargs.infileU, inargs.variableU,
                           **nio.dict_filter(vars(inargs), ['time', 'region', 'latitude', 'longitude', 'grid']))
    indataV = nio.InputData(inargs.infileV, inargs.variableV,
                           **nio.dict_filter(vars(inargs), ['time', 'region', 'latitude', 'longitude', 'grid']))

    # Calulate the new vwind #

    vwind_rot, vwind_rot_switch = rotate_vwind(indataU.data, indataV.data, inargs.north_pole, inargs.pm, anomaly=inargs.anomaly)

    # Write the output file #

    if inargs.anomaly:
        name_insert = ' anomaly'
        clim = 'Base period: %s - %s' %(inargs.anomaly[0], inargs.anomaly[1])
    else:
        name_insert = ''
	clim = ''

    history = 'Location of north pole: %s N, %s E. Prime meridian point = %s N, %s E. %s' %(str(inargs.north_pole[0]), str(inargs.north_pole[1]),
											    str(inargs.pm[0]), str(inargs.pm[1]), clim)
    indata_list = [indataU, indataV,]
    if inargs.noswitch:
        vrot_atts = {'id': 'vrot',
                     'name': 'Rotated meridional wind'+name_insert,
                     'long_name': 'Meridional wind'+name_insert+' on a rotated coordinate grid (but the poles have not yet been shifted)',
                     'units': indataU.data.units,
                     'history': history}
        outdata_list = [vwind_rot,]
        outvar_atts_list = [vrot_atts,]
        outvar_axes_list = [vwind_rot.getAxisList(),]
    else:
        vrot_atts = {'id': 'vrot',
                     'name': 'Rotated meridional wind'+name_insert,
                     'long_name': 'Meridional wind'+name_insert+' on a rotated coordinate grid (the poles have been shifted)',
                     'units': indataU.data.units,
                     'history': history}
        outdata_list = [vwind_rot_switch,]
        outvar_atts_list = [vrot_atts,]
        outvar_axes_list = [vwind_rot_switch.getAxisList(),]

    nio.write_netcdf(inargs.outfile, 'Rotated meridional wind'+name_insert,
                     indata_list,
                     outdata_list,
                     outvar_atts_list,
                     outvar_axes_list)

if __name__ == '__main__':

    extra_info ="""
example (abyss.earthsci.unimelb.edu.au):
  /usr/local/uvcdat/1.2.0rc1/bin/cdat calc_vwind_rotation.py
  /work/dbirving/datasets/Merra/data/ua_Merra_250hPa_monthly_native.nc ua
  /work/dbirving/datasets/Merra/data/va_Merra_250hPa_monthly_native.nc va
  /work/dbirving/datasets/Merra/data/processed/vrot_Merra_250hPa_monthly-anom-wrt-all_y181x360_np30-270.nc
  --north_pole 30 270
  --anomaly all all
  --grid -90.0 181 1.0 0.0 360 1.0

required improvements:
  1. There might be some funny point in the grid switch that need to be looked at. This
     might be able to be fixed by using a higher resolution grid.

author:
  Damien Irving, d.irving@student.unimelb.edu.au

"""

    description='Rotate grid and calculate new meridional wind'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info,
                                     argument_default=argparse.SUPPRESS,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("infileU", type=str, help="Input file name, containing the zonal wind")
    parser.add_argument("variableU", type=str, help="Input file variable, for the zonal wind")
    parser.add_argument("infileV", type=str, help="Input file name, containing the meridional wind")
    parser.add_argument("variableV", type=str, help="Input file variable, for the meridional wind")
    parser.add_argument("outfile", type=str, help="Output file name")

    parser.add_argument("--region", type=str, choices=nio.regions.keys(),
                        help="Region [default = entire]")
    parser.add_argument("--latitude", type=float, nargs=2, metavar=('START', 'END'),
                        help="Latitude range [default = entire]")
    parser.add_argument("--longitude", type=float, nargs=2, metavar=('START', 'END'),
                        help="Longitude range [default = entire]")
    parser.add_argument("--time", type=str, nargs=3, metavar=('START_DATE', 'END_DATE', 'MONTHS'),
                        help="Time period [default = entire]")
    parser.add_argument("--grid", type=float, nargs=6, metavar=('START_LAT', 'NLAT', 'DELTALAT', 'START_LON', 'NLON', 'DELTALON'),
                        #default=(-90.0, 73, 2.5, 0.0, 144, 2.5),
                        help="Uniform regular grid to regrid data to [default = None]")
    parser.add_argument("--north_pole", type=float, nargs=2, metavar=('LAT', 'LON'), default=[90.0, 0.0],
                        help="Location of north pole [default = (90, 0)] - (30, 270) for PSA pattern")
    parser.add_argument("--pm", type=float, nargs=2, metavar=('LAT', 'LON'), default=(0.0, 0.0),
                        help="Location of the prime meridian point")
    parser.add_argument("--anomaly", type=str, nargs=2, metavar=('START_DATE', 'END_DATE'), default=None,
                        help="""Output the anomaly timeseries (calculated from annual cycle monthly climatology). Each date can be 'all' or 'YYYY-MM-DD' [default=False]""")
    parser.add_argument("--noswitch", action="store_true", default=False,
                        help="Switch for outputing the vwind on the original grid, as opposed to the switched (or rotated) grid [default: False]")

    args = parser.parse_args()

    print 'Input files: ', args.infileU, args.infileV
    print 'Output file: ', args.outfile

    main(args)
<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class FujitsuMpi(Package):
    """Fujitsu MPI implementation only for Fujitsu compiler."""

    homepage = "https://www.fujitsu.com/us/"

    conflicts('%arm')
    conflicts('%cce')
    conflicts('%apple-clang')
    conflicts('%clang')
    conflicts('%gcc')
    conflicts('%intel')
    conflicts('%nag')
    conflicts('%pgi')
    conflicts('%xl')
    conflicts('%xl_r')

    provides('mpi@3.1:')

    def install(self, spec, prefix):
        raise InstallError(
            'Fujitsu MPI is not installable; it is vendor supplied')

    def setup_dependent_package(self, module, dependent_spec):
        self.spec.mpicc = self.prefix.bin.mpifcc
        self.spec.mpicxx = self.prefix.bin.mpiFCC
        self.spec.mpif77 = self.prefix.bin.mpifrt
        self.spec.mpifc = self.prefix.bin.mpifrt

    def setup_dependent_build_environment(self, env, dependent_spec):
        self.setup_run_environment(env)

    def setup_run_environment(self, env):
        # Because MPI are both compilers and runtimes, we set up the compilers
        # as part of run environment
        env.set('MPICC', self.prefix.bin.mpifcc)
        env.set('MPICXX', self.prefix.bin.mpiFCC)
        env.set('MPIF77', self.prefix.bin.mpifrt)
        env.set('MPIF90', self.prefix.bin.mpifrt)
<commit_msg>fujitsu-mpi: added "headers" and "libs" methods (#17253)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *
import os

class FujitsuMpi(Package):
    """Fujitsu MPI implementation only for Fujitsu compiler."""

    homepage = "https://www.fujitsu.com/us/"

    conflicts('%arm')
    conflicts('%cce')
    conflicts('%apple-clang')
    conflicts('%clang')
    conflicts('%gcc')
    conflicts('%intel')
    conflicts('%nag')
    conflicts('%pgi')
    conflicts('%xl')
    conflicts('%xl_r')

    provides('mpi@3.1:')

    def install(self, spec, prefix):
        raise InstallError(
            'Fujitsu MPI is not installable; it is vendor supplied')

    @property
    def headers(self):
        hdrs = find_headers('mpi', self.prefix.include, recursive=True)
        hdrs.directories = os.path.dirname(hdrs[0])
        return hdrs or None

    @property
    def libs(self):
        query_parameters = self.spec.last_query.extra_parameters
        libraries = ['libmpi']

        if 'cxx' in query_parameters:
            libraries = ['libmpi_cxx'] + libraries

        return find_libraries(
            libraries, root=self.prefix, shared=True, recursive=True
        )

    def setup_dependent_package(self, module, dependent_spec):
        self.spec.mpicc = self.prefix.bin.mpifcc
        self.spec.mpicxx = self.prefix.bin.mpiFCC
        self.spec.mpif77 = self.prefix.bin.mpifrt
        self.spec.mpifc = self.prefix.bin.mpifrt

    def setup_dependent_build_environment(self, env, dependent_spec):
        self.setup_run_environment(env)

    def setup_run_environment(self, env):
        # Because MPI are both compilers and runtimes, we set up the compilers
        # as part of run environment
        env.set('MPICC', self.prefix.bin.mpifcc)
        env.set('MPICXX', self.prefix.bin.mpiFCC)
        env.set('MPIF77', self.prefix.bin.mpifrt)
        env.set('MPIF90', self.prefix.bin.mpifrt)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from gospellibrary import Catalog
from datetime import date, datetime, timedelta
from itertools import cycle
from pymongo import MongoClient
from pprint import pprint
import requests
import random

def _get_month_year(start, end):

    # start_year = 1971
    start_year, start_month = start
    end_year, end_month = end

    # TODO: Make this simpler
    for year in range(start_year, end_year + 1):
        if year == start_year:
            if start_month <= 4:
                yield 4, year

            if start_month <= 10:
                yield 10, year

        if year == end_year:
            if end_month >= 4:
                yield 4, year

            if end_month >= 10:
                yield 10, year
        else:
            yield 4, year
            yield 10, year

def _clean_speaker(speaker):
    speaker = speaker.replace('\xa0', ' ')
    speaker = speaker.replace('By ', '')
    speaker = speaker.replace('President ', '')
    speaker = speaker.replace('Elder ', '')
    speaker = speaker.replace('Presented by ', '')
    speaker = speaker.strip()

    return speaker

def _clean_session(session):
    session = session.lower()
    if session in ['priesthood', 'general priesthood session',
                   'general priesthood meeting']:
        session = 'priesthood session'

    elif session == 'welfare session':
        session = 'general welfare session'

    elif session == 'saturday morning':
        session = 'saturday morning session'

    elif session in ['general women\'s meeting', 'women\'s fireside address',
                   'general relief society meeting', 'general women\'s session',
                   'general women’s meeting', 'general women’s meeting ',
                   'relief society sesquicentennial satellite broadcast']:
        session = "general young women meeting"

    session = session.replace('sesssion', 'session')
    session = session.replace('preisthood', 'priesthood')

    return session

def _get_time(year, month, session):
    first_of_month = datetime(year, month, 1)
    days_ahead = 6 - first_of_month.weekday()
    sunday = first_of_month + timedelta(days_ahead)

    def adjust_time(delta_day, hour):
        return (sunday - timedelta(delta_day)).replace(hour=hour)

    try:
        time = {"general young women meeting": adjust_time(8, 18),
                "thursday morning session": adjust_time(3, 10),
                "thursday afternoon session": adjust_time(3, 14),
                "friday morning session": adjust_time(2, 10),
                "friday afternoon session": adjust_time(2, 14),
                "saturday morning session": adjust_time(1, 10),
                "saturday afternoon session": adjust_time(1, 14),
                "priesthood session": adjust_time(1, 18),
                "sunday morning session": adjust_time(0, 10),
                "sunday afternoon session": adjust_time(0, 14),
                "general welfare session": adjust_time(0, 18),
                "tuesday morning session": adjust_time(-2, 10),
                "tuesday afternoon session": adjust_time(-2, 14)}[session.lower()]

        return time
    except:
        return None

def _get_talk_info(talk, package, year, month):
    nav_item = package.nav_items(talk.id)[0]
    nav_section = package.nav_section(nav_item.nav_section_id)[0]

    # Get talk information
    title = talk.primary_title_component
    speaker = _clean_speaker(talk.secondary_title_component)
    talk_url = talk.web_url
    talk_html = package.html(uri=talk.uri)
    preview = nav_item.preview

    # Get time information
    date = (year, month)
    session = _clean_session(nav_section.title)
    time = _get_time(date[0], date[1], session)

    # Get audio information if available
    try:
        audio = package.related_audio_items(talk.id)[0]
        audio_url = audio.media_url
        audio_size = audio.file_size
    except:
        audio_url = None
        audio_size = None

    return {'title': title,
            'speaker': speaker,
            'session': session,
            'time': time,
            'url': talk_url,
            'html': talk_html,
            'preview': preview,
            'audio_url': audio_url,
            'audio_size': audio_size}

def _valid_talk(talk):
    title = talk['title']

    if title in ['Welcome to Conference', 'The Sustaining of Church Officers']:
        return False

    if 'Church Auditing Department Report' in title or \
       'Statistical Report' in title:
        return False

    return True

def _get_speaker_image(speaker):
    speaker = speaker.lower()
    speaker = speaker.replace('\xc2', '')
    speaker = speaker.replace(' ', '-')
    speaker = speaker.replace('.', '')

    url = 'https://www.lds.org/bc/content/shared/content/images/leaders/{0}-large.jpg'.format(speaker)
    # r = requests.get(url)
    # if r.status_code == 200:
    #     return url
    # else:
    #     return None
    return url

def _new_database_version():
    client = MongoClient()
    db = client.media
    metadata = db.metadata

    new_version = Catalog().current_version()
    try:
        old_version = metadata.find({})[0]
    except IndexError:
        # This will happen the first time the value is put into
        # into the database
        metadata.insert({'db_version': new_version})
        return True

    if new_version != old_version['db_version']:
        old_version['db_version'] = new_version
        metadata.update_one({'_id': old_version['_id']},
                            {'$set': {'db_version': new_version}})
        return True
    else:
        return False

def create_database(start=(1971, 4), end=(date.today().year, date.today().month)):
    print("Creating database...")
    catalog = Catalog()

    client = MongoClient()
    db = client.media
    talks_db = db.conference_talks
    speakers_db = db.conference_speakers

    # Create main database
    for month, year in _get_month_year(start, end):
        item_uri = "/general-conference/{0}/{1:02}".format(year, month)
        print("~~~> Getting {0}".format(item_uri))

        item = catalog.item(uri=item_uri, lang='eng')
        with item.package() as package:
            talks = (_get_talk_info(subitem, package, year, month) for subitem in package.subitems())
            talks = (talk for talk in talks if talk['audio_url'] is not None)
            talks = (talk for talk in talks if talk['time'] is not None)
            talks = (talk for talk in talks if _valid_talk(talk))
            talks = (dict(talk, image=_get_speaker_image(talk['speaker'])) for talk in talks)

            talks_db.insert_many(list(talks))

    # Create database, grouped by speaker
    speakers = talks_db.aggregate([
        {
            "$group":
            {
                "_id": "$speaker",
                "talks":
                    {
                        "$push":
                            {
                                "speaker": "$speaker",
                                "title": "$title",
                                "session": "$session",
                                "time": "$time",
                                "url": "$url",
                                "html": "$html",
                                "preview": "$preview",
                                "audio_url": "$audio_url",
                                "audio_size": "$audio_size"
                            }
                    },
                "total": {"$sum": 1}
            }
        }
    ])

    speakers_db.insert(speakers)

def get_talk(speaker):
    client = MongoClient()
    db = client.media
    speakers = db.conference_speakers

    speakers = speakers.find({"_id": speaker}, {"talks": 1})
    speakers = (speaker['talks'] for speaker in speakers)
    speaker = (talk for speaker in speakers for talk in speaker)

    return list(speaker)

def get_talks(speakers):
    all_talks = [get_talk(speaker) for speaker in speakers]
    talks = [talk for talks in all_talks for talk in talks]

    return talks

def get_all_speaker_and_counts():
    client = MongoClient()
    db = client.media
    speakers = db.conference_speakers

    speakers = speakers.find({}, {"_id": 1, "total": 1})
    speakers = [(speaker['total'], speaker['_id']) for speaker in speakers]
    speakers = sorted(speakers, reverse=True)

    return speakers

def generate_id(speakers):
    client = MongoClient()
    db = client.media
    ids = db.ids

    speakers = sorted(speakers)
    id_ = ids.find_one({"speakers": speakers}, {"_id": 1})

    if id_ is not None:
        return id_["_id"]
    else:
        id_ = format(random.randint(0, 16777215), 'x')

        # Make sure I'm not duplicating a key
        while ids.find_one({"_id": id_}) is not None:
            id_ = format(random.randint(0, 16777215), 'x')

        ids.insert_one({'_id': id_, "speakers": speakers})

        return id_

def get_speakers(id_):
    client = MongoClient()
    db = client.media
    ids = db.ids

    result = ids.find_one({'_id': id_})

    if result is not None:
        return result['speakers']
    else:
        return None

def clear_database():
    client = MongoClient()
    db = client.media
    talks = db.conference_talks
    talks.remove({})

    speakers = db.conference_speakers
    speakers.remove({})

    metadata = db.metadata
    metadata.remove({})

def update_database(force=False):
    if _new_database_version() or force:
        print("Updating database to new version")
        clear_database()
        create_database()
        _new_database_version()
    else:
        print("Database is already up to date")

<commit_msg>Add optional start and end times to update_database<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from gospellibrary import Catalog
from datetime import date, datetime, timedelta
from itertools import cycle
from pymongo import MongoClient
from pprint import pprint
import requests
import random

def _get_month_year(start, end):

    # start_year = 1971
    start_year, start_month = start
    end_year, end_month = end

    # TODO: Make this simpler
    for year in range(start_year, end_year + 1):
        if year == start_year:
            if start_month <= 4:
                yield 4, year

            if start_month <= 10:
                yield 10, year

        if year == end_year:
            if end_month >= 4:
                yield 4, year

            if end_month >= 10:
                yield 10, year
        else:
            yield 4, year
            yield 10, year

def _clean_speaker(speaker):
    speaker = speaker.replace('\xa0', ' ')
    speaker = speaker.replace('By ', '')
    speaker = speaker.replace('President ', '')
    speaker = speaker.replace('Elder ', '')
    speaker = speaker.replace('Presented by ', '')
    speaker = speaker.strip()

    return speaker

def _clean_session(session):
    session = session.lower()
    if session in ['priesthood', 'general priesthood session',
                   'general priesthood meeting']:
        session = 'priesthood session'

    elif session == 'welfare session':
        session = 'general welfare session'

    elif session == 'saturday morning':
        session = 'saturday morning session'

    elif session in ['general women\'s meeting', 'women\'s fireside address',
                   'general relief society meeting', 'general women\'s session',
                   'general women’s meeting', 'general women’s meeting ',
                   'relief society sesquicentennial satellite broadcast']:
        session = "general young women meeting"

    session = session.replace('sesssion', 'session')
    session = session.replace('preisthood', 'priesthood')

    return session

def _get_time(year, month, session):
    first_of_month = datetime(year, month, 1)
    days_ahead = 6 - first_of_month.weekday()
    sunday = first_of_month + timedelta(days_ahead)

    def adjust_time(delta_day, hour):
        return (sunday - timedelta(delta_day)).replace(hour=hour)

    try:
        time = {"general young women meeting": adjust_time(8, 18),
                "thursday morning session": adjust_time(3, 10),
                "thursday afternoon session": adjust_time(3, 14),
                "friday morning session": adjust_time(2, 10),
                "friday afternoon session": adjust_time(2, 14),
                "saturday morning session": adjust_time(1, 10),
                "saturday afternoon session": adjust_time(1, 14),
                "priesthood session": adjust_time(1, 18),
                "sunday morning session": adjust_time(0, 10),
                "sunday afternoon session": adjust_time(0, 14),
                "general welfare session": adjust_time(0, 18),
                "tuesday morning session": adjust_time(-2, 10),
                "tuesday afternoon session": adjust_time(-2, 14)}[session.lower()]

        return time
    except:
        return None

def _get_talk_info(talk, package, year, month):
    nav_item = package.nav_items(talk.id)[0]
    nav_section = package.nav_section(nav_item.nav_section_id)[0]

    # Get talk information
    title = talk.primary_title_component
    speaker = _clean_speaker(talk.secondary_title_component)
    talk_url = talk.web_url
    talk_html = package.html(uri=talk.uri)
    preview = nav_item.preview

    # Get time information
    date = (year, month)
    session = _clean_session(nav_section.title)
    time = _get_time(date[0], date[1], session)

    # Get audio information if available
    try:
        audio = package.related_audio_items(talk.id)[0]
        audio_url = audio.media_url
        audio_size = audio.file_size
    except:
        audio_url = None
        audio_size = None

    return {'title': title,
            'speaker': speaker,
            'session': session,
            'time': time,
            'url': talk_url,
            'html': talk_html,
            'preview': preview,
            'audio_url': audio_url,
            'audio_size': audio_size}

def _valid_talk(talk):
    title = talk['title']

    if title in ['Welcome to Conference', 'The Sustaining of Church Officers']:
        return False

    if 'Church Auditing Department Report' in title or \
       'Statistical Report' in title:
        return False

    return True

def _get_speaker_image(speaker):
    speaker = speaker.lower()
    speaker = speaker.replace('\xc2', '')
    speaker = speaker.replace(' ', '-')
    speaker = speaker.replace('.', '')

    url = 'https://www.lds.org/bc/content/shared/content/images/leaders/{0}-large.jpg'.format(speaker)
    # r = requests.get(url)
    # if r.status_code == 200:
    #     return url
    # else:
    #     return None
    return url

def _new_database_version():
    client = MongoClient()
    db = client.media
    metadata = db.metadata

    new_version = Catalog().current_version()
    try:
        old_version = metadata.find({})[0]
    except IndexError:
        # This will happen the first time the value is put into
        # into the database
        metadata.insert({'db_version': new_version})
        return True

    if new_version != old_version['db_version']:
        old_version['db_version'] = new_version
        metadata.update_one({'_id': old_version['_id']},
                            {'$set': {'db_version': new_version}})
        return True
    else:
        return False

def create_database(start=(1971, 4), end=(date.today().year, date.today().month)):
    print("Creating database...")
    catalog = Catalog()

    client = MongoClient()
    db = client.media
    talks_db = db.conference_talks
    speakers_db = db.conference_speakers

    # Create main database
    for month, year in _get_month_year(start, end):
        item_uri = "/general-conference/{0}/{1:02}".format(year, month)
        print("~~~> Getting {0}".format(item_uri))

        item = catalog.item(uri=item_uri, lang='eng')
        with item.package() as package:
            talks = (_get_talk_info(subitem, package, year, month) for subitem in package.subitems())
            talks = (talk for talk in talks if talk['audio_url'] is not None)
            talks = (talk for talk in talks if talk['time'] is not None)
            talks = (talk for talk in talks if _valid_talk(talk))
            talks = (dict(talk, image=_get_speaker_image(talk['speaker'])) for talk in talks)

            talks_db.insert_many(list(talks))

    # Create database, grouped by speaker
    speakers = talks_db.aggregate([
        {
            "$group":
            {
                "_id": "$speaker",
                "talks":
                    {
                        "$push":
                            {
                                "speaker": "$speaker",
                                "title": "$title",
                                "session": "$session",
                                "time": "$time",
                                "url": "$url",
                                "html": "$html",
                                "preview": "$preview",
                                "audio_url": "$audio_url",
                                "audio_size": "$audio_size"
                            }
                    },
                "total": {"$sum": 1}
            }
        }
    ])

    speakers_db.insert(speakers)

def get_talk(speaker):
    client = MongoClient()
    db = client.media
    speakers = db.conference_speakers

    speakers = speakers.find({"_id": speaker}, {"talks": 1})
    speakers = (speaker['talks'] for speaker in speakers)
    speaker = (talk for speaker in speakers for talk in speaker)

    return list(speaker)

def get_talks(speakers):
    all_talks = [get_talk(speaker) for speaker in speakers]
    talks = [talk for talks in all_talks for talk in talks]

    return talks

def get_all_speaker_and_counts():
    client = MongoClient()
    db = client.media
    speakers = db.conference_speakers

    speakers = speakers.find({}, {"_id": 1, "total": 1})
    speakers = [(speaker['total'], speaker['_id']) for speaker in speakers]
    speakers = sorted(speakers, reverse=True)

    return speakers

def generate_id(speakers):
    client = MongoClient()
    db = client.media
    ids = db.ids

    speakers = sorted(speakers)
    id_ = ids.find_one({"speakers": speakers}, {"_id": 1})

    if id_ is not None:
        return id_["_id"]
    else:
        id_ = format(random.randint(0, 16777215), 'x')

        # Make sure I'm not duplicating a key
        while ids.find_one({"_id": id_}) is not None:
            id_ = format(random.randint(0, 16777215), 'x')

        ids.insert_one({'_id': id_, "speakers": speakers})

        return id_

def get_speakers(id_):
    client = MongoClient()
    db = client.media
    ids = db.ids

    result = ids.find_one({'_id': id_})

    if result is not None:
        return result['speakers']
    else:
        return None

def clear_database():
    client = MongoClient()
    db = client.media
    talks = db.conference_talks
    talks.remove({})

    speakers = db.conference_speakers
    speakers.remove({})

    metadata = db.metadata
    metadata.remove({})

def update_database(start=(1971, 4), end=(date.today().year, date.today().month), force=False):
    if _new_database_version() or force:
        print("Updating database to new version")
        clear_database()
        create_database(start, end)
        _new_database_version()
    else:
        print("Database is already up to date")

<|endoftext|>


<commit_before>from urllib import request

from IPython.utils.tz import utcnow
from bs4 import BeautifulSoup

from alexa.settings import RADIO_LINK, RADIO_LINK_BASE
from yle.models import News

def fetch_news():
    """
    Fetch current news items from yle website.
    """
    page = request.urlopen(RADIO_LINK)

    # Need to use html5lib though it's slow, lxml parser didn't work properly
    soup = BeautifulSoup(page, "html5lib")

    for el in soup.select('ul.program-list li'):
        identity = el.get('data-id')

        start = el.select('[itemprop="publication"] time')[0]\
            .get('datetime')
        name = el.select('[itemprop="name"]')[0].next_element

        News.objects.get_or_create(external_id=identity,
                                   defaults={'created': start,
                                             'title': name
                                             })

def fetch_audio_urls():
    """
    Try to get audio urls for all the items
    """
    # Only for today's news
    news = News.objects.filter(audio_url="",
                               created__gte=utcnow().replace(hour=0))
    for n in news:
        fetch_audio_url(n)

def fetch_audio_url(news: News):
    """
    Get audio url for a single news item.

    :param news: News to fetch url for.
    """
    page = request.urlopen(RADIO_LINK_BASE + news.external_id)
    soup = BeautifulSoup(page, "lxml")
    s = soup.select('div[itemprop="audio"] [itemprop="contentUrl"]')[0]
    url = s.get('href')
    news.audio_url = url
    news.save(update_fields=['audio_url'])
<commit_msg>Add debug printing<commit_after>from urllib import request

from IPython.utils.tz import utcnow
from bs4 import BeautifulSoup

from alexa.settings import RADIO_LINK, RADIO_LINK_BASE
from yle.models import News

def fetch_news():
    """
    Fetch current news items from yle website.
    """
    page = request.urlopen(RADIO_LINK)

    # Need to use html5lib though it's slow, lxml parser didn't work properly
    soup = BeautifulSoup(page, "html5lib")
    items = soup.select('ul.program-list li')
    for el in items:
        identity = el.get('data-id')

        start = el.select('[itemprop="publication"] time')[0]\
            .get('datetime')
        name = el.select('[itemprop="name"]')[0].next_element

        print("item with date", start)
        News.objects.get_or_create(external_id=identity,
                                   defaults={'created': start,
                                             'title': name
                                             })

    print("created {} items".format(len(items)))

def fetch_audio_urls():
    """
    Try to get audio urls for all the items
    """
    # Only for today's news
    news = News.objects.filter(audio_url="",
                               created__gte=utcnow().replace(hour=0))
    for n in news:
        fetch_audio_url(n)
    print("fetched urls for {} items".format(news.count()))

def fetch_audio_url(news: News):
    """
    Get audio url for a single news item.

    :param news: News to fetch url for.
    """
    page = request.urlopen(RADIO_LINK_BASE + news.external_id)
    soup = BeautifulSoup(page, "lxml")
    s = soup.select('div[itemprop="audio"] [itemprop="contentUrl"]')[0]
    url = s.get('href')
    news.audio_url = url
    news.save(update_fields=['audio_url'])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
PyBuilder configuration file
"""

import shutil
import itertools
import threading
import time
import sys

from pybuilder.core import use_plugin, after, init, task, depends
from pybuilder.utils import assert_can_execute, execute_command, read_file
from pybuilder.errors import BuildFailedException
from pybuilder.pluginhelper.external_command import ExternalCommandBuilder
import os
import fnmatch

BASE_DIR = os.path.abspath(os.path.dirname(__file__))
WEBAPP_DIR = os.path.join(BASE_DIR, 'src', 'main', 'webapp')
DJANGO_DIR = os.path.join(BASE_DIR, 'src', 'main', 'python')

use_plugin('analysis')
use_plugin('python.core')
use_plugin('python.unittest')
# use_plugin('python.coverage')  # disabled as it doesn't work out of the box with django tests anyway
use_plugin('python.flake8')
use_plugin('python.install_dependencies')
# use_plugin('pypi:pybuilder_django_enhanced_plugin')

default_task = ['clean', 'install_dependencies', 'django_makemigrations', 'django_migrate', 'django_createcachetable',
                'django_test', 'grunt', 'analyze', 'publish']

def travis_keep_alive():
    print 'TRAVIS KEEP ALIVE'
    sys.stdout.flush()
    time.sleep(5 * 60)

thread = threading.Thread(target=travis_keep_alive)
thread.setDaemon(True)
thread.start()

@init
def initialize(project):
    project.set_property('install_dependencies_upgrade', True)
    project.depends_on_requirements('requirements.txt')
    project.build_depends_on_requirements('requirements_dev.txt')

    project.set_property('django_project', 'opendatahub')
    project.set_property('django_apps', ['hub'])

    project.set_property('coverage_break_build', False)
    project.set_property('flake8_include_test_sources', True)
    project.set_property('flake8_break_build', True)
    project.set_property('flake8_verbose_output', True)

    # F401: Unused import
    # E501: long line
    # E128: visual indent
    project.set_property('flake8_ignore', 'E128')
    project.set_property('flake8_max_line_length', 120)
    project.set_property('flake8_exclude_patterns', '0001_*.py')
    project.set_property('pylint_fail_on_ids', [
        'E*',
        'F*',
        'C0103',  # invalid-name
        'W0403'  # relative-import
    ])

def custom_exec(project, logger, args, name=None, cwd=None, fail_stderr=True, fail_nonzero=True):
    cmd = args[0]
    name = name or cmd
    logger.debug('Checking availability of ' + cmd)
    assert_can_execute((cmd,), cmd, 'plugin opendatahub.' + name)
    logger.debug(name + ' has been found')

    report_file = project.expand_path('$dir_reports/{0}'.format(name))
    error_report_file = '{0}.err'.format(report_file)

    logger.info('Running {}'.format(args))
    exit_code = execute_command(args, report_file, cwd=cwd)
    report_lines = read_file(report_file)
    error_report_lines = read_file(error_report_file)

    verbose = project.get_property('verbose')
    if verbose:
        logger.info(''.join(report_lines))

    if error_report_lines or exit_code != 0:
        msg = 'Errors while running {0}, see {1}'.format(name, error_report_file)
        if verbose:
            logger.info(''.join(error_report_lines))
        if (error_report_lines and fail_stderr) or (exit_code != 0 and fail_nonzero):
            raise BuildFailedException(msg)

def django_exec(project, logger, args, **kwargs):
    django_project = project.get_property('django_project')
    django_apps = project.get_property('django_apps')
    args_ex = ['--settings={}.settings'.format(django_project), '--pythonpath=' + DJANGO_DIR]
    return custom_exec(project, logger, ['django-admin'] + args + args_ex, **kwargs)

@task('install_runtime_dependencies')
def install_bower_packages(project, logger):
    custom_exec(project, logger, ['bower', 'install', '--config.analytics=false', '--allow-root', '--no-interactive'],
                cwd=WEBAPP_DIR, fail_stderr=False)

@task('install_build_dependencies')
def install_npm_packages(project, logger):
    custom_exec(project, logger, ['npm', 'install'], cwd=WEBAPP_DIR, fail_stderr=False)

@task('install_build_dependencies')
def install_typings(project, logger):
    shutil.rmtree(os.path.join(WEBAPP_DIR, 'typings'), ignore_errors=True)
    custom_exec(project, logger, ['tsd', 'reinstall'], cwd=WEBAPP_DIR)
    custom_exec(project, logger, ['tsd', 'rebundle'], cwd=WEBAPP_DIR)

@task
@depends('install_build_dependencies', 'install_runtime_dependencies')
@after(('run_unit_tests',), only_once=True)
def grunt(project, logger):
    custom_exec(project, logger, ['grunt'], cwd=WEBAPP_DIR)

@task
@depends('prepare')
def django_test(project, logger):
    django_exec(project, logger, ['test', '--noinput', '--failfast'] + project.get_property('django_apps'),
                fail_stderr=False)

@task
@depends('prepare')
def django_makemigrations(project, logger):
    django_exec(project, logger, ['makemigrations', '--noinput'], fail_stderr=False)

@task
@depends('prepare')
def django_migrate(project, logger):
    django_exec(project, logger, ['migrate', '--noinput'], fail_stderr=False)

@task
@depends('prepare')
def django_collectstatic(project, logger):
    django_exec(project, logger, ['collectstatic', '--noinput'], fail_stderr=False)

@task
@depends('prepare')
def django_loadfixtures(project, logger):
    django_exec(project, logger, ['loadfixtures'], fail_stderr=False)

@task
@depends('prepare')
def django_createcachetable(project, logger):
    django_exec(project, logger, ['createcachetable'], fail_stderr=False)

@task
@depends('prepare')
def django_flush(project, logger):
    django_exec(project, logger, ['flush', '--noinput'], fail_stderr=False)

@task
@depends('prepare')
def django_reset_db(project, logger):
    django_exec(project, logger, ['reset_db', '--noinput'], fail_stderr=False)
    django_createcachetable(project, logger)

@task
@depends('prepare')
def django_init_db(project, logger):
    django_reset_db(project, logger)
    django_makemigrations(project, logger)
    django_migrate(project, logger)
    django_createcachetable(project, logger)
    django_loadfixtures(project, logger)

@init
def init_pylint(project):
    project.build_depends_on('pylint')

@after('prepare')
def check_pylint_availability(logger):
    logger.debug('Checking availability of pylint')
    assert_can_execute(('pylint', ), 'pylint', 'plugin python.pylint')
    logger.debug('pylint has been found')

@task('analyze')
@depends('prepare')
def execute_pylint(project, logger):
    logger.info('Executing pylint on project sources')
    verbose = project.get_property('verbose')
    failure_ids = project.get_property('pylint_fail_on_ids', [])

    command = ExternalCommandBuilder('pylint', project)
    command.use_argument('--msg-template="{msg_id}({symbol}):{path}:{line}:{obj}:{msg}"')
    result = command.run_on_production_source_files(logger, include_test_sources=True, include_scripts=True)

    if result.error_report_lines:
        msg = 'Errors while running pylint, see {0}'.format(result.error_report_file)
        if verbose:
            logger.info(''.join(result.error_report_lines))
        raise BuildFailedException(msg)

    if verbose:
        logger.info(''.join(result.report_lines))

    breaking_lines = list(
        itertools.chain(*[fnmatch.filter(result.report_lines, pattern + '(*') for pattern in failure_ids]))
    if breaking_lines:
        raise BuildFailedException(''.join(breaking_lines))
<commit_msg>oh wait I'm stupid..<commit_after># -*- coding: utf-8 -*-

"""
PyBuilder configuration file
"""

import shutil
import itertools
import threading
import time
import sys

from pybuilder.core import use_plugin, after, init, task, depends
from pybuilder.utils import assert_can_execute, execute_command, read_file
from pybuilder.errors import BuildFailedException
from pybuilder.pluginhelper.external_command import ExternalCommandBuilder
import os
import fnmatch

BASE_DIR = os.path.abspath(os.path.dirname(__file__))
WEBAPP_DIR = os.path.join(BASE_DIR, 'src', 'main', 'webapp')
DJANGO_DIR = os.path.join(BASE_DIR, 'src', 'main', 'python')

use_plugin('analysis')
use_plugin('python.core')
use_plugin('python.unittest')
# use_plugin('python.coverage')  # disabled as it doesn't work out of the box with django tests anyway
use_plugin('python.flake8')
use_plugin('python.install_dependencies')
# use_plugin('pypi:pybuilder_django_enhanced_plugin')

default_task = ['clean', 'install_dependencies', 'django_makemigrations', 'django_migrate', 'django_createcachetable',
                'django_test', 'grunt', 'analyze', 'publish']

def travis_keep_alive():
    while True:
        print 'TRAVIS KEEP ALIVE'
        sys.stdout.flush()
        time.sleep(5 * 60)

thread = threading.Thread(target=travis_keep_alive)
thread.setDaemon(True)
thread.start()

@init
def initialize(project):
    project.set_property('install_dependencies_upgrade', True)
    project.depends_on_requirements('requirements.txt')
    project.build_depends_on_requirements('requirements_dev.txt')

    project.set_property('django_project', 'opendatahub')
    project.set_property('django_apps', ['hub'])

    project.set_property('coverage_break_build', False)
    project.set_property('flake8_include_test_sources', True)
    project.set_property('flake8_break_build', True)
    project.set_property('flake8_verbose_output', True)

    # F401: Unused import
    # E501: long line
    # E128: visual indent
    project.set_property('flake8_ignore', 'E128')
    project.set_property('flake8_max_line_length', 120)
    project.set_property('flake8_exclude_patterns', '0001_*.py')
    project.set_property('pylint_fail_on_ids', [
        'E*',
        'F*',
        'C0103',  # invalid-name
        'W0403'  # relative-import
    ])

def custom_exec(project, logger, args, name=None, cwd=None, fail_stderr=True, fail_nonzero=True):
    cmd = args[0]
    name = name or cmd
    logger.debug('Checking availability of ' + cmd)
    assert_can_execute((cmd,), cmd, 'plugin opendatahub.' + name)
    logger.debug(name + ' has been found')

    report_file = project.expand_path('$dir_reports/{0}'.format(name))
    error_report_file = '{0}.err'.format(report_file)

    logger.info('Running {}'.format(args))
    exit_code = execute_command(args, report_file, cwd=cwd)
    report_lines = read_file(report_file)
    error_report_lines = read_file(error_report_file)

    verbose = project.get_property('verbose')
    if verbose:
        logger.info(''.join(report_lines))

    if error_report_lines or exit_code != 0:
        msg = 'Errors while running {0}, see {1}'.format(name, error_report_file)
        if verbose:
            logger.info(''.join(error_report_lines))
        if (error_report_lines and fail_stderr) or (exit_code != 0 and fail_nonzero):
            raise BuildFailedException(msg)

def django_exec(project, logger, args, **kwargs):
    django_project = project.get_property('django_project')
    django_apps = project.get_property('django_apps')
    args_ex = ['--settings={}.settings'.format(django_project), '--pythonpath=' + DJANGO_DIR]
    return custom_exec(project, logger, ['django-admin'] + args + args_ex, **kwargs)

@task('install_runtime_dependencies')
def install_bower_packages(project, logger):
    custom_exec(project, logger, ['bower', 'install', '--config.analytics=false', '--allow-root', '--no-interactive'],
                cwd=WEBAPP_DIR, fail_stderr=False)

@task('install_build_dependencies')
def install_npm_packages(project, logger):
    custom_exec(project, logger, ['npm', 'install'], cwd=WEBAPP_DIR, fail_stderr=False)

@task('install_build_dependencies')
def install_typings(project, logger):
    shutil.rmtree(os.path.join(WEBAPP_DIR, 'typings'), ignore_errors=True)
    custom_exec(project, logger, ['tsd', 'reinstall'], cwd=WEBAPP_DIR)
    custom_exec(project, logger, ['tsd', 'rebundle'], cwd=WEBAPP_DIR)

@task
@depends('install_build_dependencies', 'install_runtime_dependencies')
@after(('run_unit_tests',), only_once=True)
def grunt(project, logger):
    custom_exec(project, logger, ['grunt'], cwd=WEBAPP_DIR)

@task
@depends('prepare')
def django_test(project, logger):
    django_exec(project, logger, ['test', '--noinput', '--failfast'] + project.get_property('django_apps'),
                fail_stderr=False)

@task
@depends('prepare')
def django_makemigrations(project, logger):
    django_exec(project, logger, ['makemigrations', '--noinput'], fail_stderr=False)

@task
@depends('prepare')
def django_migrate(project, logger):
    django_exec(project, logger, ['migrate', '--noinput'], fail_stderr=False)

@task
@depends('prepare')
def django_collectstatic(project, logger):
    django_exec(project, logger, ['collectstatic', '--noinput'], fail_stderr=False)

@task
@depends('prepare')
def django_loadfixtures(project, logger):
    django_exec(project, logger, ['loadfixtures'], fail_stderr=False)

@task
@depends('prepare')
def django_createcachetable(project, logger):
    django_exec(project, logger, ['createcachetable'], fail_stderr=False)

@task
@depends('prepare')
def django_flush(project, logger):
    django_exec(project, logger, ['flush', '--noinput'], fail_stderr=False)

@task
@depends('prepare')
def django_reset_db(project, logger):
    django_exec(project, logger, ['reset_db', '--noinput'], fail_stderr=False)
    django_createcachetable(project, logger)

@task
@depends('prepare')
def django_init_db(project, logger):
    django_reset_db(project, logger)
    django_makemigrations(project, logger)
    django_migrate(project, logger)
    django_createcachetable(project, logger)
    django_loadfixtures(project, logger)

@init
def init_pylint(project):
    project.build_depends_on('pylint')

@after('prepare')
def check_pylint_availability(logger):
    logger.debug('Checking availability of pylint')
    assert_can_execute(('pylint', ), 'pylint', 'plugin python.pylint')
    logger.debug('pylint has been found')

@task('analyze')
@depends('prepare')
def execute_pylint(project, logger):
    logger.info('Executing pylint on project sources')
    verbose = project.get_property('verbose')
    failure_ids = project.get_property('pylint_fail_on_ids', [])

    command = ExternalCommandBuilder('pylint', project)
    command.use_argument('--msg-template="{msg_id}({symbol}):{path}:{line}:{obj}:{msg}"')
    result = command.run_on_production_source_files(logger, include_test_sources=True, include_scripts=True)

    if result.error_report_lines:
        msg = 'Errors while running pylint, see {0}'.format(result.error_report_file)
        if verbose:
            logger.info(''.join(result.error_report_lines))
        raise BuildFailedException(msg)

    if verbose:
        logger.info(''.join(result.report_lines))

    breaking_lines = list(
        itertools.chain(*[fnmatch.filter(result.report_lines, pattern + '(*') for pattern in failure_ids]))
    if breaking_lines:
        raise BuildFailedException(''.join(breaking_lines))
<|endoftext|>