<commit_before>"""
enable extensions in hgrc

[extensions]
extension = <path to>/extension.py

[hooks]
<command> = python:<extension>.<function>

"""

import sys, os, time

if sys.platform == "win32":
    import os, msvcrt
    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)

INI_JS = "ini.js"
BUILD_CONFIG = "D:\\hg\\build\\build_config"

def parse_config(path):
    try:
        f = open(path, 'r')
        lines = f.readlines()
        f.close()
    except Exception, m:
        print "loading build config failed. ", m
        return None
    ret = {}
    cursor = None
    def set_cursor(dict):
        if dict != None:
            parent = dict.has_key('_parent') and ret.has_key(dict['_parent']) \
                and ret[ dict.pop('_parent') ] or ret

            parent[ dict.pop('_name') ] = dict
            if dict.has_key('short'):
                for s in dict.pop('short'):
                    if s:
                        parent[s] = dict
        return {}
    for line in lines:
        if "#" in line:
            continue
        if "[" in line and "]" in line:
            cursor = set_cursor(cursor)
            if ":" in line:
                parent, key = line.split(":", 1)
                cursor["_name"] = key.strip(" []\n\r")
                cursor["_parent"] = parent.strip(" []\n\r")
            else:
                cursor["_name"] = line.strip(" []\n\r")
        if "=" in line and cursor != None :
            key, value = line.split("=", 1)
            if "," in value:
                cursor[ key.strip(" \n\r") ] = [v.strip(" \n\r") for v in value.split(',')]
            else:
                cursor[ key.strip(" \n\r") ] = value.strip(" \n\r")
    set_cursor(cursor)
    return ret

def app_revision(ui, repo, **kwargs):
    """sets mercurial_revision in ini.js. will be displayed only if revision_number is not set"""
    import os, sys, re
    import mercurial.commands
    from mercurial.node import short
    heads = repo.heads()
    if len(heads) != 1:
        print "abort. ensure that there is only one head in the repository"
        return
    head = heads[0]
    rev = repo.changelog.rev(head)
    short_hash = short(head)
    branch = repo.dirstate.branch()
    revision = "%s:%s, %s" % (rev, short_hash, branch)
    for dirpath, dirnames, filenames in os.walk(repo.root):
        if INI_JS in filenames:
            f = open(os.path.join(dirpath, INI_JS), 'rb')
            content = f.read()
            f.close()
            f = open(os.path.join(dirpath, INI_JS), 'wb')
            f.write(re.sub(
                re.compile("mercurial_revision:([^\n]*)\n"),
                "mercurial_revision: \"%s\"\n" % revision,
                content ) )
            f.close()
            break

def create_path(path):
    if not os.path.exists(path): os.makedirs(path)

def run_build_script(ui, repo, core_version=0, type=None, **opts):
    """
    make a build from a configuration file.
    usage: hg build-app [-l <log range> <core version> <type, e.g. nightly>

    requires a configuration file

    config file path defined in BUILD_CONFIG

    format example

    [general]
    server      = chrisk@is.oslo.opera.com
    scp_session =

    [core 2.2]
    short       = 2.2,
    branch      = protocol-4

    [core 2.2 : nightly]
    short       = n, ce
    name        = 0.7 alpha-4-snapshot
    local-repo  = D:\apache-root\dragonfly.opera.com\app\core-2-2
    local-zip   = D:\apache-root\dragonfly.opera.com\app\core-2-2\zips\Opera-Dragonfly-%s-%s.zip
    local-log   = D:\apache-root\dragonfly.opera.com\app\core-2-2\logs\%s.%s.log
    remote-repo = /var/www/dragonfly.opera.com/app/test/

    usage hg build-app [-l sart:end] 2.2 ce

    meaning build application log from xx to xx for core 2.2 as a cutting-edge ( or nightly ) build

    a log of the build commands will be created in the same repo as the config file
    """

    import cPickle

    if opts['command_log']:
        path = os.path.join(os.path.split(BUILD_CONFIG)[0], "BUILD_LOG")
        if os.path.exists(path):
            l = open(path, 'r')
            sys.stdout.write(l.read())
            l.close()
            return

    if opts['show_config_file']:
        if os.path.exists(BUILD_CONFIG):
            f = open(BUILD_CONFIG, 'r')
            sys.stdout.write(f.read())
            f.close()
            return

    if opts['log_history']:
        path = os.path.join(os.path.split(BUILD_CONFIG)[0], "BUILD_LOG_PICKLE")
        f = open(path, "rb")
        try:
            build_log = cPickle.load(f)
            for key in build_log: print key, ': ', build_log[key]
        except:
            pass
        f.close()
        return

    if opts['config_file']:
        print "path to config file: ", BUILD_CONFIG
        print "path to config file set in: ", __file__
        return

    log_entry = "hg build-app" +  \
        ( 'log' in opts and ( " -l " + opts['log'] ) or "" ) + \
        " " + (core_version and core_version or "")+ \
        " " + (type and type or "")
    import mercurial.commands
    from mercurial.node import short
    heads = repo.heads()
    if len(heads) != 1:
        print "abort. ensure that there is only one head in the repository"
        return
    build_config = parse_config(BUILD_CONFIG)
    if not build_config:
        print "no or broken build config file"
        return
    os.chdir(repo.root)

    path = os.path.join(os.path.split(BUILD_CONFIG)[0], "BUILD_LOG_PICKLE")
    f = open(path, "rb")

    try:
        build_log = cPickle.load(f)
        build_log['read-counter'] +=1
    except:
        build_log = {'title': 'logs', 'read-counter': 0}
    f.close()

    head = heads[0]
    root = repo.root
    sys.path.insert(0, os.path.join(root, 'tools' ))

    import dfbuild

    core_version = core_version in build_config and build_config[core_version] or None
    type = core_version and type in core_version and core_version[type] or None
    if not type:
        print "abort. the command arguments have no according entries in the config file"
        return

    print "update to", core_version["branch"]
    if mercurial.commands.update(ui, repo, rev=core_version["branch"]) != 0:
        print "abort. hg update failed"
        return
    rev = repo.changelog.rev(head)
    sort_hash = short(head)
    revision = "%d:%s, %s" % (rev, sort_hash, core_version["branch"])

    if not 'id' in type:
        print "each entriy in the build config must have an id"
        return

    if not type['id'] in build_log:
        build_log[type['id']] = []
    build_log[type['id']].append(sort_hash)

    print "make build, revision:", revision
    try:
        sys.argv = \
        [
            __file__,
            os.path.join(root, 'src'),
            type["local-repo"],
            '-t',
            '-s',
            '-d',
            '-m',
            '-p',
            '-k', '$dfversion$=' + type['name'],
            '-k', '$revdate$=' + revision
        ]
        dfbuild.main()
    except Exception, msg:
        print "abort. making build failed. ", msg
        return
    print "build created"
    print "make zip"
    path = type["local-zip"] % (rev, sort_hash)
    create_path(os.path.split(path)[0])
    try:
        sys.argv = \
        [
            __file__,
            os.path.join(root, 'src'),
            path,
            '-c',
            '-s',
            '-d',
            '-k', '$dfversion$=' + type['name'],
            '-k', '$revdate$=' + revision
        ]
        dfbuild.main()
    except Exception, msg:
        print "abort. making zip failed. ", msg
        return
    print "zip created"
    print "make log"

    if not opts['log'] and len(build_log[type['id']]):
        opts['log'] = "tip:%s" % ( build_log[type['id']][-1] )

    if opts['log']:
        print "log", opts['log']
        path = type["local-log"] % (rev, sort_hash)
        create_path(os.path.split(path)[0])
        f = open(path, 'w')
        store_stdout = sys.stdout
        sys.stdout = f
        mercurial.commands.log\
        (
            ui, repo,
            rev=[opts['log']],
            copies=None,
            date='',
            only_branch=[core_version["branch"]],
            no_merges=None,
            only_merges=None,
            keyword=[],
            style="changelog"
        )
        sys.stdout.flush()
        sys.stdout.close()
        sys.stdout = store_stdout
    path = os.path.join(os.path.split(BUILD_CONFIG)[0], "BUILD_LOG")
    f = open(path, os.path.exists(path) and "a" or "w")
    f.write(time.strftime("%d.%m.%y %H:%M", time.localtime()) + ': ' + log_entry + "; rev: " + revision)
    f.write("\n") # FWIW os.linesep should be editor linesep

    path = os.path.join(os.path.split(BUILD_CONFIG)[0], "BUILD_LOG_PICKLE")
    f = open(path, "wb")
    cPickle.dump(build_log, f)
    f.close()

    print "done"

cmdtable = \
{
  # cmd name function call
  "app-revision": \
  (
    app_revision,
    # see mercurial/fancyopts.py for all of the command
    # flag options.
    [],
    "sets mercurial_revision in ini.js"
  ),
  "build-app": \
  (
    run_build_script,
    # see mercurial/fancyopts.py for all of the command
    # flag options.
    [('l', 'log', '', 'log range'),
    ('f', 'config-file', None, 'path to config file'),
    ('s', 'show-config-file', None, 'show config file'),
    ('o', 'command-log', None, 'show the log of the last used parameters'),
    ('i', 'log-history', None, 'show the log of the last created logs')],
    "make a build, a zip and a log file"
  )
}

"""
def app_test_node(ui, repo):
    pass

    import os, sys

    import mercurial.commands
    from mercurial.node import short

    heads = repo.heads()
    if len(heads) != 1:
        print "abort. ensure that there is only one head in the repository"
        return

    head = heads[0]

    rev = repo.changelog.rev(head)
    sort_hash = short(head)
    changes = repo.changelog.read(head)

    print rev, short_hash, changes

    import os, sys

    from mercurial.node import short

    heads = repo.heads()
    root = repo.root

    rev = repo.changelog.rev(heads[0])

    changes = repo.changelog.read(heads[0])

    print rev, changes

def decon_path(path):
    path = os.path.normpath(path)
    ret = []
    while os.path.sep in path:
        path, part = os.path.split(path)
        path = path.rstrip(os.path.sep)
        ret.append(part)
    ret.append(path)
    return ret[::-1]

def get_relative_path(path_1, path_2):
    path_1_dec = decon_path(path_1)
    path_2_dec = decon_path(path_2)
    match = 0
    while len(path_1_dec) > match \
        and len(path_2_dec) > match \
        and path_1_dec[match] == path_2_dec[match]: match += 1
    if match:
        delta = len(path_1_dec) - match
        prefix = delta and ( delta * "../" ) or "./"
        return prefix + "/".join(path_2_dec[match:])
    return None

def app_test(ui, repo, core_version = 0, type = None, **opts):

    print 'core_version: ', core_version, 'type: ', type, 'opts: ', opts
    print repo.root, os.getcwd()
    os.chdir(repo.root)
    print repo.root, os.getcwd()
    print get_relative_path(os.getcwd(), CUTTING_EDGE_BUILD)

class file_obj(object):

    def __init__(self):
        self.content = ""
        self.closed = False
        self.cursor = 0

    def write(self, data):
        self.content += data
        return None

    def read(self):
        pos = self.cursor
        self.cursor = len(self.content)
        return self.content[pos:]

    def flush(self):
        pass

"""

<commit_msg>added views-inspection.js; fixed automatic log creation in hgappext.py<commit_after>"""
enable extensions in hgrc

[extensions]
extension = <path to>/extension.py

[hooks]
<command> = python:<extension>.<function>

"""

import sys, os, time

if sys.platform == "win32":
    import os, msvcrt
    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)

INI_JS = "ini.js"
BUILD_CONFIG = "D:\\hg\\build\\build_config"

def parse_config(path):
    try:
        f = open(path, 'r')
        lines = f.readlines()
        f.close()
    except Exception, m:
        print "loading build config failed. ", m
        return None
    ret = {}
    cursor = None
    def set_cursor(dict):
        if dict != None:
            parent = dict.has_key('_parent') and ret.has_key(dict['_parent']) \
                and ret[ dict.pop('_parent') ] or ret

            parent[ dict.pop('_name') ] = dict
            if dict.has_key('short'):
                for s in dict.pop('short'):
                    if s:
                        parent[s] = dict
        return {}
    for line in lines:
        if "#" in line:
            continue
        if "[" in line and "]" in line:
            cursor = set_cursor(cursor)
            if ":" in line:
                parent, key = line.split(":", 1)
                cursor["_name"] = key.strip(" []\n\r")
                cursor["_parent"] = parent.strip(" []\n\r")
            else:
                cursor["_name"] = line.strip(" []\n\r")
        if "=" in line and cursor != None :
            key, value = line.split("=", 1)
            if "," in value:
                cursor[ key.strip(" \n\r") ] = [v.strip(" \n\r") for v in value.split(',')]
            else:
                cursor[ key.strip(" \n\r") ] = value.strip(" \n\r")
    set_cursor(cursor)
    return ret

def app_revision(ui, repo, **kwargs):
    """sets mercurial_revision in ini.js. will be displayed only if revision_number is not set"""
    import os, sys, re
    import mercurial.commands
    from mercurial.node import short
    heads = repo.heads()
    if len(heads) != 1:
        print "abort. ensure that there is only one head in the repository"
        return
    head = heads[0]
    rev = repo.changelog.rev(head)
    short_hash = short(head)
    branch = repo.dirstate.branch()
    revision = "%s:%s, %s" % (rev, short_hash, branch)
    for dirpath, dirnames, filenames in os.walk(repo.root):
        if INI_JS in filenames:
            f = open(os.path.join(dirpath, INI_JS), 'rb')
            content = f.read()
            f.close()
            f = open(os.path.join(dirpath, INI_JS), 'wb')
            f.write(re.sub(
                re.compile("mercurial_revision:([^\n]*)\n"),
                "mercurial_revision: \"%s\"\n" % revision,
                content ) )
            f.close()
            break

def create_path(path):
    if not os.path.exists(path): os.makedirs(path)

def run_build_script(ui, repo, core_version=0, type=None, **opts):
    """
    make a build from a configuration file.
    usage: hg build-app [-l <log range> <core version> <type, e.g. nightly>

    requires a configuration file

    config file path defined in BUILD_CONFIG

    format example

    [general]
    server      = chrisk@is.oslo.opera.com
    scp_session =

    [core 2.2]
    short       = 2.2,
    branch      = protocol-4

    [core 2.2 : nightly]
    short       = n, ce
    name        = 0.7 alpha-4-snapshot
    local-repo  = D:\apache-root\dragonfly.opera.com\app\core-2-2
    local-zip   = D:\apache-root\dragonfly.opera.com\app\core-2-2\zips\Opera-Dragonfly-%s-%s.zip
    local-log   = D:\apache-root\dragonfly.opera.com\app\core-2-2\logs\%s.%s.log
    remote-repo = /var/www/dragonfly.opera.com/app/test/

    usage hg build-app [-l sart:end] 2.2 ce

    meaning build application log from xx to xx for core 2.2 as a cutting-edge ( or nightly ) build

    a log of the build commands will be created in the same repo as the config file
    """

    import cPickle

    if opts['command_log']:
        path = os.path.join(os.path.split(BUILD_CONFIG)[0], "BUILD_LOG")
        if os.path.exists(path):
            l = open(path, 'r')
            sys.stdout.write(l.read())
            l.close()
            return

    if opts['show_config_file']:
        if os.path.exists(BUILD_CONFIG):
            f = open(BUILD_CONFIG, 'r')
            sys.stdout.write(f.read())
            f.close()
            return

    if opts['log_history']:
        path = os.path.join(os.path.split(BUILD_CONFIG)[0], "BUILD_LOG_PICKLE")
        f = open(path, "rb")
        try:
            build_log = cPickle.load(f)
            for key in build_log: print key, ': ', build_log[key]
        except:
            pass
        f.close()
        return

    if opts['config_file']:
        print "path to config file: ", BUILD_CONFIG
        print "path to config file set in: ", __file__
        return

    log_entry = "hg build-app" +  \
        ( 'log' in opts and ( " -l " + opts['log'] ) or "" ) + \
        " " + (core_version and core_version or "")+ \
        " " + (type and type or "")
    import mercurial.commands
    from mercurial.node import short
    heads = repo.heads()
    if len(heads) != 1:
        print "abort. ensure that there is only one head in the repository"
        return
    build_config = parse_config(BUILD_CONFIG)
    if not build_config:
        print "no or broken build config file"
        return
    os.chdir(repo.root)

    path = os.path.join(os.path.split(BUILD_CONFIG)[0], "BUILD_LOG_PICKLE")
    f = open(path, "rb")

    try:
        build_log = cPickle.load(f)
        build_log['read-counter'] +=1
    except:
        build_log = {'title': 'logs', 'read-counter': 0}
    f.close()

    head = heads[0]
    root = repo.root
    sys.path.insert(0, os.path.join(root, 'tools' ))

    import dfbuild

    core_version = core_version in build_config and build_config[core_version] or None
    type = core_version and type in core_version and core_version[type] or None
    if not type:
        print "abort. the command arguments have no according entries in the config file"
        return

    print "update to", core_version["branch"]
    if mercurial.commands.update(ui, repo, rev=core_version["branch"]) != 0:
        print "abort. hg update failed"
        return
    rev = repo.changelog.rev(head)
    sort_hash = short(head)
    revision = "%d:%s, %s" % (rev, sort_hash, core_version["branch"])

    if not 'id' in type:
        print "each entriy in the build config must have an id"
        return

    if not type['id'] in build_log:
        build_log[type['id']] = []
    build_log[type['id']].append(sort_hash)

    print "make build, revision:", revision
    try:
        sys.argv = \
        [
            __file__,
            os.path.join(root, 'src'),
            type["local-repo"],
            '-t',
            '-s',
            '-d',
            '-m',
            '-p',
            '-k', '$dfversion$=' + type['name'],
            '-k', '$revdate$=' + revision
        ]
        dfbuild.main()
    except Exception, msg:
        print "abort. making build failed. ", msg
        return
    print "build created"
    print "make zip"
    path = type["local-zip"] % (rev, sort_hash)
    create_path(os.path.split(path)[0])
    try:
        sys.argv = \
        [
            __file__,
            os.path.join(root, 'src'),
            path,
            '-c',
            '-s',
            '-d',
            '-k', '$dfversion$=' + type['name'],
            '-k', '$revdate$=' + revision
        ]
        dfbuild.main()
    except Exception, msg:
        print "abort. making zip failed. ", msg
        return
    print "zip created"
    print "make log"

    if not opts['log'] and len(build_log[type['id']]) > 1:
        opts['log'] = "tip:%s" % ( build_log[type['id']][-2] )

    if opts['log']:
        print "log", opts['log']
        path = type["local-log"] % (rev, sort_hash)
        create_path(os.path.split(path)[0])
        f = open(path, 'w')
        store_stdout = sys.stdout
        sys.stdout = f
        mercurial.commands.log\
        (
            ui, repo,
            rev=[opts['log']],
            copies=None,
            date='',
            only_branch=[core_version["branch"]],
            no_merges=None,
            only_merges=None,
            keyword=[],
            style="changelog"
        )
        sys.stdout.flush()
        sys.stdout.close()
        sys.stdout = store_stdout
    path = os.path.join(os.path.split(BUILD_CONFIG)[0], "BUILD_LOG")
    f = open(path, os.path.exists(path) and "a" or "w")
    f.write(time.strftime("%d.%m.%y %H:%M", time.localtime()) + ': ' + log_entry + "; rev: " + revision)
    f.write("\n") # FWIW os.linesep should be editor linesep

    path = os.path.join(os.path.split(BUILD_CONFIG)[0], "BUILD_LOG_PICKLE")
    f = open(path, "wb")
    cPickle.dump(build_log, f)
    f.close()

    print "done"

cmdtable = \
{
  # cmd name function call
  "app-revision": \
  (
    app_revision,
    # see mercurial/fancyopts.py for all of the command
    # flag options.
    [],
    "sets mercurial_revision in ini.js"
  ),
  "build-app": \
  (
    run_build_script,
    # see mercurial/fancyopts.py for all of the command
    # flag options.
    [('l', 'log', '', 'log range'),
    ('f', 'config-file', None, 'path to config file'),
    ('s', 'show-config-file', None, 'show config file'),
    ('o', 'command-log', None, 'show the log of the last used parameters'),
    ('i', 'log-history', None, 'show the log of the last created logs')],
    "make a build, a zip and a log file"
  )
}

"""
def app_test_node(ui, repo):
    pass

    import os, sys

    import mercurial.commands
    from mercurial.node import short

    heads = repo.heads()
    if len(heads) != 1:
        print "abort. ensure that there is only one head in the repository"
        return

    head = heads[0]

    rev = repo.changelog.rev(head)
    sort_hash = short(head)
    changes = repo.changelog.read(head)

    print rev, short_hash, changes

    import os, sys

    from mercurial.node import short

    heads = repo.heads()
    root = repo.root

    rev = repo.changelog.rev(heads[0])

    changes = repo.changelog.read(heads[0])

    print rev, changes

def decon_path(path):
    path = os.path.normpath(path)
    ret = []
    while os.path.sep in path:
        path, part = os.path.split(path)
        path = path.rstrip(os.path.sep)
        ret.append(part)
    ret.append(path)
    return ret[::-1]

def get_relative_path(path_1, path_2):
    path_1_dec = decon_path(path_1)
    path_2_dec = decon_path(path_2)
    match = 0
    while len(path_1_dec) > match \
        and len(path_2_dec) > match \
        and path_1_dec[match] == path_2_dec[match]: match += 1
    if match:
        delta = len(path_1_dec) - match
        prefix = delta and ( delta * "../" ) or "./"
        return prefix + "/".join(path_2_dec[match:])
    return None

def app_test(ui, repo, core_version = 0, type = None, **opts):

    print 'core_version: ', core_version, 'type: ', type, 'opts: ', opts
    print repo.root, os.getcwd()
    os.chdir(repo.root)
    print repo.root, os.getcwd()
    print get_relative_path(os.getcwd(), CUTTING_EDGE_BUILD)

class file_obj(object):

    def __init__(self):
        self.content = ""
        self.closed = False
        self.cursor = 0

    def write(self, data):
        self.content += data
        return None

    def read(self):
        pos = self.cursor
        self.cursor = len(self.content)
        return self.content[pos:]

    def flush(self):
        pass

"""

<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys

infile = sys.argv[1]
outfile = sys.argv[2]
params = sys.argv[3:]

fi = open(infile, 'r')
fo = open(outfile, 'w')

while fi:
  line = fi.readline()
  if not line:
    break
  for param in params:
    terms = param.split(':')
    # horribel hack
    if terms[0].find('{') == 0 and terms[0].find('}') == len(terms[0]) - 1:
      terms[0] = '$' + terms[0]
    if line.find(terms[0]) >= 0:
      line = line.replace(terms[0], terms[1])
  fo.write(line)

fi.close()
fo.close()

<commit_msg>use startswith and endswith<commit_after>#!/usr/bin/env python

import sys

infile = sys.argv[1]
outfile = sys.argv[2]
params = sys.argv[3:]

fi = open(infile, 'r')
fo = open(outfile, 'w')

while fi:
  line = fi.readline()
  if not line:
    break
  for param in params:
    terms = param.split(':')
    # horribel hack
    if terms[0].startswith('{') and terms[0].endswith('}'):
      terms[0] = '$' + terms[0]
    if line.find(terms[0]) >= 0:
      line = line.replace(terms[0], terms[1])
  fo.write(line)

fi.close()
fo.close()

<|endoftext|>


<commit_before>"""Train Encoder/Decoder models for Multi-label Classification tasks."""

import os
import tensorflow as tf
import tensorflow.compat.v1 as tf1
import anna.model.metrics as metrics
import anna.data.utils as datautils

class Trainer:
    """
    Trains Multi-label Classification models. Generalizes most common models
    using:
        - Encoder: Takes the input and creates a tensor representation (e.g.
                   average word embeddings, RNN, etc.)
        - Decoder: Builds the output and loss functions (e.g. feedforward,
                   classifier chains, RNN, etc.)
    """

    def __init__(self,
                 data_dir,
                 labels,
                 encoder,
                 decoder,
                 folder_name="model",
                 name="unnamed",
                 batch_size=64,
                 learning_rate=0.001,
                 grad_clip=0.,
                 decay_rate=1.,
                 decay_steps=0,
                 *args,
                 **kwargs):
        """
        Trains Multi-label Classification models.

        Args:
            data_dir (str): path to the data folder
            labels (list[str]): list of possible labels
            encoder (Encoder): transforms the input text into numbers
            decoder (Decoder): takes the encoded input and produces labels
            folder_name (str): name of the folder where to save the model
            name (str): name for the model (used to save checkpoints/summaries)
            batch_size (int): batch size for training
            learning_rate (float): training learning rate
            grad_clip (float): maximum norm for gradients when optimizing
            decay_rate (float): the factor to decay the learning rate
            decay_steps (int): how many steps to wait for each decay
        """
        session_config = tf1.ConfigProto()
        session_config.gpu_options.per_process_gpu_memory_fraction = 1
        config = tf.estimator.RunConfig(
            session_config=session_config,
            keep_checkpoint_max=1
        )
        model_dir = os.path.join(data_dir, folder_name, name)
        self.batch_size = batch_size
        self.estimator = tf.estimator.Estimator(
            model_fn=model_fn,
            model_dir=model_dir,
            config=config,
            params={
                "encoder": encoder,
                "decoder": decoder,
                "label_vocab": labels,
                "learning_rate": learning_rate,
                "grad_clip": grad_clip,
                "decay_rate": decay_rate,
                "decay_steps": decay_steps
            })

    def train(self, docs_path, test_docs_path=None,
              val_size=500, shuffle=10000, epochs=10, eval_every=100):
        """
        Train model on `docs`, and run evaluations on `test_docs`.

        The datasets are expected to produce a tuple of:
            - dict(str -> tf.Tensor): having tokenized `title` and `text`
            - tf.Tensor: the list of labels as strings

        Args:
            docs_path (str): the path to the documents for training
            test_docs_path (str): the path to the documents for evaluation
            val_size (int): size of the validation set, in nr of docs
            shuffle (int): size of the buffer use to shuffle the training set
            epochs (int): max number of epochs to run
            eval_every (int): number of training steps to wait for each eval
        """

        def train_input():
            docs = tf.data.TFRecordDataset([docs_path])\
                          .map(datautils.parse_example)
            return input_fn(docs.skip(val_size),
                            batch_size=self.batch_size,
                            shuffle=shuffle,
                            repeat=epochs)

        def val_input():
            docs = tf.data.TFRecordDataset([docs_path])\
                          .map(datautils.parse_example)
            return input_fn(docs.take(val_size), batch_size=self.batch_size)

        def test_input():
            test_docs = tf.data.TFRecordDataset([test_docs_path])\
                               .map(datautils.parse_example)
            return input_fn(test_docs, batch_size=self.batch_size)

        hooks = []
        if val_size:
            hooks.append(tf.estimator.experimental.InMemoryEvaluatorHook(
                self.estimator,
                val_input,
                name="val",
                every_n_iter=eval_every))
        if test_docs_path:
            hooks.append(tf.estimator.experimental.InMemoryEvaluatorHook(
                self.estimator,
                test_input,
                name="test",
                every_n_iter=eval_every))

        self.estimator.train(input_fn=train_input, hooks=hooks)

def input_fn(docs, batch_size=64, shuffle=None, repeat=None):
    with tf.name_scope("input_processing"):
        def pad_empty(doc, labels):
            doc = {k: tf.cond(tf.equal(tf.shape(v)[0], 0),
                              lambda: tf.constant([""]),
                              lambda: v)
                   for k, v in doc.items()}
            return doc, labels

        def mask(doc, labels):
            ret = {k + "_mask": tf.ones_like(v, dtype=tf.float32)
                   for k, v in doc.items()}
            ret.update(doc)
            return ret, labels

        docs = docs.map(pad_empty)
        docs = docs.map(mask)

        if repeat:
            docs = docs.repeat(repeat)

        if shuffle:
            docs = docs.shuffle(buffer_size=shuffle)

        pads = ({
                    "title": [None],
                    "title_mask": [None],
                    "text": [None],
                    "text_mask": [None]
                }, [None])
        return docs.padded_batch(batch_size, padded_shapes=pads)

def model_fn(features, labels, mode, params):
    encoder = params["encoder"]
    decoder = params["decoder"]
    label_vocab = params["label_vocab"]
    lr = params["learning_rate"]
    clipping = params["grad_clip"]
    decay_rate = params["decay_rate"]
    decay_steps = params["decay_steps"]

    with tf.name_scope("expected_output"):
        if labels is not None:
            labels = label_idx_to_hot(labels, label_vocab)

    with tf.name_scope("model"):
        mem, mem_len, mem_fixed = encoder(features, mode)
        predictions, loss = decoder(mem, mem_len, mem_fixed, labels, mode)

    eval_metric_ops = None
    if mode in [tf.estimator.ModeKeys.EVAL, tf.estimator.ModeKeys.TRAIN]:
        eval_metric_ops = metrics.create(labels, predictions, label_vocab)

    train_op = None
    if mode == tf.estimator.ModeKeys.TRAIN:
        train_op = create_optimizer(loss, lr, clipping, decay_rate, decay_steps)

    pred = {"predictions": predictions}
    return tf.estimator.EstimatorSpec(mode,
                                      loss=loss,
                                      train_op=train_op,
                                      eval_metric_ops=eval_metric_ops,
                                      predictions=pred)

def label_idx_to_hot(labels, vocab):
    """
    Coverts the labels from a list of ids to a multi-hot vector.

    Args:
        labels (tf.Tensor): expected labels as a list of ids.
          [batch, nr_positive_labels]
        vocab (list[str]): vocabulary of labels

    Returns:
        labels (tf.Tensor): expected labels as a list of ids.
          [batch, nr_labels]
    """
    # Convert strings to ids (pads become -1)
    # (batch, n_positive_labels)
    labels = tf.contrib.lookup.index_table_from_tensor(
        mapping=vocab,
        default_value=-1).lookup(labels)

    # Turn indexes to one-hot vectors (pads become all 0)
    # (batch, n_positive_labels, n_classes)
    labels = tf.one_hot(labels, len(vocab), dtype=tf.float32)

    # Turn labels into fixed-sized 1/0 vector
    # (batch, n_classes)
    return tf.reduce_sum(labels, 1)

def create_optimizer(loss, learning_rate, max_norm, decay_rate, decay_steps):
    """
    Creates an optimizing operation for the given loss function.

    Args:
        loss (tf.Tensor): a loss function to optimize against
        learning_rate (float): the learning rate for the optimizer
        max_norm (float): maximum norm for any gradient when doing the update
        decay_rate (float): the factor to decay the learning rate
        decay_steps (int): how many steps to wait for each decay

    Returns:
        updater (tf.Tensor): operation that updates a single step of the network
    """
    global_step = tf1.train.get_or_create_global_step()

    if decay_rate < 1.:
        learning_rate = tf.train.exponential_decay(
            learning_rate, global_step, decay_steps, decay_rate, staircase=True)

    tf1.summary.scalar("misc/learning_rate", learning_rate)

    opt = tf.contrib.opt.LazyAdamOptimizer(learning_rate=learning_rate)
    grad, var = zip(*opt.compute_gradients(loss))

    if max_norm > 0.:
        grad = clip_gradients(grad, max_norm)

    return opt.apply_gradients(zip(grad, var), global_step=global_step)

def clip_gradients(gradients, max_norm):
    """
    Clips gradients to the maximum `max_norm`.

    Args:
        gradients: the gradients to be applied when optimizing
        max_norm: the maximum norm for each gradient

    Returns:
        gradients: the gradients to be applied when optimizing
    """
    gradients, norm = tf.clip_by_global_norm(gradients, max_norm)

    tf1.summary.scalar("misc/grad_norm", norm)
    tf1.summary.scalar("misc/clipped_gradient", tf.linalg.global_norm(gradients))

    return gradients
<commit_msg>trainer: don't keep checkpoints and avoid fixing gpu memory<commit_after>"""Train Encoder/Decoder models for Multi-label Classification tasks."""

import os
import tensorflow as tf
import tensorflow.compat.v1 as tf1
import anna.model.metrics as metrics
import anna.data.utils as datautils

class Trainer:
    """
    Trains Multi-label Classification models. Generalizes most common models
    using:
        - Encoder: Takes the input and creates a tensor representation (e.g.
                   average word embeddings, RNN, etc.)
        - Decoder: Builds the output and loss functions (e.g. feedforward,
                   classifier chains, RNN, etc.)
    """

    def __init__(self,
                 data_dir,
                 labels,
                 encoder,
                 decoder,
                 folder_name="model",
                 name="unnamed",
                 batch_size=64,
                 learning_rate=0.001,
                 grad_clip=0.,
                 decay_rate=1.,
                 decay_steps=0,
                 *args,
                 **kwargs):
        """
        Trains Multi-label Classification models.

        Args:
            data_dir (str): path to the data folder
            labels (list[str]): list of possible labels
            encoder (Encoder): transforms the input text into numbers
            decoder (Decoder): takes the encoded input and produces labels
            folder_name (str): name of the folder where to save the model
            name (str): name for the model (used to save checkpoints/summaries)
            batch_size (int): batch size for training
            learning_rate (float): training learning rate
            grad_clip (float): maximum norm for gradients when optimizing
            decay_rate (float): the factor to decay the learning rate
            decay_steps (int): how many steps to wait for each decay
        """
        session_config = tf1.ConfigProto()
        session_config.gpu_options.allow_growth = True
        config = tf.estimator.RunConfig(
            session_config=session_config,
            keep_checkpoint_max=0
        )
        model_dir = os.path.join(data_dir, folder_name, name)
        self.batch_size = batch_size
        self.estimator = tf.estimator.Estimator(
            model_fn=model_fn,
            model_dir=model_dir,
            config=config,
            params={
                "encoder": encoder,
                "decoder": decoder,
                "label_vocab": labels,
                "learning_rate": learning_rate,
                "grad_clip": grad_clip,
                "decay_rate": decay_rate,
                "decay_steps": decay_steps
            })

    def train(self, docs_path, test_docs_path=None,
              val_size=500, shuffle=10000, epochs=10, eval_every=100):
        """
        Train model on `docs`, and run evaluations on `test_docs`.

        The datasets are expected to produce a tuple of:
            - dict(str -> tf.Tensor): having tokenized `title` and `text`
            - tf.Tensor: the list of labels as strings

        Args:
            docs_path (str): the path to the documents for training
            test_docs_path (str): the path to the documents for evaluation
            val_size (int): size of the validation set, in nr of docs
            shuffle (int): size of the buffer use to shuffle the training set
            epochs (int): max number of epochs to run
            eval_every (int): number of training steps to wait for each eval
        """

        def train_input():
            docs = tf.data.TFRecordDataset([docs_path])\
                          .map(datautils.parse_example)
            return input_fn(docs.skip(val_size),
                            batch_size=self.batch_size,
                            shuffle=shuffle,
                            repeat=epochs)

        def val_input():
            docs = tf.data.TFRecordDataset([docs_path])\
                          .map(datautils.parse_example)
            return input_fn(docs.take(val_size), batch_size=self.batch_size)

        def test_input():
            test_docs = tf.data.TFRecordDataset([test_docs_path])\
                               .map(datautils.parse_example)
            return input_fn(test_docs, batch_size=self.batch_size)

        hooks = []
        if val_size:
            hooks.append(tf.estimator.experimental.InMemoryEvaluatorHook(
                self.estimator,
                val_input,
                name="val",
                every_n_iter=eval_every))
        if test_docs_path:
            hooks.append(tf.estimator.experimental.InMemoryEvaluatorHook(
                self.estimator,
                test_input,
                name="test",
                every_n_iter=eval_every))

        self.estimator.train(input_fn=train_input, hooks=hooks)

def input_fn(docs, batch_size=64, shuffle=None, repeat=None):
    with tf.name_scope("input_processing"):
        def pad_empty(doc, labels):
            doc = {k: tf.cond(tf.equal(tf.shape(v)[0], 0),
                              lambda: tf.constant([""]),
                              lambda: v)
                   for k, v in doc.items()}
            return doc, labels

        def mask(doc, labels):
            ret = {k + "_mask": tf.ones_like(v, dtype=tf.float32)
                   for k, v in doc.items()}
            ret.update(doc)
            return ret, labels

        docs = docs.map(pad_empty)
        docs = docs.map(mask)

        if repeat:
            docs = docs.repeat(repeat)

        if shuffle:
            docs = docs.shuffle(buffer_size=shuffle)

        pads = ({
                    "title": [None],
                    "title_mask": [None],
                    "text": [None],
                    "text_mask": [None]
                }, [None])
        return docs.padded_batch(batch_size, padded_shapes=pads)

def model_fn(features, labels, mode, params):
    encoder = params["encoder"]
    decoder = params["decoder"]
    label_vocab = params["label_vocab"]
    lr = params["learning_rate"]
    clipping = params["grad_clip"]
    decay_rate = params["decay_rate"]
    decay_steps = params["decay_steps"]

    with tf.name_scope("expected_output"):
        if labels is not None:
            labels = label_idx_to_hot(labels, label_vocab)

    with tf.name_scope("model"):
        mem, mem_len, mem_fixed = encoder(features, mode)
        predictions, loss = decoder(mem, mem_len, mem_fixed, labels, mode)

    eval_metric_ops = None
    if mode in [tf.estimator.ModeKeys.EVAL, tf.estimator.ModeKeys.TRAIN]:
        eval_metric_ops = metrics.create(labels, predictions, label_vocab)

    train_op = None
    if mode == tf.estimator.ModeKeys.TRAIN:
        train_op = create_optimizer(loss, lr, clipping, decay_rate, decay_steps)

    pred = {"predictions": predictions}
    return tf.estimator.EstimatorSpec(mode,
                                      loss=loss,
                                      train_op=train_op,
                                      eval_metric_ops=eval_metric_ops,
                                      predictions=pred)

def label_idx_to_hot(labels, vocab):
    """
    Coverts the labels from a list of ids to a multi-hot vector.

    Args:
        labels (tf.Tensor): expected labels as a list of ids.
          [batch, nr_positive_labels]
        vocab (list[str]): vocabulary of labels

    Returns:
        labels (tf.Tensor): expected labels as a list of ids.
          [batch, nr_labels]
    """
    # Convert strings to ids (pads become -1)
    # (batch, n_positive_labels)
    labels = tf.contrib.lookup.index_table_from_tensor(
        mapping=vocab,
        default_value=-1).lookup(labels)

    # Turn indexes to one-hot vectors (pads become all 0)
    # (batch, n_positive_labels, n_classes)
    labels = tf.one_hot(labels, len(vocab), dtype=tf.float32)

    # Turn labels into fixed-sized 1/0 vector
    # (batch, n_classes)
    return tf.reduce_sum(labels, 1)

def create_optimizer(loss, learning_rate, max_norm, decay_rate, decay_steps):
    """
    Creates an optimizing operation for the given loss function.

    Args:
        loss (tf.Tensor): a loss function to optimize against
        learning_rate (float): the learning rate for the optimizer
        max_norm (float): maximum norm for any gradient when doing the update
        decay_rate (float): the factor to decay the learning rate
        decay_steps (int): how many steps to wait for each decay

    Returns:
        updater (tf.Tensor): operation that updates a single step of the network
    """
    global_step = tf1.train.get_or_create_global_step()

    if decay_rate < 1.:
        learning_rate = tf.train.exponential_decay(
            learning_rate, global_step, decay_steps, decay_rate, staircase=True)

    tf1.summary.scalar("misc/learning_rate", learning_rate)

    opt = tf.contrib.opt.LazyAdamOptimizer(learning_rate=learning_rate)
    grad, var = zip(*opt.compute_gradients(loss))

    if max_norm > 0.:
        grad = clip_gradients(grad, max_norm)

    return opt.apply_gradients(zip(grad, var), global_step=global_step)

def clip_gradients(gradients, max_norm):
    """
    Clips gradients to the maximum `max_norm`.

    Args:
        gradients: the gradients to be applied when optimizing
        max_norm: the maximum norm for each gradient

    Returns:
        gradients: the gradients to be applied when optimizing
    """
    gradients, norm = tf.clip_by_global_norm(gradients, max_norm)

    tf1.summary.scalar("misc/grad_norm", norm)
    tf1.summary.scalar("misc/clipped_gradient", tf.linalg.global_norm(gradients))

    return gradients
<|endoftext|>


<commit_before>"""Model generic, a dual polarized antenna element."""
# TobiaC 2015-12-02

import numpy
import matplotlib.pyplot as plt
import matplotlib.dates
from antpat.reps.sphgridfun import pntsonsphere, tvecfun
from antpat.io.feko_ffe import FEKOffe
from antpat.radfarfield import RadFarField

class DualPolElem(object):
    """Main class for a dual-pol antenna element. It can be constructed
    from two generic representations: two radiation far-fields (two
    single pol antennas) or a tangential (Jones) matrix.
    """

    def __init__(self, *args):
        if len(args) == 0:
            self.tmfd = None
        elif len(args) == 1:
            #"tmfd" stands for tangential matrix field on directions
            self.tmfd = args[0]
            self.radFFp = None
            self.radFFq = None
        elif len(args) == 2:
            self.tmfd = None
            self.radFFp = args[0]
            self.radFFq = args[1]
        else:
            raise RuntimeError("Not more than two arguments")
        self.basis = None

    def getfreqs(self):
        """Get Frequencies"""
        if self.tmfd is None:
            return self.radFFp.getfreqs()
        else:
            return self.tmfd.getfreqs()

    def getJonesPat(self, freqval):
        """Return the dual-pol antenna elements Jones pattern for a
        given frequency."""
        THETA, PHI, p_E_th, p_E_ph = self.radFFp.getFFongrid(freqval)
        THETA, PHI, q_E_th, q_E_ph = self.radFFq.getFFongrid(freqval)

        Jones = numpy.zeros(p_E_th.shape+(2, 2), dtype=complex)
        Jones[..., 0, 0] = p_E_th
        Jones[..., 0, 1] = p_E_ph
        Jones[..., 1, 0] = q_E_th
        Jones[..., 1, 1] = q_E_ph
        return THETA, PHI, Jones

    def getJonesAlong(self, freqval, theta_phi_view):
        theta_view, phi_view = theta_phi_view
        (theta_build, phi_build) = self.view2build_coords(theta_view, phi_view)
        if self.tmfd is None:
            p_E_th, p_E_ph = self.radFFp.getFFalong_build(freqval,
                                                          (theta_build, phi_build))
            q_E_th, q_E_ph = self.radFFq.getFFalong_build(freqval,
                                                          (theta_build, phi_build))
            Jones = numpy.zeros(p_E_th.shape+(2, 2), dtype=complex)
            Jones[..., 0, 0] = p_E_th
            Jones[..., 0, 1] = p_E_ph
            Jones[..., 1, 0] = q_E_th
            Jones[..., 1, 1] = q_E_ph
        else:
            Jones = self.tmfd.getJonesAlong(freqval,
                                            (theta_build, phi_build))
            if self.basis is not None:
                p_E_th = Jones[..., 0, 0]
                p_E_ph = Jones[..., 0, 1]
                q_E_th = Jones[..., 1, 0]
                q_E_ph = Jones[..., 1, 1]
                p_E_th, p_E_ph = tvecfun.transfVecField2RotBasis(self.basis,
                                                                 (theta_build,
                                                                  phi_build),
                                                                 (p_E_th, p_E_ph))
                q_E_th, q_E_ph = tvecfun.transfVecField2RotBasis(self.basis,
                                                                 (theta_build,
                                                                  phi_build),
                                                                 (q_E_th, q_E_ph))
                Jones[..., 0, 0] = p_E_th
                Jones[..., 0, 1] = p_E_ph
                Jones[..., 1, 0] = q_E_th
                Jones[..., 1, 1] = q_E_ph
        return Jones

    def getFFalong(self, freqval,  theta_phi_view, polchan=0):
        jones = self.getJonesAlong(freqval,  theta_phi_view)
        E_th = jones[..., polchan, 0].squeeze()
        E_ph = jones[..., polchan, 1].squeeze()
        return E_th, E_ph

    def view2build_coords(self, theta_view, phi_view):
        """Get the corresponding directions in the build frame."""
        if self.basis is not None:
            (theta_build, phi_build) = pntsonsphere.rotToFrame(
                                           numpy.transpose(self.basis),
                                           theta_view, phi_view)
        else:
            (theta_build, phi_build) = (theta_view, phi_view)
        return (theta_build, phi_build)

    def rotateframe(self, rotMat):
        """Rotate the frame of antenna. This is a 'passive' rotation: it
        does not rotate the field, but when evaluated in some
        direction the direction given will be rotated to the frame so
        as to appear as if it were rotated.

        The basis or rotation matrix is to be considered as acting on
        the antenna, i.e.

              view_crds=rotMat*build_crds

        assuming the the antenna has not been rotated already. If it has then
        the inputted rotation is added to the current rotation, so that

              view_crds=rotMat*rotMat_0*build_crds

        where rotMat_0 is previous rotation state (could be aggregate of many).
        """
        if self.basis is None:
            self.basis = rotMat
        else:
            self.basis = numpy.matmul(rotMat, self.basis)

    def load_ffes(self, filename_p, filename_q):
        """Load a pair of FFE and make them correspond to this DualPolElem
        object. First file will be pol-channel p and second q."""
        ffefile_p = FEKOffe(filename_p)
        tvf_p = tvecfun.TVecFields()
        tvf_q = tvecfun.TVecFields()
        tvf_p.load_ffe(filename_p)
        tvf_q.load_ffe(filename_q)
        self.radFFp = RadFarField(tvf_p)
        self.radFFq = RadFarField(tvf_q)

    def load_ffe(self, filename, request_p=None, request_q=None):
        #FIX: This not the most efficient way to do this as it does two passes over feko file.
        ffefile = FEKOffe(filename)
        if request_p is None and request_q is None:
            if len(ffefile.Requests) == 2:
                requests = list(ffefile.Requests)
                requests.sort()  # # FIXME: Not sure how to order requests
                request_p = requests[0]
                request_q = requests[1]
            else:
                raise RuntimeError(
                    "File contains multiple FFs (specify one): "
                    + ','.join(ffefile.Requests))
        print("Request_p= "+request_p)
        print("Request_q= "+request_q)
        tvf_p = tvecfun.TVecFields()
        tvf_q = tvecfun.TVecFields()
        tvf_p.load_ffe(filename, request_p)
        tvf_q.load_ffe(filename, request_q)
        self.radFFp = RadFarField(tvf_p)
        self.radFFq = RadFarField(tvf_q)

    def plotJonesPat3D(self, freq=0.0, vcoord='sph',
                       projection='equirectangular', cmplx_rep='AbsAng'):
        """Plot the Jones pattern as two single pol antenna patterns."""
        theta_rad, phi_rad, JonesPat = self.getJonesPat(freq)
        Ep = numpy.squeeze(JonesPat[..., 0, :])
        Eq = numpy.squeeze(JonesPat[..., 1, :])
        tvecfun.plotvfonsph(theta_rad, phi_rad, numpy.squeeze(Ep[..., 0]),
                            numpy.squeeze(Ep[..., 1]), freq, vcoord,
                            projection, cmplx_rep, vfname='p-chan:'+self.radFFp.name)
        tvecfun.plotvfonsph(theta_rad, phi_rad, numpy.squeeze(Eq[..., 0]),
                            numpy.squeeze(Eq[..., 1]), freq, vcoord,
                            projection, cmplx_rep, vfname='q-chan:'+self.radFFp.name)

def plot_polcomp_dynspec(tims, frqs, jones):
    """Plot dynamic power spectra of each polarization component."""
    #fig, (ax0, ax1) = plt.subplots(nrows=2)
    p_ch = numpy.abs(jones[:, :, 0, 0].squeeze())**2 + \
        numpy.abs(jones[:, :, 0, 1].squeeze())**2
    q_ch = numpy.abs(jones[:, :, 1, 1].squeeze())**2 + \
        numpy.abs(jones[:, :, 1, 0].squeeze())**2
    ftims = matplotlib.dates.date2num(tims)
    dynspecunit = 'flux arb.'
    # In dB
    dBunit = False
    if dBunit:
        p_ch = 10*numpy.log10(p_ch)
        q_ch = 10*numpy.log10(q_ch)
        dynspecunit += ' dB'
    dynspecunit += ' unit'
    plt.figure()
    plt.subplot(211)
    plt.pcolormesh(numpy.asarray(tims), frqs, p_ch)
    plt.title('p-channel')
    #plt.clim(0, 1.0)
    plt.colorbar().set_label(dynspecunit)
    plt.subplot(212)
    plt.pcolormesh(numpy.asarray(tims), frqs, q_ch)
    plt.title('q-channel')
    #plt.clim(0, 1.0)
    plt.xlabel('Time')
    plt.ylabel('Frequency')
    plt.colorbar().set_label(dynspecunit)
    plt.show()

def jones2gIXR(jones):
    U, s, V = numpy.linalg.svd(jones)
    g = (s[..., 0]+s[..., 1])*0.5
    cnd = s[..., 0]/s[..., 1]
    IXRJ = ((1+cnd)/(1-cnd))**2
    return g, IXRJ

def ampgain2intensitygain(g):
    pass

def IXRJ2IXRM(IXRJ):
    """Convert Jones IXR to Mueller IXR. See Carozzi2011."""
    return (1+IXRJ)/(1*numpy.sqrt(IXRJ))
<commit_msg>Add: new function that computes intrumental Stokes from a Jones matrix<commit_after>"""Model generic, a dual polarized antenna element."""
# TobiaC 2015-12-02

import numpy
import matplotlib.pyplot as plt
import matplotlib.dates
from antpat.reps.sphgridfun import pntsonsphere, tvecfun
from antpat.io.feko_ffe import FEKOffe
from antpat.radfarfield import RadFarField

class DualPolElem(object):
    """Main class for a dual-pol antenna element. It can be constructed
    from two generic representations: two radiation far-fields (two
    single pol antennas) or a tangential (Jones) matrix.
    """

    def __init__(self, *args):
        if len(args) == 0:
            self.tmfd = None
        elif len(args) == 1:
            #"tmfd" stands for tangential matrix field on directions
            self.tmfd = args[0]
            self.radFFp = None
            self.radFFq = None
        elif len(args) == 2:
            self.tmfd = None
            self.radFFp = args[0]
            self.radFFq = args[1]
        else:
            raise RuntimeError("Not more than two arguments")
        self.basis = None

    def getfreqs(self):
        """Get Frequencies"""
        if self.tmfd is None:
            return self.radFFp.getfreqs()
        else:
            return self.tmfd.getfreqs()

    def getJonesPat(self, freqval):
        """Return the dual-pol antenna elements Jones pattern for a
        given frequency."""
        THETA, PHI, p_E_th, p_E_ph = self.radFFp.getFFongrid(freqval)
        THETA, PHI, q_E_th, q_E_ph = self.radFFq.getFFongrid(freqval)

        Jones = numpy.zeros(p_E_th.shape+(2, 2), dtype=complex)
        Jones[..., 0, 0] = p_E_th
        Jones[..., 0, 1] = p_E_ph
        Jones[..., 1, 0] = q_E_th
        Jones[..., 1, 1] = q_E_ph
        return THETA, PHI, Jones

    def getJonesAlong(self, freqval, theta_phi_view):
        theta_view, phi_view = theta_phi_view
        (theta_build, phi_build) = self.view2build_coords(theta_view, phi_view)
        if self.tmfd is None:
            p_E_th, p_E_ph = self.radFFp.getFFalong_build(freqval,
                                                          (theta_build, phi_build))
            q_E_th, q_E_ph = self.radFFq.getFFalong_build(freqval,
                                                          (theta_build, phi_build))
            Jones = numpy.zeros(p_E_th.shape+(2, 2), dtype=complex)
            Jones[..., 0, 0] = p_E_th
            Jones[..., 0, 1] = p_E_ph
            Jones[..., 1, 0] = q_E_th
            Jones[..., 1, 1] = q_E_ph
        else:
            Jones = self.tmfd.getJonesAlong(freqval,
                                            (theta_build, phi_build))
            if self.basis is not None:
                p_E_th = Jones[..., 0, 0]
                p_E_ph = Jones[..., 0, 1]
                q_E_th = Jones[..., 1, 0]
                q_E_ph = Jones[..., 1, 1]
                p_E_th, p_E_ph = tvecfun.transfVecField2RotBasis(self.basis,
                                                                 (theta_build,
                                                                  phi_build),
                                                                 (p_E_th, p_E_ph))
                q_E_th, q_E_ph = tvecfun.transfVecField2RotBasis(self.basis,
                                                                 (theta_build,
                                                                  phi_build),
                                                                 (q_E_th, q_E_ph))
                Jones[..., 0, 0] = p_E_th
                Jones[..., 0, 1] = p_E_ph
                Jones[..., 1, 0] = q_E_th
                Jones[..., 1, 1] = q_E_ph
        return Jones

    def getFFalong(self, freqval,  theta_phi_view, polchan=0):
        jones = self.getJonesAlong(freqval,  theta_phi_view)
        E_th = jones[..., polchan, 0].squeeze()
        E_ph = jones[..., polchan, 1].squeeze()
        return E_th, E_ph

    def view2build_coords(self, theta_view, phi_view):
        """Get the corresponding directions in the build frame."""
        if self.basis is not None:
            (theta_build, phi_build) = pntsonsphere.rotToFrame(
                                           numpy.transpose(self.basis),
                                           theta_view, phi_view)
        else:
            (theta_build, phi_build) = (theta_view, phi_view)
        return (theta_build, phi_build)

    def rotateframe(self, rotMat):
        """Rotate the frame of antenna. This is a 'passive' rotation: it
        does not rotate the field, but when evaluated in some
        direction the direction given will be rotated to the frame so
        as to appear as if it were rotated.

        The basis or rotation matrix is to be considered as acting on
        the antenna, i.e.

              view_crds=rotMat*build_crds

        assuming the the antenna has not been rotated already. If it has then
        the inputted rotation is added to the current rotation, so that

              view_crds=rotMat*rotMat_0*build_crds

        where rotMat_0 is previous rotation state (could be aggregate of many).
        """
        if self.basis is None:
            self.basis = rotMat
        else:
            self.basis = numpy.matmul(rotMat, self.basis)

    def load_ffes(self, filename_p, filename_q):
        """Load a pair of FFE and make them correspond to this DualPolElem
        object. First file will be pol-channel p and second q."""
        ffefile_p = FEKOffe(filename_p)
        tvf_p = tvecfun.TVecFields()
        tvf_q = tvecfun.TVecFields()
        tvf_p.load_ffe(filename_p)
        tvf_q.load_ffe(filename_q)
        self.radFFp = RadFarField(tvf_p)
        self.radFFq = RadFarField(tvf_q)

    def load_ffe(self, filename, request_p=None, request_q=None):
        #FIX: This not the most efficient way to do this as it does two passes over feko file.
        ffefile = FEKOffe(filename)
        if request_p is None and request_q is None:
            if len(ffefile.Requests) == 2:
                requests = list(ffefile.Requests)
                requests.sort()  # # FIXME: Not sure how to order requests
                request_p = requests[0]
                request_q = requests[1]
            else:
                raise RuntimeError(
                    "File contains multiple FFs (specify one): "
                    + ','.join(ffefile.Requests))
        print("Request_p= "+request_p)
        print("Request_q= "+request_q)
        tvf_p = tvecfun.TVecFields()
        tvf_q = tvecfun.TVecFields()
        tvf_p.load_ffe(filename, request_p)
        tvf_q.load_ffe(filename, request_q)
        self.radFFp = RadFarField(tvf_p)
        self.radFFq = RadFarField(tvf_q)

    def plotJonesPat3D(self, freq=0.0, vcoord='sph',
                       projection='equirectangular', cmplx_rep='AbsAng'):
        """Plot the Jones pattern as two single pol antenna patterns."""
        theta_rad, phi_rad, JonesPat = self.getJonesPat(freq)
        Ep = numpy.squeeze(JonesPat[..., 0, :])
        Eq = numpy.squeeze(JonesPat[..., 1, :])
        tvecfun.plotvfonsph(theta_rad, phi_rad, numpy.squeeze(Ep[..., 0]),
                            numpy.squeeze(Ep[..., 1]), freq, vcoord,
                            projection, cmplx_rep, vfname='p-chan:'+self.radFFp.name)
        tvecfun.plotvfonsph(theta_rad, phi_rad, numpy.squeeze(Eq[..., 0]),
                            numpy.squeeze(Eq[..., 1]), freq, vcoord,
                            projection, cmplx_rep, vfname='q-chan:'+self.radFFp.name)

def plot_polcomp_dynspec(tims, frqs, jones):
    """Plot dynamic power spectra of each polarization component."""
    #fig, (ax0, ax1) = plt.subplots(nrows=2)
    p_ch = numpy.abs(jones[:, :, 0, 0].squeeze())**2 + \
        numpy.abs(jones[:, :, 0, 1].squeeze())**2
    q_ch = numpy.abs(jones[:, :, 1, 1].squeeze())**2 + \
        numpy.abs(jones[:, :, 1, 0].squeeze())**2
    ftims = matplotlib.dates.date2num(tims)
    dynspecunit = 'flux arb.'
    # In dB
    dBunit = False
    if dBunit:
        p_ch = 10*numpy.log10(p_ch)
        q_ch = 10*numpy.log10(q_ch)
        dynspecunit += ' dB'
    dynspecunit += ' unit'
    plt.figure()
    plt.subplot(211)
    plt.pcolormesh(numpy.asarray(tims), frqs, p_ch)
    plt.title('p-channel')
    #plt.clim(0, 1.0)
    plt.colorbar().set_label(dynspecunit)
    plt.subplot(212)
    plt.pcolormesh(numpy.asarray(tims), frqs, q_ch)
    plt.title('q-channel')
    #plt.clim(0, 1.0)
    plt.xlabel('Time')
    plt.ylabel('Frequency')
    plt.colorbar().set_label(dynspecunit)
    plt.show()

def jones2gIXR(jones):
    U, s, V = numpy.linalg.svd(jones)
    g = (s[..., 0]+s[..., 1])*0.5
    cnd = s[..., 0]/s[..., 1]
    IXRJ = ((1+cnd)/(1-cnd))**2
    return g, IXRJ

def instrumentalStokes(jones):
    """\
    Compute the intrumental Stokes parameters from a Jones matrix

    The instrumental Stokes is computed from the hermitian square
    of the Jones matrix.

    Parameters
    ----------
    jones : array
        A stack of Jones matrices with the last two indices pertaining
        to the Jones msatrix proper.

    Returns
    -------
    si : array
        Stokes I.
    sq : array
        Stokes Q.
    su : array
        Stokes U.
    sv : array
        Stokes V.
    """
    cov = numpy.matmul(jones, numpy.conjugate(numpy.swapaxes(jones, -2, -1)))
    cov_xx = cov[..., 0, 0]
    cov_xy = cov[..., 0, 1]
    cov_yx = cov[..., 1, 0]
    cov_yy = cov[..., 1, 1]
    si = +1*numpy.real(cov_xx + cov_yy)
    sq = +1*numpy.real(cov_xx - cov_yy)
    su = +1*numpy.real(cov_xy + cov_yx)
    sv = -1*numpy.imag(cov_xy - cov_yx)
    return si, sq, su, sv

def ampgain2intensitygain(g):
    pass

def IXRJ2IXRM(IXRJ):
    """Convert Jones IXR to Mueller IXR. See Carozzi2011."""
    return (1+IXRJ)/(1*numpy.sqrt(IXRJ))
<|endoftext|>


<commit_before>"""
Django settings for apel_rest project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '6^&2c@p@nld52qs4nfp1%99plh&*69&@@9%no3sgq6v+ci&g73'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False

TEMPLATE_DEBUG = False

ALLOWED_HOSTS = ['*']

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'rest_framework',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.RemoteUserMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

# AUTHENTICATION_BACKENDS = (
#     'django.contrib.auth.backends.RemoteUserBackend',
# )

ROOT_URLCONF = 'apel_rest.urls'

WSGI_APPLICATION = 'apel_rest.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_ROOT = '/var/www/html/static'
STATIC_URL = '/static/'

# debugging stuff
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s]: %(message)s",
            'datefmt': "%d/%b/%Y %H:%M:%S"
        },
        'simple': {
            'format': '%(levelname)s: %(message)s'
        },
    },
    'handlers': {
        'file': {
            'class': 'logging.FileHandler',
            'filename': '/var/www/html/apel_rest.log',
            'formatter': 'verbose'
        },
        'console': {
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        },
    },
    'loggers': {
        'django': {
            'handlers': ['console'],
            'propagate': True,
            'level': 'INFO',
        },
        'api': {
            'handlers': ['console'],
            'level': 'INFO',
            #'level': 'DEBUG',
        },
    }
}

# XML stuff
REST_FRAMEWORK = {
    'DEFAULT_PARSER_CLASSES': (
        'rest_framework_xml.parsers.XMLParser',
    ),
    # 'DEFAULT_RENDERER_CLASSES': (
    #     'rest_framework_xml.renderers.XMLRenderer',
    # ),
}

# QPATH defines where the rest api will store
# incoming messages for later processing
QPATH = '/tmp/django'
<commit_msg>Remove file log handler<commit_after>"""
Django settings for apel_rest project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '6^&2c@p@nld52qs4nfp1%99plh&*69&@@9%no3sgq6v+ci&g73'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False

TEMPLATE_DEBUG = False

ALLOWED_HOSTS = ['*']

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'rest_framework',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.RemoteUserMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

# AUTHENTICATION_BACKENDS = (
#     'django.contrib.auth.backends.RemoteUserBackend',
# )

ROOT_URLCONF = 'apel_rest.urls'

WSGI_APPLICATION = 'apel_rest.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_ROOT = '/var/www/html/static'
STATIC_URL = '/static/'

# debugging stuff
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s]: %(message)s",
            'datefmt': "%d/%b/%Y %H:%M:%S"
        },
        'simple': {
            'format': '%(levelname)s: %(message)s'
        },
    },
    'handlers': {
        # 'file': {
        #     'class': 'logging.FileHandler',
        #     'filename': './apel_rest.log',
        #     'formatter': 'verbose'
        # },
        'console': {
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        },
    },
    'loggers': {
        'django': {
            'handlers': ['console'],
            'propagate': True,
            'level': 'INFO',
        },
        'api': {
            'handlers': ['console'],
            'level': 'INFO',
            #'level': 'DEBUG',
        },
    }
}

# XML stuff
REST_FRAMEWORK = {
    'DEFAULT_PARSER_CLASSES': (
        'rest_framework_xml.parsers.XMLParser',
    ),
    # 'DEFAULT_RENDERER_CLASSES': (
    #     'rest_framework_xml.renderers.XMLRenderer',
    # ),
}

# QPATH defines where the rest api will store
# incoming messages for later processing
QPATH = '/tmp/django'
<|endoftext|>


<commit_before>"""
Wallabag API accesses.
"""
from enum import Enum
import json
import re
import requests
import time
import conf
from conf import Configs

MINIMUM_API_VERSION = 2, 1, 1
MINIMUM_API_VERSION_HR = "2.1.1"

class OAuthException(Exception):
    pass

class Error(Enum):
    undefined = -1
    ok = 0
    dns_error = 1
    http_bad_request = 400
    http_unauthorized = 401
    http_forbidden = 403
    http_not_found = 404
    unknown_error = 999

class ApiMethod(Enum):
    add_entry = "/api/entries"
    delete_entry = "/api/entries/{0}"
    get_entry = "/api/entries/{0}"
    update_entry = "/api/entries/{0}"
    entry_exists = "/api/entries/exists"
    list_entries = "/api/entries"
    token = "/oauth/v2/token"
    version = "/api/version"

class Response:
    http_code = 0
    error = Error.undefined
    error_text = ""
    error_description = ""

    response = ""

    def __init__(self, status_code, http_response):
        self.http_code = status_code
        self.response = http_response

        # DNS not found
        if self.http_code == 0:
            self.error = Error.dns_error
            self.error_text = "Name or service not known."
        # 400 bad request
        elif self.http_code == 400:
            self.error = Error.http_bad_request
            errors = json.loads(self.response)
            if 'error' in errors:
                self.error_text = errors['error']
            if 'error_description' in errors:
                self.error_description = errors['error_description']
        # 401 unauthorized
        elif self.http_code == 401:
            self.error = Error.http_unauthorized
            errors = json.loads(self.response)
            if 'error' in errors:
                self.error_text = errors['error']
            if 'error_description' in errors:
                self.error_description = errors['error_description']
        # 403 forbidden
        elif self.http_code == 403:
            self.error = Error.http_forbidden
            self.error_text = "403: Could not reach API due to rights issues."
        # 404 not found
        elif self.http_code == 404:
            self.error = Error.http_not_found
            self.error_text = "404: API was not found."
        # 200 okay
        elif self.http_code == 200:
            self.error = Error.ok
        # unknown Error
        else:
            self.error = Error.unknown_error
            self.error_text = "An unknown error occured."

    def is_rersponse_status_ok(self):
        return self.http_code == 200

    def hasError(self):
        return self.error != Error.ok

def __get_api_url(api_method, different_url=None):
    if api_method in ApiMethod:
        if different_url != None:
            return different_url + api_method.value
        return Configs.serverurl + api_method.value
    return None

def __get_authorization_header():
    success, token_or_error = get_token()
    if not success:
        e = OAuthException
        e.text = token_or_error
        raise e
    else:
        return {'Authorization': "Bearer {0}".format(token_or_error)}

def __request_delete(url, headers=None):
    ret = None
    request = None

    try:
        request = requests.delete(url, headers=headers)
        ret = Response(request.status_code, request.text)
    # dns error
    except (requests.exceptions.ConnectionError, requests.exceptions.MissingSchema):
        ret = Response(0, None)
    return ret

def __request_get(url, headers=None, params=None):
    ret = None
    request = None

    try:
        request = requests.get(url, headers=headers, params=params)
        ret = Response(request.status_code, request.text)
    # dns error
    except (requests.exceptions.ConnectionError, requests.exceptions.MissingSchema):
        ret = Response(0, None)
    return ret

def __request_post(url, headers=None, data=None):
    ret = None
    request = None

    try:
        request = requests.post(url, data=data, headers=headers)
        ret = Response(request.status_code, request.text)
    # dns error
    except (requests.exceptions.ConnectionError, requests.exceptions.MissingSchema):
        ret = Response(0, None)
    return ret

def __request_patch(url, headers=None, data=None):
    ret = None
    request = None

    try:
        request = requests.patch(url, data=data, headers=headers)
        ret = Response(request.status_code, request.text)
    # dns error
    except (requests.exceptions.ConnectionError, requests.exceptions.MissingSchema):
        ret = Response(0, None)
    return ret

def is_valid_url(url):
    response = __request_get(url)
    return not response.hasError()

def is_minimum_version(version_response):
    versionstring = version_response.response

    if not re.compile('"\\d+\\.\\d+\\.\\d+"').match(versionstring):
        return False

    ver = versionstring.strip('"').split('.')

    x = int(ver[0])
    y = int(ver[1])
    z = int(ver[2])
    tx, ty, tz = MINIMUM_API_VERSION

    if x > tx:
        return True
    elif x < tx:
        return False
    else:
        if y > ty:
            return True
        elif y < ty:
            return False
        else:
            return z >= tz

def api_version(different_url=None):
    url = __get_api_url(ApiMethod.version, different_url)
    response = __request_get(url)
    return response

def api_token():
    url = __get_api_url(ApiMethod.token)
    data = dict()
    data['grant_type'] = "password"
    data['client_id'] = Configs.client
    data['client_secret'] = Configs.secret
    data['username'] = Configs.username
    data['password'] = Configs.password

    response = __request_get(url, params=data)
    return response

def api_add_entry(targeturl, title=None, star=False, read=False):
    url = __get_api_url(ApiMethod.add_entry)
    header = __get_authorization_header()
    data = dict()
    data['url'] = targeturl
    if title != None:
        data['title'] = title
    if star:
        data['starred'] = 1
    if read:
        data['archive'] = 1
    response = __request_post(url, header, data)
    return response

def api_delete_entry(entry_id):
    url = __get_api_url(ApiMethod.delete_entry).format(entry_id)
    header = __get_authorization_header()

    response = __request_delete(url, header)
    return response

def api_entry_exists(url):
    apiurl = __get_api_url(ApiMethod.entry_exists)
    header = __get_authorization_header()
    data = dict()
    data['url'] = url
    response = __request_get(apiurl, headers=header, params=data)
    return response

def api_get_entry(entry_id):
    url = __get_api_url(ApiMethod.get_entry).format(entry_id)
    header = __get_authorization_header()

    response = __request_get(url, header)
    return response

def api_list_entries(count, filter_read=None, filter_starred=None, oldest=False):
    url = __get_api_url(ApiMethod.list_entries)
    header = __get_authorization_header()
    params = dict()

    params['perPage'] = count

    if oldest:
        params['order'] = "asc"

    if filter_read != None:
        if filter_read:
            params['archive'] = 1
        else:
            params['archive'] = 0

    if filter_starred != None:
        if filter_starred:
            params['starred'] = 1
        else:
            params['starred'] = 0

    response = __request_get(url, headers=header, params=params)
    return response

def get_token(force_creation=False):
    """
    Returns a valid oauth token. Creates a new one if the old token is expired.

    Parameters:
    -----------
    force_creation [optional]bool
        Enforces the creation even if the old token is valid.

    Returns:
    --------
    bool
        Getting the token was successful.
    string
        A valid token or an error message.
    """
    if conf.is_token_expired() or force_creation:
        response = api_token()
        if not response.hasError():
            content = json.loads(response.response)
            Configs.access_token = content['access_token']
            Configs.expires = time.time() + content['expires_in']
            conf.save()
            return True, Configs.access_token
        else:
            if response.error_description == "":
                return False, response.error_text
            else:
                return False, "{0} - {1}".format(response.error_text, response.error_description)
    else:
        return True, Configs.access_token
<commit_msg>Added method for updating existing entries<commit_after>"""
Wallabag API accesses.
"""
from enum import Enum
import json
import re
import requests
import time
import conf
from conf import Configs

MINIMUM_API_VERSION = 2, 1, 1
MINIMUM_API_VERSION_HR = "2.1.1"

class OAuthException(Exception):
    pass

class Error(Enum):
    undefined = -1
    ok = 0
    dns_error = 1
    http_bad_request = 400
    http_unauthorized = 401
    http_forbidden = 403
    http_not_found = 404
    unknown_error = 999

class ApiMethod(Enum):
    add_entry = "/api/entries"
    delete_entry = "/api/entries/{0}"
    get_entry = "/api/entries/{0}"
    update_entry = "/api/entries/{0}"
    entry_exists = "/api/entries/exists"
    list_entries = "/api/entries"
    token = "/oauth/v2/token"
    version = "/api/version"

class Response:
    http_code = 0
    error = Error.undefined
    error_text = ""
    error_description = ""

    response = ""

    def __init__(self, status_code, http_response):
        self.http_code = status_code
        self.response = http_response

        # DNS not found
        if self.http_code == 0:
            self.error = Error.dns_error
            self.error_text = "Name or service not known."
        # 400 bad request
        elif self.http_code == 400:
            self.error = Error.http_bad_request
            errors = json.loads(self.response)
            if 'error' in errors:
                self.error_text = errors['error']
            if 'error_description' in errors:
                self.error_description = errors['error_description']
        # 401 unauthorized
        elif self.http_code == 401:
            self.error = Error.http_unauthorized
            errors = json.loads(self.response)
            if 'error' in errors:
                self.error_text = errors['error']
            if 'error_description' in errors:
                self.error_description = errors['error_description']
        # 403 forbidden
        elif self.http_code == 403:
            self.error = Error.http_forbidden
            self.error_text = "403: Could not reach API due to rights issues."
        # 404 not found
        elif self.http_code == 404:
            self.error = Error.http_not_found
            self.error_text = "404: API was not found."
        # 200 okay
        elif self.http_code == 200:
            self.error = Error.ok
        # unknown Error
        else:
            self.error = Error.unknown_error
            self.error_text = "An unknown error occured."

    def is_rersponse_status_ok(self):
        return self.http_code == 200

    def hasError(self):
        return self.error != Error.ok

def __get_api_url(api_method, different_url=None):
    if api_method in ApiMethod:
        if different_url != None:
            return different_url + api_method.value
        return Configs.serverurl + api_method.value
    return None

def __get_authorization_header():
    success, token_or_error = get_token()
    if not success:
        e = OAuthException
        e.text = token_or_error
        raise e
    else:
        return {'Authorization': "Bearer {0}".format(token_or_error)}

def __request_delete(url, headers=None):
    ret = None
    request = None

    try:
        request = requests.delete(url, headers=headers)
        ret = Response(request.status_code, request.text)
    # dns error
    except (requests.exceptions.ConnectionError, requests.exceptions.MissingSchema):
        ret = Response(0, None)
    return ret

def __request_get(url, headers=None, params=None):
    ret = None
    request = None

    try:
        request = requests.get(url, headers=headers, params=params)
        ret = Response(request.status_code, request.text)
    # dns error
    except (requests.exceptions.ConnectionError, requests.exceptions.MissingSchema):
        ret = Response(0, None)
    return ret

def __request_post(url, headers=None, data=None):
    ret = None
    request = None

    try:
        request = requests.post(url, data=data, headers=headers)
        ret = Response(request.status_code, request.text)
    # dns error
    except (requests.exceptions.ConnectionError, requests.exceptions.MissingSchema):
        ret = Response(0, None)
    return ret

def __request_patch(url, headers=None, data=None):
    ret = None
    request = None

    try:
        request = requests.patch(url, data=data, headers=headers)
        ret = Response(request.status_code, request.text)
    # dns error
    except (requests.exceptions.ConnectionError, requests.exceptions.MissingSchema):
        ret = Response(0, None)
    return ret

def is_valid_url(url):
    response = __request_get(url)
    return not response.hasError()

def is_minimum_version(version_response):
    versionstring = version_response.response

    if not re.compile('"\\d+\\.\\d+\\.\\d+"').match(versionstring):
        return False

    ver = versionstring.strip('"').split('.')

    x = int(ver[0])
    y = int(ver[1])
    z = int(ver[2])
    tx, ty, tz = MINIMUM_API_VERSION

    if x > tx:
        return True
    elif x < tx:
        return False
    else:
        if y > ty:
            return True
        elif y < ty:
            return False
        else:
            return z >= tz

def api_version(different_url=None):
    url = __get_api_url(ApiMethod.version, different_url)
    response = __request_get(url)
    return response

def api_token():
    url = __get_api_url(ApiMethod.token)
    data = dict()
    data['grant_type'] = "password"
    data['client_id'] = Configs.client
    data['client_secret'] = Configs.secret
    data['username'] = Configs.username
    data['password'] = Configs.password

    response = __request_get(url, params=data)
    return response

def api_add_entry(targeturl, title=None, star=False, read=False):
    url = __get_api_url(ApiMethod.add_entry)
    header = __get_authorization_header()
    data = dict()
    data['url'] = targeturl
    if title != None:
        data['title'] = title
    if star:
        data['starred'] = 1
    if read:
        data['archive'] = 1
    response = __request_post(url, header, data)
    return response

def api_delete_entry(entry_id):
    url = __get_api_url(ApiMethod.delete_entry).format(entry_id)
    header = __get_authorization_header()

    response = __request_delete(url, header)
    return response

def api_entry_exists(url):
    apiurl = __get_api_url(ApiMethod.entry_exists)
    header = __get_authorization_header()
    data = dict()
    data['url'] = url
    response = __request_get(apiurl, headers=header, params=data)
    return response

def api_get_entry(entry_id):
    url = __get_api_url(ApiMethod.get_entry).format(entry_id)
    header = __get_authorization_header()

    response = __request_get(url, header)
    return response

def api_update_entry(entry_id, new_title=None, star=None, read=None):
    url = __get_api_url(ApiMethod.update_entry).format(entry_id)
    header = __get_authorization_header()
    data = dict()
    data['starred'] = 1
    if new_title != None:
        data['title'] = new_title
    if star != None:
        if not star:
            data['starred'] = 0
        else:
            data["starred"] = 1
    if read != None:
        if not read:
            data['archive'] = 0
        else:
            data['archive'] = 1
    response = __request_patch(url, header, data)
    return response

def api_list_entries(count, filter_read=None, filter_starred=None, oldest=False):
    url = __get_api_url(ApiMethod.list_entries)
    header = __get_authorization_header()
    params = dict()

    params['perPage'] = count

    if oldest:
        params['order'] = "asc"

    if filter_read != None:
        if filter_read:
            params['archive'] = 1
        else:
            params['archive'] = 0

    if filter_starred != None:
        if filter_starred:
            params['starred'] = 1
        else:
            params['starred'] = 0

    response = __request_get(url, headers=header, params=params)
    return response

def get_token(force_creation=False):
    """
    Returns a valid oauth token. Creates a new one if the old token is expired.

    Parameters:
    -----------
    force_creation [optional]bool
        Enforces the creation even if the old token is valid.

    Returns:
    --------
    bool
        Getting the token was successful.
    string
        A valid token or an error message.
    """
    if conf.is_token_expired() or force_creation:
        response = api_token()
        if not response.hasError():
            content = json.loads(response.response)
            Configs.access_token = content['access_token']
            Configs.expires = time.time() + content['expires_in']
            conf.save()
            return True, Configs.access_token
        else:
            if response.error_description == "":
                return False, response.error_text
            else:
                return False, "{0} - {1}".format(response.error_text, response.error_description)
    else:
        return True, Configs.access_token
<|endoftext|>


<commit_before>"""
Wallabag API accesses.
"""
from enum import Enum
import json
import re
import requests
import time
import conf
from conf import Configs

MINIMUM_API_VERSION = 2, 1, 1
MINIMUM_API_VERSION_HR = "2.1.1"

class Error(Enum):
    undefined = -1
    ok = 0
    dns_error = 1
    http_not_found = 404
    http_forbidden = 403
    http_bad_request = 400
    unknown_error = 999

class ApiMethod(Enum):
    add_entry = "/api/entries"
    token = "/oauth/v2/token"
    version = "/api/version"

class Response:
    http_code = 0
    error = Error.undefined
    error_text = ""
    error_description = ""

    response = ""

    def __init__(self, status_code, http_response):
        self.http_code = status_code
        self.response = http_response

        # DNS not found
        if self.http_code == 0:
            self.error = Error.dns_error
            self.error_text = "Name or service not known."
        # 400 bad request
        elif self.http_code == 400:
            self.error = Error.http_bad_request
            errors = json.loads(self.response)
            if 'error' in errors:
                self.error_text = errors['error']
            if 'error_description' in errors:
                self.error_description = errors['error_description']
        # 404 not found
        elif self.http_code == 404:
            self.error = Error.http_not_found
            self.error_text = "404: API was not found."
        # 403 forbidden
        elif self.http_code == 403:
            self.error = Error.http_forbidden
            self.error_text = "403: Could not reach API due to rights issues."
        # 200 okay
        elif self.http_code == 200:
            self.error = Error.ok
        # unknown Error
        else:
            self.error = Error.unknown_error
            self.error_text = "An unknown error occured."

    def is_rersponse_status_ok(self):
        return self.http_code == 200

    def hasError(self):
        return self.error != Error.ok

def __get_api_url(api_method, different_url=None):
    if api_method in ApiMethod:
        if different_url != None:
            return different_url + api_method.value
        return Configs.serverurl + api_method.value
    return None

def __request_get(url, **data):
    ret = None
    request = None

    try:
        request = requests.get(url, data)
        ret = Response(request.status_code, request.text)
    # dns error
    except requests.exceptions.ConnectionError:
        ret = Response(0, None)
    return ret

def __request_post(url, headers=None, data=None):
    ret = None
    request = None

    try:
        request = requests.post(url, data=data, headers=headers)
        ret = Response(request.status_code, request.text)
    # dns error
    except requests.exceptions.ConnectionError:
        ret = Response(0, None)
    return ret

def is_minimum_version(version_response):
    versionstring = version_response.response

    if not re.compile('"\\d+\\.\\d+\\.\\d+"').match(versionstring):
        return False

    ver = versionstring.strip('"').split('.')

    x = int(ver[0])
    y = int(ver[1])
    z = int(ver[2])
    tx, ty, tz = MINIMUM_API_VERSION

    if x > tx:
        return True
    elif x < tx:
        return False
    else:
        if y > ty:
            return True
        elif y < ty:
            return False
        else:
            return z >= tz

def api_version(different_url=None):
    url = __get_api_url(ApiMethod.version, different_url)
    response = __request_get(url)
    return response

def api_token():
    url = __get_api_url(ApiMethod.token)
    data = dict()
    data['grant_type'] = "password"
    data['client_id'] = Configs.client
    data['client_secret'] = Configs.secret
    data['username'] = Configs.username
    data['password'] = Configs.password

    response = __request_get(url, **data)
    return response

def api_add_entry(url):
    pass

def get_token(force_creation=False):
    """
    Returns a valid oauth token. Creates a new one if the old token is expired.

    Parameters:
    -----------
    force_creation [optional]bool
        Enforces the creation even if the old token is valid.

    Returns:
    --------
    bool
        Getting the token was successful.
    string
        A valid token or an error message.
    """
    if conf.is_token_expired() or force_creation:
        response = api_token()
        if not response.hasError():
            content = json.loads(response.response)
            Configs.access_token = content['access_token']
            Configs.expires = time.time() + content['expires_in']
            conf.save()
            return True, Configs.access_token
        else:
            return False, response.error_text
    else:
        return True, Configs.access_token
<commit_msg>Added function that creates an oauth2 http header<commit_after>"""
Wallabag API accesses.
"""
from enum import Enum
import json
import re
import requests
import time
import conf
from conf import Configs

MINIMUM_API_VERSION = 2, 1, 1
MINIMUM_API_VERSION_HR = "2.1.1"

class Error(Enum):
    undefined = -1
    ok = 0
    dns_error = 1
    http_not_found = 404
    http_forbidden = 403
    http_bad_request = 400
    unknown_error = 999

class ApiMethod(Enum):
    add_entry = "/api/entries"
    token = "/oauth/v2/token"
    version = "/api/version"

class Response:
    http_code = 0
    error = Error.undefined
    error_text = ""
    error_description = ""

    response = ""

    def __init__(self, status_code, http_response):
        self.http_code = status_code
        self.response = http_response

        # DNS not found
        if self.http_code == 0:
            self.error = Error.dns_error
            self.error_text = "Name or service not known."
        # 400 bad request
        elif self.http_code == 400:
            self.error = Error.http_bad_request
            errors = json.loads(self.response)
            if 'error' in errors:
                self.error_text = errors['error']
            if 'error_description' in errors:
                self.error_description = errors['error_description']
        # 404 not found
        elif self.http_code == 404:
            self.error = Error.http_not_found
            self.error_text = "404: API was not found."
        # 403 forbidden
        elif self.http_code == 403:
            self.error = Error.http_forbidden
            self.error_text = "403: Could not reach API due to rights issues."
        # 200 okay
        elif self.http_code == 200:
            self.error = Error.ok
        # unknown Error
        else:
            self.error = Error.unknown_error
            self.error_text = "An unknown error occured."

    def is_rersponse_status_ok(self):
        return self.http_code == 200

    def hasError(self):
        return self.error != Error.ok

def __get_api_url(api_method, different_url=None):
    if api_method in ApiMethod:
        if different_url != None:
            return different_url + api_method.value
        return Configs.serverurl + api_method.value
    return None

def __get_authorization_header():
    success, token = get_token()
    if not success:
        return None
    else:
        return {'Authorization': "Bearer {0}".format(token)}

def __request_get(url, **data):
    ret = None
    request = None

    try:
        request = requests.get(url, data)
        ret = Response(request.status_code, request.text)
    # dns error
    except (requests.exceptions.ConnectionError, requests.exceptions.MissingSchema):
        ret = Response(0, None)
    return ret

def __request_post(url, headers=None, data=None):
    ret = None
    request = None

    try:
        request = requests.post(url, data=data, headers=headers)
        ret = Response(request.status_code, request.text)
    # dns error
    except (requests.exceptions.ConnectionError, requests.exceptions.MissingSchema):
        ret = Response(0, None)
    return ret

def is_minimum_version(version_response):
    versionstring = version_response.response

    if not re.compile('"\\d+\\.\\d+\\.\\d+"').match(versionstring):
        return False

    ver = versionstring.strip('"').split('.')

    x = int(ver[0])
    y = int(ver[1])
    z = int(ver[2])
    tx, ty, tz = MINIMUM_API_VERSION

    if x > tx:
        return True
    elif x < tx:
        return False
    else:
        if y > ty:
            return True
        elif y < ty:
            return False
        else:
            return z >= tz

def api_version(different_url=None):
    url = __get_api_url(ApiMethod.version, different_url)
    response = __request_get(url)
    return response

def api_token():
    url = __get_api_url(ApiMethod.token)
    data = dict()
    data['grant_type'] = "password"
    data['client_id'] = Configs.client
    data['client_secret'] = Configs.secret
    data['username'] = Configs.username
    data['password'] = Configs.password

    response = __request_get(url, **data)
    return response

def api_add_entry(url):
    pass

def get_token(force_creation=False):
    """
    Returns a valid oauth token. Creates a new one if the old token is expired.

    Parameters:
    -----------
    force_creation [optional]bool
        Enforces the creation even if the old token is valid.

    Returns:
    --------
    bool
        Getting the token was successful.
    string
        A valid token or an error message.
    """
    if conf.is_token_expired() or force_creation:
        response = api_token()
        if not response.hasError():
            content = json.loads(response.response)
            Configs.access_token = content['access_token']
            Configs.expires = time.time() + content['expires_in']
            conf.save()
            return True, Configs.access_token
        else:
            return False, response.error_text
    else:
        return True, Configs.access_token
<|endoftext|>