<commit_before>#!/usr/bin/env python3
# -*- coding: utf8 -*-

import sys
import os
import re
import json
import shlex
import urllib.request
import codecs

reader = codecs.getreader("utf-8")
return_code = 0

class c:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    MAYBE_FAIL = '\033[96m'
    FAIL = '\033[91m'
    END = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

def header(app_path):
    print(c.UNDERLINE + c.HEADER + c.BOLD +
          "YUNOHOST APP PACKAGE LINTER\n", c.END,
    "App packaging documentation: https://yunohost.org/#/packaging_apps\n",
    "App package example: https://github.com/YunoHost/example_ynh\n",
    "Checking " + c.BOLD + app_path + c.END + " package\n")

def print_right(str):
    print(c.OKGREEN + "✔", str, c.END)

def print_warning(str):
    print(c.WARNING + "!", str, c.END)

def print_wrong(str, reliable=True):
    if reliable:
        global return_code
        return_code = 1
        print(c.FAIL + "✘", str, c.END)
    else:
        print(c.MAYBE_FAIL + "?", str, c.END)

def urlopen(url):
    try:
        conn = urllib.request.urlopen(url)
    except urllib.error.HTTPError as e:
        return {'content': '', 'code': e.code}
    except urllib.error.URLError as e:
        print('URLError')
    return {'content': conn.read().decode('UTF8'), 'code': 200}

def check_files_exist(app_path):
    """
    Check files exist
    'backup' and 'restore' scripts are mandatory
    """

    print(c.BOLD + c.HEADER + ">>>> MISSING FILES <<<<" + c.END)
    fnames = ("manifest.json", "scripts/install", "scripts/remove",
              "scripts/upgrade", "scripts/backup", "scripts/restore", "LICENSE",
              "README.md")

    for nbr, fname in enumerate(fnames):
        if not check_file_exist(app_path + "/" + fname):
            if nbr != 4 and nbr != 5:
                print_wrong(fname)
            else:
                print_warning(fname)

def check_file_exist(file_path):
    return 1 if os.path.isfile(file_path) and os.stat(file_path).st_size > 0 else 0

def read_file(file_path):
    f = open(file_path)
    # remove every comments and empty lines from the file content to avoid
    # false positives
    file = shlex.shlex(f, False)
    #file = filter(None, re.sub("#.*[^\n]", "", f.read()).splitlines())
    return file

def check_source_management(app_path):
    print(c.BOLD + c.HEADER + "\n>>>> SOURCES MANAGEMENT <<<<" + c.END)
    DIR = os.path.join(app_path, "sources")
    # Check if there is more than six files on 'sources' folder
    if os.path.exists(os.path.join(app_path, "sources")) and \
     len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))]) > 5:
        print_warning("[YEP-3.3] Upstream app sources shouldn't be stored on this "
                      "'sources' folder of this git repository as a copy/paste."
                      "\nAt installation, the package should download sources "
                      "from upstream via 'ynh_setup_source'.\nSee "
                      "https://dev.yunohost.org/issues/201#Conclusion-chart")

def is_license_mention_in_readme(path):
    readme_path = os.path.join(path, 'README.md')
    if os.path.isfile(readme_path):
        return "LICENSE" in open(readme_path).read()
    return False

def check_manifest(path):
    manifest = os.path.join(path, 'manifest.json')
    if not os.path.exists(manifest):
        return
    print(c.BOLD + c.HEADER + "\n>>>> MANIFEST <<<<" + c.END)
    """
    Check if there is no comma syntax issue
    """

    try:
        with open(manifest, encoding='utf-8') as data_file:
            manifest = json.loads(data_file.read())
    except:
        print_wrong("[YEP-2.1] Syntax (comma) or encoding issue with manifest.json. "
                    "Can't check file.")

    fields = ("name", "id", "packaging_format", "description", "url", "version",
              "license", "maintainer", "requirements", "multi_instance",
              "services", "arguments")

    for field in fields:
        if field not in manifest:
            print_warning("[YEP-2.1] \"" + field + "\" field is missing")

    """
    Check values in keys
    """

    if "packaging_format" not in manifest:
        print_wrong("[YEP-2.1] \"packaging_format\" key is missing")
    elif not isinstance(manifest["packaging_format"], int):
        print_wrong("[YEP-2.1] \"packaging_format\": value isn't an integer type")
    elif manifest["packaging_format"] != 1:
        print_wrong("[YEP-2.1] \"packaging_format\" field: current format value is '1'")

    # YEP 1.1 Name is app
    if "id" in manifest:
        if not re.match('^[a-z1-9]((_|-)?[a-z1-9])+$', manifest["id"]):
            print_wrong("[YEP-1.1] 'id' field '%s' should respect this regex "
                        "'^[a-z1-9]((_|-)?[a-z1-9])+$'")

    if "name" in manifest:
        if len(manifest["name"]) > 22 :
            print_warning("[YEP-1.1] The 'name' field shouldn't be too long to be"
                          " able to be with one line in the app list. The most "
                          "current bigger name is actually compound of 22 characters.")

    # YEP 1.2 Put the app in a weel known repo
    if "id" in manifest:
        official_list_url = "https://raw.githubusercontent.com/YunoHost/apps/master/official.json"
        official_list = json.loads(urlopen(official_list_url)['content'])
        community_list_url = "https://raw.githubusercontent.com/YunoHost/apps/master/community.json"
        community_list = json.loads(urlopen(community_list_url)['content'])
        if manifest["id"] not in official_list and manifest["id"] not in community_list:
            print_warning("[YEP-1.2] This app is not registered in official or community applications")

    # YEP 1.3 License
    if "license" in manifest:
        for license in manifest['license'].replace('&', ',').split(','):
            code_license = '' + license + ''
            link = "https://spdx.org/licenses/"
            if license == "nonfree":
                print_warning("[YEP-1.3] The correct value for non free license in license"
                              " field is 'non-free' and not 'nonfree'")
                license = "non-free"
            if license in ["free", "non-free", "dep-non-free"]:
                if not is_license_mention_in_readme(path):
                    print_warning("[YEP-1.3] The use of '%s' in license field implies to "
                                  "write something about the license in your "
                                  "README.md" % (license))
                if license == "non-free" or "dep-non-free":
                    print_warning("[YEP-1.3] 'non-free' apps can't be officialised."
                                  "The way to integrate them or not is always "
                                  "in discussion notably about apps with non "
                                  "free dependencies")
            elif code_license not in urlopen(link)['content']:
                print_warning("[YEP-1.3] The license '%s' is not registered in "
                            "https://spdx.org/licenses/ . It can be a typo error."
                            "If no, you should write free or non-free in place and"
                            " gice more explanation in README.md" % (license))

    # YEP 1.4 Inform if we continue to maintain the app
    # YEP 1.5 Update regularly the app status
    # YEP 1.6 Check regularly the evolution of the upstream

    # YEP 1.7 - Add an app to the YunoHost-Apps organization
    if "id" in manifest:
        repo = "https://github.com/YunoHost-Apps/%s_ynh" % (manifest["id"])
        is_not_added_to_org =  urlopen(repo)['code'] == 404

        if is_not_added_to_org:
            print_warning("[YEP-1.7] You should add your app in the "
                          "YunoHost-Apps organisation.")

    # YEP 1.8 Publish test request
    # YEP 1.9 Document app
    if "description" in manifest and "name" in manifest:
        if manifest["description"] == manifest["name"]:
            print_warning("[YEP-1.9] You should write a good description of the"
                          "app (1 line is enough).")
    #TODO test a specific template in README.md

    # YEP 1.10 Garder un historique de version propre

    # YEP 1.11 Cancelled

    # YEP 2.1
    if "multi_instance" in manifest and manifest["multi_instance"] != 1 and manifest["multi_instance"] != 0:
        print_wrong(
            "[YEP-2.1] \"multi_instance\" field must be boolean type values 'true' or 'false' and not string type")

    if "services" in manifest:
        services = ("nginx", "php5-fpm", "mysql", "uwsgi", "metronome",
                    "postfix", "dovecot")  # , "rspamd", "rmilter")

        for service in manifest["services"]:
            if service not in services:
                print_warning("[YEP-2.1]" + service + " service may not exist")

    if "install" in manifest["arguments"]:
        types = ("domain", "path", "password", "user", "admin")

        for nbr, typ in enumerate(types):
            for install_arg in manifest["arguments"]["install"]:
                if typ == install_arg["name"]:
                    if "type" not in install_arg:
                        print_wrong("[YEP-2.1] You should specify the type of the key with %s" % (typ))

def check_script(path, script_name, script_nbr):

    script_path = path + "/scripts/" + script_name

    if check_file_exist(script_path) == 0:
        return

    print(c.BOLD + c.HEADER + "\n>>>>",
           script_name.upper(), "SCRIPT <<<<" + c.END)

    check_non_helpers_usage(read_file(script_path))
    if script_nbr < 5:
        check_verifications_done_before_modifying_system(read_file(script_path))
        check_set_usage(script_name, read_file(script_path))
        #check_arg_retrieval(script.copy())

def check_verifications_done_before_modifying_system(script):
    """
    Check if verifications are done before modifying the system
    """
    ok = True
    modify_cmd = ''
    cmds = ("cp", "mkdir", "rm", "chown", "chmod", "apt-get", "apt", "service",
           "find", "sed", "mysql", "swapon", "mount", "dd", "mkswap", "useradd")
    cmds_before_exit = []
    is_exit = False
    for cmd in script:
        if "ynh_die" == cmd or "exit " == cmd:
            is_exit = True
            break
        cmds_before_exit.append(cmd)

    if not is_exit:
        return

    for cmd in cmds_before_exit:
        if "ynh_die" == cmd or "exit " == cmd:
            break
        if not ok or cmd in cmds:
            modify_cmd = cmd
            ok = False
            break

    if not ok:
        print_wrong("[YEP-2.4] 'ynh_die' or 'exit' command is executed with system modification before (cmd '%s').\n"
        "This system modification is an issue if a verification exit the script.\n"
        "You should move this verification before any system modification." % (modify_cmd) , False)

def check_non_helpers_usage(script):
    """
    check if deprecated commands are used and propose helpers:
    - 'yunohost app setting' –> ynh_app_setting_(set,get,delete)
    - 'exit' –> 'ynh_die'
    """

    ok = True
    #TODO
    #for line_nbr, cmd in script:
    #    if "yunohost app setting" in cmd:
    #        print_wrong("[YEP-2.11] Line {}: 'yunohost app setting' command is deprecated,"
    #                    " please use helpers ynh_app_setting_(set,get,delete)."
    #                    .format(line_nbr + 1))
    #        ok = False

    if not ok:
        print("Helpers documentation: "
                    "https://yunohost.org/#/packaging_apps_helpers\n"
                    "code: https://github.com/YunoHost/yunohost/…helpers")

    if "exit" in script:
        print_wrong("[YEP-2.4] 'exit' command shouldn't be used."
                    "Use 'ynh_die' helper instead.")

def check_set_usage(script_name, script):
    present = False

    if script_name in ["backup", "remove"]:
        present = "ynh_abort_if_errors" in script or "set -eu" in script
    else:
        present = "ynh_abort_if_errors" in script

    if script_name == "remove":
        # Remove script shouldn't use set -eu or ynh_abort_if_errors
        if present:
            print_wrong("[YEP-2.4] set -eu or ynh_abort_if_errors is present. "
                        "If there is a crash it could put yunohost system in "
                        "invalidated states. For details, look at "
                        "https://dev.yunohost.org/issues/419")
    else:
        if not present:
            print_wrong("[YEP-2.4] ynh_abort_if_errors is missing. For details,"
                        "look at https://dev.yunohost.org/issues/419")

def check_arg_retrieval(script):
    """
    Check arguments retrival from manifest is done with env var $YNH_APP_ARG_* and not with arg $1
    env var was found till line ~30 on scripts. Stop file checking at L30: This could avoid wrong positives
    Check only from '$1' to '$10' as 10 arg retrieval is already a lot.
    """
    present = False

    for cmd in script:
        if cmd =='$' and script.get_token() in [str(x) for x in range(1, 10)]:
            present = True
            break

    if present:
        print_wrong("Argument retrieval from manifest with $1 is deprecated. You may use $YNH_APP_ARG_*")
        print_wrong("For more details see: https://yunohost.org/#/packaging_apps_arguments_management_en")

if __name__ == '__main__':
    os.system("clear")

    if len(sys.argv) != 2:
        print("Give one app package path.")
        exit()

    # "or" trick to always be 1 if 1 is present:
    # 1 or 0 = 1
    # 1 or 1 = 1
    # 0 or 1 = 1
    # 0 or 0 = 0

    app_path = sys.argv[1]
    header(app_path)
    check_files_exist(app_path)
    check_source_management(app_path)
    check_manifest(app_path) # + "/manifest.json")

    scripts = ["install", "remove", "upgrade", "backup", "restore"]
    for (dirpath, dirnames, filenames) in os.walk(os.path.join(app_path, "scripts")):
        for filename in filenames:
            if filename not in scripts and filename[-4:] != ".swp":
                scripts.append(filename)

    for script_nbr, script in enumerate(scripts):
        check_script(app_path, script, script_nbr)

    sys.exit(return_code)
<commit_msg>Fix license test + improve message<commit_after>#!/usr/bin/env python3
# -*- coding: utf8 -*-

import sys
import os
import re
import json
import shlex
import urllib.request
import codecs

reader = codecs.getreader("utf-8")
return_code = 0

class c:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    MAYBE_FAIL = '\033[96m'
    FAIL = '\033[91m'
    END = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

def header(app_path):
    print(c.UNDERLINE + c.HEADER + c.BOLD +
          "YUNOHOST APP PACKAGE LINTER\n", c.END,
    "App packaging documentation: https://yunohost.org/#/packaging_apps\n",
    "App package example: https://github.com/YunoHost/example_ynh\n",
    "Checking " + c.BOLD + app_path + c.END + " package\n")

def print_right(str):
    print(c.OKGREEN + "✔", str, c.END)

def print_warning(str):
    print(c.WARNING + "!", str, c.END)

def print_wrong(str, reliable=True):
    if reliable:
        global return_code
        return_code = 1
        print(c.FAIL + "✘", str, c.END)
    else:
        print(c.MAYBE_FAIL + "?", str, c.END)

def urlopen(url):
    try:
        conn = urllib.request.urlopen(url)
    except urllib.error.HTTPError as e:
        return {'content': '', 'code': e.code}
    except urllib.error.URLError as e:
        print('URLError')
    return {'content': conn.read().decode('UTF8'), 'code': 200}

def check_files_exist(app_path):
    """
    Check files exist
    'backup' and 'restore' scripts are mandatory
    """

    print(c.BOLD + c.HEADER + ">>>> MISSING FILES <<<<" + c.END)
    fnames = ("manifest.json", "scripts/install", "scripts/remove",
              "scripts/upgrade", "scripts/backup", "scripts/restore", "LICENSE",
              "README.md")

    for nbr, fname in enumerate(fnames):
        if not check_file_exist(app_path + "/" + fname):
            if nbr != 4 and nbr != 5:
                print_wrong(fname)
            else:
                print_warning(fname)

def check_file_exist(file_path):
    return 1 if os.path.isfile(file_path) and os.stat(file_path).st_size > 0 else 0

def read_file(file_path):
    f = open(file_path)
    # remove every comments and empty lines from the file content to avoid
    # false positives
    file = shlex.shlex(f, False)
    #file = filter(None, re.sub("#.*[^\n]", "", f.read()).splitlines())
    return file

def check_source_management(app_path):
    print(c.BOLD + c.HEADER + "\n>>>> SOURCES MANAGEMENT <<<<" + c.END)
    DIR = os.path.join(app_path, "sources")
    # Check if there is more than six files on 'sources' folder
    if os.path.exists(os.path.join(app_path, "sources")) and \
     len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))]) > 5:
        print_warning("[YEP-3.3] Upstream app sources shouldn't be stored on this "
                      "'sources' folder of this git repository as a copy/paste."
                      "\nAt installation, the package should download sources "
                      "from upstream via 'ynh_setup_source'.\nSee "
                      "https://dev.yunohost.org/issues/201#Conclusion-chart")

def is_license_mention_in_readme(path):
    readme_path = os.path.join(path, 'README.md')
    if os.path.isfile(readme_path):
        return "LICENSE" in open(readme_path).read()
    return False

def check_manifest(path):
    manifest = os.path.join(path, 'manifest.json')
    if not os.path.exists(manifest):
        return
    print(c.BOLD + c.HEADER + "\n>>>> MANIFEST <<<<" + c.END)
    """
    Check if there is no comma syntax issue
    """

    try:
        with open(manifest, encoding='utf-8') as data_file:
            manifest = json.loads(data_file.read())
    except:
        print_wrong("[YEP-2.1] Syntax (comma) or encoding issue with manifest.json. "
                    "Can't check file.")

    fields = ("name", "id", "packaging_format", "description", "url", "version",
              "license", "maintainer", "requirements", "multi_instance",
              "services", "arguments")

    for field in fields:
        if field not in manifest:
            print_warning("[YEP-2.1] \"" + field + "\" field is missing")

    """
    Check values in keys
    """

    if "packaging_format" not in manifest:
        print_wrong("[YEP-2.1] \"packaging_format\" key is missing")
    elif not isinstance(manifest["packaging_format"], int):
        print_wrong("[YEP-2.1] \"packaging_format\": value isn't an integer type")
    elif manifest["packaging_format"] != 1:
        print_wrong("[YEP-2.1] \"packaging_format\" field: current format value is '1'")

    # YEP 1.1 Name is app
    if "id" in manifest:
        if not re.match('^[a-z1-9]((_|-)?[a-z1-9])+$', manifest["id"]):
            print_wrong("[YEP-1.1] 'id' field '%s' should respect this regex "
                        "'^[a-z1-9]((_|-)?[a-z1-9])+$'")

    if "name" in manifest:
        if len(manifest["name"]) > 22 :
            print_warning("[YEP-1.1] The 'name' field shouldn't be too long to be"
                          " able to be with one line in the app list. The most "
                          "current bigger name is actually compound of 22 characters.")

    # YEP 1.2 Put the app in a weel known repo
    if "id" in manifest:
        official_list_url = "https://raw.githubusercontent.com/YunoHost/apps/master/official.json"
        official_list = json.loads(urlopen(official_list_url)['content'])
        community_list_url = "https://raw.githubusercontent.com/YunoHost/apps/master/community.json"
        community_list = json.loads(urlopen(community_list_url)['content'])
        if manifest["id"] not in official_list and manifest["id"] not in community_list:
            print_warning("[YEP-1.2] This app is not registered in official or community applications")

    # YEP 1.3 License
    if "license" in manifest:
        for license in manifest['license'].replace('&', ',').split(','):
            code_license = '' + license + ''
            link = "https://spdx.org/licenses/"
            if license == "nonfree":
                print_warning("[YEP-1.3] The correct value for non free license in license"
                              " field is 'non-free' and not 'nonfree'")
                license = "non-free"
            if license in ["free", "non-free", "dep-non-free"]:
                if not is_license_mention_in_readme(path):
                    print_warning("[YEP-1.3] The use of '%s' in license field implies to "
                                  "write something about the license in your "
                                  "README.md" % (license))
                if license in ["non-free", "dep-non-free"]:
                    print_warning("[YEP-1.3] 'non-free' apps can't be officialized."
                                  "Their integration is still being discussed,"
                                  "especially for apps with non-free dependencies")
            elif code_license not in urlopen(link)['content']:
                print_warning("[YEP-1.3] The license '%s' is not registered in "
                            "https://spdx.org/licenses/ . It can be a typo error."
                            "If no, you should write free or non-free in place and"
                            " gice more explanation in README.md" % (license))

    # YEP 1.4 Inform if we continue to maintain the app
    # YEP 1.5 Update regularly the app status
    # YEP 1.6 Check regularly the evolution of the upstream

    # YEP 1.7 - Add an app to the YunoHost-Apps organization
    if "id" in manifest:
        repo = "https://github.com/YunoHost-Apps/%s_ynh" % (manifest["id"])
        is_not_added_to_org =  urlopen(repo)['code'] == 404

        if is_not_added_to_org:
            print_warning("[YEP-1.7] You should add your app in the "
                          "YunoHost-Apps organisation.")

    # YEP 1.8 Publish test request
    # YEP 1.9 Document app
    if "description" in manifest and "name" in manifest:
        if manifest["description"] == manifest["name"]:
            print_warning("[YEP-1.9] You should write a good description of the"
                          "app (1 line is enough).")
    #TODO test a specific template in README.md

    # YEP 1.10 Garder un historique de version propre

    # YEP 1.11 Cancelled

    # YEP 2.1
    if "multi_instance" in manifest and manifest["multi_instance"] != 1 and manifest["multi_instance"] != 0:
        print_wrong(
            "[YEP-2.1] \"multi_instance\" field must be boolean type values 'true' or 'false' and not string type")

    if "services" in manifest:
        services = ("nginx", "php5-fpm", "mysql", "uwsgi", "metronome",
                    "postfix", "dovecot")  # , "rspamd", "rmilter")

        for service in manifest["services"]:
            if service not in services:
                print_warning("[YEP-2.1]" + service + " service may not exist")

    if "install" in manifest["arguments"]:
        types = ("domain", "path", "password", "user", "admin")

        for nbr, typ in enumerate(types):
            for install_arg in manifest["arguments"]["install"]:
                if typ == install_arg["name"]:
                    if "type" not in install_arg:
                        print_wrong("[YEP-2.1] You should specify the type of the key with %s" % (typ))

def check_script(path, script_name, script_nbr):

    script_path = path + "/scripts/" + script_name

    if check_file_exist(script_path) == 0:
        return

    print(c.BOLD + c.HEADER + "\n>>>>",
           script_name.upper(), "SCRIPT <<<<" + c.END)

    check_non_helpers_usage(read_file(script_path))
    if script_nbr < 5:
        check_verifications_done_before_modifying_system(read_file(script_path))
        check_set_usage(script_name, read_file(script_path))
        #check_arg_retrieval(script.copy())

def check_verifications_done_before_modifying_system(script):
    """
    Check if verifications are done before modifying the system
    """
    ok = True
    modify_cmd = ''
    cmds = ("cp", "mkdir", "rm", "chown", "chmod", "apt-get", "apt", "service",
           "find", "sed", "mysql", "swapon", "mount", "dd", "mkswap", "useradd")
    cmds_before_exit = []
    is_exit = False
    for cmd in script:
        if "ynh_die" == cmd or "exit " == cmd:
            is_exit = True
            break
        cmds_before_exit.append(cmd)

    if not is_exit:
        return

    for cmd in cmds_before_exit:
        if "ynh_die" == cmd or "exit " == cmd:
            break
        if not ok or cmd in cmds:
            modify_cmd = cmd
            ok = False
            break

    if not ok:
        print_wrong("[YEP-2.4] 'ynh_die' or 'exit' command is executed with system modification before (cmd '%s').\n"
        "This system modification is an issue if a verification exit the script.\n"
        "You should move this verification before any system modification." % (modify_cmd) , False)

def check_non_helpers_usage(script):
    """
    check if deprecated commands are used and propose helpers:
    - 'yunohost app setting' –> ynh_app_setting_(set,get,delete)
    - 'exit' –> 'ynh_die'
    """

    ok = True
    #TODO
    #for line_nbr, cmd in script:
    #    if "yunohost app setting" in cmd:
    #        print_wrong("[YEP-2.11] Line {}: 'yunohost app setting' command is deprecated,"
    #                    " please use helpers ynh_app_setting_(set,get,delete)."
    #                    .format(line_nbr + 1))
    #        ok = False

    if not ok:
        print("Helpers documentation: "
                    "https://yunohost.org/#/packaging_apps_helpers\n"
                    "code: https://github.com/YunoHost/yunohost/…helpers")

    if "exit" in script:
        print_wrong("[YEP-2.4] 'exit' command shouldn't be used."
                    "Use 'ynh_die' helper instead.")

def check_set_usage(script_name, script):
    present = False

    if script_name in ["backup", "remove"]:
        present = "ynh_abort_if_errors" in script or "set -eu" in script
    else:
        present = "ynh_abort_if_errors" in script

    if script_name == "remove":
        # Remove script shouldn't use set -eu or ynh_abort_if_errors
        if present:
            print_wrong("[YEP-2.4] set -eu or ynh_abort_if_errors is present. "
                        "If there is a crash it could put yunohost system in "
                        "invalidated states. For details, look at "
                        "https://dev.yunohost.org/issues/419")
    else:
        if not present:
            print_wrong("[YEP-2.4] ynh_abort_if_errors is missing. For details,"
                        "look at https://dev.yunohost.org/issues/419")

def check_arg_retrieval(script):
    """
    Check arguments retrival from manifest is done with env var $YNH_APP_ARG_* and not with arg $1
    env var was found till line ~30 on scripts. Stop file checking at L30: This could avoid wrong positives
    Check only from '$1' to '$10' as 10 arg retrieval is already a lot.
    """
    present = False

    for cmd in script:
        if cmd =='$' and script.get_token() in [str(x) for x in range(1, 10)]:
            present = True
            break

    if present:
        print_wrong("Argument retrieval from manifest with $1 is deprecated. You may use $YNH_APP_ARG_*")
        print_wrong("For more details see: https://yunohost.org/#/packaging_apps_arguments_management_en")

if __name__ == '__main__':
    os.system("clear")

    if len(sys.argv) != 2:
        print("Give one app package path.")
        exit()

    # "or" trick to always be 1 if 1 is present:
    # 1 or 0 = 1
    # 1 or 1 = 1
    # 0 or 1 = 1
    # 0 or 0 = 0

    app_path = sys.argv[1]
    header(app_path)
    check_files_exist(app_path)
    check_source_management(app_path)
    check_manifest(app_path) # + "/manifest.json")

    scripts = ["install", "remove", "upgrade", "backup", "restore"]
    for (dirpath, dirnames, filenames) in os.walk(os.path.join(app_path, "scripts")):
        for filename in filenames:
            if filename not in scripts and filename[-4:] != ".swp":
                scripts.append(filename)

    for script_nbr, script in enumerate(scripts):
        check_script(app_path, script, script_nbr)

    sys.exit(return_code)
<|endoftext|>


<commit_before>import math
import sys
import random
import os

class Structure_SFold(object):
	"""docstring for Structure_SFold"""
	def __init__(self, arg):
		super(Structure_SFold, self).__init__()
		self.count = int(arg[1])
		self.boltzmann_weight = float(arg[2])
		self.energy = float(arg[3])
		self.pairs = []

	def add_pair(self, arg):
		super(Structure_SFold, self).__init__()
		self.pairs.append(int(arg[0])-1)
		self.pairs.append(int(arg[1])-1)

	def __str__(self):
		return (str(self.count) + ": weight - " + str(self.boltzmann_weight) + ", energy - " + str(self.energy) + "\n" + str(self.pairs))

def get_file():
	if len(sys.argv) != 3:
		print("Incorrect number of arguments.")
		sys.exit(1)
	input_file = sys.argv[1]
	seq_file = sys.argv[2]
	return input_file, seq_file

def parse_seq_file(seq_file):
	file = open(seq_file)
	data = file.read().splitlines()
	seq = ""
	for i in range(len(data)):
		if i > 0 and data[i] != "":
			seq += data[i]
	return seq

def parse_sfold_file(input_file):
	file = open(input_file)
	data = file.read().splitlines()

	for i in range(len(data)):
		data[i] = data[i].split()

	#a list containing the structures
	structures = []

	i = 3
	current = -1
	while i < len(data):
		#Create new structure
		if data[i][0] == "Structure":
			current += 1
			temp = Structure_SFold(data[i])
			structures.append(temp)
		else:
			structures[current].add_pair(data[i])
		i += 1
	return structures

def MetropolisHastings(structures, seq):

	#samples the structure set and recursively calls if duplicate found
	def sample(a, structures):
		b = random.choice(structures)
		if a == b:
			sample(a,structures)
		return b

	def generate_dictionaries(a,b):
		pairsA = {}
		pairsB = {}
		for i in range(0,len(a.pairs),2):
			pairsA[a.pairs[i]] = a.pairs[i+1]
			pairsA[a.pairs[i+1]] = a.pairs[i]
		for i in range(0,len(b.pairs),2):
			pairsB[b.pairs[i]] = b.pairs[i+1]
			pairsB[b.pairs[i+1]] = b.pairs[i]
		return pairsA, pairsB

	#handles ties between pairs in the structures
	def handle_ties(key,a,b,t):
		t += 1
		if random.randint(0,1) == 0:
			other = b[key]
			b.pop(key)
			b.pop(other)
			if a[key] in b:
				t = handle_ties(a[key],a,b,t)
		else:
			other = a[key]
			a.pop(key)
			a.pop(other)
			if b[key] in a:
				t = handle_ties(b[key],b,a,t)
		return t

	def helper(a,b,t):
		for key in a.keys():
			if key in b:
				t = handle_ties(key,a,b,t)
				t = helper(a,b,t)
				break
		return t

	#recursively check if there are duplicates in the dictionaries of pairs
	def remove_duplicates(a,b):
		t = 0
		for key in a.keys():
			if key in b:
				t = handle_ties(key,a,b,t)
				t = helper(a,b,t)
				break
		return t

	def trim(a):
		for key in a.keys():
			if a[key] in a.keys():
				if key < a[key]:
					a.pop(a[key])
				else:
					a.pop(key)
				trim(a)
				break

	def backtrack(a,b):
		trim(a)
		trim(b)
		keysA = list(a)
		keysB = list(b)
		keysA.sort()
		keysB.sort()
		answer = []
		i = 0
		j = 0
		while(i < len(keysA) and j <len(keysB)):
			if keysA[i] < keysB[j]:
				answer.append(keysA[i])
				answer.append(a[keysA[i]])
				i += 1
			else:
				answer.append(keysB[j])
				answer.append(b[keysB[j]])
				j += 1
		while i < len(keysA):
			answer.append(keysA[i])
			answer.append(a[keysA[i]])
			i += 1
		while j < len(keysB):
			answer.append(keysB[j])
			answer.append(b[keysB[j]])
			j += 1
		return answer

	def check(answer):
		zest = set()
		for item in answer:
			if item in zest:
				print(str(item) + "is already in the set.")
			else:
				zest.add(item)

	#combines structure a and structure b
	def combine(a,b):
		t = remove_duplicates(a,b)
		answer = backtrack(a,b)
		check(answer)
		return answer, t

	def HotKnots(pairs):
		dot_bracket = " \"" + BracketedStructureFromPairs(pairs, len(seq)) +"\""
		cmd = '$HOTKNOTS/bin/computeEnergy -s ' + seq + dot_bracket + ' > $RESEARCH/output_files/hotknots_out.txt'
		relative_path = os.getcwd()
		os.chdir(relative_path+"/HotKnots_v2.0/bin")
		os.system(cmd)
		os.chdir(relative_path)
		energy_file = "output_files/hotknots_out.txt"
		file = open(energy_file,'r',encoding='iso-8859-15')
		data = file.read().splitlines()
		line = data[len(data)-1]
		line = line.split()
		if line[1] == "the":
			print("HotKnots failed")
		free_energy = float(line[1])
		free_energy_wo_dangling = float(line[2])
		return  free_energy

	#call nupack and return the resulting energy
	def get_structure_energy(pairs):
		cmd = '$NUPACKHOME/bin/energy -pseudo $RESEARCH/input_files/nupack_in > $RESEARCH/output_files/nupack_out.txt'
		dot_bracket = StructureFromPairs(pairs,len(seq))
		if generate_inFile(dot_bracket) != 0:
			print("ERROR: Failed to generate nupack_in.in file")
			sys.exit(1)
		os.system(cmd)
		energy_file = "output_files/nupack_out.txt"
		file = open(energy_file)
		data = file.read().splitlines()
		if len(data) <= 2:
			print("length of Nupack file is 0")
			sys.exit(1)
		if (data[len(data)-2] == "% Energy (kcal/mol):"):
			# print("Nupack Succeeded")
			return float(data[len(data)-1])
		else:
			# print("Nupack failed, establishing large energy value (10**8)")
			return float(10**8)

	def generate_inFile(dot_bracket):
		nupack_file = "input_files/nupack_in.in"
		file = open(nupack_file, "w")
		file.write(seq+"\n")
		file.write(dot_bracket)
		file.close()
		return 0

	def BracketedStructureFromPairs(pairs,L):
		struct = list('.'*L)
		stack0 = []
		stack1 = []
		s0_min = 0
		s1_min = 0
		s0_max = L
		s1_max = L
		current = 0
		for i in range(0,len(pairs), 2):
			if current == 0:
				if pairs[i] > s0_min and pairs[i+1] < s0_max:
					stack0.append((pairs[i],pairs[i+1]))
				else:
					stack1.append((pairs[i],pairs[i+1]))
					if pairs[i+1] > s0_max:
						s1_max = pairs[i+1]
					if pairs[i] < s0_min:
						s1_min = pairs[i]
					current = 1
			if current == 1:
				if pairs[i] > s1_min and pairs[i+1] < s1_max:
					stack1.append((pairs[i],pairs[i+1]))
				else:
					stack0.append((pairs[i],pairs[i+1]))
					if pairs[i+1] > s1_max:
						s0_max = pairs[i+1]
					if pairs[i] < s1_min:
						s0_min = pairs[i]
					current = 0
		for pair in stack0:
			struct[pair[0]] = "("
			struct[pair[1]] = ")"
		for pair in stack1:
			struct[pair[0]] = "["
			struct[pair[1]] = "]"
		return ''.join(struct)

	def StructureFromPairs(pairs, L):
	    struct = list('.' * L)
	    for i in range(0, len(pairs), 2):
	        struct[pairs[i]] = '('
	        struct[pairs[i+1]] = ')'

	    return ''.join(struct)

	def adjust_counts(current,counts):
		for (i,j) in enumerate(current):
			if (i,j) in counts:
				counts[(i,j)] += 1.0
			else:
				counts[(i,j)] = 1.0

	#Step 1: finding a,b sample for current
	try:
		a = sample(None, structures)
		b = sample(a,structures)
	except IndexError:
		print("ERROR: Structures list is empty.")
		sys.exit(1)

	# ###TESTS
	# print (len(a.pairs), a)
	# print("")
	# print(len(b.pairs), b)

	#Step 2: Combine a,b into a new structure, current
	#t_current = # of coin flips to break ties
	pairsA, pairsB = generate_dictionaries(a,b)
	current, t_current = combine(pairsA,pairsB)
	# print(current)
	# print(t_current)

	# energy_current = get_structure_energy(current)
	energy_current = HotKnots(current)

	#Step 3: Set sample-list = {}
	sample_list = {}

	#Step 4: Define Burn-in rate to repeat steps 5-7
	#follow with fixed sampling period
	burnin = 1000
	count = 0
	exists = 0

	counts = {}
	acceptance_rate = 0.0

	print("beginning Burn in iterations")
	while (count < burnin):
		# if(count % (burnin/20)) == 0:
		# 	print("working on iteration " + str(count))

		#Step 5: Sample/combine new pair (i, j) into a new structure, proposal
		#t_proposal = # of coins flips required to break ties
		try:
			i = sample(None, structures)
			j = sample(i, structures)
		except IndexError:
			print("ERROR: Structures list is empty.")
			return 1

		pairsI, pairsJ = generate_dictionaries(i,j)
		proposal, t_proposal = combine(pairsI,pairsJ)

		#TO-DO HotKnots Implementation
		energy_proposal = HotKnots(proposal)
		# print("CHECKING FOR DUPLICATES: " + str(len(proposal) == len(set(proposal))) )
		#Step 6: Calculate Transition Functions
		#T(proposal|current) = T(proposal) = P(S_i)*P(S_j)*(0.5)**t_proposal
		#T(current|proposal) = T(current) = P(S_a)*P(S_b)*(0.5)**t_current

		Transition_proposal = i.boltzmann_weight * j.boltzmann_weight * 0.5**t_proposal
		Transition_current = a.boltzmann_weight * b.boltzmann_weight * 0.5**t_current
		# print(Transition_proposal,Transition_current)
		# print(Transition_proposal, Transition_current)
		#Step 7: Compute Probability/Select whether to accept
		R = float(13807E-23) * float(6.022E23)
		# T = 37.0
		T = float(275.928)

		# print("energies:" , str(energy_proposal),str(energy_current))
		temp = (Transition_current * math.exp((-1.0 * energy_proposal)/(R * T)))
		temp2 =  (Transition_proposal * math.exp((-1.0* energy_current)/(R * T)))
		ans = temp/ temp2
		# print("temp: " + str(ans))
		Probability = min(1.0, ans )
		# print(Probability)
		if Probability == 1:
			# print("accepting proposal")
			a = i
			b = j
			current = proposal
			t_current = t_proposal
			energy_current = energy_proposal
			acceptance_rate += 1.0
		else:
			x = random.random()
			# print(x, Probability)
			if Probability > x:
				# print("accepting proposal after failing")
				a = i
				b = j
				current = proposal
				t_current = t_proposal
				energy_current = energy_proposal
				acceptance_rate += 1.0
			else:
				# print("doesn't accept proposal")
				pass
		adjust_counts(current,counts)
		count += 1

	centroid = []
	keys = list(counts)
	keys.sort()
	for key in keys:
		if (counts[key]/count) >= 0.5:
			centroid.append(key)

	print(centroid)

	# return current
	return current, BracketedStructureFromPairs(current,len(seq))

def main():
	input_file, seq_file = get_file()
	seq = parse_seq_file(seq_file)
	parsed_sfold = parse_sfold_file(input_file)
	current, dot_bracket = MetropolisHastings(parsed_sfold, seq)
	# current = MetropolisHastings(parsed_sfold, seq)
	print(current)
	print(dot_bracket)
	return 0

if __name__ == "__main__":
	sys.exit(main())<commit_msg>fixed centroid structure and formatted output<commit_after>import math
import sys
import random
import os

class Structure_SFold(object):
	"""docstring for Structure_SFold"""
	def __init__(self, arg):
		super(Structure_SFold, self).__init__()
		self.count = int(arg[1])
		self.boltzmann_weight = float(arg[2])
		self.energy = float(arg[3])
		self.pairs = []

	def add_pair(self, arg):
		super(Structure_SFold, self).__init__()
		self.pairs.append(int(arg[0])-1)
		self.pairs.append(int(arg[1])-1)

	def __str__(self):
		return (str(self.count) + ": weight - " + str(self.boltzmann_weight) + ", energy - " + str(self.energy) + "\n" + str(self.pairs))

def get_file():
	if len(sys.argv) != 3:
		print("Incorrect number of arguments.")
		sys.exit(1)
	input_file = sys.argv[1]
	seq_file = sys.argv[2]
	return input_file, seq_file

def parse_seq_file(seq_file):
	file = open(seq_file)
	data = file.read().splitlines()
	seq = ""
	for i in range(len(data)):
		if i > 0 and data[i] != "":
			seq += data[i]
	return seq

def parse_sfold_file(input_file):
	file = open(input_file)
	data = file.read().splitlines()

	for i in range(len(data)):
		data[i] = data[i].split()

	#a list containing the structures
	structures = []

	i = 3
	current = -1
	while i < len(data):
		#Create new structure
		if data[i][0] == "Structure":
			current += 1
			temp = Structure_SFold(data[i])
			structures.append(temp)
		else:
			structures[current].add_pair(data[i])
		i += 1
	return structures

def MetropolisHastings(structures, seq):

	#samples the structure set and recursively calls if duplicate found
	def sample(a, structures):
		b = random.choice(structures)
		if a == b:
			sample(a,structures)
		return b

	def generate_dictionaries(a,b):
		pairsA = {}
		pairsB = {}
		for i in range(0,len(a.pairs),2):
			pairsA[a.pairs[i]] = a.pairs[i+1]
			pairsA[a.pairs[i+1]] = a.pairs[i]
		for i in range(0,len(b.pairs),2):
			pairsB[b.pairs[i]] = b.pairs[i+1]
			pairsB[b.pairs[i+1]] = b.pairs[i]
		return pairsA, pairsB

	#handles ties between pairs in the structures
	def handle_ties(key,a,b,t):
		t += 1
		if random.randint(0,1) == 0:
			other = b[key]
			b.pop(key)
			b.pop(other)
			if a[key] in b:
				t = handle_ties(a[key],a,b,t)
		else:
			other = a[key]
			a.pop(key)
			a.pop(other)
			if b[key] in a:
				t = handle_ties(b[key],b,a,t)
		return t

	def helper(a,b,t):
		for key in a.keys():
			if key in b:
				t = handle_ties(key,a,b,t)
				t = helper(a,b,t)
				break
		return t

	#recursively check if there are duplicates in the dictionaries of pairs
	def remove_duplicates(a,b):
		t = 0
		for key in a.keys():
			if key in b:
				t = handle_ties(key,a,b,t)
				t = helper(a,b,t)
				break
		return t

	def trim(a):
		for key in a.keys():
			if a[key] in a.keys():
				if key < a[key]:
					a.pop(a[key])
				else:
					a.pop(key)
				trim(a)
				break

	def backtrack(a,b):
		trim(a)
		trim(b)
		keysA = list(a)
		keysB = list(b)
		keysA.sort()
		keysB.sort()
		answer = []
		i = 0
		j = 0
		while(i < len(keysA) and j <len(keysB)):
			if keysA[i] < keysB[j]:
				answer.append(keysA[i])
				answer.append(a[keysA[i]])
				i += 1
			else:
				answer.append(keysB[j])
				answer.append(b[keysB[j]])
				j += 1
		while i < len(keysA):
			answer.append(keysA[i])
			answer.append(a[keysA[i]])
			i += 1
		while j < len(keysB):
			answer.append(keysB[j])
			answer.append(b[keysB[j]])
			j += 1
		return answer

	def check(answer):
		zest = set()
		for item in answer:
			if item in zest:
				print(str(item) + "is already in the set.")
			else:
				zest.add(item)

	#combines structure a and structure b
	def combine(a,b):
		t = remove_duplicates(a,b)
		answer = backtrack(a,b)
		check(answer)
		return answer, t

	def HotKnots(pairs):
		dot_bracket = " \"" + BracketedStructureFromPairs(pairs, len(seq)) +"\""
		cmd = '$HOTKNOTS/bin/computeEnergy -s ' + seq + dot_bracket + ' > $RESEARCH/output_files/hotknots_out.txt'
		relative_path = os.getcwd()
		os.chdir(relative_path+"/HotKnots_v2.0/bin")
		os.system(cmd)
		os.chdir(relative_path)
		energy_file = "output_files/hotknots_out.txt"
		file = open(energy_file,'r',encoding='iso-8859-15')
		data = file.read().splitlines()
		line = data[len(data)-1]
		line = line.split()
		if line[1] == "the":
			print("HotKnots failed")
		free_energy = float(line[1])
		free_energy_wo_dangling = float(line[2])
		return  free_energy

	#call nupack and return the resulting energy
	def get_structure_energy(pairs):
		cmd = '$NUPACKHOME/bin/energy -pseudo $RESEARCH/input_files/nupack_in > $RESEARCH/output_files/nupack_out.txt'
		dot_bracket = StructureFromPairs(pairs,len(seq))
		if generate_inFile(dot_bracket) != 0:
			print("ERROR: Failed to generate nupack_in.in file")
			sys.exit(1)
		os.system(cmd)
		energy_file = "output_files/nupack_out.txt"
		file = open(energy_file)
		data = file.read().splitlines()
		if len(data) <= 2:
			print("length of Nupack file is 0")
			sys.exit(1)
		if (data[len(data)-2] == "% Energy (kcal/mol):"):
			# print("Nupack Succeeded")
			return float(data[len(data)-1])
		else:
			# print("Nupack failed, establishing large energy value (10**8)")
			return float(10**8)

	def generate_inFile(dot_bracket):
		nupack_file = "input_files/nupack_in.in"
		file = open(nupack_file, "w")
		file.write(seq+"\n")
		file.write(dot_bracket)
		file.close()
		return 0

	def BracketedStructureFromPairs(pairs,L):
		struct = list('.'*L)
		stack0 = []
		stack1 = []
		s0_min = 0
		s1_min = 0
		s0_max = L
		s1_max = L
		current = 0
		for i in range(0,len(pairs), 2):
			if current == 0:
				for pair in stack0:
					if (pairs[i] < pair[0] and (pairs[i+1] > pair[0] and pairs[i+1] < pair[1])) \
                    	or ((pairs[i] > pair[0] and pairs[i] < pair[1]) and pairs[i+1] > pair[1]):
						stack1.append((pairs[i],pairs[i+1]))
						current = 1
						break
				if current == 0:
					stack0.append((pairs[i],pairs[i+1]))
			if current == 1:
				for pair in stack1:
					if (pairs[i] < pair[0] and (pairs[i+1] > pair[0] and pairs[i+1] < pair[1])) \
                    	or ((pairs[i] > pair[0] and pairs[i] < pair[1]) and pairs[i+1] > pair[1]):
						stack0.append((pairs[i],pairs[i+1]))
						current = 0
						break
				if current == 1:
					stack1.append((pairs[i],pairs[i+1]))
		for pair in stack0:
			struct[pair[0]] = "("
			struct[pair[1]] = ")"
		for pair in stack1:
			struct[pair[0]] = "["
			struct[pair[1]] = "]"
		return ''.join(struct)

	def StructureFromPairs(pairs, L):
	    struct = list('.' * L)
	    for i in range(0, len(pairs), 2):
	        struct[pairs[i]] = '('
	        struct[pairs[i+1]] = ')'

	    return ''.join(struct)

	def adjust_counts(current,counts):
		for i in range(0,len(current),2):
			if (current[i],current[i+1]) in counts:
				counts[(current[i],current[i+1])] += 1.0
			else:
				counts[(current[i],current[i+1])] = 1.0

	#Step 1: finding a,b sample for current
	try:
		a = sample(None, structures)
		b = sample(a,structures)
	except IndexError:
		print("ERROR: Structures list is empty.")
		sys.exit(1)

	# ###TESTS
	# print (len(a.pairs), a)
	# print("")
	# print(len(b.pairs), b)

	#Step 2: Combine a,b into a new structure, current
	#t_current = # of coin flips to break ties
	pairsA, pairsB = generate_dictionaries(a,b)
	current, t_current = combine(pairsA,pairsB)
	# print(current)
	# print(t_current)

	# energy_current = get_structure_energy(current)
	energy_current = HotKnots(current)

	#Step 3: Set sample-list = {}
	sample_list = {}

	#Step 4: Define Burn-in rate to repeat steps 5-7
	#follow with fixed sampling period
	burnin = 1000
	count = 0
	exists = 0

	counts = {}
	acceptance_rate = 0.0

	print("beginning Burn in iterations")
	while (count < burnin):
		if(count % (burnin/20)) == 0:
			print("working on iteration " + str(count))
			# print(BracketedStructureFromPairs(current, len(seq)))

		#Step 5: Sample/combine new pair (i, j) into a new structure, proposal
		#t_proposal = # of coins flips required to break ties
		try:
			i = sample(None, structures)
			j = sample(i, structures)
		except IndexError:
			print("ERROR: Structures list is empty.")
			return 1

		pairsI, pairsJ = generate_dictionaries(i,j)
		proposal, t_proposal = combine(pairsI,pairsJ)

		#TO-DO HotKnots Implementation
		energy_proposal = HotKnots(proposal)
		# print("CHECKING FOR DUPLICATES: " + str(len(proposal) == len(set(proposal))) )
		#Step 6: Calculate Transition Functions
		#T(proposal|current) = T(proposal) = P(S_i)*P(S_j)*(0.5)**t_proposal
		#T(current|proposal) = T(current) = P(S_a)*P(S_b)*(0.5)**t_current

		Transition_proposal = i.boltzmann_weight * j.boltzmann_weight * 0.5**t_proposal
		Transition_current = a.boltzmann_weight * b.boltzmann_weight * 0.5**t_current
		# print(Transition_proposal,Transition_current)
		# print(Transition_proposal, Transition_current)
		#Step 7: Compute Probability/Select whether to accept
		R = float(13807E-23) * float(6.022E23)
		# T = 37.0
		T = float(275.928)

		# print("energies:" , str(energy_proposal),str(energy_current))
		temp = (Transition_current * math.exp((-1.0 * energy_proposal)/(R * T)))
		temp2 =  (Transition_proposal * math.exp((-1.0* energy_current)/(R * T)))
		ans = temp/ temp2
		# print("temp: " + str(ans))
		Probability = min(1.0, ans )
		# print(Probability)
		if Probability == 1:
			# print("accepting proposal")
			a = i
			b = j
			current = proposal
			t_current = t_proposal
			energy_current = energy_proposal
			acceptance_rate += 1.0
		else:
			x = random.random()
			# print(x, Probability)
			if Probability > x:
				# print("accepting proposal after failing")
				a = i
				b = j
				current = proposal
				t_current = t_proposal
				energy_current = energy_proposal
				acceptance_rate += 1.0
			else:
				# print("doesn't accept proposal")
				pass
		adjust_counts(current,counts)
		count += 1

	centroid = []
	keys = list(counts)
	keys.sort()
	for key in keys:
		if (counts[key]/count) >= 0.5:
			centroid.append(key)
	print("\nCentroid Structure")
	print("Length:\n",len(centroid))
	print("Pairs:\n",centroid)

	# return current
	return current, BracketedStructureFromPairs(current,len(seq))

def main():
	input_file, seq_file = get_file()
	seq = parse_seq_file(seq_file)
	parsed_sfold = parse_sfold_file(input_file)
	current, dot_bracket = MetropolisHastings(parsed_sfold, seq)
	# current = MetropolisHastings(parsed_sfold, seq)
	print("\nCurrent Structure")
	print("Length:\n",len(current)/2)
	print("Pairs:\n",current)
	print("Dot Bracket:\n",dot_bracket)
	return 0

if __name__ == "__main__":
	sys.exit(main())<|endoftext|>


<commit_before>import sys
import numpy as np

from flare import gp, otf, kernels
from flare.mgp.mgp import MappedGaussianProcess
from flare.ase.calculator import FLARE_Calculator
import flare.mc_simple as mc_simple

# ---------- create gaussian process model -------------------
kernel = mc_simple.two_plus_three_body_mc
kernel_grad = mc_simple.two_plus_three_body_mc_grad
hyps = np.array([0.1, 1., 0.001, 1, 0.06])
two_cut = 5.0
three_cut = 5.0
cutoffs = np.array([two_cut, three_cut])
hyp_labels = ['sig2', 'ls2', 'sig3', 'ls3', 'noise']
opt_algorithm = 'BFGS'

gp_model = gp.GaussianProcess(kernel, kernel_grad, hyps, cutoffs,
                              hyp_labels=hyp_labels,
                              opt_algorithm=opt_algorithm, par=False)

# ----------- create mapped gaussian process ------------------
struc_params = {'species': [47, 53],
                'cube_lat': np.eye(3) * 100,
                'mass_dict': {'0': 27, '1': 16}}

# grid parameters
lower_cut = 2.5
grid_num_2 = 16
grid_num_3 = 16
grid_params = {'bounds_2': [[lower_cut], [two_cut]],
               'bounds_3': [[lower_cut, lower_cut, 0],
                            [three_cut, three_cut, np.pi]],
               'grid_num_2': grid_num_2,
               'grid_num_3': [grid_num_3, grid_num_3, grid_num_3],
               'svd_rank_2': None,
               'svd_rank_3': None,
               'bodies': [2, 3],
               'load_grid': None,
               'update': True}

mgp_model = MappedGaussianProcess(gp_model, grid_params, struc_params,
                                 mean_only=True, lmp_file_name='agi.mgp')

# ------------ create ASE's flare calculator -----------------------
flare_calc = FLARE_Calculator(gp_model, mgp_model, par=False, use_mapping=True)
<commit_msg>ase uses predict.py and move is_std_in_bound out<commit_after>import sys
import numpy as np

from flare import otf, kernels
from flare.gp import GaussianProcess
from flare.mgp.mgp import MappedGaussianProcess
from flare.ase.calculator import FLARE_Calculator
import flare.mc_simple as mc_simple

# ---------- create gaussian process model -------------------
kernel = mc_simple.two_plus_three_body_mc
kernel_grad = mc_simple.two_plus_three_body_mc_grad
energy_force_kernel = mc_simple.two_plus_three_mc_force_en
energy_kernel = mc_simple.two_plus_three_mc_en

hyps = np.array([0.1, 1., 0.001, 1, 0.06])
two_cut = 5.0
three_cut = 5.0
cutoffs = np.array([two_cut, three_cut])
hyp_labels = ['sig2', 'ls2', 'sig3', 'ls3', 'noise']
opt_algorithm = 'BFGS'

gp_model = GaussianProcess(kernel, kernel_grad, hyps, cutoffs,
                           energy_kernel=energy_kernel,
                           energy_force_kernel=energy_force_kernel,
                           hyp_labels=hyp_labels,
                           opt_algorithm=opt_algorithm, par=False)

# ----------- create mapped gaussian process ------------------
struc_params = {'species': [47, 53],
                'cube_lat': np.eye(3) * 100,
                'mass_dict': {'0': 27, '1': 16}}

# grid parameters
lower_cut = 2.5
grid_num_2 = 16
grid_num_3 = 16
grid_params = {'bounds_2': [[lower_cut], [two_cut]],
               'bounds_3': [[lower_cut, lower_cut, 0],
                            [three_cut, three_cut, np.pi]],
               'grid_num_2': grid_num_2,
               'grid_num_3': [grid_num_3, grid_num_3, grid_num_3],
               'svd_rank_2': None,
               'svd_rank_3': None,
               'bodies': [2, 3],
               'load_grid': None,
               'update': True}

mgp_model = MappedGaussianProcess(gp_model, grid_params, struc_params,
                                 mean_only=True, lmp_file_name='agi.mgp')

# ------------ create ASE's flare calculator -----------------------
flare_calc = FLARE_Calculator(gp_model, mgp_model, par=False, use_mapping=True)
<|endoftext|>


<commit_before>import unittest

from common import get_context

class TestCase(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        # raise unittest.SkipTest('NYI')
        cls.ctx = get_context(require=430)

    def test_compute_shader(self):
        if self.ctx.version_code < 430:
            self.skipTest('OpenGL 4.3 is not supported')

        src = '''
            #version 430
            layout (local_size_x = 1, local_size_y = 1, local_size_z = 1) in;
            void main() {
            }
        '''

        compute_shader = self.ctx.compute_shader(src)
        compute_shader.release()

if __name__ == '__main__':
    unittest.main()
<commit_msg>Greatly improve ComputeShader testcase<commit_after>import unittest

import moderngl
from common import get_context

class TestCase(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        # raise unittest.SkipTest('NYI')
        cls.ctx = get_context(require=430)

    def test_compute_shader(self):
        if self.ctx.version_code < 430:
            self.skipTest('OpenGL 4.3 is not supported')

        src = '''
            #version 430
            layout (local_size_x = 1, local_size_y = 1, local_size_z = 1) in;

            struct Something
            {
                vec4 pos;
            };

            layout(std430, binding=0) buffer something_in
            {
                Something entries[];
            } In;

            // output into the buffer bound to 1
            layout(std430, binding=1) buffer something_out
            {
                Something entries[];
            } Out;

            uniform float multiplier;

            void main() {
                int x = int(gl_GlobalInvocationID);
                Something entry = In.entries[x];

                Out.entries[x] = Something(entry.pos * multiplier);
            }
        '''

        self.buffer_in = self.ctx.buffer(reserve=16)
        self.buffer_out = self.ctx.buffer(reserve=16)

        self.buffer_in.bind_to_storage_buffer(binding=0)
        self.buffer_out.bind_to_storage_buffer(binding=1)

        compute_shader = self.ctx.compute_shader(src)

        self.assertIsInstance(compute_shader.glo, int)
        self.assertIsNotNone(compute_shader.mglo)
        compute_shader.extra = {'name': 'test'}
        self.assertEqual(compute_shader.extra, {'name': 'test'})
        self.assertIsInstance(compute_shader.ctx, moderngl.Context)

        compute_shader['multiplier'] = 1.0
        compute_shader['multiplier'].value = 1.0
        compute_shader.run()
        self.assertIsNotNone(compute_shader.get('multiplier', None))
        self.assertEqual(compute_shader, compute_shader)
        self.assertEqual([i for i in compute_shader], ['multiplier'])
        compute_shader.release()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>"""
4. Many-to-one relationships

To define a many-to-one relationship, use ``ForeignKey()`` .
"""

from django.core import meta

class Reporter(meta.Model):
    first_name = meta.CharField(maxlength=30)
    last_name = meta.CharField(maxlength=30)

    def __repr__(self):
        return "%s %s" % (self.first_name, self.last_name)

class Article(meta.Model):
    headline = meta.CharField(maxlength=100)
    pub_date = meta.DateField()
    reporter = meta.ForeignKey(Reporter)

    def __repr__(self):
        return self.headline

API_TESTS = """
# Create a Reporter.
>>> r = reporters.Reporter(first_name='John', last_name='Smith')
>>> r.save()

# Create an Article.
>>> from datetime import datetime
>>> a = articles.Article(id=None, headline="This is a test", pub_date=datetime(2005, 7, 27), reporter=r)
>>> a.save()

>>> a.reporter_id
1

>>> a.get_reporter()
John Smith

# Article objects have access to their related Reporter objects.
>>> r = a.get_reporter()
>>> r.first_name, r.last_name
('John', 'Smith')

# Create an Article via the Reporter object.
>>> new_article = r.add_article(headline="John's second story", pub_date=datetime(2005, 7, 29))
>>> new_article
John's second story
>>> new_article.reporter_id
1

# Reporter objects have access to their related Article objects.
>>> r.get_article_list(order_by=['pub_date'])
[This is a test, John's second story]

>>> r.get_article(headline__startswith='This')
This is a test

>>> r.get_article_count()
2

# The API automatically follows relationships as far as you need.
# Use double underscores to separate relationships.
# This works as many levels deep as you want. There's no limit.
# Find all Articles for any Reporter whose first name is "John".
>>> articles.get_list(reporter__first_name__exact='John', order_by=['pub_date'])
[This is a test, John's second story]

# Find all Articles for the Reporter whose ID is 1.
>>> articles.get_list(reporter__id__exact=1, order_by=['pub_date'])
[This is a test, John's second story]

# Note you need two underscores between "reporter" and "id" -- not one.
>>> articles.get_list(reporter_id__exact=1)
Traceback (most recent call last):
    ...
TypeError: got unexpected keyword argument 'reporter_id__exact'

# "pk" shortcut syntax works in a related context, too.
>>> articles.get_list(reporter__pk=1, order_by=['pub_date'])
[This is a test, John's second story]

# You can also instantiate an Article by passing
# the Reporter's ID instead of a Reporter object.
>>> a3 = articles.Article(id=None, headline="This is a test", pub_date=datetime(2005, 7, 27), reporter_id=r.id)
>>> a3.save()
>>> a3.reporter_id
1
>>> a3.get_reporter()
John Smith

# Similarly, the reporter ID can be a string.
>>> a4 = articles.Article(id=None, headline="This is a test", pub_date=datetime(2005, 7, 27), reporter_id="1")
>>> a4.save()
>>> a4.get_reporter()
John Smith
"""
<commit_msg>Added an EmailField to the many_to_one API tests<commit_after>"""
4. Many-to-one relationships

To define a many-to-one relationship, use ``ForeignKey()`` .
"""

from django.core import meta

class Reporter(meta.Model):
    first_name = meta.CharField(maxlength=30)
    last_name = meta.CharField(maxlength=30)
    email = meta.EmailField()

    def __repr__(self):
        return "%s %s" % (self.first_name, self.last_name)

class Article(meta.Model):
    headline = meta.CharField(maxlength=100)
    pub_date = meta.DateField()
    reporter = meta.ForeignKey(Reporter)

    def __repr__(self):
        return self.headline

API_TESTS = """
# Create a Reporter.
>>> r = reporters.Reporter(first_name='John', last_name='Smith', email='john@example.com')
>>> r.save()

# Create an Article.
>>> from datetime import datetime
>>> a = articles.Article(id=None, headline="This is a test", pub_date=datetime(2005, 7, 27), reporter=r)
>>> a.save()

>>> a.reporter_id
1

>>> a.get_reporter()
John Smith

# Article objects have access to their related Reporter objects.
>>> r = a.get_reporter()
>>> r.first_name, r.last_name
('John', 'Smith')

# Create an Article via the Reporter object.
>>> new_article = r.add_article(headline="John's second story", pub_date=datetime(2005, 7, 29))
>>> new_article
John's second story
>>> new_article.reporter_id
1

# Reporter objects have access to their related Article objects.
>>> r.get_article_list(order_by=['pub_date'])
[This is a test, John's second story]

>>> r.get_article(headline__startswith='This')
This is a test

>>> r.get_article_count()
2

# The API automatically follows relationships as far as you need.
# Use double underscores to separate relationships.
# This works as many levels deep as you want. There's no limit.
# Find all Articles for any Reporter whose first name is "John".
>>> articles.get_list(reporter__first_name__exact='John', order_by=['pub_date'])
[This is a test, John's second story]

# Find all Articles for the Reporter whose ID is 1.
>>> articles.get_list(reporter__id__exact=1, order_by=['pub_date'])
[This is a test, John's second story]

# Note you need two underscores between "reporter" and "id" -- not one.
>>> articles.get_list(reporter_id__exact=1)
Traceback (most recent call last):
    ...
TypeError: got unexpected keyword argument 'reporter_id__exact'

# "pk" shortcut syntax works in a related context, too.
>>> articles.get_list(reporter__pk=1, order_by=['pub_date'])
[This is a test, John's second story]

# You can also instantiate an Article by passing
# the Reporter's ID instead of a Reporter object.
>>> a3 = articles.Article(id=None, headline="This is a test", pub_date=datetime(2005, 7, 27), reporter_id=r.id)
>>> a3.save()
>>> a3.reporter_id
1
>>> a3.get_reporter()
John Smith

# Similarly, the reporter ID can be a string.
>>> a4 = articles.Article(id=None, headline="This is a test", pub_date=datetime(2005, 7, 27), reporter_id="1")
>>> a4.save()
>>> a4.get_reporter()
John Smith
"""
<|endoftext|>


<commit_before>from __future__ import absolute_import
from decimal import Decimal
import time
from types import TracebackType
import traceback
import sys

import pytest

from concurrent.futures import Future, as_completed, TimeoutError
from threadloop import ThreadLoop
from threadloop.exceptions import ThreadNotStartedError
from tornado import gen, ioloop

@pytest.yield_fixture
def threadloop():
    with ThreadLoop() as threadloop:
        yield threadloop

def test_coroutine_returns_future(threadloop):

    @gen.coroutine
    def coroutine():
        raise gen.Return("Hello World")

    future = threadloop.submit(coroutine)

    assert (
        isinstance(future, Future),
        "expected a concurrent.futures.Future"
    )

    assert future.result() == "Hello World"

def test_propogates_arguments(threadloop):

    @gen.coroutine
    def coroutine(message, adjective="Shady"):
        raise gen.Return("Hello %s %s" % (adjective, message))

    future = threadloop.submit(coroutine, "World")
    assert future.result() == "Hello Shady World"

    future = threadloop.submit(coroutine, "World", adjective="Cloudy")
    assert future.result() == "Hello Cloudy World"

def test_coroutine_exception_propagates(threadloop):

    class TestException(Exception):
        pass

    @gen.coroutine
    def coroutine():
        raise TestException()

    with pytest.raises(TestException):
        future = threadloop.submit(coroutine)
        future.result()

def test_coroutine_exception_contains_exc_info(threadloop):

    class TestException(Exception):
        pass

    @gen.coroutine
    def coroutine():
        raise TestException('something went wrong')

    try:
        threadloop.submit(coroutine).result()
    except Exception:
        e, tb = sys.exc_info()[1:]

        assert isinstance(e, TestException)
        assert isinstance(tb, TracebackType)

        tb_out = traceback.extract_tb(tb)
        assert "raise TestException('something went wrong')" in str(tb_out)

        return

    assert False, "should have thrown exception"

def test_use_existing_ioloop():
    io_loop = ioloop.IOLoop.current()
    threadloop = ThreadLoop(io_loop)

    assert threadloop._io_loop is io_loop

    @gen.coroutine
    def coroutine():
        raise gen.Return("Hello World")

    with threadloop:
        future = threadloop.submit(coroutine)
        assert future.result() == "Hello World"

def test_start_must_be_called_before_submit():
    threadloop = ThreadLoop()

    @gen.coroutine
    def coroutine():
        raise gen.Return("Hello World")

    with pytest.raises(ThreadNotStartedError):
        threadloop.submit(coroutine)

def test_submits_coroutines_concurrently(threadloop):

    @gen.coroutine
    def coroutine1():
        yield gen.sleep(.1)
        raise gen.Return('coroutine1')

    @gen.coroutine
    def coroutine2():
        yield gen.sleep(.1)
        raise gen.Return('coroutine2')

    @gen.coroutine
    def coroutine3():
        yield gen.sleep(.1)
        raise gen.Return('coroutine3')

    start = time.time()

    future1 = threadloop.submit(coroutine1)
    future2 = threadloop.submit(coroutine2)
    future3 = threadloop.submit(coroutine3)

    result1 = future1.result()
    result2 = future2.result()
    result3 = future3.result()

    end = time.time() - start

    # round to float with precision of 1, eg 0.3
    took = float(round(Decimal(str(end)), 1))

    # should only take ~100 ms to finish both
    # instead of ~300ms if they were executed serially
    assert took == .1

    assert result1 == 'coroutine1'
    assert result2 == 'coroutine2'
    assert result3 == 'coroutine3'

def test_as_completed(threadloop):

    @gen.coroutine
    def coroutine1():
        yield gen.sleep(.02)
        raise gen.Return('coroutine1')

    @gen.coroutine
    def coroutine2():
        yield gen.sleep(.03)
        raise gen.Return('coroutine2')

    @gen.coroutine
    def coroutine3():
        yield gen.sleep(.01)
        raise gen.Return('coroutine3')

    @gen.coroutine
    def coroutine4():
        yield gen.sleep(.04)
        raise gen.Return('coroutine4')

    futures = []
    futures.append(threadloop.submit(coroutine1))
    futures.append(threadloop.submit(coroutine2))
    futures.append(threadloop.submit(coroutine3))
    futures.append(threadloop.submit(coroutine4))

    i = 0
    for future in as_completed(futures):
        i = i + 1

        # make sure futures finish in the expected order
        if i == 1:
            assert future.result() == "coroutine3"
        elif i == 2:
            assert future.result() == "coroutine1"
        elif i == 3:
            assert future.result() == "coroutine2"
        elif i == 4:
            assert future.result() == "coroutine4"

    assert i == 4, "expected 4 completed futures"

def test_timeout(threadloop):

    @gen.coroutine
    def too_long():
        yield gen.sleep(5)  # 5 sec task
        raise gen.Return('that was too long')

    start = time.time()
    future = threadloop.submit(too_long)

    with pytest.raises(TimeoutError):
        future.result(timeout=.001)

    end = time.time() - start
    took = float(round(Decimal(str(end)), 1))

    assert took <= .002

def test_block_until_thread_is_ready():

    threadloop = ThreadLoop()

    assert not threadloop.is_ready()

    threadloop.start()

    assert threadloop.is_ready()
<commit_msg>test is_ready<commit_after>from __future__ import absolute_import
from decimal import Decimal
import time
from types import TracebackType
import traceback
import sys

import pytest

from concurrent.futures import Future, as_completed, TimeoutError
from threadloop import ThreadLoop
from threadloop.exceptions import ThreadNotStartedError
from tornado import gen, ioloop

@pytest.yield_fixture
def threadloop():
    with ThreadLoop() as threadloop:
        yield threadloop

def test_coroutine_returns_future(threadloop):

    @gen.coroutine
    def coroutine():
        raise gen.Return("Hello World")

    future = threadloop.submit(coroutine)

    assert (
        isinstance(future, Future),
        "expected a concurrent.futures.Future"
    )

    assert future.result() == "Hello World"

def test_propogates_arguments(threadloop):

    @gen.coroutine
    def coroutine(message, adjective="Shady"):
        raise gen.Return("Hello %s %s" % (adjective, message))

    future = threadloop.submit(coroutine, "World")
    assert future.result() == "Hello Shady World"

    future = threadloop.submit(coroutine, "World", adjective="Cloudy")
    assert future.result() == "Hello Cloudy World"

def test_coroutine_exception_propagates(threadloop):

    class TestException(Exception):
        pass

    @gen.coroutine
    def coroutine():
        raise TestException()

    with pytest.raises(TestException):
        future = threadloop.submit(coroutine)
        future.result()

def test_coroutine_exception_contains_exc_info(threadloop):

    class TestException(Exception):
        pass

    @gen.coroutine
    def coroutine():
        raise TestException('something went wrong')

    try:
        threadloop.submit(coroutine).result()
    except Exception:
        e, tb = sys.exc_info()[1:]

        assert isinstance(e, TestException)
        assert isinstance(tb, TracebackType)

        tb_out = traceback.extract_tb(tb)
        assert "raise TestException('something went wrong')" in str(tb_out)

        return

    assert False, "should have thrown exception"

def test_use_existing_ioloop():
    io_loop = ioloop.IOLoop.current()
    threadloop = ThreadLoop(io_loop)

    assert threadloop._io_loop is io_loop

    @gen.coroutine
    def coroutine():
        raise gen.Return("Hello World")

    with threadloop:
        future = threadloop.submit(coroutine)
        assert future.result() == "Hello World"

def test_start_must_be_called_before_submit():
    threadloop = ThreadLoop()

    @gen.coroutine
    def coroutine():
        raise gen.Return("Hello World")

    with pytest.raises(ThreadNotStartedError):
        threadloop.submit(coroutine)

def test_submits_coroutines_concurrently(threadloop):

    @gen.coroutine
    def coroutine1():
        yield gen.sleep(.1)
        raise gen.Return('coroutine1')

    @gen.coroutine
    def coroutine2():
        yield gen.sleep(.1)
        raise gen.Return('coroutine2')

    @gen.coroutine
    def coroutine3():
        yield gen.sleep(.1)
        raise gen.Return('coroutine3')

    start = time.time()

    future1 = threadloop.submit(coroutine1)
    future2 = threadloop.submit(coroutine2)
    future3 = threadloop.submit(coroutine3)

    result1 = future1.result()
    result2 = future2.result()
    result3 = future3.result()

    end = time.time() - start

    # round to float with precision of 1, eg 0.3
    took = float(round(Decimal(str(end)), 1))

    # should only take ~100 ms to finish both
    # instead of ~300ms if they were executed serially
    assert took == .1

    assert result1 == 'coroutine1'
    assert result2 == 'coroutine2'
    assert result3 == 'coroutine3'

def test_as_completed(threadloop):

    @gen.coroutine
    def coroutine1():
        yield gen.sleep(.02)
        raise gen.Return('coroutine1')

    @gen.coroutine
    def coroutine2():
        yield gen.sleep(.03)
        raise gen.Return('coroutine2')

    @gen.coroutine
    def coroutine3():
        yield gen.sleep(.01)
        raise gen.Return('coroutine3')

    @gen.coroutine
    def coroutine4():
        yield gen.sleep(.04)
        raise gen.Return('coroutine4')

    futures = []
    futures.append(threadloop.submit(coroutine1))
    futures.append(threadloop.submit(coroutine2))
    futures.append(threadloop.submit(coroutine3))
    futures.append(threadloop.submit(coroutine4))

    i = 0
    for future in as_completed(futures):
        i = i + 1

        # make sure futures finish in the expected order
        if i == 1:
            assert future.result() == "coroutine3"
        elif i == 2:
            assert future.result() == "coroutine1"
        elif i == 3:
            assert future.result() == "coroutine2"
        elif i == 4:
            assert future.result() == "coroutine4"

    assert i == 4, "expected 4 completed futures"

def test_timeout(threadloop):

    @gen.coroutine
    def too_long():
        yield gen.sleep(5)  # 5 sec task
        raise gen.Return('that was too long')

    start = time.time()
    future = threadloop.submit(too_long)

    with pytest.raises(TimeoutError):
        future.result(timeout=.001)

    end = time.time() - start
    took = float(round(Decimal(str(end)), 1))

    assert took <= .002

def test_block_until_thread_is_ready():

    threadloop = ThreadLoop()

    assert not threadloop.is_ready()

    threadloop.start()

    assert threadloop.is_ready()

def test_is_not_ready_when_ready_hasnt_been_sent():

    threadloop = ThreadLoop()
    threadloop._thread = True  # fake the Thread being set

    assert not threadloop.is_ready()
<|endoftext|>


<commit_before>import json
import pytest

from dcicutils.env_utils import is_stg_or_prd_env, CGAP_ENV_PRODUCTION_BLUE_NEW, CGAP_ENV_PRODUCTION_GREEN_NEW
from dcicutils.ff_utils import patch_metadata
from ..types.workflow import _wfoutput_bucket_for_env

pytestmark = [pytest.mark.setone, pytest.mark.working]

@pytest.fixture()
def input_json(workflow):
    # use workflow that should always be in webdev
    item = {
      "app_name": "hi-c-processing-bam",
      "parameters": {
      },
      "output_bucket": "elasticbeanstalk-fourfront-webdev-wfoutput",
      "tag": "0.2.5",
      "config": {
          "log_bucket": "tibanna-output"
      },
      "workflow_uuid": "023bfb3e-9a8b-42b9-a9d4-216079526f68",
      "input_files": [
        {
          "object_key": [
            "4DNFI9H51IRL.bam",
            "4DNFIP16HHGH.bam"
          ],
          "bucket_name": "elasticbeanstalk-fourfront-webdev-wfoutput",
          "workflow_argument_name": "input_bams",
          "uuid": [
            "68f38e45-8c66-41e2-99ab-b0b2fcd20d45",
            "7420a20a-aa77-4ea8-b0b0-32a8e80c9bcb"
          ]
        },
        {
          "object_key":"4DNFI823LSII.chrom.sizes",
          "bucket_name": "elasticbeanstalk-fourfront-webprod-files",
          "workflow_argument_name": "chromsize",
          "uuid": "4a6d10ee-2edb-4402-a98f-0edb1d58f5e9"
        }
      ],
      "metadata_only": True,
      "output_files": [
              {"workflow_argument_name": "annotated_bam",
               "uuid": "68f38e45-8c66-41e2-99ab-b0b2fcd20d45"
              },
              {"workflow_argument_name": "filtered_pairs",
               "uuid": "7054061b-e87d-4ca4-9693-d186348f5206"
              }
      ]
    }

    return item

@pytest.fixture()
def workflow(testapp, software, award, lab):
    # ensure we always use uuid that's on fourfront-webdev
    workflow_uuid = '023bfb3e-9a8b-42b9-a9d4-216079526f68'
    return workflow_uuid

@pytest.mark.xfail
def test_pseudo_run(testapp, input_json):
    # this test can be problematic; uncomment the following line to disable it
    # assert False

    res = testapp.post_json('/WorkflowRun/pseudo-run', input_json)
    assert(res)

    # cleanup
    output = json.loads(res.json['output'])
    patch_metadata({'status':'deleted'}, output['ff_meta']['uuid'], ff_env='fourfront-webdev')

def test_workflow_for_env():

    # These tests will want to become more abstract sometime, but for transition they test that
    # we're getting obvious values we expect. -kmp 1-Apr-2020

    # Fourfront prod environments

    assert _wfoutput_bucket_for_env('fourfront-webprod') == 'elasticbeanstalk-fourfront-webprod-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-webprod2') == 'elasticbeanstalk-fourfront-webprod-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-blue') == 'elasticbeanstalk-fourfront-webprod-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-green') == 'elasticbeanstalk-fourfront-webprod-wfoutput'

    # Other (non-prod) Fourfront environments

    assert _wfoutput_bucket_for_env('fourfront-mastertest') == 'elasticbeanstalk-fourfront-mastertest-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-webdev') == 'elasticbeanstalk-fourfront-webdev-wfoutput'

    # CGAP prod environments

    assert _wfoutput_bucket_for_env('fourfront-cgap') == 'elasticbeanstalk-fourfront-cgap-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-cgap-green') == 'elasticbeanstalk-fourfront-cgap-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-cgap-blue') == 'elasticbeanstalk-fourfront-cgap-wfoutput'

    # There is a bug in is_stg_or_prd_env that needs to be fixed before these tests can run right:
    if is_stg_or_prd_env(CGAP_ENV_PRODUCTION_BLUE_NEW) and is_stg_or_prd_env(CGAP_ENV_PRODUCTION_GREEN_NEW):
        assert _wfoutput_bucket_for_env('cgap-blue') == 'elasticbeanstalk-fourfront-cgap-wfoutput'
        assert _wfoutput_bucket_for_env('cgap-green') == 'elasticbeanstalk-fourfront-cgap-wfoutput'

    # Other (non-prod) CGAP environments

    assert _wfoutput_bucket_for_env('fourfront-cgapwolf') == 'elasticbeanstalk-fourfront-cgapwolf-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-cgaptest') == 'elasticbeanstalk-fourfront-cgaptest-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-yellow') == 'elasticbeanstalk-fourfront-yellow-wfoutput'

<commit_msg>purge deleted test wfr<commit_after>import json
import pytest

from dcicutils.env_utils import is_stg_or_prd_env, CGAP_ENV_PRODUCTION_BLUE_NEW, CGAP_ENV_PRODUCTION_GREEN_NEW
from dcicutils.ff_utils import patch_metadata
from ..types.workflow import _wfoutput_bucket_for_env

pytestmark = [pytest.mark.setone, pytest.mark.working]

@pytest.fixture()
def input_json(workflow):
    # use workflow that should always be in webdev
    item = {
      "app_name": "hi-c-processing-bam",
      "parameters": {
      },
      "output_bucket": "elasticbeanstalk-fourfront-webdev-wfoutput",
      "tag": "0.2.5",
      "config": {
          "log_bucket": "tibanna-output"
      },
      "workflow_uuid": "023bfb3e-9a8b-42b9-a9d4-216079526f68",
      "input_files": [
        {
          "object_key": [
            "4DNFI9H51IRL.bam",
            "4DNFIP16HHGH.bam"
          ],
          "bucket_name": "elasticbeanstalk-fourfront-webdev-wfoutput",
          "workflow_argument_name": "input_bams",
          "uuid": [
            "68f38e45-8c66-41e2-99ab-b0b2fcd20d45",
            "7420a20a-aa77-4ea8-b0b0-32a8e80c9bcb"
          ]
        },
        {
          "object_key":"4DNFI823LSII.chrom.sizes",
          "bucket_name": "elasticbeanstalk-fourfront-webprod-files",
          "workflow_argument_name": "chromsize",
          "uuid": "4a6d10ee-2edb-4402-a98f-0edb1d58f5e9"
        }
      ],
      "metadata_only": True,
      "output_files": [
              {"workflow_argument_name": "annotated_bam",
               "uuid": "68f38e45-8c66-41e2-99ab-b0b2fcd20d45"
              },
              {"workflow_argument_name": "filtered_pairs",
               "uuid": "7054061b-e87d-4ca4-9693-d186348f5206"
              }
      ]
    }

    return item

@pytest.fixture()
def workflow(testapp, software, award, lab):
    # ensure we always use uuid that's on fourfront-webdev
    workflow_uuid = '023bfb3e-9a8b-42b9-a9d4-216079526f68'
    return workflow_uuid

@pytest.mark.xfail
def test_pseudo_run(testapp, input_json):
    # this test can be problematic; uncomment the following line to disable it
    # assert False

    res = testapp.post_json('/WorkflowRun/pseudo-run', input_json)
    assert(res)

    # cleanup
    output = json.loads(res.json['output'])
    patch_metadata({'status':'deleted'}, output['ff_meta']['uuid'], ff_env='fourfront-webdev')
    purge_metadata(output['ff_meta']['uuid'], ff_env='fourfront-webdev')

def test_workflow_for_env():

    # These tests will want to become more abstract sometime, but for transition they test that
    # we're getting obvious values we expect. -kmp 1-Apr-2020

    # Fourfront prod environments

    assert _wfoutput_bucket_for_env('fourfront-webprod') == 'elasticbeanstalk-fourfront-webprod-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-webprod2') == 'elasticbeanstalk-fourfront-webprod-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-blue') == 'elasticbeanstalk-fourfront-webprod-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-green') == 'elasticbeanstalk-fourfront-webprod-wfoutput'

    # Other (non-prod) Fourfront environments

    assert _wfoutput_bucket_for_env('fourfront-mastertest') == 'elasticbeanstalk-fourfront-mastertest-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-webdev') == 'elasticbeanstalk-fourfront-webdev-wfoutput'

    # CGAP prod environments

    assert _wfoutput_bucket_for_env('fourfront-cgap') == 'elasticbeanstalk-fourfront-cgap-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-cgap-green') == 'elasticbeanstalk-fourfront-cgap-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-cgap-blue') == 'elasticbeanstalk-fourfront-cgap-wfoutput'

    # There is a bug in is_stg_or_prd_env that needs to be fixed before these tests can run right:
    if is_stg_or_prd_env(CGAP_ENV_PRODUCTION_BLUE_NEW) and is_stg_or_prd_env(CGAP_ENV_PRODUCTION_GREEN_NEW):
        assert _wfoutput_bucket_for_env('cgap-blue') == 'elasticbeanstalk-fourfront-cgap-wfoutput'
        assert _wfoutput_bucket_for_env('cgap-green') == 'elasticbeanstalk-fourfront-cgap-wfoutput'

    # Other (non-prod) CGAP environments

    assert _wfoutput_bucket_for_env('fourfront-cgapwolf') == 'elasticbeanstalk-fourfront-cgapwolf-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-cgaptest') == 'elasticbeanstalk-fourfront-cgaptest-wfoutput'
    assert _wfoutput_bucket_for_env('fourfront-yellow') == 'elasticbeanstalk-fourfront-yellow-wfoutput'

<|endoftext|>