<commit_before>import json
import os.path

from flask import Flask, abort, redirect

from common import file, page, console, post_header

rss = None
menu_list = None
template_config = None
page_name_list = list()
cache_index = dict()
cache_post = dict()

app = Flask(__name__)

console.log("info", "Loading configuration")

system_config = json.loads(file.read_file("./config/system.json"))

if system_config["Author_Image"] == "" and system_config["Author_Name"] != "":
    import urllib.request
    r = {"entry": [{"hash": ""}]}
    console.log("info", "Get Gravatar URL")
    try:
        r = urllib.request.urlopen(
            "https://en.gravatar.com/{0}.json".format(system_config["Author_Name"])).read().decode('utf-8')
    except urllib2.HTTPError:
        console.log("Error", "Get Gravatar URL error")
        pass
    req = json.loads(r)
    gravatar_hash = req["entry"][0]["hash"]
    system_config["Author_Image"] = "https://secure.gravatar.com/avatar/{0}".format(gravatar_hash)
    file.write_file("./config/system.json", json.dumps(system_config))

page_list = json.loads(file.read_file("./config/page.json"))

if os.path.exists("./config/menu.json"):
    menu_list = json.loads(file.read_file("./config/menu.json"))

for item in page_list:
    page_name_list.append(item["name"])

if os.path.exists("./document/rss.xml"):
    rss = file.read_file("document/rss.xml")

if os.path.exists("./templates/{0}/config.json".format(system_config["Theme"])):
    template_config = json.loads(file.read_file("./templates/{0}/config.json".format(system_config["Theme"])))

system_config["API_Password"] = None
page_list = list(map(post_header.add_post_header, page_list))
menu_list = list(map(post_header.add_post_header, menu_list))
console.log("Success", "load the configuration file successfully")

# Subscribe
@app.route("/rss/", strict_slashes=False)
@app.route("/feed/", strict_slashes=False)
def load_rss():
    if rss is None:
        abort(404)
    return rss, 200, {'Content-Type': 'text/xml; charset=utf-8'}

@app.route("/static/")
def static_file():
    abort(400)

@app.route("/")
@app.route("/index/", strict_slashes=False)
@app.route('/index/p/<int:page_index>/', strict_slashes=False)
def index_route(page_index=1):
    page_url = "/index/p/{0}/".format(page_index)
    if page_url in cache_index:
        console.log("info", "Get cache Success: {0}".format(page_url))
        return cache_index[page_url]

    console.log("info", "Trying to build: {0}".format(page_url))
    result, row = page.build_index(page_index, system_config, page_list, menu_list,
                                   False, template_config)

    console.log("info", "Writing to cache: {0}".format(page_url))
    if len(cache_index) >= 100:
        page_keys = sorted(cache_index.keys())
        console.log("info", "Delete cache: {0}".format(page_keys[0]))
        del cache_index[page_keys[0]]
    cache_index[page_url] = result
    console.log("Success", "Get success: {0}".format(page_url))

    return result

@app.route("/<file_name>/")
@app.route("/<file_name>")
def redirect_301(file_name):
    if file_name in page_name_list or os.path.exists("document/{0}.md".format(file_name)):
        return redirect("/post/{0}".format(file_name), code=301)
    abort(404)

@app.route("/post/<file_name>")
@app.route("/post/<file_name>/")
def post_route(file_name=None):
    if file_name is None or not os.path.exists("document/{0}.md".format(file_name)):
        abort(404)
    page_url = "/post/{0}/".format(file_name)
    if page_url in cache_post:
        console.log("info", "Get cache Success: {0}".format(page_url))
        return cache_post[page_url]
    page_info = None
    if file_name in page_name_list:
        this_page_index = page_name_list.index(file_name)
        page_info = page_list[this_page_index]
    result = page.build_page(file_name, system_config, page_info, menu_list,
                             False,
                             template_config)
    console.log("info", "Writing to cache: {0}".format(page_url))
    if len(cache_post) >= 100:
        page_keys = sorted(cache_post.keys())
        console.log("info", "Delete cache: {0}".format(page_keys[0]))
        del cache_post[page_keys[0]]
    cache_post[page_url] = result
    console.log("Success", "Get success: {0}".format(page_url))

    return result

<commit_msg>修改页面路由<commit_after>import json
import os.path

from flask import Flask, abort, redirect

from common import file, page, console, post_header

rss = None
menu_list = None
template_config = None
page_name_list = list()
cache_index = dict()
cache_post = dict()

app = Flask(__name__)

console.log("info", "Loading configuration")

system_config = json.loads(file.read_file("./config/system.json"))

if system_config["Author_Image"] == "" and system_config["Author_Name"] != "":
    import urllib.request
    r = {"entry": [{"hash": ""}]}
    console.log("info", "Get Gravatar URL")
    try:
        r = urllib.request.urlopen(
            "https://en.gravatar.com/{0}.json".format(system_config["Author_Name"])).read().decode('utf-8')
    except urllib2.HTTPError:
        console.log("Error", "Get Gravatar URL error")
        pass
    req = json.loads(r)
    gravatar_hash = req["entry"][0]["hash"]
    system_config["Author_Image"] = "https://secure.gravatar.com/avatar/{0}".format(gravatar_hash)
    file.write_file("./config/system.json", json.dumps(system_config))

page_list = json.loads(file.read_file("./config/page.json"))

if os.path.exists("./config/menu.json"):
    menu_list = json.loads(file.read_file("./config/menu.json"))

for item in page_list:
    page_name_list.append(item["name"])

if os.path.exists("./document/rss.xml"):
    rss = file.read_file("document/rss.xml")

if os.path.exists("./templates/{0}/config.json".format(system_config["Theme"])):
    template_config = json.loads(file.read_file("./templates/{0}/config.json".format(system_config["Theme"])))

system_config["API_Password"] = None
page_list = list(map(post_header.add_post_header, page_list))
menu_list = list(map(post_header.add_post_header, menu_list))
console.log("Success", "load the configuration file successfully")

# Subscribe
@app.route("/rss/", strict_slashes=False)
@app.route("/feed/", strict_slashes=False)
def load_rss():
    if rss is None:
        abort(404)
    return rss, 200, {'Content-Type': 'text/xml; charset=utf-8'}

@app.route("/static/")
def static_file():
    abort(400)

@app.route("/")
@app.route("/index/", strict_slashes=False)
@app.route('/index/p/<int:page_index>/', strict_slashes=False)
def index_route(page_index=1):
    page_url = "/index/p/{0}/".format(page_index)
    if page_url in cache_index:
        console.log("info", "Get cache Success: {0}".format(page_url))
        return cache_index[page_url]

    console.log("info", "Trying to build: {0}".format(page_url))
    result, row = page.build_index(page_index, system_config, page_list, menu_list,
                                   False, template_config)

    console.log("info", "Writing to cache: {0}".format(page_url))
    if len(cache_index) >= 100:
        page_keys = sorted(cache_index.keys())
        console.log("info", "Delete cache: {0}".format(page_keys[0]))
        del cache_index[page_keys[0]]
    cache_index[page_url] = result
    console.log("Success", "Get success: {0}".format(page_url))

    return result

@app.route("/<file_name>", strict_slashes=False)
def redirect_301(file_name):
    if file_name in page_name_list or os.path.exists("document/{0}.md".format(file_name)):
        return redirect("/post/{0}/".format(file_name), code=301)
    abort(404)

@app.route("/post/<file_name>")
@app.route("/post/<file_name>/")
def post_route(file_name=None):
    if file_name is None or not os.path.exists("document/{0}.md".format(file_name)):
        abort(404)
    page_url = "/post/{0}/".format(file_name)
    if page_url in cache_post:
        console.log("info", "Get cache Success: {0}".format(page_url))
        return cache_post[page_url]
    page_info = None
    if file_name in page_name_list:
        this_page_index = page_name_list.index(file_name)
        page_info = page_list[this_page_index]
    result = page.build_page(file_name, system_config, page_info, menu_list,
                             False,
                             template_config)
    console.log("info", "Writing to cache: {0}".format(page_url))
    if len(cache_post) >= 100:
        page_keys = sorted(cache_post.keys())
        console.log("info", "Delete cache: {0}".format(page_keys[0]))
        del cache_post[page_keys[0]]
    cache_post[page_url] = result
    console.log("Success", "Get success: {0}".format(page_url))

    return result

<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

# ==================================================
# open
# --------------------------------------------------

from __future__ import print_function
from __future__ import absolute_import
import os
import tct
import sys

params = tct.readjson(sys.argv[1])
facts = tct.readjson(params['factsfile'])
milestones = tct.readjson(params['milestonesfile'])
resultfile = params['resultfile']
result = tct.readjson(resultfile)
toolname = params['toolname']
toolname_pure = params['toolname_pure']
toolchain_name = facts['toolchain_name']
workdir = params['workdir']
loglist = result['loglist'] = result.get('loglist', [])
exitcode = CONTINUE = 0

# ==================================================
# Make a copy of milestones for later inspection?
# --------------------------------------------------

if 0 or milestones.get('debug_always_make_milestones_snapshot'):
    tct.make_snapshot_of_milestones(params['milestonesfile'], sys.argv[1])

# ==================================================
# Helper functions
# --------------------------------------------------

deepget = tct.deepget

def lookup(D, *keys, **kwdargs):
    result = deepget(D, *keys, **kwdargs)
    loglist.append((keys, result))
    return result

# ==================================================
# define
# --------------------------------------------------

publish_params_json_file = None
publish_params_data = {}
xeq_name_cnt = 0

# ==================================================
# example - just for demonstration
# --------------------------------------------------

example_list = [
    {
        "publish_dir": "/ALL/dummy_webroot/typo3cms/extensions/blog/blog/8.7.0",
        "publish_dir_buildinfo": "/ALL/dummy_webroot/typo3cms/extensions/blog/blog/8.7.0/_buildinfo",
        "publish_html_done": 1,
        "publish_language_dir": "/ALL/dummy_webroot/typo3cms/extensions/blog",
        "publish_package_file": "/ALL/dummy_webroot/typo3cms/extensions/blog/packages/blog-8.7.0-default.zip",
        "publish_packages_xml_file": "/ALL/dummy_webroot/typo3cms/extensions/blog/packages/packages.xml",
        "publish_project_dir": "/ALL/dummy_webroot/typo3cms/extensions/blog",
        "publish_project_parent_dir": "/ALL/dummy_webroot/typo3cms/extensions",
        "publish_removed_old": 1,
        "webroot_abspath": "/ALL/dummy_webroot",
    },
    {
        "publish_dir": "typo3cms/extensions/sphinx/fr-fr/2.5.1",
        "publish_dir_buildinfo": "typo3cms/extensions/sphinx/fr-fr/2.5.1/_buildinfo",
        "publish_language_dir": "typo3cms/extensions/sphinx",
        "publish_package_file": "typo3cms/extensions/sphinx/packages/sphinx-2.5.1-fr-fr.zip",
        "publish_packages_xml_file": "typo3cms/extensions/sphinx/packages/packages.xml",
        "publish_project_dir": "typo3cms/extensions/sphinx",
        "publish_project_parent_dir": "typo3cms/extensions",
        "todo_update_stable_symlink": 1
    }]

# ==================================================
# Check params
# --------------------------------------------------

if exitcode == CONTINUE:
    loglist.append('CHECK PARAMS')

    publish_html_done = lookup(milestones, "publish_html_done")
    if not publish_html_done:
        exitcode = 22

    publish_dir = lookup(milestones, "publish_dir")
    publish_language_dir = lookup(milestones, "publish_language_dir")
    publish_project_dir = lookup(milestones, "publish_project_dir")
    publish_project_parent_dir = lookup(milestones, "publish_project_parent_dir")
    webroot_abspath = lookup(milestones, "webroot_abspath")

    if not (publish_dir and publish_language_dir and publish_project_dir and
            publish_project_parent_dir and webroot_abspath):
        exitcode = 22

if exitcode == CONTINUE:
    publish_dir_buildinfo = lookup(milestones, "publish_dir_buildinfo", default='')
    publish_package_file = lookup(milestones, "publish_package_file", default='')
    publish_packages_xml_file = lookup(milestones, "publish_packages_xml_file", default='')
    ter_extension = lookup(milestones, 'buildsettings', 'ter_extension')

if exitcode == CONTINUE:
    loglist.append('PARAMS are ok')
else:
    loglist.append('Bad PARAMS or nothing to do')

# ==================================================
# work
# --------------------------------------------------

len_webroot_abspath = len(webroot_abspath)

def fixparm(fpath):
    result = fpath
    if fpath.startswith(webroot_abspath):
        result = fpath[len_webroot_abspath:]
    return result.strip('/')

if exitcode == CONTINUE:
    publish_params_data['publish_dir'] = fixparm(publish_dir)
    publish_params_data['publish_dir_buildinfo'] = fixparm(publish_dir_buildinfo)
    publish_params_data['publish_language_dir'] = fixparm(publish_language_dir)
    publish_params_data['publish_package_file'] = fixparm(publish_package_file)
    publish_params_data['publish_packages_xml_file'] = fixparm(publish_packages_xml_file)
    publish_params_data['publish_project_dir'] = fixparm(publish_project_dir)
    publish_params_data['publish_project_parent_dir'] = fixparm(publish_project_parent_dir)
    publish_params_data['todo_update_stable_symlink'] = 1 if ter_extension else 0

    publish_params_json_file = os.path.join(webroot_abspath, 'publish-params.json')

    if os.path.exists(publish_params_json_file):
        publish_params_existing_data = tct.readjson(publish_params_json_file)
    else:
        publish_params_existing_data = []

    publish_params_existing_data.append(publish_params_data)
    tct.writejson(publish_params_existing_data, publish_params_json_file)

# ==================================================
# Set MILESTONE
# --------------------------------------------------

if publish_params_json_file:
    result['MILESTONES'].append({'publish_params_json_file': publish_params_json_file})

if publish_params_data:
    result['MILESTONES'].append({'publish_params_data': publish_params_data})

# ==================================================
# save result
# --------------------------------------------------

tct.writejson(result, resultfile)

# ==================================================
# Return with proper exitcode
# --------------------------------------------------

sys.exit(exitcode)
<commit_msg>Improve run_10-Create-publish-params-json.py<commit_after>#!/usr/bin/env python
# coding: utf-8

# ==================================================
# open
# --------------------------------------------------

from __future__ import print_function
from __future__ import absolute_import

import os
import sys
import tct

from tct import deepget

params = tct.readjson(sys.argv[1])
facts = tct.readjson(params['factsfile'])
milestones = tct.readjson(params['milestonesfile'])
resultfile = params['resultfile']
result = tct.readjson(resultfile)
toolname = params['toolname']
toolname_pure = params['toolname_pure']
toolchain_name = facts['toolchain_name']
workdir = params['workdir']
loglist = result['loglist'] = result.get('loglist', [])
exitcode = CONTINUE = 0

# ==================================================
# Make a copy of milestones for later inspection?
# --------------------------------------------------

if 0 or milestones.get('debug_always_make_milestones_snapshot'):
    tct.make_snapshot_of_milestones(params['milestonesfile'], sys.argv[1])

# ==================================================
# Helper functions
# --------------------------------------------------

def lookup(D, *keys, **kwdargs):
    result = deepget(D, *keys, **kwdargs)
    loglist.append((keys, result))
    return result

# ==================================================
# define
# --------------------------------------------------

publish_params_json_file = None
publish_params_data = {}
xeq_name_cnt = 0

# ==================================================
# example - just for demonstration
# --------------------------------------------------

example_list = [
    {
        "publish_dir": "/ALL/dummy_webroot/typo3cms/extensions/blog/blog/8.7.0",
        "publish_dir_buildinfo": "/ALL/dummy_webroot/typo3cms/extensions/blog/blog/8.7.0/_buildinfo",
        "publish_html_done": 1,
        "publish_language_dir": "/ALL/dummy_webroot/typo3cms/extensions/blog",
        "publish_package_file": "/ALL/dummy_webroot/typo3cms/extensions/blog/packages/blog-8.7.0-default.zip",
        "publish_packages_xml_file": "/ALL/dummy_webroot/typo3cms/extensions/blog/packages/packages.xml",
        "publish_project_dir": "/ALL/dummy_webroot/typo3cms/extensions/blog",
        "publish_project_parent_dir": "/ALL/dummy_webroot/typo3cms/extensions",
        "publish_removed_old": 1,
        "webroot_abspath": "/ALL/dummy_webroot",
    },
    {
        "publish_dir": "typo3cms/extensions/sphinx/fr-fr/2.5.1",
        "publish_dir_buildinfo": "typo3cms/extensions/sphinx/fr-fr/2.5.1/_buildinfo",
        "publish_language_dir": "typo3cms/extensions/sphinx",
        "publish_package_file": "typo3cms/extensions/sphinx/packages/sphinx-2.5.1-fr-fr.zip",
        "publish_packages_xml_file": "typo3cms/extensions/sphinx/packages/packages.xml",
        "publish_project_dir": "typo3cms/extensions/sphinx",
        "publish_project_parent_dir": "typo3cms/extensions",
        "todo_update_stable_symlink": 1
    }]

# ==================================================
# Check params
# --------------------------------------------------

if exitcode == CONTINUE:
    loglist.append('CHECK PARAMS')

    publish_html_done = lookup(milestones, "publish_html_done")
    if not publish_html_done:
        exitcode = 22

    publish_dir = lookup(milestones, "publish_dir", default=None)
    publish_language_dir = lookup(milestones, "publish_language_dir",
                                  default=None)
    publish_project_dir = lookup(milestones, "publish_project_dir",
                                 default=None)
    publish_project_parent_dir = lookup(milestones,
                                        "publish_project_parent_dir",
                                        default=None)
    TheProjectWebroot = lookup(milestones, "TheProjectWebroot", default=None)
    #webroot_abspath = lookup(milestones, "webroot_abspath")

    if not (1
            and publish_dir
            and publish_language_dir
            and publish_project_dir
            and publish_project_parent_dir
            and TheProjectWebroot):
        CONTINUE = -2

if exitcode == CONTINUE:
    publish_dir_buildinfo = lookup(milestones, "publish_dir_buildinfo",
                                   default='')
    publish_package_file = lookup(milestones, "publish_package_file",
                                  default='')
    publish_packages_xml_file = lookup(milestones, "publish_packages_xml_file",
                                       default='')
    ter_extension = lookup(milestones, 'buildsettings', 'ter_extension')

if exitcode == CONTINUE:
    loglist.append('PARAMS are ok')
else:
    loglist.append('Bad PARAMS or nothing to do')

# ==================================================
# work
# --------------------------------------------------

if exitcode == CONTINUE:
    len_TheProjectWebroot = len(TheProjectWebroot)

    def fixparm(fpath):
        result = fpath
        if fpath.startswith(TheProjectWebroot):
            result = fpath[len_TheProjectWebroot:]
        return result.strip('/')

if exitcode == CONTINUE:
    publish_params_data['publish_dir'] = fixparm(publish_dir)
    publish_params_data['publish_dir_buildinfo'] = fixparm(publish_dir_buildinfo)
    publish_params_data['publish_language_dir'] = fixparm(publish_language_dir)
    publish_params_data['publish_package_file'] = fixparm(publish_package_file)
    publish_params_data['publish_packages_xml_file'] = fixparm(publish_packages_xml_file)
    publish_params_data['publish_project_dir'] = fixparm(publish_project_dir)
    publish_params_data['publish_project_parent_dir'] = fixparm(publish_project_parent_dir)
    publish_params_data['todo_update_stable_symlink'] = 1 if ter_extension else 0

    publish_params_json_file = os.path.join(TheProjectWebroot, 'publish-params.json')

    if os.path.exists(publish_params_json_file):
        publish_params_existing_data = tct.readjson(publish_params_json_file)
    else:
        publish_params_existing_data = []

    publish_params_existing_data.append(publish_params_data)
    tct.writejson(publish_params_existing_data, publish_params_json_file)

# ==================================================
# Set MILESTONE
# --------------------------------------------------

if publish_params_json_file:
    result['MILESTONES'].append({'publish_params_json_file':
                                 publish_params_json_file})

if publish_params_data:
    result['MILESTONES'].append({'publish_params_data':
                                 publish_params_data})

# ==================================================
# save result
# --------------------------------------------------

tct.writejson(result, resultfile)

# ==================================================
# Return with proper exitcode
# --------------------------------------------------

sys.exit(exitcode)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2014-2018 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test mining RPCs

- getmininginfo
- getblocktemplate proposal mode
- submitblock"""

import copy
from decimal import Decimal

from test_framework.blocktools import create_coinbase
from test_framework.messages import (
    CBlock,
    CBlockHeader,
    BLOCK_HEADER_SIZE
)
from test_framework.mininode import (
    P2PDataStore,
)
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
    assert_equal,
    assert_raises_rpc_error,
    bytes_to_hex_str as b2x,
)
from test_framework.script import CScriptNum

def assert_template(node, block, expect, rehash=True):
    if rehash:
        block.hashMerkleRoot = block.calc_merkle_root()
    rsp = node.getblocktemplate(template_request={'data': b2x(block.serialize()), 'mode': 'proposal', 'rules': ['segwit']})
    assert_equal(rsp, expect)

class MiningTest(BitcoinTestFramework):
    def set_test_params(self):
        self.num_nodes = 2
        self.setup_clean_chain = False

    def run_test(self):
        node = self.nodes[0]

        def assert_submitblock(block, result_str_1, result_str_2=None):
            block.solve()
            result_str_2 = result_str_2 or 'duplicate-invalid'
            assert_equal(result_str_1, node.submitblock(hexdata=b2x(block.serialize())))
            assert_equal(result_str_2, node.submitblock(hexdata=b2x(block.serialize())))

        self.log.info('getmininginfo')
        mining_info = node.getmininginfo()
        assert_equal(mining_info['blocks'], 200)
        assert_equal(mining_info['chain'], 'regtest')
        assert_equal(mining_info['currentblocktx'], 0)
        assert_equal(mining_info['currentblockweight'], 0)
        assert_equal(mining_info['difficulty'], Decimal('4.656542373906925E-10'))
        assert_equal(mining_info['networkhashps'], Decimal('0.003333333333333334'))
        assert_equal(mining_info['pooledtx'], 0)

        # Mine a block to leave initial block download
        node.generatetoaddress(1, node.get_deterministic_priv_key().address)
        tmpl = node.getblocktemplate({'rules': ['segwit']})
        self.log.info("getblocktemplate: Test capability advertised")
        assert 'proposal' in tmpl['capabilities']
        assert 'coinbasetxn' not in tmpl

        next_height = int(tmpl["height"])
        coinbase_tx = create_coinbase(height=next_height)
        # sequence numbers must not be max for nLockTime to have effect
        coinbase_tx.vin[0].nSequence = 2 ** 32 - 2
        coinbase_tx.rehash()

        # round-trip the encoded bip34 block height commitment
        assert_equal(CScriptNum.decode(coinbase_tx.vin[0].scriptSig), next_height)
        # round-trip negative and multi-byte CScriptNums to catch python regression
        assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(1500))), 1500)
        assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1500))), -1500)
        assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1))), -1)

        block = CBlock()
        block.nVersion = tmpl["version"]
        block.hashPrevBlock = int(tmpl["previousblockhash"], 16)
        block.nTime = tmpl["curtime"]
        block.nBits = int(tmpl["bits"], 16)
        block.nNonce = 0
        block.vtx = [coinbase_tx]

        self.log.info("getblocktemplate: segwit rule must be set")
        assert_raises_rpc_error(-8, "getblocktemplate must be called with the segwit rule set", node.getblocktemplate)

        self.log.info("getblocktemplate: Test valid block")
        assert_template(node, block, None)

        self.log.info("submitblock: Test block decode failure")
        assert_raises_rpc_error(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15]))

        self.log.info("getblocktemplate: Test bad input hash for coinbase transaction")
        bad_block = copy.deepcopy(block)
        bad_block.vtx[0].vin[0].prevout.hash += 1
        bad_block.vtx[0].rehash()
        assert_template(node, bad_block, 'bad-cb-missing')

        self.log.info("submitblock: Test invalid coinbase transaction")
        assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize()))

        self.log.info("getblocktemplate: Test truncated final transaction")
        assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal', 'rules': ['segwit']})

        self.log.info("getblocktemplate: Test duplicate transaction")
        bad_block = copy.deepcopy(block)
        bad_block.vtx.append(bad_block.vtx[0])
        assert_template(node, bad_block, 'bad-txns-duplicate')
        assert_submitblock(bad_block, 'bad-txns-duplicate', 'bad-txns-duplicate')

        self.log.info("getblocktemplate: Test invalid transaction")
        bad_block = copy.deepcopy(block)
        bad_tx = copy.deepcopy(bad_block.vtx[0])
        bad_tx.vin[0].prevout.hash = 255
        bad_tx.rehash()
        bad_block.vtx.append(bad_tx)
        assert_template(node, bad_block, 'bad-txns-inputs-missingorspent')
        assert_submitblock(bad_block, 'bad-txns-inputs-missingorspent')

        self.log.info("getblocktemplate: Test nonfinal transaction")
        bad_block = copy.deepcopy(block)
        bad_block.vtx[0].nLockTime = 2 ** 32 - 1
        bad_block.vtx[0].rehash()
        assert_template(node, bad_block, 'bad-txns-nonfinal')
        assert_submitblock(bad_block, 'bad-txns-nonfinal')

        self.log.info("getblocktemplate: Test bad tx count")
        # The tx count is immediately after the block header
        bad_block_sn = bytearray(block.serialize())
        assert_equal(bad_block_sn[BLOCK_HEADER_SIZE], 1)
        bad_block_sn[BLOCK_HEADER_SIZE] += 1
        assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal', 'rules': ['segwit']})

        self.log.info("getblocktemplate: Test bad bits")
        bad_block = copy.deepcopy(block)
        bad_block.nBits = 469762303  # impossible in the real world
        assert_template(node, bad_block, 'bad-diffbits')

        self.log.info("getblocktemplate: Test bad merkle root")
        bad_block = copy.deepcopy(block)
        bad_block.hashMerkleRoot += 1
        assert_template(node, bad_block, 'bad-txnmrklroot', False)
        assert_submitblock(bad_block, 'bad-txnmrklroot', 'bad-txnmrklroot')

        self.log.info("getblocktemplate: Test bad timestamps")
        bad_block = copy.deepcopy(block)
        bad_block.nTime = 2 ** 31 - 1
        assert_template(node, bad_block, 'time-too-new')
        assert_submitblock(bad_block, 'time-too-new', 'time-too-new')
        bad_block.nTime = 0
        assert_template(node, bad_block, 'time-too-old')
        assert_submitblock(bad_block, 'time-too-old', 'time-too-old')

        self.log.info("getblocktemplate: Test not best block")
        bad_block = copy.deepcopy(block)
        bad_block.hashPrevBlock = 123
        assert_template(node, bad_block, 'inconclusive-not-best-prevblk')
        assert_submitblock(bad_block, 'prev-blk-not-found', 'prev-blk-not-found')

        self.log.info('submitheader tests')
        assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='xx' * BLOCK_HEADER_SIZE))
        assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='ff' * (BLOCK_HEADER_SIZE-2)))
        assert_raises_rpc_error(-25, 'Must submit previous header', lambda: node.submitheader(hexdata='ff' * BLOCK_HEADER_SIZE))

        block.nTime += 1
        block.solve()

        def chain_tip(b_hash, *, status='headers-only', branchlen=1):
            return {'hash': b_hash, 'height': 202, 'branchlen': branchlen, 'status': status}

        assert chain_tip(block.hash) not in node.getchaintips()
        node.submitheader(hexdata=b2x(block.serialize()))
        assert chain_tip(block.hash) in node.getchaintips()
        node.submitheader(hexdata=b2x(CBlockHeader(block).serialize()))  # Noop
        assert chain_tip(block.hash) in node.getchaintips()

        bad_block_root = copy.deepcopy(block)
        bad_block_root.hashMerkleRoot += 2
        bad_block_root.solve()
        assert chain_tip(bad_block_root.hash) not in node.getchaintips()
        node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize()))
        assert chain_tip(bad_block_root.hash) in node.getchaintips()
        # Should still reject invalid blocks, even if we have the header:
        assert_equal(node.submitblock(hexdata=b2x(bad_block_root.serialize())), 'bad-txnmrklroot')
        assert_equal(node.submitblock(hexdata=b2x(bad_block_root.serialize())), 'bad-txnmrklroot')
        assert chain_tip(bad_block_root.hash) in node.getchaintips()
        # We know the header for this invalid block, so should just return early without error:
        node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize()))
        assert chain_tip(bad_block_root.hash) in node.getchaintips()

        bad_block_lock = copy.deepcopy(block)
        bad_block_lock.vtx[0].nLockTime = 2**32 - 1
        bad_block_lock.vtx[0].rehash()
        bad_block_lock.hashMerkleRoot = bad_block_lock.calc_merkle_root()
        bad_block_lock.solve()
        assert_equal(node.submitblock(hexdata=b2x(bad_block_lock.serialize())), 'bad-txns-nonfinal')
        assert_equal(node.submitblock(hexdata=b2x(bad_block_lock.serialize())), 'duplicate-invalid')
        # Build a "good" block on top of the submitted bad block
        bad_block2 = copy.deepcopy(block)
        bad_block2.hashPrevBlock = bad_block_lock.sha256
        bad_block2.solve()
        assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block2).serialize())))

        # Should reject invalid header right away
        bad_block_time = copy.deepcopy(block)
        bad_block_time.nTime = 1
        bad_block_time.solve()
        assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block_time).serialize())))

        # Should ask for the block from a p2p node, if they announce the header as well:
        node.add_p2p_connection(P2PDataStore())
        node.p2p.wait_for_getheaders(timeout=5)  # Drop the first getheaders
        node.p2p.send_blocks_and_test(blocks=[block], node=node)
        # Must be active now:
        assert chain_tip(block.hash, status='active', branchlen=0) in node.getchaintips()

        # Building a few blocks should give the same results
        node.generatetoaddress(10, node.get_deterministic_priv_key().address)
        assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block_time).serialize())))
        assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block2).serialize())))
        node.submitheader(hexdata=b2x(CBlockHeader(block).serialize()))
        node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize()))
        assert_equal(node.submitblock(hexdata=b2x(block.serialize())), 'duplicate')  # valid

if __name__ == '__main__':
    MiningTest().main()
<commit_msg>submitheader: more directly test missing prev block header<commit_after>#!/usr/bin/env python3
# Copyright (c) 2014-2018 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test mining RPCs

- getmininginfo
- getblocktemplate proposal mode
- submitblock"""

import copy
from decimal import Decimal

from test_framework.blocktools import create_coinbase
from test_framework.messages import (
    CBlock,
    CBlockHeader,
    BLOCK_HEADER_SIZE
)
from test_framework.mininode import (
    P2PDataStore,
)
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
    assert_equal,
    assert_raises_rpc_error,
    bytes_to_hex_str as b2x,
)
from test_framework.script import CScriptNum

def assert_template(node, block, expect, rehash=True):
    if rehash:
        block.hashMerkleRoot = block.calc_merkle_root()
    rsp = node.getblocktemplate(template_request={'data': b2x(block.serialize()), 'mode': 'proposal', 'rules': ['segwit']})
    assert_equal(rsp, expect)

class MiningTest(BitcoinTestFramework):
    def set_test_params(self):
        self.num_nodes = 2
        self.setup_clean_chain = False

    def run_test(self):
        node = self.nodes[0]

        def assert_submitblock(block, result_str_1, result_str_2=None):
            block.solve()
            result_str_2 = result_str_2 or 'duplicate-invalid'
            assert_equal(result_str_1, node.submitblock(hexdata=b2x(block.serialize())))
            assert_equal(result_str_2, node.submitblock(hexdata=b2x(block.serialize())))

        self.log.info('getmininginfo')
        mining_info = node.getmininginfo()
        assert_equal(mining_info['blocks'], 200)
        assert_equal(mining_info['chain'], 'regtest')
        assert_equal(mining_info['currentblocktx'], 0)
        assert_equal(mining_info['currentblockweight'], 0)
        assert_equal(mining_info['difficulty'], Decimal('4.656542373906925E-10'))
        assert_equal(mining_info['networkhashps'], Decimal('0.003333333333333334'))
        assert_equal(mining_info['pooledtx'], 0)

        # Mine a block to leave initial block download
        node.generatetoaddress(1, node.get_deterministic_priv_key().address)
        tmpl = node.getblocktemplate({'rules': ['segwit']})
        self.log.info("getblocktemplate: Test capability advertised")
        assert 'proposal' in tmpl['capabilities']
        assert 'coinbasetxn' not in tmpl

        next_height = int(tmpl["height"])
        coinbase_tx = create_coinbase(height=next_height)
        # sequence numbers must not be max for nLockTime to have effect
        coinbase_tx.vin[0].nSequence = 2 ** 32 - 2
        coinbase_tx.rehash()

        # round-trip the encoded bip34 block height commitment
        assert_equal(CScriptNum.decode(coinbase_tx.vin[0].scriptSig), next_height)
        # round-trip negative and multi-byte CScriptNums to catch python regression
        assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(1500))), 1500)
        assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1500))), -1500)
        assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1))), -1)

        block = CBlock()
        block.nVersion = tmpl["version"]
        block.hashPrevBlock = int(tmpl["previousblockhash"], 16)
        block.nTime = tmpl["curtime"]
        block.nBits = int(tmpl["bits"], 16)
        block.nNonce = 0
        block.vtx = [coinbase_tx]

        self.log.info("getblocktemplate: segwit rule must be set")
        assert_raises_rpc_error(-8, "getblocktemplate must be called with the segwit rule set", node.getblocktemplate)

        self.log.info("getblocktemplate: Test valid block")
        assert_template(node, block, None)

        self.log.info("submitblock: Test block decode failure")
        assert_raises_rpc_error(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15]))

        self.log.info("getblocktemplate: Test bad input hash for coinbase transaction")
        bad_block = copy.deepcopy(block)
        bad_block.vtx[0].vin[0].prevout.hash += 1
        bad_block.vtx[0].rehash()
        assert_template(node, bad_block, 'bad-cb-missing')

        self.log.info("submitblock: Test invalid coinbase transaction")
        assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize()))

        self.log.info("getblocktemplate: Test truncated final transaction")
        assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal', 'rules': ['segwit']})

        self.log.info("getblocktemplate: Test duplicate transaction")
        bad_block = copy.deepcopy(block)
        bad_block.vtx.append(bad_block.vtx[0])
        assert_template(node, bad_block, 'bad-txns-duplicate')
        assert_submitblock(bad_block, 'bad-txns-duplicate', 'bad-txns-duplicate')

        self.log.info("getblocktemplate: Test invalid transaction")
        bad_block = copy.deepcopy(block)
        bad_tx = copy.deepcopy(bad_block.vtx[0])
        bad_tx.vin[0].prevout.hash = 255
        bad_tx.rehash()
        bad_block.vtx.append(bad_tx)
        assert_template(node, bad_block, 'bad-txns-inputs-missingorspent')
        assert_submitblock(bad_block, 'bad-txns-inputs-missingorspent')

        self.log.info("getblocktemplate: Test nonfinal transaction")
        bad_block = copy.deepcopy(block)
        bad_block.vtx[0].nLockTime = 2 ** 32 - 1
        bad_block.vtx[0].rehash()
        assert_template(node, bad_block, 'bad-txns-nonfinal')
        assert_submitblock(bad_block, 'bad-txns-nonfinal')

        self.log.info("getblocktemplate: Test bad tx count")
        # The tx count is immediately after the block header
        bad_block_sn = bytearray(block.serialize())
        assert_equal(bad_block_sn[BLOCK_HEADER_SIZE], 1)
        bad_block_sn[BLOCK_HEADER_SIZE] += 1
        assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal', 'rules': ['segwit']})

        self.log.info("getblocktemplate: Test bad bits")
        bad_block = copy.deepcopy(block)
        bad_block.nBits = 469762303  # impossible in the real world
        assert_template(node, bad_block, 'bad-diffbits')

        self.log.info("getblocktemplate: Test bad merkle root")
        bad_block = copy.deepcopy(block)
        bad_block.hashMerkleRoot += 1
        assert_template(node, bad_block, 'bad-txnmrklroot', False)
        assert_submitblock(bad_block, 'bad-txnmrklroot', 'bad-txnmrklroot')

        self.log.info("getblocktemplate: Test bad timestamps")
        bad_block = copy.deepcopy(block)
        bad_block.nTime = 2 ** 31 - 1
        assert_template(node, bad_block, 'time-too-new')
        assert_submitblock(bad_block, 'time-too-new', 'time-too-new')
        bad_block.nTime = 0
        assert_template(node, bad_block, 'time-too-old')
        assert_submitblock(bad_block, 'time-too-old', 'time-too-old')

        self.log.info("getblocktemplate: Test not best block")
        bad_block = copy.deepcopy(block)
        bad_block.hashPrevBlock = 123
        assert_template(node, bad_block, 'inconclusive-not-best-prevblk')
        assert_submitblock(bad_block, 'prev-blk-not-found', 'prev-blk-not-found')

        self.log.info('submitheader tests')
        assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='xx' * BLOCK_HEADER_SIZE))
        assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='ff' * (BLOCK_HEADER_SIZE-2)))
        assert_raises_rpc_error(-25, 'Must submit previous header', lambda: node.submitheader(hexdata=super(CBlock, bad_block).serialize().hex()))

        block.nTime += 1
        block.solve()

        def chain_tip(b_hash, *, status='headers-only', branchlen=1):
            return {'hash': b_hash, 'height': 202, 'branchlen': branchlen, 'status': status}

        assert chain_tip(block.hash) not in node.getchaintips()
        node.submitheader(hexdata=b2x(block.serialize()))
        assert chain_tip(block.hash) in node.getchaintips()
        node.submitheader(hexdata=b2x(CBlockHeader(block).serialize()))  # Noop
        assert chain_tip(block.hash) in node.getchaintips()

        bad_block_root = copy.deepcopy(block)
        bad_block_root.hashMerkleRoot += 2
        bad_block_root.solve()
        assert chain_tip(bad_block_root.hash) not in node.getchaintips()
        node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize()))
        assert chain_tip(bad_block_root.hash) in node.getchaintips()
        # Should still reject invalid blocks, even if we have the header:
        assert_equal(node.submitblock(hexdata=b2x(bad_block_root.serialize())), 'bad-txnmrklroot')
        assert_equal(node.submitblock(hexdata=b2x(bad_block_root.serialize())), 'bad-txnmrklroot')
        assert chain_tip(bad_block_root.hash) in node.getchaintips()
        # We know the header for this invalid block, so should just return early without error:
        node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize()))
        assert chain_tip(bad_block_root.hash) in node.getchaintips()

        bad_block_lock = copy.deepcopy(block)
        bad_block_lock.vtx[0].nLockTime = 2**32 - 1
        bad_block_lock.vtx[0].rehash()
        bad_block_lock.hashMerkleRoot = bad_block_lock.calc_merkle_root()
        bad_block_lock.solve()
        assert_equal(node.submitblock(hexdata=b2x(bad_block_lock.serialize())), 'bad-txns-nonfinal')
        assert_equal(node.submitblock(hexdata=b2x(bad_block_lock.serialize())), 'duplicate-invalid')
        # Build a "good" block on top of the submitted bad block
        bad_block2 = copy.deepcopy(block)
        bad_block2.hashPrevBlock = bad_block_lock.sha256
        bad_block2.solve()
        assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block2).serialize())))

        # Should reject invalid header right away
        bad_block_time = copy.deepcopy(block)
        bad_block_time.nTime = 1
        bad_block_time.solve()
        assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block_time).serialize())))

        # Should ask for the block from a p2p node, if they announce the header as well:
        node.add_p2p_connection(P2PDataStore())
        node.p2p.wait_for_getheaders(timeout=5)  # Drop the first getheaders
        node.p2p.send_blocks_and_test(blocks=[block], node=node)
        # Must be active now:
        assert chain_tip(block.hash, status='active', branchlen=0) in node.getchaintips()

        # Building a few blocks should give the same results
        node.generatetoaddress(10, node.get_deterministic_priv_key().address)
        assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block_time).serialize())))
        assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block2).serialize())))
        node.submitheader(hexdata=b2x(CBlockHeader(block).serialize()))
        node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize()))
        assert_equal(node.submitblock(hexdata=b2x(block.serialize())), 'duplicate')  # valid

if __name__ == '__main__':
    MiningTest().main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from datetime import datetime, timedelta
from flask import (abort, Flask, session, redirect, url_for, flash, g, request,
                   jsonify, render_template)
from flask_assets import Environment
from flask_babel import gettext
from flask_wtf.csrf import CSRFProtect, CSRFError
from os import path

import i18n
import template_filters
import version

from crypto_util import CryptoUtil
from db import db
from journalist_app import account, admin, api, main, col
from journalist_app.utils import get_source, logged_in
from models import Journalist
from store import Storage

import typing
# https://www.python.org/dev/peps/pep-0484/#runtime-or-type-checking
if typing.TYPE_CHECKING:
    # flake8 can not understand type annotation yet.
    # That is why all type annotation relative import
    # statements has to be marked as noqa.
    # http://flake8.pycqa.org/en/latest/user/error-codes.html?highlight=f401
    from sdconfig import SDConfig  # noqa: F401

_insecure_api_views = ['api.get_endpoints', 'api.get_token']
_insecure_views = ['main.login', 'main.select_logo', 'static']

def create_app(config):
    # type: (SDConfig) -> Flask
    app = Flask(__name__,
                template_folder=config.JOURNALIST_TEMPLATES_DIR,
                static_folder=path.join(config.SECUREDROP_ROOT, 'static'))

    app.config.from_object(config.JournalistInterfaceFlaskConfig)
    app.sdconfig = config

    CSRFProtect(app)
    Environment(app)

    if config.DATABASE_ENGINE == "sqlite":
        db_uri = (config.DATABASE_ENGINE + ":///" +
                  config.DATABASE_FILE)
    else:
        db_uri = (
            config.DATABASE_ENGINE + '://' +
            config.DATABASE_USERNAME + ':' +
            config.DATABASE_PASSWORD + '@' +
            config.DATABASE_HOST + '/' +
            config.DATABASE_NAME
        )
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
    app.config['SQLALCHEMY_DATABASE_URI'] = db_uri
    db.init_app(app)

    app.storage = Storage(config.STORE_DIR,
                          config.TEMP_DIR,
                          config.JOURNALIST_KEY)

    app.crypto_util = CryptoUtil(
        scrypt_params=config.SCRYPT_PARAMS,
        scrypt_id_pepper=config.SCRYPT_ID_PEPPER,
        scrypt_gpg_pepper=config.SCRYPT_GPG_PEPPER,
        securedrop_root=config.SECUREDROP_ROOT,
        word_list=config.WORD_LIST,
        nouns_file=config.NOUNS,
        adjectives_file=config.ADJECTIVES,
        gpg_key_dir=config.GPG_KEY_DIR,
    )

    @app.errorhandler(CSRFError)
    def handle_csrf_error(e):
        # render the message first to ensure it's localized.
        msg = gettext('You have been logged out due to inactivity')
        session.clear()
        flash(msg, 'error')
        return redirect(url_for('main.login'))

    i18n.setup_app(config, app)

    app.jinja_env.trim_blocks = True
    app.jinja_env.lstrip_blocks = True
    app.jinja_env.globals['version'] = version.__version__
    if hasattr(config, 'CUSTOM_HEADER_IMAGE'):
        app.jinja_env.globals['header_image'] = \
            config.CUSTOM_HEADER_IMAGE  # type: ignore
        app.jinja_env.globals['use_custom_header_image'] = True
    else:
        app.jinja_env.globals['header_image'] = 'logo.png'
        app.jinja_env.globals['use_custom_header_image'] = False

    app.jinja_env.filters['rel_datetime_format'] = \
        template_filters.rel_datetime_format
    app.jinja_env.filters['filesizeformat'] = template_filters.filesizeformat

    @app.before_request
    def setup_g():
        """Store commonly used values in Flask's special g object"""
        if 'expires' in session and datetime.utcnow() >= session['expires']:
            session.clear()
            flash(gettext('You have been logged out due to inactivity'),
                  'error')

        session['expires'] = datetime.utcnow() + \
            timedelta(minutes=getattr(config,
                                      'SESSION_EXPIRATION_MINUTES',
                                      120))

        uid = session.get('uid', None)
        if uid:
            g.user = Journalist.query.get(uid)

        g.locale = i18n.get_locale(config)
        g.text_direction = i18n.get_text_direction(g.locale)
        g.html_lang = i18n.locale_to_rfc_5646(g.locale)
        g.locales = i18n.get_locale2name()

        if str(request.endpoint).split('.')[0] == 'api':
            if request.endpoint not in _insecure_api_views:
                abort(403, 'API token is invalid or expired.')
        else:  # We are not using the API
            if request.endpoint not in _insecure_views and not logged_in():
                return redirect(url_for('main.login'))

        if request.method == 'POST':
            filesystem_id = request.form.get('filesystem_id')
            if filesystem_id:
                g.filesystem_id = filesystem_id
                g.source = get_source(filesystem_id)

    app.register_blueprint(main.make_blueprint(config))
    app.register_blueprint(account.make_blueprint(config),
                           url_prefix='/account')
    app.register_blueprint(admin.make_blueprint(config), url_prefix='/admin')
    app.register_blueprint(col.make_blueprint(config), url_prefix='/col')
    app.register_blueprint(api.make_blueprint(config), url_prefix='/api/v1')

    @app.errorhandler(403)
    def forbidden(message):
        if request.headers['Content-Type'] == 'application/json':
            response = jsonify({'error': 'forbidden',
                                'message': 'Not authorized'})
            return response, 403
        else:
            return render_template('403.html'), 403

    return app
<commit_msg>Journalist API: Use decorator for protected API endpoints<commit_after># -*- coding: utf-8 -*-

from datetime import datetime, timedelta
from flask import (abort, Flask, session, redirect, url_for, flash, g, request,
                   jsonify, render_template)
from flask_assets import Environment
from flask_babel import gettext
from flask_wtf.csrf import CSRFProtect, CSRFError
from os import path

import i18n
import template_filters
import version

from crypto_util import CryptoUtil
from db import db
from journalist_app import account, admin, api, main, col
from journalist_app.utils import get_source, logged_in
from models import Journalist
from store import Storage

import typing
# https://www.python.org/dev/peps/pep-0484/#runtime-or-type-checking
if typing.TYPE_CHECKING:
    # flake8 can not understand type annotation yet.
    # That is why all type annotation relative import
    # statements has to be marked as noqa.
    # http://flake8.pycqa.org/en/latest/user/error-codes.html?highlight=f401
    from sdconfig import SDConfig  # noqa: F401

_insecure_api_views = ['api.get_endpoints', 'api.get_token']
_insecure_views = ['main.login', 'main.select_logo', 'static']

def create_app(config):
    # type: (SDConfig) -> Flask
    app = Flask(__name__,
                template_folder=config.JOURNALIST_TEMPLATES_DIR,
                static_folder=path.join(config.SECUREDROP_ROOT, 'static'))

    app.config.from_object(config.JournalistInterfaceFlaskConfig)
    app.sdconfig = config

    CSRFProtect(app)
    Environment(app)

    if config.DATABASE_ENGINE == "sqlite":
        db_uri = (config.DATABASE_ENGINE + ":///" +
                  config.DATABASE_FILE)
    else:
        db_uri = (
            config.DATABASE_ENGINE + '://' +
            config.DATABASE_USERNAME + ':' +
            config.DATABASE_PASSWORD + '@' +
            config.DATABASE_HOST + '/' +
            config.DATABASE_NAME
        )
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
    app.config['SQLALCHEMY_DATABASE_URI'] = db_uri
    db.init_app(app)

    app.storage = Storage(config.STORE_DIR,
                          config.TEMP_DIR,
                          config.JOURNALIST_KEY)

    app.crypto_util = CryptoUtil(
        scrypt_params=config.SCRYPT_PARAMS,
        scrypt_id_pepper=config.SCRYPT_ID_PEPPER,
        scrypt_gpg_pepper=config.SCRYPT_GPG_PEPPER,
        securedrop_root=config.SECUREDROP_ROOT,
        word_list=config.WORD_LIST,
        nouns_file=config.NOUNS,
        adjectives_file=config.ADJECTIVES,
        gpg_key_dir=config.GPG_KEY_DIR,
    )

    @app.errorhandler(CSRFError)
    def handle_csrf_error(e):
        # render the message first to ensure it's localized.
        msg = gettext('You have been logged out due to inactivity')
        session.clear()
        flash(msg, 'error')
        return redirect(url_for('main.login'))

    i18n.setup_app(config, app)

    app.jinja_env.trim_blocks = True
    app.jinja_env.lstrip_blocks = True
    app.jinja_env.globals['version'] = version.__version__
    if hasattr(config, 'CUSTOM_HEADER_IMAGE'):
        app.jinja_env.globals['header_image'] = \
            config.CUSTOM_HEADER_IMAGE  # type: ignore
        app.jinja_env.globals['use_custom_header_image'] = True
    else:
        app.jinja_env.globals['header_image'] = 'logo.png'
        app.jinja_env.globals['use_custom_header_image'] = False

    app.jinja_env.filters['rel_datetime_format'] = \
        template_filters.rel_datetime_format
    app.jinja_env.filters['filesizeformat'] = template_filters.filesizeformat

    @app.before_request
    def setup_g():
        """Store commonly used values in Flask's special g object"""
        if 'expires' in session and datetime.utcnow() >= session['expires']:
            session.clear()
            flash(gettext('You have been logged out due to inactivity'),
                  'error')

        session['expires'] = datetime.utcnow() + \
            timedelta(minutes=getattr(config,
                                      'SESSION_EXPIRATION_MINUTES',
                                      120))

        uid = session.get('uid', None)
        if uid:
            g.user = Journalist.query.get(uid)

        g.locale = i18n.get_locale(config)
        g.text_direction = i18n.get_text_direction(g.locale)
        g.html_lang = i18n.locale_to_rfc_5646(g.locale)
        g.locales = i18n.get_locale2name()

        if str(request.endpoint).split('.')[0] == 'api':
            pass  # We use the @token_required decorator for the API endpoints
        else:  # We are not using the API
            if request.endpoint not in _insecure_views and not logged_in():
                return redirect(url_for('main.login'))

        if request.method == 'POST':
            filesystem_id = request.form.get('filesystem_id')
            if filesystem_id:
                g.filesystem_id = filesystem_id
                g.source = get_source(filesystem_id)

    app.register_blueprint(main.make_blueprint(config))
    app.register_blueprint(account.make_blueprint(config),
                           url_prefix='/account')
    app.register_blueprint(admin.make_blueprint(config), url_prefix='/admin')
    app.register_blueprint(col.make_blueprint(config), url_prefix='/col')
    app.register_blueprint(api.make_blueprint(config), url_prefix='/api/v1')

    @app.errorhandler(403)
    def forbidden(message):
        if request.headers['Content-Type'] == 'application/json':
            response = jsonify({'error': 'forbidden',
                                'message': 'Not authorized'})
            return response, 403
        else:
            return render_template('403.html'), 403

    return app
<|endoftext|>


<commit_before>import sys
import yaml

import rethinkdb as r
from rethinkdb.errors import RqlDriverError, RqlRuntimeError
import socket

def createTable(dbname, tablename, conn):
    ''' Create a rethinkDB table '''
    print("Creating table: %s") % tablename
    try:
        r.db(dbname).table_create(tablename).run(conn)
    except (RqlDriverError, RqlRuntimeError, socket.error) as e:
        print("RethinkDB Error: %s") % e.message
        print("Table %s not created") % tablename

if len(sys.argv) < 2:
    print("Hey, thats not how you launch this...")
    print("%s <config file> [--travis]") % sys.argv[0]
    sys.exit(1)

# Open Config File and Parse Config Data
configfile = sys.argv[1]
cfh = open(configfile, "r")
config = yaml.safe_load(cfh)
cfh.close()

if sys.argv[2]:
    if sys.argv[2] == "--travis":
        travis = True
    else:
        print("Didn't understand second command line argument.."
              "Assuming non-Travis environment")

# Establish Connection
host = config['rethink_host']
port = config['rethink_port']
database = config['rethink_db']
auth_key = config['rethink_authkey']
try:
    if auth_key and travis == False:
        conn = r.connect(host, port, auth_key=auth_key).repl()
    else:
        conn = r.connect(host, port).repl()
except (RqlDriverError, RqlRuntimeError, socket.error) as e:
    print("RethinkDB Error on Connection: %s") % e.message
    sys.exit(1)

if travis == True:
    try:
        r.db('rethinkdb').table('cluster_config').get('auth').update({'auth_key' : auth_key}).run(conn)
    except (RqlDriverError, RqlRuntimeError, socket.error) as e:
        print("RethinkDB Error setting auth key: %s") % e.message

result = r.db_list().run(conn)
if database in result:
    print("Database %s already exists exiting") % database
    sys.exit(0)

r.db_create(database).run(conn)
tables = [
    'monitors',
    'reactions',
    'users',
    'history',
    'events',
    'subscription_history',
    'dc1queue',
    'dc2queue'
]
for name in tables:
    createTable(database, name, conn)
print "Database created!"

#######################
### Database Tables ###
#######################

"""
monitors - Stores monitors
reactions - Stores reactions
users - Stores user data
history - Stores historical tracking for monitors and reactions
events - Stores unique events for monitors
subscription_history - Stores subscription history for tracking signups
dc1queue - Queue for monitors and reactions in datacenter 1
dc2queue - Queue for monitors and reactions in datacenter 2
"""
<commit_msg>Adding output comment<commit_after>import sys
import yaml

import rethinkdb as r
from rethinkdb.errors import RqlDriverError, RqlRuntimeError
import socket

def createTable(dbname, tablename, conn):
    ''' Create a rethinkDB table '''
    print("Creating table: %s") % tablename
    try:
        r.db(dbname).table_create(tablename).run(conn)
    except (RqlDriverError, RqlRuntimeError, socket.error) as e:
        print("RethinkDB Error: %s") % e.message
        print("Table %s not created") % tablename

if len(sys.argv) < 2:
    print("Hey, thats not how you launch this...")
    print("%s <config file> [--travis]") % sys.argv[0]
    sys.exit(1)

# Open Config File and Parse Config Data
configfile = sys.argv[1]
cfh = open(configfile, "r")
config = yaml.safe_load(cfh)
cfh.close()

if sys.argv[2]:
    if sys.argv[2] == "--travis":
        travis = True
    else:
        print("Didn't understand second command line argument.."
              "Assuming non-Travis environment")

# Establish Connection
host = config['rethink_host']
port = config['rethink_port']
database = config['rethink_db']
auth_key = config['rethink_authkey']
try:
    if auth_key and travis == False:
        conn = r.connect(host, port, auth_key=auth_key).repl()
    else:
        conn = r.connect(host, port).repl()
except (RqlDriverError, RqlRuntimeError, socket.error) as e:
    print("RethinkDB Error on Connection: %s") % e.message
    sys.exit(1)

if travis == True:
    print("Setting RethinkDB auth key")
    try:
        r.db('rethinkdb').table('cluster_config').get('auth').update({'auth_key' : auth_key}).run(conn)
    except (RqlDriverError, RqlRuntimeError, socket.error) as e:
        print("RethinkDB Error setting auth key: %s") % e.message

result = r.db_list().run(conn)
if database in result:
    print("Database %s already exists exiting") % database
    sys.exit(0)

r.db_create(database).run(conn)
tables = [
    'monitors',
    'reactions',
    'users',
    'history',
    'events',
    'subscription_history',
    'dc1queue',
    'dc2queue'
]
for name in tables:
    createTable(database, name, conn)
print "Database created!"

#######################
### Database Tables ###
#######################

"""
monitors - Stores monitors
reactions - Stores reactions
users - Stores user data
history - Stores historical tracking for monitors and reactions
events - Stores unique events for monitors
subscription_history - Stores subscription history for tracking signups
dc1queue - Queue for monitors and reactions in datacenter 1
dc2queue - Queue for monitors and reactions in datacenter 2
"""
<|endoftext|>


<commit_before><commit_msg>quando situacao_empenho = esf, mostra colunas sobre finalizados<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
import warnings

from hvac.exceptions import InvalidPath

from ansible.module_utils.hashivault import hashivault_argspec
from ansible.module_utils.hashivault import hashivault_auth_client
from ansible.module_utils.hashivault import hashivault_init
from ansible.module_utils.hashivault import hashiwrapper

ANSIBLE_METADATA = {'status': ['stableinterface'], 'supported_by': 'community', 'version': '1.1'}
DOCUMENTATION = '''
---
module: hashivault_delete
version_added: "3.4.0"
short_description: Hashicorp Vault delete module
description:
    - Module to delete from Hashicorp Vault.
options:
    version:
        description:
            - version of the kv engine (int)
        default: 1
    mount_point:
        description:
            - secret mount point
        default: secret
    secret:
        description:
            - secret to delete.
    permanent:
        description:
            - delete all versions and metadata for a given secret (only effective for kv engine version 2).
        default: false
extends_documentation_fragment: hashivault
'''
EXAMPLES = '''
---
- hosts: localhost
  tasks:
    - hashivault_delete:
        secret: giant
'''

def main():
    argspec = hashivault_argspec()
    argspec['version'] = dict(required=False, type='int', default=1)
    argspec['mount_point'] = dict(required=False, type='str', default='secret')
    argspec['secret'] = dict(required=True, type='str')
    argspec['permanent'] = dict(required=False, type='bool', default=False)
    module = hashivault_init(argspec)
    result = hashivault_delete(module.params)
    if result.get('failed'):
        module.fail_json(**result)
    else:
        module.exit_json(**result)

@hashiwrapper
def hashivault_delete(params):
    result = {"changed": False, "rc": 0}
    client = hashivault_auth_client(params)
    version = params.get('version')
    mount_point = params.get('mount_point')
    secret = params.get('secret')
    permanent = params.get('permanent')
    if secret.startswith('/'):
        secret = secret.lstrip('/')
        mount_point = ''
    if mount_point:
        secret_path = '%s/%s' % (mount_point, secret)
    else:
        secret_path = secret

    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        returned_data = None
        try:
            if version == 2:
                if permanent:
                    returned_data = client.secrets.kv.v2.delete_metadata_and_all_versions(secret, mount_point=mount_point)
                else:
                    returned_data = client.secrets.kv.v2.delete_latest_version_of_secret(secret, mount_point=mount_point)
            else:
                returned_data = client.delete(secret_path)
        except InvalidPath:
            pass
        except Exception as e:
            result['rc'] = 1
            result['failed'] = True
            error_string = "%s(%s)" % (e.__class__.__name__, e)
            result['msg'] = u"Error %s deleting %s" % (error_string, secret_path)
            return result
        if returned_data:
            result['data'] = returned_data.text
        result['msg'] = u"Secret %s deleted" % secret_path
    result['changed'] = True
    return result

if __name__ == '__main__':
    main()
<commit_msg>hashivault_delete: fixing linting error<commit_after>#!/usr/bin/env python
import warnings

from hvac.exceptions import InvalidPath

from ansible.module_utils.hashivault import hashivault_argspec
from ansible.module_utils.hashivault import hashivault_auth_client
from ansible.module_utils.hashivault import hashivault_init
from ansible.module_utils.hashivault import hashiwrapper

ANSIBLE_METADATA = {'status': ['stableinterface'], 'supported_by': 'community', 'version': '1.1'}
DOCUMENTATION = '''
---
module: hashivault_delete
version_added: "3.4.0"
short_description: Hashicorp Vault delete module
description:
    - Module to delete from Hashicorp Vault.
options:
    version:
        description:
            - version of the kv engine (int)
        default: 1
    mount_point:
        description:
            - secret mount point
        default: secret
    secret:
        description:
            - secret to delete.
    permanent:
        description:
            - delete all versions and metadata for a given secret (only effective for kv engine version 2).
        default: false
extends_documentation_fragment: hashivault
'''
EXAMPLES = '''
---
- hosts: localhost
  tasks:
    - hashivault_delete:
        secret: giant
'''

def main():
    argspec = hashivault_argspec()
    argspec['version'] = dict(required=False, type='int', default=1)
    argspec['mount_point'] = dict(required=False, type='str', default='secret')
    argspec['secret'] = dict(required=True, type='str')
    argspec['permanent'] = dict(required=False, type='bool', default=False)
    module = hashivault_init(argspec)
    result = hashivault_delete(module.params)
    if result.get('failed'):
        module.fail_json(**result)
    else:
        module.exit_json(**result)

@hashiwrapper
def hashivault_delete(params):
    result = {"changed": False, "rc": 0}
    client = hashivault_auth_client(params)
    version = params.get('version')
    mount_point = params.get('mount_point')
    secret = params.get('secret')
    permanent = params.get('permanent')
    if secret.startswith('/'):
        secret = secret.lstrip('/')
        mount_point = ''
    if mount_point:
        secret_path = '%s/%s' % (mount_point, secret)
    else:
        secret_path = secret

    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        returned_data = None
        try:
            if version == 2:
                if permanent:
                    returned_data = client.secrets.kv.v2.delete_metadata_and_all_versions(
                        secret,
                        mount_point=mount_point
                    )
                else:
                    returned_data = client.secrets.kv.v2.delete_latest_version_of_secret(
                        secret,
                        mount_point=mount_point
                    )
            else:
                returned_data = client.delete(secret_path)
        except InvalidPath:
            pass
        except Exception as e:
            result['rc'] = 1
            result['failed'] = True
            error_string = "%s(%s)" % (e.__class__.__name__, e)
            result['msg'] = u"Error %s deleting %s" % (error_string, secret_path)
            return result
        if returned_data:
            result['data'] = returned_data.text
        result['msg'] = u"Secret %s deleted" % secret_path
    result['changed'] = True
    return result

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import Adafruit_GPIO as GPIO
import adafruit_spi_modified as SPI
import os
import Queue
import settings
import time
import threading
import traceback
import sys

BASE_PATH = os.path.dirname(os.path.realpath(__file__))
UPPER_PATH = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
DEVICES_PATH = "%s/Hosts/" % (BASE_PATH )
THIRTYBIRDS_PATH = "%s/thirtybirds_2_0" % (UPPER_PATH )

sys.path.append(BASE_PATH)
sys.path.append(UPPER_PATH)

from thirtybirds_2_0.Network.manager import init as network_init

class MCP3008s(object):
    def __init__(self, spi_clock_pin, miso_pin, mosi_pin, chip_select_pins):
        self.gpio = GPIO.get_platform_gpio()
        self.chip_select_pins = chip_select_pins
        self._spi = SPI.BitBang(self.gpio, spi_clock_pin, mosi_pin, miso_pin)
        self._spi.set_clock_hz(1000000)
        self._spi.set_mode(0)
        self._spi.set_bit_order(SPI.MSBFIRST)
        for chip_select_pin in chip_select_pins:
            self.gpio.setup(chip_select_pin, GPIO.OUT)
            self.gpio.set_high(chip_select_pin)

    def read(self, chip_select_pin, adc_number):
        assert 0 <= adc_number <= 7, 'ADC number must be a value of 0-7!'
        # Build a single channel read command.
        # For example channel zero = 0b11000000
        command = 0b11 << 6                  # Start bit, single channel read
        command |= (adc_number & 0x07) << 3  # Channel number (in 3 bits)
        # Note the bottom 3 bits of command are 0, this is to account for the
        # extra clock to do the conversion, and the low null bit returned at
        # the start of the response.
        resp = self._spi.transfer(chip_select_pin,[command, 0x0, 0x0])
        # Parse out the 10 bits of response data and return it.
        result = (resp[0] & 0x01) << 9
        result |= (resp[1] & 0xFF) << 1
        result |= (resp[2] & 0x80) >> 7
        return result & 0x3FF

    def scan_all(self):
        adcs = []
        for chip_select_pin in self.chip_select_pins:
            channels = []
            for adc_number in range(8):
                channels.append(self.read(chip_select_pin, adc_number))
            adcs.append(channels)
        return adcs

class Potentiometers(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.noise_threshold = 5
        self.spi_clock_pin = 11
        self.miso_pin = 9
        self.mosi_pin = 10
        self.chip_select_pins = [8,7,12,16,20,21]
        self.potentiometers_layout  = [
            [
                "",
                "voice_1_overtone_2_harmonic",
                "voice_1_overtone_2_volume",
                "voice_1_overtone_1_fine",
                "voice_1_overtone_1_harmonic",
                "voice_1_overtone_1_volume",
                "voice_1_root_fine",
                "voice_1_root_harmonic",
            ],
            [
                "",
                "",
                "voice_1_formant_front_back",
                "voice_1_formant_open_close",
                "voice_1_formant_pitch",
                "voice_1_formant_volume",
                "voice_1_overtone_2_fine",
                ""
            ],
            [
                "voice_2_root_harmonic",
                "voice_2_root_fine",
                "voice_2_overtone_1_volume",
                "voice_2_overtone_1_harmonic",
                "voice_2_overtone_1_fine",
                "voice_2_overtone_2_volume",
                "voice_2_overtone_2_harmonic",
                ""
            ],
            [
                "",
                "",
                "voice_2_formant_front_back",
                "voice_2_formant_open_close",
                "voice_2_formant_pitch",
                "voice_2_formant_volume",
                "voice_2_overtone_2_fine",
                ""
            ],
            [
                "voice_3_root_harmonic",
                "voice_3_root_fine",
                "voice_3_overtone_1_volume",
                "voice_3_overtone_1_harmonic",
                "voice_3_overtone_1_fine",
                "voice_3_overtone_2_volume",
                "voice_3_overtone_2_harmonic",
                ""
            ],
            [
                "",
                "",
                "voice_3_formant_front_back",
                "voice_3_formant_open_close",
                "voice_3_formant_pitch",
                "voice_3_formant_volume",
                "voice_3_overtone_2_fine",
                ""
            ]
        ]
        self.potentiometer_last_value  = [
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0]
        ]
        self.mcp3008s = MCP3008s(self.spi_clock_pin, self.miso_pin, self.mosi_pin, self.chip_select_pins)

    def run(self):
        while True:
            all_adc_values =  self.mcp3008s.scan_all()
            #print all_adc_values
            for adc in range(len(all_adc_values)):
                for channel in range(8):
                    adc_value = 1023 - all_adc_values[adc][channel]
                    if abs(adc_value - self.potentiometer_last_value[adc][channel] ) > self.noise_threshold:
                        print adc, channel, self.potentiometers_layout[adc][channel], adc_value
                    self.potentiometer_last_value[adc][channel] = adc_value
            time.sleep(0.1)

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.network.thirtybirds.subscribe_to_topic("door_closed")
        self.potentiometers = Potentiometers() # self.network.thirtybirds.send
        self.potentiometers.daemon = True
        self.potentiometers.start()
    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        if len(msg) > 0:
            msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):
        while True:
            try:
                topic, msg = self.queue.get(True)
                if topic == "door_closed":
                    door.add_to_queue(msg)
                    self.network.thirtybirds.send("door_status", True)

            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

<commit_msg>sending slide pot values to controller only on change<commit_after>import Adafruit_GPIO as GPIO
import adafruit_spi_modified as SPI
import os
import Queue
import settings
import time
import threading
import traceback
import sys

BASE_PATH = os.path.dirname(os.path.realpath(__file__))
UPPER_PATH = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
DEVICES_PATH = "%s/Hosts/" % (BASE_PATH )
THIRTYBIRDS_PATH = "%s/thirtybirds_2_0" % (UPPER_PATH )

sys.path.append(BASE_PATH)
sys.path.append(UPPER_PATH)

from thirtybirds_2_0.Network.manager import init as network_init

class MCP3008s(object):
    def __init__(self, spi_clock_pin, miso_pin, mosi_pin, chip_select_pins):
        self.gpio = GPIO.get_platform_gpio()
        self.chip_select_pins = chip_select_pins
        self._spi = SPI.BitBang(self.gpio, spi_clock_pin, mosi_pin, miso_pin)
        self._spi.set_clock_hz(1000000)
        self._spi.set_mode(0)
        self._spi.set_bit_order(SPI.MSBFIRST)
        for chip_select_pin in chip_select_pins:
            self.gpio.setup(chip_select_pin, GPIO.OUT)
            self.gpio.set_high(chip_select_pin)

    def read(self, chip_select_pin, adc_number):
        assert 0 <= adc_number <= 7, 'ADC number must be a value of 0-7!'
        # Build a single channel read command.
        # For example channel zero = 0b11000000
        command = 0b11 << 6                  # Start bit, single channel read
        command |= (adc_number & 0x07) << 3  # Channel number (in 3 bits)
        # Note the bottom 3 bits of command are 0, this is to account for the
        # extra clock to do the conversion, and the low null bit returned at
        # the start of the response.
        resp = self._spi.transfer(chip_select_pin,[command, 0x0, 0x0])
        # Parse out the 10 bits of response data and return it.
        result = (resp[0] & 0x01) << 9
        result |= (resp[1] & 0xFF) << 1
        result |= (resp[2] & 0x80) >> 7
        return result & 0x3FF

    def scan_all(self):
        adcs = []
        for chip_select_pin in self.chip_select_pins:
            channels = []
            for adc_number in range(8):
                channels.append(self.read(chip_select_pin, adc_number))
            adcs.append(channels)
        return adcs

class Potentiometers(threading.Thread):
    def __init__(self, network_send_ref):
        threading.Thread.__init__(self)
        self.network_send_ref = network_send_ref
        self.noise_threshold = 10
        self.spi_clock_pin = 11
        self.miso_pin = 9
        self.mosi_pin = 10
        self.chip_select_pins = [8,7,12,16,20,21]
        self.potentiometers_layout  = [
            [
                "",
                "voice_1_overtone_2_harmonic",
                "voice_1_overtone_2_volume",
                "voice_1_overtone_1_fine",
                "voice_1_overtone_1_harmonic",
                "voice_1_overtone_1_volume",
                "voice_1_root_fine",
                "voice_1_root_harmonic",
            ],
            [
                "",
                "",
                "voice_1_formant_front_back",
                "voice_1_formant_open_close",
                "voice_1_formant_pitch",
                "voice_1_formant_volume",
                "voice_1_overtone_2_fine",
                ""
            ],
            [
                "voice_2_root_harmonic",
                "voice_2_root_fine",
                "voice_2_overtone_1_volume",
                "voice_2_overtone_1_harmonic",
                "voice_2_overtone_1_fine",
                "voice_2_overtone_2_volume",
                "voice_2_overtone_2_harmonic",
                ""
            ],
            [
                "",
                "",
                "voice_2_formant_front_back",
                "voice_2_formant_open_close",
                "voice_2_formant_pitch",
                "voice_2_formant_volume",
                "voice_2_overtone_2_fine",
                ""
            ],
            [
                "voice_3_root_harmonic",
                "voice_3_root_fine",
                "voice_3_overtone_1_volume",
                "voice_3_overtone_1_harmonic",
                "voice_3_overtone_1_fine",
                "voice_3_overtone_2_volume",
                "voice_3_overtone_2_harmonic",
                ""
            ],
            [
                "",
                "",
                "voice_3_formant_front_back",
                "voice_3_formant_open_close",
                "voice_3_formant_pitch",
                "voice_3_formant_volume",
                "voice_3_overtone_2_fine",
                ""
            ]
        ]
        self.potentiometer_last_value  = [
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0]
        ]
        self.mcp3008s = MCP3008s(self.spi_clock_pin, self.miso_pin, self.mosi_pin, self.chip_select_pins)

    def run(self):
        while True:
            all_adc_values =  self.mcp3008s.scan_all()
            #print all_adc_values
            for adc in range(len(all_adc_values)):
                for channel in range(8):
                    adc_value = 1023 - all_adc_values[adc][channel]
                    if abs(adc_value - self.potentiometer_last_value[adc][channel] ) > self.noise_threshold:
                        self.network_send_ref(self.potentiometers_layout[adc][channel], adc_value/1023.0)
                        print adc, channel, self.potentiometers_layout[adc][channel], adc_value
                    self.potentiometer_last_value[adc][channel] = adc_value
            time.sleep(0.1)

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.network.thirtybirds.subscribe_to_topic("door_closed")
        self.potentiometers = Potentiometers(self.network.thirtybirds.send) # self.network.thirtybirds.send
        self.potentiometers.daemon = True
        self.potentiometers.start()
    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        if len(msg) > 0:
            msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):
        while True:
            try:
                topic, msg = self.queue.get(True)
                if topic == "door_closed":
                    door.add_to_queue(msg)
                    self.network.thirtybirds.send("door_status", True)

            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

<|endoftext|>