<commit_before>from app import app, db
from app.models import User, Post
from datetime import datetime
from flask.ext.script import Manager

manager = Manager(app)

@manager.command
def init():
    dropdb()
    initdb()
    filldb()

@manager.command
def initdb():
    print('Initializing database...'),
    db.create_all()
    print('done!')

@manager.command
def filldb():
    print('Filling database...'),

    admin = User(u'admin', u'admin')
    db.session.add(admin)
    db.session.commit()

    post = Post(
        title=u'Hello, world!',
        body=POST_1,
        author_id=admin.id,
        visible=True,
    )
    db.session.add(post)
    post.published = datetime(2011, 06, 13)
    post.edit(post.title, post.body, True)
    post = Post(
        title=u'Random Words 1',
        body=POST_5,
        author_id=admin.id,
        visible=True,
    )
    db.session.add(post)
    post.published = datetime(2012, 8, 15)
    post.edit(post.title, post.body, True)
    post = Post(
        title=u'Random Words 2',
        body=POST_2,
        author_id=admin.id,
        visible=True,
    )
    db.session.add(post)
    post.published = datetime(2012, 12, 24)
    post.edit(post.title, post.body, True)
    post = Post(
        title=u'Picture of a cat!',
        body=POST_3,
        author_id=admin.id,
        visible=True,
    )
    db.session.add(post)
    post = Post(
        title=u'Some code',
        body=POST_4,
        author_id=admin.id,
        visible=True,
    )
    db.session.add(post)
    post = Post(
        title=u'Random Words 3',
        body=POST_5,
        author_id=admin.id,
        visible=True,
    )
    db.session.add(post)
    db.session.commit()
    print('done!')

@manager.command
def dropdb():
    print('Dropping database...'),
    db.drop_all()
    print('done!')

POST_1 = u"""
First blog post.
"""

POST_2 = u"""
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean vel ipsum
lectus. Pellentesque tempus enim sed leo imperdiet non lobortis nulla
sollicitudin. Maecenas arcu orci, interdum eu rhoncus ut, blandit id felis.
Mauris consectetur dui at felis ultricies tempus. Quisque molestie convallis
lectus vitae viverra. Duis lobortis ultrices turpis, nec eleifend est
venenatis nec. Sed sed lorem quis metus eleifend ullamcorper. Ut semper
nulla a arcu ornare **condimentum**.

> Ut et lacus ac lacus pulvinar accumsan quis eget lacus. Integer id nibh non
> eros tincidunt bibendum. Aenean diam lectus, tempus sed consequat
> consectetur, posuere non ipsum.
> > Donec vitae eleifend est. Donec at elit mi. Maecenas tempor nulla gravida
> > quam volutpat varius. Vivamus malesuada viverra mauris sed dapibus.
> > Aliquam erat volutpat.

Aliquam neque metus, posuere vitae condimentum ut, fermentum quis diam.
*Nulla facilisi*. Proin sapien felis, tristique eu venenatis at,
**accumsan** non dui. Vestibulum ante ipsum primis in faucibus orci luctus et
ultrices posuere cubilia.
"""

POST_3 = u"""
Maecenas ut gravida nisi. Aenean feugiat orci non quam vehicula accumsan.
Nullam scelerisque elementum sollicitudin. Sed vel tellus nisi, non tincidunt
augue. Aliquam at nulla ut sem mollis tincidunt.

![Cat](http://i.imgur.com/5wTeD7p.jpg)

Nam quis urna est. Duis vel tincidunt quam. Vivamus odio tortor, suscipit vel
pretium quis, imperdiet quis dolor. Integer molestie enim nec risus malesuada
imperdiet. Donec pellentesque justo id sem tempor varius. Etiam ut tincidunt
lorem. Nullam a tellus sem.

Vestibulum a neque sed quam pharetra interdum. Quisque euismod dictum ipsum.
Vivamus tincidunt mi at tellus pharetra placerat. Sed sed sem nisi, sit amet
ultrices neque. Quisque eget turpis et sapien luctus auctor in ac magna.
Etiam rhoncus commodo molestie.
"""

POST_4 = u"""
How to run a *Hello World* program, code from [Flask](http://flask.pocoo.org).

```python
from flask import Flask
app = Flask(__name__)

@app.route("/")
def hello():
    return "Hello World!"

if __name__ == "__main__":
    app.run()
```

Vestibulum a neque sed quam pharetra interdum. Quisque euismod dictum ipsum.
Vivamus tincidunt mi at tellus pharetra placerat. Sed sed sem nisi, sit amet
ultrices neque. Quisque eget turpis et sapien luctus auctor in ac magna.
"""

POST_5 = u"""
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean vel ipsum
lectus. Pellentesque tempus enim sed leo imperdiet non lobortis nulla
sollicitudin. Maecenas arcu orci, interdum eu rhoncus ut, blandit id felis.
Mauris consectetur dui at felis ultricies tempus. Quisque molestie convallis
lectus vitae viverra. Duis lobortis ultrices turpis, nec eleifend est
venenatis nec.

+ Quisque
+ Venenatis

Sed sed lorem quis metus eleifend ullamcorper. Ut semper nulla a arcu ornare
condimentum. Ut et lacus ac lacus pulvinar accumsan quis eget lacus. Integer
id nibh non eros tincidunt bibendum. Aenean diam lectus, tempus sed consequat
consectetur, posuere non ipsum. Donec vitae eleifend est. Donec at elit mi.
Maecenas tempor nulla gravida quam volutpat varius.

Vivamus malesuada viverra mauris sed dapibus. Aliquam erat volutpat. Aliquam
neque metus, posuere vitae condimentum ut, fermentum quis diam. Nulla
facilisi. Proin sapien felis, tristique eu venenatis at, accumsan non dui.
Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere
cubilia.
"""

if __name__ == '__main__':
    manager.run()
<commit_msg>rename variables<commit_after>from app import app, db
from app.models import User, Post
from datetime import datetime
from flask.ext.script import Manager

manager = Manager(app)

@manager.command
def init():
    dropdb()
    initdb()
    filldb()

@manager.command
def initdb():
    print('Initializing database...'),
    db.create_all()
    print('done!')

@manager.command
def filldb():
    print('Filling database...'),

    admin = User(u'admin', u'admin')
    db.session.add(admin)
    db.session.commit()

    post = Post(
        title=u'Hello, world!',
        markup=POST_1,
        author_id=admin.id,
        visible=True,
    )
    db.session.add(post)
    post.created = datetime(2011, 06, 13)
    post.update(post.title, post.markup, True)
    post = Post(
        title=u'Random Words 1',
        markup=POST_5,
        author_id=admin.id,
        visible=True,
    )
    db.session.add(post)
    post.created = datetime(2012, 8, 15)
    post.update(post.title, post.markup, True)
    post = Post(
        title=u'Random Words 2',
        markup=POST_2,
        author_id=admin.id,
        visible=True,
    )
    db.session.add(post)
    post.created = datetime(2012, 12, 24)
    post.update(post.title, post.markup, True)
    post = Post(
        title=u'Picture of a cat!',
        markup=POST_3,
        author_id=admin.id,
        visible=True,
    )
    db.session.add(post)
    post = Post(
        title=u'Some code',
        markup=POST_4,
        author_id=admin.id,
        visible=True,
    )
    db.session.add(post)
    post = Post(
        title=u'Random Words 3',
        markup=POST_5,
        author_id=admin.id,
        visible=True,
    )
    db.session.add(post)
    db.session.commit()
    print('done!')

@manager.command
def dropdb():
    print('Dropping database...'),
    db.drop_all()
    print('done!')

POST_1 = u"""
First blog post.
"""

POST_2 = u"""
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean vel ipsum
lectus. Pellentesque tempus enim sed leo imperdiet non lobortis nulla
sollicitudin. Maecenas arcu orci, interdum eu rhoncus ut, blandit id felis.
Mauris consectetur dui at felis ultricies tempus. Quisque molestie convallis
lectus vitae viverra. Duis lobortis ultrices turpis, nec eleifend est
venenatis nec. Sed sed lorem quis metus eleifend ullamcorper. Ut semper
nulla a arcu ornare **condimentum**.

> Ut et lacus ac lacus pulvinar accumsan quis eget lacus. Integer id nibh non
> eros tincidunt bibendum. Aenean diam lectus, tempus sed consequat
> consectetur, posuere non ipsum.
> > Donec vitae eleifend est. Donec at elit mi. Maecenas tempor nulla gravida
> > quam volutpat varius. Vivamus malesuada viverra mauris sed dapibus.
> > Aliquam erat volutpat.

Aliquam neque metus, posuere vitae condimentum ut, fermentum quis diam.
*Nulla facilisi*. Proin sapien felis, tristique eu venenatis at,
**accumsan** non dui. Vestibulum ante ipsum primis in faucibus orci luctus et
ultrices posuere cubilia.
"""

POST_3 = u"""
Maecenas ut gravida nisi. Aenean feugiat orci non quam vehicula accumsan.
Nullam scelerisque elementum sollicitudin. Sed vel tellus nisi, non tincidunt
augue. Aliquam at nulla ut sem mollis tincidunt.

![Cat](http://i.imgur.com/5wTeD7p.jpg)

Nam quis urna est. Duis vel tincidunt quam. Vivamus odio tortor, suscipit vel
pretium quis, imperdiet quis dolor. Integer molestie enim nec risus malesuada
imperdiet. Donec pellentesque justo id sem tempor varius. Etiam ut tincidunt
lorem. Nullam a tellus sem.

Vestibulum a neque sed quam pharetra interdum. Quisque euismod dictum ipsum.
Vivamus tincidunt mi at tellus pharetra placerat. Sed sed sem nisi, sit amet
ultrices neque. Quisque eget turpis et sapien luctus auctor in ac magna.
Etiam rhoncus commodo molestie.
"""

POST_4 = u"""
How to run a *Hello World* program, code from [Flask](http://flask.pocoo.org).

```python
from flask import Flask
app = Flask(__name__)

@app.route("/")
def hello():
    return "Hello World!"

if __name__ == "__main__":
    app.run()
```

Vestibulum a neque sed quam pharetra interdum. Quisque euismod dictum ipsum.
Vivamus tincidunt mi at tellus pharetra placerat. Sed sed sem nisi, sit amet
ultrices neque. Quisque eget turpis et sapien luctus auctor in ac magna.
"""

POST_5 = u"""
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean vel ipsum
lectus. Pellentesque tempus enim sed leo imperdiet non lobortis nulla
sollicitudin. Maecenas arcu orci, interdum eu rhoncus ut, blandit id felis.
Mauris consectetur dui at felis ultricies tempus. Quisque molestie convallis
lectus vitae viverra. Duis lobortis ultrices turpis, nec eleifend est
venenatis nec.

+ Quisque
+ Venenatis

Sed sed lorem quis metus eleifend ullamcorper. Ut semper nulla a arcu ornare
condimentum. Ut et lacus ac lacus pulvinar accumsan quis eget lacus. Integer
id nibh non eros tincidunt bibendum. Aenean diam lectus, tempus sed consequat
consectetur, posuere non ipsum. Donec vitae eleifend est. Donec at elit mi.
Maecenas tempor nulla gravida quam volutpat varius.

Vivamus malesuada viverra mauris sed dapibus. Aliquam erat volutpat. Aliquam
neque metus, posuere vitae condimentum ut, fermentum quis diam. Nulla
facilisi. Proin sapien felis, tristique eu venenatis at, accumsan non dui.
Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere
cubilia.
"""

if __name__ == '__main__':
    manager.run()
<|endoftext|>


<commit_before>import torch
import hyperchamber as hc
import numpy as np
import inspect
from operator import itemgetter
from hypergan.train_hooks.base_train_hook import BaseTrainHook
from torch.nn.parameter import Parameter
from torch.autograd import Variable
from torch.autograd import grad as torch_grad

class AdversarialNormTrainHook(BaseTrainHook):
    def __init__(self, gan=None, config=None, trainer=None):
        super().__init__(config=config, gan=gan, trainer=trainer)
        self.d_loss = None
        self.g_loss = None
        self.gamma = torch.Tensor([self.config.gamma]).float()[0].cuda()#self.gan.configurable_param(self.config.gamma or 1.0)
        self.relu = torch.nn.ReLU()
        self.target = [Parameter(x, requires_grad=True) for x in self.gan.discriminator_real_inputs()]
        #self.x_mod_target = torch.zeros_like(self.target[0])
        #self.g_mod_target = torch.zeros_like(self.target[0])

    def forward(self, d_loss, g_loss):
        if self.config.mode == "real" or self.config.mode is None:
            for target, data in zip(self.target, self.gan.discriminator_real_inputs()):
                target.data = data.clone()
            d_fake = self.gan.d_fake
            d_real = self.gan.forward_discriminator(self.target)
            loss, norm, mod_target = self.regularize_adversarial_norm(d_real, d_fake, self.target)
            #if mod_target is not None:
            #    self.x_mod_target = mod_target[0]
        elif self.config.mode == "fake":
            for target, data in zip(self.target, self.gan.discriminator_fake_inputs()):
                target.data = data.clone()
            d_fake = self.gan.forward_discriminator(self.target)
            d_real = self.gan.d_real
            loss, norm, mod_target = self.regularize_adversarial_norm(d_real, d_fake, self.target)
            if mod_target is not None:
                norm = ((mod_target[0] - self.gan.g) ** 2)
                #self.g_mod_target = mod_target[0]

        if loss is None:
            return [None, None]

        if self.config.loss:
          if "g" in self.config.loss:
              self.g_loss = self.gamma * norm.mean()
              self.gan.add_metric('gn_g', self.g_loss)
          if "d" in self.config.loss:
              self.d_loss = self.gamma * norm.mean()
              self.gan.add_metric('gn_d', self.d_loss)
        else:
            self.d_loss = self.gamma * norm.mean()
            self.gan.add_metric('gn_d', self.d_loss)

        return [self.d_loss, self.g_loss]

    def regularize_adversarial_norm(self, d1_logits, d2_logits, target):
        loss = self.gan.loss.forward_adversarial_norm(d1_logits, d2_logits)

        if loss == 0:
            return [None, None, None]

        d1_grads = torch_grad(outputs=loss, inputs=target, retain_graph=True, create_graph=True)
        d1_norm = [torch.norm(_d1_grads.view(-1),p=2,dim=0) for _d1_grads in d1_grads]
        reg_d1 = d1_norm[0]
        for d1 in d1_norm[1:]:
            reg_d1 = reg_d1 + d1
        mod_target = [_d1 + _t for _d1, _t in zip(d1_grads, target)]

        return loss, reg_d1, mod_target
<commit_msg>[fix] better formulation<commit_after>import torch
import hyperchamber as hc
import numpy as np
import inspect
from operator import itemgetter
from hypergan.train_hooks.base_train_hook import BaseTrainHook
from torch.nn.parameter import Parameter
from torch.autograd import Variable
from torch.autograd import grad as torch_grad

class AdversarialNormTrainHook(BaseTrainHook):
    def __init__(self, gan=None, config=None, trainer=None):
        super().__init__(config=config, gan=gan, trainer=trainer)
        self.d_loss = None
        self.g_loss = None
        self.gamma = torch.Tensor([self.config.gamma]).float()[0].cuda()#self.gan.configurable_param(self.config.gamma or 1.0)
        self.relu = torch.nn.ReLU()
        self.target = [Parameter(x, requires_grad=True) for x in self.gan.discriminator_real_inputs()]
        #self.x_mod_target = torch.zeros_like(self.target[0])
        #self.g_mod_target = torch.zeros_like(self.target[0])

    def forward(self, d_loss, g_loss):
        if self.config.mode == "real" or self.config.mode is None:
            for target, data in zip(self.target, self.gan.discriminator_real_inputs()):
                target.data = data.clone()
            d_fake = self.gan.d_fake
            d_real = self.gan.forward_discriminator(self.target)
            loss, norm, mod_target = self.regularize_adversarial_norm(d_real, d_fake, self.target)
            #if mod_target is not None:
            #    self.x_mod_target = mod_target[0]
        elif self.config.mode == "fake":
            for target, data in zip(self.target, self.gan.discriminator_fake_inputs()):
                target.data = data.clone()
            d_fake = self.gan.forward_discriminator(self.target)
            d_real = self.gan.d_real
            loss, norm, mod_target = self.regularize_adversarial_norm(d_real, d_fake, self.target)
            if mod_target is not None:
                norm = self.relu(self.config.offset-((mod_target[0] - self.gan.g) ** 2))

        if loss is None:
            return [None, None]

        if self.config.loss:
          if "g" in self.config.loss:
              self.g_loss = self.gamma * norm.mean()
              self.gan.add_metric('gn_g', self.g_loss)
          if "d" in self.config.loss:
              self.d_loss = self.gamma * norm.mean()
              self.gan.add_metric('gn_d', self.d_loss)
        else:
            self.d_loss = self.gamma * norm.mean()
            self.gan.add_metric('gn_d', self.d_loss)

        return [self.d_loss, self.g_loss]

    def regularize_adversarial_norm(self, d1_logits, d2_logits, target):
        loss = self.gan.loss.forward_adversarial_norm(d1_logits, d2_logits)

        if loss == 0:
            return [None, None, None]

        d1_grads = torch_grad(outputs=loss, inputs=target, retain_graph=True, create_graph=True)
        d1_norm = [torch.norm(_d1_grads.view(-1),p=2,dim=0) for _d1_grads in d1_grads]
        reg_d1 = d1_norm[0]
        for d1 in d1_norm[1:]:
            reg_d1 = reg_d1 + d1
        mod_target = [_d1 + _t for _d1, _t in zip(d1_grads, target)]

        return loss, reg_d1, mod_target
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os, signal, sys, time, platform, subprocess
THIS_SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))

exitBadUsage = 2
close_fds = (os.name == "posix") # would be nice to use this everywhere, but it's broken on Windows (http://docs.python.org/library/subprocess.html)

(CRASHED, TIMED_OUT, NORMAL, ABNORMAL, NONE) = range(5)

def getSignalName(num, default=None):
    for p in dir(signal):
        if p.startswith("SIG") and not p.startswith("SIG_"):
            if getattr(signal, p) == num:
                return p
    return default

class rundata(object):
  def __init__(self, sta, rc, msg, elapsedtime, killed, crashinfo, out, err):
    self.sta = sta
    self.rc = rc
    self.msg = msg
    self.elapsedtime = elapsedtime
    self.killed = killed
    self.crashinfo = crashinfo
    self.out = out
    self.err = err

def xpkill(p):
    '''Based on mozilla-central/source/build/automation.py.in'''
    if hasattr(p, "kill"): # only available in python 2.6+
        try:
            p.kill()
        except WindowsError:
            if p.poll() == 0:
                # Verify that the process is really killed.
                p.kill()
    elif os.name == "nt": # Windows
        pidString = str(p.pid)
        if platform.release() == "2000":
            # Windows 2000 needs 'kill.exe' from the
            #'Windows 2000 Resource Kit tools'. (See bug 475455.)
            try:
                subprocess.Popen(["kill", "-f", pidString]).wait()
            except:
                print("Missing 'kill' utility to kill process with pid=%s. Kill it manually!" % pidString)
        else:
            # Windows XP and later.
            subprocess.Popen(["taskkill", "/F", "/PID", pidString]).wait()
            assert False, 'We should no longer hit this since Python 2.6.5 is on MozillaBuild 1.5.1, already released for 1 year.'
    else:
        os.kill(p.pid, signal.SIGKILL)

def timed_run(commandWithArgs, timeout, logPrefix, input=None):
    '''If logPrefix is None, uses pipes instead of files for all output.'''

    if not isinstance(commandWithArgs, list):
        raise TypeError, "commandWithArgs should be a list (of strings)."
    if not isinstance(timeout, int):
        raise TypeError, "timeout should be an int."

    useLogFiles = isinstance(logPrefix, str)

    commandWithArgs[0] = os.path.expanduser(commandWithArgs[0])
    progname = commandWithArgs[0].split(os.path.sep)[-1]

    if progname == "firefox" and os.name == "posix":
        # Running the |firefox| shell script makes our time-out kills useless,
        # prevents us from knowing the correct pid for crashes (needed on Leopard),
        # and screws with exit codes.
        print "I think you want firefox-bin!"
        sys.exit(exitBadUsage)

    starttime = time.time()

    if useLogFiles:
        childStdOut = file(logPrefix + "-out", 'w')
        childStdErr = file(logPrefix + "-err", 'w')

    try:
        child = subprocess.Popen(
            commandWithArgs,
            stdin = (None         if (input == None) else subprocess.PIPE),
            stderr = (childStdErr if useLogFiles else subprocess.PIPE),
            stdout = (childStdOut if useLogFiles else subprocess.PIPE),
            close_fds = close_fds
        )
    except OSError, e:
        print "Tried to run:"
        print "  " + repr(commandWithArgs)
        print "but got this error:"
        print "  " + str(e)
        sys.exit(2)

    if input != None:
        child.stdin.write(input)
        child.stdin.close()

    sta = NONE
    msg = ''

    killed = False

    # It would be nice to have a timeout with less polling, but apparently that's hard
    # http://mail.python.org/pipermail/python-bugs-list/2009-April/075008.html
    # http://bugs.python.org/issue5673
    # http://benjamin.smedbergs.us/blog/2006-11-09/adventures-in-python-launching-subprocesses/
    # http://benjamin.smedbergs.us/blog/2006-12-11/killableprocesspy/

    # This part is a bit like subprocess.communicate, but with a timeout
    while 1:
        rc = child.poll()
        elapsedtime = time.time() - starttime
        if rc == None:
            if elapsedtime > timeout and not killed:
                xpkill(child)
                # but continue looping, because maybe kill takes a few seconds or maybe it's busy crashing!
                killed = True
            else:
                time.sleep(0.010)
        else:
            break

    crashinfo = None

    if killed and (os.name != "posix" or rc == -signal.SIGKILL):
        msg = 'TIMED OUT'
        sta = TIMED_OUT
    elif rc == 0:
        msg = 'NORMAL'
        sta = NORMAL
    elif rc > 0:
        msg = 'ABNORMAL return code ' + str(rc)
        sta = ABNORMAL
    else:
        # rc < 0
        # The program was terminated by a signal, which usually indicates a crash.
        # Mac/Linux only!
        signum = -rc
        msg = 'CRASHED signal %d (%s)' % (signum, getSignalName(signum, "Unknown signal"))
        sta = CRASHED
        crashinfo = grabCrashLog(progname, commandWithArgs[0], child.pid, logPrefix, signum)

    if useLogFiles:
        # Am I supposed to do this?
        childStdOut.close()
        childStdErr.close()

    return rundata(
        sta,
        rc,
        msg,
        elapsedtime,
        killed,
        crashinfo,
        logPrefix + "-out" if useLogFiles else child.stdout.read(),
        logPrefix + "-err" if useLogFiles else child.stderr.read()
    )

def grabCrashLog(progname, progfullname, crashedPID, logPrefix, signum):
    if progname == "valgrind":
        return
    useLogFiles = isinstance(logPrefix, str)
    if useLogFiles:
        if os.path.exists(logPrefix + "-crash"):
            os.remove(logPrefix + "-crash")
        if os.path.exists(logPrefix + "-core"):
            os.remove(logPrefix + "-core")

    # On Mac and Linux, look for a core file.
    # In addition to the one-time setup mentioned below for each OS,
    # use the bash command "ulimit -c unlimited" before each invocation.
    coreFilename = None
    if platform.system() == "Darwin":
        # Assuming you ran: mkdir -p /cores/
        coreFilename = "/cores/core." + str(crashedPID)
    elif platform.system() == "Linux":
        # Assuming you ran: echo 1 | sudo tee /proc/sys/kernel/core_uses_pid
        # (I want to require this because otherwise multi-threading jsfunfuzz would result in core file collisions.)
        coreFilename = "core." + str(crashedPID)
    if coreFilename and os.path.exists(coreFilename):
        # Run gdb and move the core file.
        # Tip: gdb gives more info for (debug with intact build dir > debug > opt with frame pointers > opt)
        gdbCommandFile = os.path.join(THIS_SCRIPT_DIRECTORY, "gdb-quick.txt")
        gdbArgs = ["gdb", "-n", "-batch", "-x", gdbCommandFile, progfullname, coreFilename]
        print " ".join(gdbArgs)
        child = subprocess.call(
            gdbArgs,
            stdin =  None,
            stderr = subprocess.STDOUT,
            stdout = file(logPrefix + "-crash", 'w') if useLogFiles else None,
            close_fds = close_fds
        )
        if useLogFiles:
            os.rename(coreFilename, logPrefix + "-core")
            subprocess.call(["gzip", logPrefix + "-core"])
            return logPrefix + "-crash"
        else:
            print "I don't know what to do with a core file when logPrefix is null"

    # On Mac, look for a crash log generated by Mac OS X Crash Reporter
    if platform.system() == "Darwin":
        found = False
        loops = 0
        while not found:
            # Find a crash log for the right process name and pid, preferring
            # newer crash logs (which sort last).
            crashLogDir = "~/Library/Logs/CrashReporter/" if platform.mac_ver()[0].startswith("10.5") else "~/Library/Logs/DiagnosticReports/"
            crashLogDir = os.path.expanduser(crashLogDir)
            try:
                crashLogs = os.listdir(crashLogDir)
            except (OSError, IOError), e:
                # Maybe this is the first crash ever on this computer, and the directory doesn't exist yet.
                crashLogs = []
            crashLogs = filter(lambda s: s.startswith(progname + "_"), crashLogs)
            crashLogs.sort(reverse=True)
            for fn in crashLogs:
                fullfn = os.path.join(crashLogDir, fn)
                try:
                    c = file(fullfn)
                    firstLine = c.readline()
                    c.close()
                    if firstLine.rstrip().endswith("[" + str(crashedPID) + "]"):
                        if useLogFiles:
                            os.rename(fullfn, logPrefix + "-crash")
                            return logPrefix + "-crash"
                        else:
                            return fullfn
                            #return open(fullfn).read()

                except (OSError, IOError), e:
                    # Maybe the log was rotated out between when we got the list
                    # of files and when we tried to open this file.  If so, it's
                    # clearly not The One.
                    pass
            if not found:
                # print "[grabCrashLog] Waiting for the crash log to appear..."
                time.sleep(0.100)
                loops += 1
                if loops > 1000:
                    # I suppose this might happen if the process corrupts itself so much that
                    # the crash reporter gets confused about the process name, for example.
                    print "grabCrashLog waited a long time, but a crash log for " + progname + " [" + str(crashedPID) + "] never appeared!"
                    break
<commit_msg>Make ntr.py wait 100 seconds to get a crash log only if process name starts with firefox, and 6 seconds otherwise.<commit_after>#!/usr/bin/env python

import os, signal, sys, time, platform, subprocess
THIS_SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))

exitBadUsage = 2
close_fds = (os.name == "posix") # would be nice to use this everywhere, but it's broken on Windows (http://docs.python.org/library/subprocess.html)

(CRASHED, TIMED_OUT, NORMAL, ABNORMAL, NONE) = range(5)

def getSignalName(num, default=None):
    for p in dir(signal):
        if p.startswith("SIG") and not p.startswith("SIG_"):
            if getattr(signal, p) == num:
                return p
    return default

class rundata(object):
  def __init__(self, sta, rc, msg, elapsedtime, killed, crashinfo, out, err):
    self.sta = sta
    self.rc = rc
    self.msg = msg
    self.elapsedtime = elapsedtime
    self.killed = killed
    self.crashinfo = crashinfo
    self.out = out
    self.err = err

def xpkill(p):
    '''Based on mozilla-central/source/build/automation.py.in'''
    if hasattr(p, "kill"): # only available in python 2.6+
        try:
            p.kill()
        except WindowsError:
            if p.poll() == 0:
                # Verify that the process is really killed.
                p.kill()
    elif os.name == "nt": # Windows
        pidString = str(p.pid)
        if platform.release() == "2000":
            # Windows 2000 needs 'kill.exe' from the
            #'Windows 2000 Resource Kit tools'. (See bug 475455.)
            try:
                subprocess.Popen(["kill", "-f", pidString]).wait()
            except:
                print("Missing 'kill' utility to kill process with pid=%s. Kill it manually!" % pidString)
        else:
            # Windows XP and later.
            subprocess.Popen(["taskkill", "/F", "/PID", pidString]).wait()
            assert False, 'We should no longer hit this since Python 2.6.5 is on MozillaBuild 1.5.1, already released for 1 year.'
    else:
        os.kill(p.pid, signal.SIGKILL)

def timed_run(commandWithArgs, timeout, logPrefix, input=None):
    '''If logPrefix is None, uses pipes instead of files for all output.'''

    if not isinstance(commandWithArgs, list):
        raise TypeError, "commandWithArgs should be a list (of strings)."
    if not isinstance(timeout, int):
        raise TypeError, "timeout should be an int."

    useLogFiles = isinstance(logPrefix, str)

    commandWithArgs[0] = os.path.expanduser(commandWithArgs[0])
    progname = commandWithArgs[0].split(os.path.sep)[-1]

    if progname == "firefox" and os.name == "posix":
        # Running the |firefox| shell script makes our time-out kills useless,
        # prevents us from knowing the correct pid for crashes (needed on Leopard),
        # and screws with exit codes.
        print "I think you want firefox-bin!"
        sys.exit(exitBadUsage)

    starttime = time.time()

    if useLogFiles:
        childStdOut = file(logPrefix + "-out", 'w')
        childStdErr = file(logPrefix + "-err", 'w')

    try:
        child = subprocess.Popen(
            commandWithArgs,
            stdin = (None         if (input == None) else subprocess.PIPE),
            stderr = (childStdErr if useLogFiles else subprocess.PIPE),
            stdout = (childStdOut if useLogFiles else subprocess.PIPE),
            close_fds = close_fds
        )
    except OSError, e:
        print "Tried to run:"
        print "  " + repr(commandWithArgs)
        print "but got this error:"
        print "  " + str(e)
        sys.exit(2)

    if input != None:
        child.stdin.write(input)
        child.stdin.close()

    sta = NONE
    msg = ''

    killed = False

    # It would be nice to have a timeout with less polling, but apparently that's hard
    # http://mail.python.org/pipermail/python-bugs-list/2009-April/075008.html
    # http://bugs.python.org/issue5673
    # http://benjamin.smedbergs.us/blog/2006-11-09/adventures-in-python-launching-subprocesses/
    # http://benjamin.smedbergs.us/blog/2006-12-11/killableprocesspy/

    # This part is a bit like subprocess.communicate, but with a timeout
    while 1:
        rc = child.poll()
        elapsedtime = time.time() - starttime
        if rc == None:
            if elapsedtime > timeout and not killed:
                xpkill(child)
                # but continue looping, because maybe kill takes a few seconds or maybe it's busy crashing!
                killed = True
            else:
                time.sleep(0.010)
        else:
            break

    crashinfo = None

    if killed and (os.name != "posix" or rc == -signal.SIGKILL):
        msg = 'TIMED OUT'
        sta = TIMED_OUT
    elif rc == 0:
        msg = 'NORMAL'
        sta = NORMAL
    elif rc > 0:
        msg = 'ABNORMAL return code ' + str(rc)
        sta = ABNORMAL
    else:
        # rc < 0
        # The program was terminated by a signal, which usually indicates a crash.
        # Mac/Linux only!
        signum = -rc
        msg = 'CRASHED signal %d (%s)' % (signum, getSignalName(signum, "Unknown signal"))
        sta = CRASHED
        crashinfo = grabCrashLog(progname, commandWithArgs[0], child.pid, logPrefix, signum)

    if useLogFiles:
        # Am I supposed to do this?
        childStdOut.close()
        childStdErr.close()

    return rundata(
        sta,
        rc,
        msg,
        elapsedtime,
        killed,
        crashinfo,
        logPrefix + "-out" if useLogFiles else child.stdout.read(),
        logPrefix + "-err" if useLogFiles else child.stderr.read()
    )

def grabCrashLog(progname, progfullname, crashedPID, logPrefix, signum):
    if progname == "valgrind":
        return
    useLogFiles = isinstance(logPrefix, str)
    if useLogFiles:
        if os.path.exists(logPrefix + "-crash"):
            os.remove(logPrefix + "-crash")
        if os.path.exists(logPrefix + "-core"):
            os.remove(logPrefix + "-core")

    # On Mac and Linux, look for a core file.
    # In addition to the one-time setup mentioned below for each OS,
    # use the bash command "ulimit -c unlimited" before each invocation.
    coreFilename = None
    if platform.system() == "Darwin":
        # Assuming you ran: mkdir -p /cores/
        coreFilename = "/cores/core." + str(crashedPID)
    elif platform.system() == "Linux":
        # Assuming you ran: echo 1 | sudo tee /proc/sys/kernel/core_uses_pid
        # (I want to require this because otherwise multi-threading jsfunfuzz would result in core file collisions.)
        coreFilename = "core." + str(crashedPID)
    if coreFilename and os.path.exists(coreFilename):
        # Run gdb and move the core file.
        # Tip: gdb gives more info for (debug with intact build dir > debug > opt with frame pointers > opt)
        gdbCommandFile = os.path.join(THIS_SCRIPT_DIRECTORY, "gdb-quick.txt")
        gdbArgs = ["gdb", "-n", "-batch", "-x", gdbCommandFile, progfullname, coreFilename]
        print " ".join(gdbArgs)
        child = subprocess.call(
            gdbArgs,
            stdin =  None,
            stderr = subprocess.STDOUT,
            stdout = file(logPrefix + "-crash", 'w') if useLogFiles else None,
            close_fds = close_fds
        )
        if useLogFiles:
            os.rename(coreFilename, logPrefix + "-core")
            subprocess.call(["gzip", logPrefix + "-core"])
            return logPrefix + "-crash"
        else:
            print "I don't know what to do with a core file when logPrefix is null"

    # On Mac, look for a crash log generated by Mac OS X Crash Reporter
    if platform.system() == "Darwin":
        found = False
        loops = 0
        maxLoops = 500 if progname.startswith("firefox") else 30
        while not found:
            # Find a crash log for the right process name and pid, preferring
            # newer crash logs (which sort last).
            crashLogDir = "~/Library/Logs/CrashReporter/" if platform.mac_ver()[0].startswith("10.5") else "~/Library/Logs/DiagnosticReports/"
            crashLogDir = os.path.expanduser(crashLogDir)
            try:
                crashLogs = os.listdir(crashLogDir)
            except (OSError, IOError), e:
                # Maybe this is the first crash ever on this computer, and the directory doesn't exist yet.
                crashLogs = []
            crashLogs = filter(lambda s: s.startswith(progname + "_"), crashLogs)
            crashLogs.sort(reverse=True)
            for fn in crashLogs:
                fullfn = os.path.join(crashLogDir, fn)
                try:
                    c = file(fullfn)
                    firstLine = c.readline()
                    c.close()
                    if firstLine.rstrip().endswith("[" + str(crashedPID) + "]"):
                        if useLogFiles:
                            os.rename(fullfn, logPrefix + "-crash")
                            return logPrefix + "-crash"
                        else:
                            return fullfn
                            #return open(fullfn).read()

                except (OSError, IOError), e:
                    # Maybe the log was rotated out between when we got the list
                    # of files and when we tried to open this file.  If so, it's
                    # clearly not The One.
                    pass
            if not found:
                # print "[grabCrashLog] Waiting for the crash log to appear..."
                time.sleep(0.200)
                loops += 1
                if loops > maxLoops:
                    # I suppose this might happen if the process corrupts itself so much that
                    # the crash reporter gets confused about the process name, for example.
                    print "grabCrashLog waited a long time, but a crash log for " + progname + " [" + str(crashedPID) + "] never appeared!"
                    break
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import json
from typing import Any

from cryptography.fernet import InvalidToken as InvalidFernetToken
from sqlalchemy import Boolean, Column, Integer, String, Text
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import synonym

from airflow.models.base import ID_LEN, Base
from airflow.models.crypto import get_fernet
from airflow.utils.db import provide_session
from airflow.utils.log.logging_mixin import LoggingMixin

class Variable(Base, LoggingMixin):
    __tablename__ = "variable"
    __NO_DEFAULT_SENTINEL = object()

    id = Column(Integer, primary_key=True)
    key = Column(String(ID_LEN), unique=True)
    _val = Column('val', Text)
    is_encrypted = Column(Boolean, unique=False, default=False)

    def __repr__(self):
        # Hiding the value
        return '{} : {}'.format(self.key, self._val)

    def get_val(self):
        log = LoggingMixin().log
        if self._val and self.is_encrypted:
            try:
                fernet = get_fernet()
                return fernet.decrypt(bytes(self._val, 'utf-8')).decode()
            except InvalidFernetToken:
                log.error("Can't decrypt _val for key=%s, invalid token or value", self.key)
                return None
            except Exception:
                log.error("Can't decrypt _val for key=%s, FERNET_KEY configuration missing", self.key)
                return None
        else:
            return self._val

    def set_val(self, value):
        if value:
            fernet = get_fernet()
            self._val = fernet.encrypt(bytes(value, 'utf-8')).decode()
            self.is_encrypted = fernet.is_encrypted

    @declared_attr
    def val(cls):
        return synonym('_val',
                       descriptor=property(cls.get_val, cls.set_val))

    @classmethod
    def setdefault(cls, key, default, deserialize_json=False):
        """
        Like a Python builtin dict object, setdefault returns the current value
        for a key, and if it isn't there, stores the default value and returns it.

        :param key: Dict key for this Variable
        :type key: str
        :param default: Default value to set and return if the variable
            isn't already in the DB
        :type default: Mixed
        :param deserialize_json: Store this as a JSON encoded value in the DB
            and un-encode it when retrieving a value
        :return: Mixed
        """
        obj = Variable.get(key, default_var=None,
                           deserialize_json=deserialize_json)
        if obj is None:
            if default is not None:
                Variable.set(key, default, serialize_json=deserialize_json)
                return default
            else:
                raise ValueError('Default Value must be set')
        else:
            return obj

    @classmethod
    @provide_session
    def get(
        cls,
        key: str,
        default_var: Any = __NO_DEFAULT_SENTINEL,
        deserialize_json: bool = False,
        session=None
    ):
        obj = session.query(cls).filter(cls.key == key).first()
        if obj is None:
            if default_var is not cls.__NO_DEFAULT_SENTINEL:
                return default_var
            else:
                raise KeyError('Variable {} does not exist'.format(key))
        else:
            if deserialize_json:
                return json.loads(obj.val)
            else:
                return obj.val

    @classmethod
    @provide_session
    def set(
        cls,
        key: str,
        value: Any,
        serialize_json: bool = False,
        session=None
    ):

        if serialize_json:
            stored_value = json.dumps(value, indent=2)
        else:
            stored_value = str(value)

        Variable.delete(key)
        session.add(Variable(key=key, val=stored_value))  # type: ignore
        session.flush()

    @classmethod
    @provide_session
    def delete(cls, key, session=None):
        session.query(cls).filter(cls.key == key).delete()

    def rotate_fernet_key(self):
        fernet = get_fernet()
        if self._val and self.is_encrypted:
            self._val = fernet.rotate(self._val.encode('utf-8')).decode()
<commit_msg>[AIRFLOW-5463] Use same session to delete and add variable in set (#6807)<commit_after># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import json
from typing import Any

from cryptography.fernet import InvalidToken as InvalidFernetToken
from sqlalchemy import Boolean, Column, Integer, String, Text
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import synonym

from airflow.models.base import ID_LEN, Base
from airflow.models.crypto import get_fernet
from airflow.utils.db import provide_session
from airflow.utils.log.logging_mixin import LoggingMixin

class Variable(Base, LoggingMixin):
    __tablename__ = "variable"
    __NO_DEFAULT_SENTINEL = object()

    id = Column(Integer, primary_key=True)
    key = Column(String(ID_LEN), unique=True)
    _val = Column('val', Text)
    is_encrypted = Column(Boolean, unique=False, default=False)

    def __repr__(self):
        # Hiding the value
        return '{} : {}'.format(self.key, self._val)

    def get_val(self):
        log = LoggingMixin().log
        if self._val and self.is_encrypted:
            try:
                fernet = get_fernet()
                return fernet.decrypt(bytes(self._val, 'utf-8')).decode()
            except InvalidFernetToken:
                log.error("Can't decrypt _val for key=%s, invalid token or value", self.key)
                return None
            except Exception:
                log.error("Can't decrypt _val for key=%s, FERNET_KEY configuration missing", self.key)
                return None
        else:
            return self._val

    def set_val(self, value):
        if value:
            fernet = get_fernet()
            self._val = fernet.encrypt(bytes(value, 'utf-8')).decode()
            self.is_encrypted = fernet.is_encrypted

    @declared_attr
    def val(cls):
        return synonym('_val',
                       descriptor=property(cls.get_val, cls.set_val))

    @classmethod
    def setdefault(cls, key, default, deserialize_json=False):
        """
        Like a Python builtin dict object, setdefault returns the current value
        for a key, and if it isn't there, stores the default value and returns it.

        :param key: Dict key for this Variable
        :type key: str
        :param default: Default value to set and return if the variable
            isn't already in the DB
        :type default: Mixed
        :param deserialize_json: Store this as a JSON encoded value in the DB
            and un-encode it when retrieving a value
        :return: Mixed
        """
        obj = Variable.get(key, default_var=None,
                           deserialize_json=deserialize_json)
        if obj is None:
            if default is not None:
                Variable.set(key, default, serialize_json=deserialize_json)
                return default
            else:
                raise ValueError('Default Value must be set')
        else:
            return obj

    @classmethod
    @provide_session
    def get(
        cls,
        key: str,
        default_var: Any = __NO_DEFAULT_SENTINEL,
        deserialize_json: bool = False,
        session=None
    ):
        obj = session.query(cls).filter(cls.key == key).first()
        if obj is None:
            if default_var is not cls.__NO_DEFAULT_SENTINEL:
                return default_var
            else:
                raise KeyError('Variable {} does not exist'.format(key))
        else:
            if deserialize_json:
                return json.loads(obj.val)
            else:
                return obj.val

    @classmethod
    @provide_session
    def set(
        cls,
        key: str,
        value: Any,
        serialize_json: bool = False,
        session=None
    ):

        if serialize_json:
            stored_value = json.dumps(value, indent=2)
        else:
            stored_value = str(value)

        Variable.delete(key, session=session)
        session.add(Variable(key=key, val=stored_value))  # type: ignore
        session.flush()

    @classmethod
    @provide_session
    def delete(cls, key, session=None):
        session.query(cls).filter(cls.key == key).delete()

    def rotate_fernet_key(self):
        fernet = get_fernet()
        if self._val and self.is_encrypted:
            self._val = fernet.rotate(self._val.encode('utf-8')).decode()
<|endoftext|>


<commit_before># Copyright 2020 The StackStorm Authors.
# Copyright 2019 Extreme Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import os
import sys
import signal

import eventlet
from eventlet.green import subprocess

from st2tests.base import IntegrationTestCase
from st2tests.fixturesloader import get_fixtures_base_path

__all__ = ["ServiceSetupLogLevelFilteringTestCase"]

BASE_DIR = os.path.dirname(os.path.abspath(__file__))

FIXTURES_DIR = get_fixtures_base_path()

ST2_CONFIG_INFO_LL_PATH = os.path.join(
    FIXTURES_DIR, "conf/st2.tests.api.info_log_level.conf"
)
ST2_CONFIG_INFO_LL_PATH = os.path.abspath(ST2_CONFIG_INFO_LL_PATH)

ST2_CONFIG_DEBUG_LL_PATH = os.path.join(
    FIXTURES_DIR, "conf/st2.tests.api.debug_log_level.conf"
)
ST2_CONFIG_DEBUG_LL_PATH = os.path.abspath(ST2_CONFIG_DEBUG_LL_PATH)

ST2_CONFIG_AUDIT_LL_PATH = os.path.join(
    FIXTURES_DIR, "conf/st2.tests.api.audit_log_level.conf"
)
ST2_CONFIG_AUDIT_LL_PATH = os.path.abspath(ST2_CONFIG_AUDIT_LL_PATH)

ST2_CONFIG_SYSTEM_DEBUG_PATH = os.path.join(
    FIXTURES_DIR, "conf/st2.tests.api.system_debug_true.conf"
)
ST2_CONFIG_SYSTEM_DEBUG_PATH = os.path.abspath(ST2_CONFIG_SYSTEM_DEBUG_PATH)

ST2_CONFIG_SYSTEM_LL_DEBUG_PATH = os.path.join(
    FIXTURES_DIR, "conf/st2.tests.api.system_debug_true_logging_debug.conf"
)

PYTHON_BINARY = sys.executable

ST2API_BINARY = os.path.join(BASE_DIR, "../../../st2api/bin/st2api")
ST2API_BINARY = os.path.abspath(ST2API_BINARY)

CMD = [PYTHON_BINARY, ST2API_BINARY, "--config-file"]

class ServiceSetupLogLevelFilteringTestCase(IntegrationTestCase):
    def test_system_info_is_logged_on_startup(self):
        # Verify INFO level service start up messages
        process = self._start_process(config_path=ST2_CONFIG_INFO_LL_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(3)
        process.send_signal(signal.SIGKILL)

        # Verify first 4 environment related log messages
        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[:4])
        self.assertIn("INFO [-] Using Python:", stdout)
        self.assertIn("INFO [-] Using fs encoding:", stdout)
        self.assertIn("INFO [-] Using config files:", stdout)
        self.assertIn("INFO [-] Using logging config:", stdout)

    def test_warning_is_emitted_on_non_utf8_encoding(self):
        env = os.environ.copy()
        env["LC_ALL"] = "invalid"
        process = self._start_process(config_path=ST2_CONFIG_INFO_LL_PATH, env=env)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(3)
        process.send_signal(signal.SIGKILL)

        # Verify first 4 environment related log messages
        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n"))
        self.assertIn("WARNING [-] Detected a non utf-8 locale / encoding", stdout)
        self.assertIn("fs encoding: ascii", stdout)
        self.assertIn("unknown locale: invalid", stdout)

    def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self):
        # 1. INFO log level - audit messages should not be included
        process = self._start_process(config_path=ST2_CONFIG_INFO_LL_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(3)
        process.send_signal(signal.SIGKILL)

        # First 4 log lines are debug messages about the environment which are always logged
        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[4:])

        self.assertIn("INFO [-]", stdout)
        self.assertNotIn("DEBUG [-]", stdout)
        self.assertNotIn("AUDIT [-]", stdout)

        # 2. DEBUG log level - audit messages should be included
        process = self._start_process(config_path=ST2_CONFIG_DEBUG_LL_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(5)
        process.send_signal(signal.SIGKILL)

        # First 4 log lines are debug messages about the environment which are always logged
        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[4:])

        self.assertIn("INFO [-]", stdout)
        self.assertIn("DEBUG [-]", stdout)
        self.assertIn("AUDIT [-]", stdout)

        # 3. AUDIT log level - audit messages should be included
        process = self._start_process(config_path=ST2_CONFIG_AUDIT_LL_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(5)
        process.send_signal(signal.SIGKILL)

        # First 4 log lines are debug messages about the environment which are always logged
        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[4:])

        self.assertNotIn("INFO [-]", stdout)
        self.assertNotIn("DEBUG [-]", stdout)
        self.assertIn("AUDIT [-]", stdout)

        # 2. INFO log level but system.debug set to True
        process = self._start_process(config_path=ST2_CONFIG_SYSTEM_DEBUG_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(5)
        process.send_signal(signal.SIGKILL)

        # First 4 log lines are debug messages about the environment which are always logged
        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[4:])

        self.assertIn("INFO [-]", stdout)
        self.assertIn("DEBUG [-]", stdout)
        self.assertIn("AUDIT [-]", stdout)

    def test_kombu_heartbeat_tick_log_messages_are_excluded(self):
        # 1. system.debug = True config option is set, verify heartbeat_tick message is not logged
        process = self._start_process(config_path=ST2_CONFIG_SYSTEM_LL_DEBUG_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(5)
        process.send_signal(signal.SIGKILL)

        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n"))
        self.assertNotIn("heartbeat_tick", stdout)

        # 2. system.debug = False, log level is set to debug
        process = self._start_process(config_path=ST2_CONFIG_DEBUG_LL_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(5)
        process.send_signal(signal.SIGKILL)

        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n"))
        self.assertNotIn("heartbeat_tick", stdout)

    def _start_process(self, config_path, env=None):
        cmd = CMD + [config_path]
        cwd = os.path.abspath(os.path.join(BASE_DIR, "../../../"))
        cwd = os.path.abspath(cwd)
        process = subprocess.Popen(
            cmd,
            env=env or os.environ.copy(),
            cwd=cwd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=False,
            preexec_fn=os.setsid,
        )
        return process
<commit_msg>Update affected tests.<commit_after># Copyright 2020 The StackStorm Authors.
# Copyright 2019 Extreme Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import os
import sys
import signal

import eventlet
from eventlet.green import subprocess

from st2tests.base import IntegrationTestCase
from st2tests.fixturesloader import get_fixtures_base_path

__all__ = ["ServiceSetupLogLevelFilteringTestCase"]

BASE_DIR = os.path.dirname(os.path.abspath(__file__))

FIXTURES_DIR = get_fixtures_base_path()

ST2_CONFIG_INFO_LL_PATH = os.path.join(
    FIXTURES_DIR, "conf/st2.tests.api.info_log_level.conf"
)
ST2_CONFIG_INFO_LL_PATH = os.path.abspath(ST2_CONFIG_INFO_LL_PATH)

ST2_CONFIG_DEBUG_LL_PATH = os.path.join(
    FIXTURES_DIR, "conf/st2.tests.api.debug_log_level.conf"
)
ST2_CONFIG_DEBUG_LL_PATH = os.path.abspath(ST2_CONFIG_DEBUG_LL_PATH)

ST2_CONFIG_AUDIT_LL_PATH = os.path.join(
    FIXTURES_DIR, "conf/st2.tests.api.audit_log_level.conf"
)
ST2_CONFIG_AUDIT_LL_PATH = os.path.abspath(ST2_CONFIG_AUDIT_LL_PATH)

ST2_CONFIG_SYSTEM_DEBUG_PATH = os.path.join(
    FIXTURES_DIR, "conf/st2.tests.api.system_debug_true.conf"
)
ST2_CONFIG_SYSTEM_DEBUG_PATH = os.path.abspath(ST2_CONFIG_SYSTEM_DEBUG_PATH)

ST2_CONFIG_SYSTEM_LL_DEBUG_PATH = os.path.join(
    FIXTURES_DIR, "conf/st2.tests.api.system_debug_true_logging_debug.conf"
)

PYTHON_BINARY = sys.executable

ST2API_BINARY = os.path.join(BASE_DIR, "../../../st2api/bin/st2api")
ST2API_BINARY = os.path.abspath(ST2API_BINARY)

CMD = [PYTHON_BINARY, ST2API_BINARY, "--config-file"]

class ServiceSetupLogLevelFilteringTestCase(IntegrationTestCase):
    def test_system_info_is_logged_on_startup(self):
        # Verify INFO level service start up messages
        process = self._start_process(config_path=ST2_CONFIG_INFO_LL_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(3)
        process.send_signal(signal.SIGKILL)

        # Verify first 4 environment related log messages
        stdout = process.stdout.read().decode("utf-8")
        self.assertIn("INFO [-] Using Python:", stdout)
        self.assertIn("INFO [-] Using fs encoding:", stdout)
        self.assertIn("INFO [-] Using config files:", stdout)
        self.assertIn("INFO [-] Using logging config:", stdout)

    def test_warning_is_emitted_on_non_utf8_encoding(self):
        env = os.environ.copy()
        env["LC_ALL"] = "invalid"
        process = self._start_process(config_path=ST2_CONFIG_INFO_LL_PATH, env=env)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(3)
        process.send_signal(signal.SIGKILL)

        # Verify first 4 environment related log messages
        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n"))
        self.assertIn("WARNING [-] Detected a non utf-8 locale / encoding", stdout)
        self.assertIn("fs encoding: ascii", stdout)
        self.assertIn("unknown locale: invalid", stdout)

    def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self):
        # 1. INFO log level - audit messages should not be included
        process = self._start_process(config_path=ST2_CONFIG_INFO_LL_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(3)
        process.send_signal(signal.SIGKILL)

        # First 4 log lines are debug messages about the environment which are always logged
        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[4:])

        self.assertIn("INFO [-]", stdout)
        self.assertNotIn("DEBUG [-]", stdout)
        self.assertNotIn("AUDIT [-]", stdout)

        # 2. DEBUG log level - audit messages should be included
        process = self._start_process(config_path=ST2_CONFIG_DEBUG_LL_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(5)
        process.send_signal(signal.SIGKILL)

        # First 4 log lines are debug messages about the environment which are always logged
        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[4:])

        self.assertIn("INFO [-]", stdout)
        self.assertIn("DEBUG [-]", stdout)
        self.assertIn("AUDIT [-]", stdout)

        # 3. AUDIT log level - audit messages should be included
        process = self._start_process(config_path=ST2_CONFIG_AUDIT_LL_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(5)
        process.send_signal(signal.SIGKILL)

        # First 4 log lines are debug messages about the environment which are always logged
        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[4:])

        self.assertIn("INFO [-]", stdout)
        self.assertNotIn("DEBUG [-]", stdout)
        self.assertIn("AUDIT [-]", stdout)

        # 2. INFO log level but system.debug set to True
        process = self._start_process(config_path=ST2_CONFIG_SYSTEM_DEBUG_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(5)
        process.send_signal(signal.SIGKILL)

        # First 4 log lines are debug messages about the environment which are always logged
        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[4:])

        self.assertIn("INFO [-]", stdout)
        self.assertIn("DEBUG [-]", stdout)
        self.assertIn("AUDIT [-]", stdout)

    def test_kombu_heartbeat_tick_log_messages_are_excluded(self):
        # 1. system.debug = True config option is set, verify heartbeat_tick message is not logged
        process = self._start_process(config_path=ST2_CONFIG_SYSTEM_LL_DEBUG_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(5)
        process.send_signal(signal.SIGKILL)

        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n"))
        self.assertNotIn("heartbeat_tick", stdout)

        # 2. system.debug = False, log level is set to debug
        process = self._start_process(config_path=ST2_CONFIG_DEBUG_LL_PATH)
        self.add_process(process=process)

        # Give it some time to start up
        eventlet.sleep(5)
        process.send_signal(signal.SIGKILL)

        stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n"))
        self.assertNotIn("heartbeat_tick", stdout)

    def _start_process(self, config_path, env=None):
        cmd = CMD + [config_path]
        cwd = os.path.abspath(os.path.join(BASE_DIR, "../../../"))
        cwd = os.path.abspath(cwd)
        process = subprocess.Popen(
            cmd,
            env=env or os.environ.copy(),
            cwd=cwd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=False,
            preexec_fn=os.setsid,
        )
        return process
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""Manager for Flask-Script, containing backend-related commands."""

import sys
import os
from flask import render_template
from flask_debugtoolbar import DebugToolbarExtension
from flask_migrate import Migrate, MigrateCommand
from flask_script import Manager

from stuffrapp import create_app
from database import db

# Manager setup
################

if not os.environ['STUFFR_SETTINGS']:
    print('Set the STUFFR_SETTINGS environment variable before using this tool.')
    sys.exit(1)

# Manager is given a different app from runserver that doesn't touch
# the database during initialization
manager_app = create_app(config_override={
    'STUFFR_INITIALIZE_DATABASE': False,
    'STUFFR_CREATE_TABLES': False
})
manager = Manager(manager_app)

# Alembic
migrate = Migrate(manager_app, db)
manager.add_command('db', MigrateCommand)

@manager.command
def runserver():
    """Run the Flask debug server."""
    # manager_app does not do database initialization
    debug_app = create_app()
    DebugToolbarExtension(debug_app)

    @debug_app.route('/')
    def debug_root():
        """Serve index.html when using the debug server."""
        # Using render_template so flask_debugtoolbar can do its thing
        return render_template('index.html')

    debug_app.run(debug_app.config['STUFFR_DEBUG_HOST'],
                  debug_app.config['STUFFR_DEBUG_PORT'])

if __name__ == "__main__":
    manager.run()
<commit_msg>Added command to list all routes<commit_after>#!/usr/bin/env python3
"""Manager for Flask-Script, containing backend-related commands."""

import sys
import os
from flask import render_template
from flask_debugtoolbar import DebugToolbarExtension
from flask_migrate import Migrate, MigrateCommand
from flask_script import Manager

from stuffrapp import create_app
from database import db

# Manager setup
################

if not os.environ['STUFFR_SETTINGS']:
    print('Set the STUFFR_SETTINGS environment variable before using this tool.')
    sys.exit(1)

# Manager is given a different app from runserver that doesn't touch
# the database during initialization
manager_app = create_app(config_override={
    'STUFFR_INITIALIZE_DATABASE': False,
    'STUFFR_CREATE_TABLES': False
})
manager = Manager(manager_app)

# Alembic
migrate = Migrate(manager_app, db)
manager.add_command('db', MigrateCommand)

@manager.command
def runserver():
    """Run the Flask debug server."""
    # manager_app does not do database initialization
    debug_app = create_app()
    DebugToolbarExtension(debug_app)

    @debug_app.route('/')
    def debug_root():
        """Serve index.html when using the debug server."""
        # Using render_template so flask_debugtoolbar can do its thing
        return render_template('index.html')

    debug_app.run(debug_app.config['STUFFR_DEBUG_HOST'],
                  debug_app.config['STUFFR_DEBUG_PORT'])

@manager.command
def listroutes():
    """List all views defined by the app."""
    for rule in sorted(manager_app.url_map.iter_rules(),
                       key=lambda r: r.endpoint):
        endpoint = rule.endpoint
        methods = ', '.join(r for r in rule.methods if r not in ['OPTIONS', 'HEAD'])
        path = rule.rule
        print(f'{endpoint}: ({methods}) {path}')

if __name__ == "__main__":
    manager.run()
<|endoftext|>


<commit_before>from django.db import models

class Building(models.Model):
    name = models.CharField(max_length=10)

    def __unicode__(self):
        return u"Building: %s" % self.name

class Device(models.Model):
    building = models.ForeignKey('Building')
    name = models.CharField(max_length=10)

    def __unicode__(self):
        return u"device '%s' in building %s" % (self.name, self.building)

class Port(models.Model):
    device = models.ForeignKey('Device')
    number = models.CharField(max_length=10)

    def __unicode__(self):
        return u"%s/%s" % (self.device.name, self.number)

class Connection(models.Model):
    start = models.ForeignKey(Port, related_name='connection_start',
            unique=True)
    end = models.ForeignKey(Port, related_name='connection_end', unique=True)

    def __unicode__(self):
        return u"%s to %s" % (self.start, self.end)

__test__ = {'API_TESTS': """
Regression test for bug #7110. When using select_related(), we must query the
Device and Building tables using two different aliases (each) in order to
differentiate the start and end Connection fields. The net result is that both
the "connections = ..." queries here should give the same results.

>>> b=Building.objects.create(name='101')
>>> dev1=Device.objects.create(name="router", building=b)
>>> dev2=Device.objects.create(name="switch", building=b)
>>> dev3=Device.objects.create(name="server", building=b)
>>> port1=Port.objects.create(number='4',device=dev1)
>>> port2=Port.objects.create(number='7',device=dev2)
>>> port3=Port.objects.create(number='1',device=dev3)
>>> c1=Connection.objects.create(start=port1, end=port2)
>>> c2=Connection.objects.create(start=port2, end=port3)

>>> connections=Connection.objects.filter(start__device__building=b, end__device__building=b).order_by('id')
>>> [(c.id, unicode(c.start), unicode(c.end)) for c in connections]
[(1, u'router/4', u'switch/7'), (2, u'switch/7', u'server/1')]

>>> connections=Connection.objects.filter(start__device__building=b, end__device__building=b).select_related().order_by('id')
>>> [(c.id, unicode(c.start), unicode(c.end)) for c in connections]
[(1, u'router/4', u'switch/7'), (2, u'switch/7', u'server/1')]

# This final query should only join seven tables (port, device and building
# twice each, plus connection once).
>>> connections.query.count_active_tables()
7

"""}
<commit_msg>Renamed a field in a test case that was preventing the tests from  running in Oracle.<commit_after>from django.db import models

class Building(models.Model):
    name = models.CharField(max_length=10)

    def __unicode__(self):
        return u"Building: %s" % self.name

class Device(models.Model):
    building = models.ForeignKey('Building')
    name = models.CharField(max_length=10)

    def __unicode__(self):
        return u"device '%s' in building %s" % (self.name, self.building)

class Port(models.Model):
    device = models.ForeignKey('Device')
    port_number = models.CharField(max_length=10)

    def __unicode__(self):
        return u"%s/%s" % (self.device.name, self.port_number)

class Connection(models.Model):
    start = models.ForeignKey(Port, related_name='connection_start',
            unique=True)
    end = models.ForeignKey(Port, related_name='connection_end', unique=True)

    def __unicode__(self):
        return u"%s to %s" % (self.start, self.end)

__test__ = {'API_TESTS': """
Regression test for bug #7110. When using select_related(), we must query the
Device and Building tables using two different aliases (each) in order to
differentiate the start and end Connection fields. The net result is that both
the "connections = ..." queries here should give the same results.

>>> b=Building.objects.create(name='101')
>>> dev1=Device.objects.create(name="router", building=b)
>>> dev2=Device.objects.create(name="switch", building=b)
>>> dev3=Device.objects.create(name="server", building=b)
>>> port1=Port.objects.create(port_number='4',device=dev1)
>>> port2=Port.objects.create(port_number='7',device=dev2)
>>> port3=Port.objects.create(port_number='1',device=dev3)
>>> c1=Connection.objects.create(start=port1, end=port2)
>>> c2=Connection.objects.create(start=port2, end=port3)

>>> connections=Connection.objects.filter(start__device__building=b, end__device__building=b).order_by('id')
>>> [(c.id, unicode(c.start), unicode(c.end)) for c in connections]
[(1, u'router/4', u'switch/7'), (2, u'switch/7', u'server/1')]

>>> connections=Connection.objects.filter(start__device__building=b, end__device__building=b).select_related().order_by('id')
>>> [(c.id, unicode(c.start), unicode(c.end)) for c in connections]
[(1, u'router/4', u'switch/7'), (2, u'switch/7', u'server/1')]

# This final query should only join seven tables (port, device and building
# twice each, plus connection once).
>>> connections.query.count_active_tables()
7

"""}
<|endoftext|>


<commit_before>#!/bin/python

#from subprocess import run
from subprocess import Popen, PIPE
import random
from copy import deepcopy
import operator
from functools import reduce
from urllib import request, parse
import json
import requests
import datetime
import os

from Settings import getSettings

gekkoDIR = 'TBD'

def getURL(path):
    pass
#return random.choice(gekkoURLs)+path
epochToString = lambda D: datetime.datetime.utcfromtimestamp(D).strftime(DateFormat)

def initializeGekko(): # not used yet.
    CMD = ['node', gekkoDIR + '/gekko', '--ui']
    D = Popen(CMD, stdin=PIPE, stdout=PIPE, stderr=PIPE)

def httpPost(URL, data={}):
    try:

        Request = requests.post(URL, json=data)
        Response = json.loads(Request.text)

    except ConnectionRefusedError:
        print("Error: Gekko comm error! Check your local Gekko instance.")
        exit()
    except Exception as e:
        print("Error: config failure")
        print(URL)
        print(data)
        raise e

    return Response

def getAllScanset():
    globalconf = getSettings('Global')
    base = random.choice(globalconf.GekkoURLs)
    URL = base + '/api/scansets'

    RESP = httpPost(URL)

    return RESP['datasets']

def getAvailableDataset(exchange_source=None):

    DataSetPack = getAllScanset()

    scanset = []
    for s in DataSetPack:
        Valid = True
        for k in "exchange currency asset".split(" "):
            if exchange_source and s[k] != exchange_source[k]:
                Valid = False
                continue
        if Valid:
            scanset.append(s)

    if len(scanset) == 0:
        if exchange_source:
            raise "scanset not available: %s" % exchange_source
        else:
            raise "no scanset available! check Gekko candle database."

    for EXCHANGE in scanset:
        ranges = EXCHANGE['ranges']
        range_spans = [x['to']-x['from'] for x in ranges]
        LONGEST = range_spans.index(max(range_spans))
        EXCHANGE['max_span'] = range_spans[LONGEST]
        EXCHANGE['max_span_index'] = LONGEST

    exchange_longest_spans = [ x['max_span'] for x in scanset ]
    best_exchange = exchange_longest_spans.index(max(exchange_longest_spans))

    chosenScansetRange = scanset[ best_exchange]['ranges'][scanset[best_exchange]['max_span_index'] ]

    specKeys = ['exchange', 'currency', 'asset']

    chosenScansetSpecifications = {K:scanset[best_exchange][K] for K in scanset[best_exchange] if K in specKeys}

    return chosenScansetSpecifications, chosenScansetRange

def loadHostsFile(HostsFilePath):
    remoteGekkos=[]
    if os.path.isfile(HostsFilePath):
        H = open(HostsFilePath).read().split('\n')
        for W in H:
            if W and not '=' in W and not '[' in W:
                remoteGekkos.append("http://%s:3000" % W)

    return remoteGekkos

def runBacktest(GekkoInstanceUrl, TradeSetting, Database, DateRange, candleSize=10, gekko_config=None):
    gekko_config = createConfig(TradeSetting, Database, DateRange, candleSize, gekko_config)
    url = GekkoInstanceUrl+'/api/backtest'
    result = httpPost(url, gekko_config)
    # sometime report is False(not dict)
    if type(result['report']) is bool:
        print("Warning: report not found, probable Gekko fail!")
        print(DateRange)

        # That fail is so rare that has no impact.. still happens randomly;
        return {'relativeProfit':0, 'market':0, 'trades':0, 'sharpe':0} # fake backtest report

    #rProfit = result['report']['relativeProfit']
    #nbTransactions = result['report']['trades']
    #market = result['report']['market']
    return result['report']

def firePaperTrader(GekkoInstanceUrl, TradeSetting, Exchange, Currency, Asset):

    TradeMethod = list(TradeSetting.keys())[0]
    true = True
    false= False

    CONFIG = {
        "market":{
            "type":"leech",
            "from":"2017-09-13T15:42:00Z" # TIME ATM;
        },
        "mode":"realtime",
        "watch":{
            "exchange": Exchange,
            "currency": Currency,
            "asset": Asset
        },
        "tradingAdvisor":{
            "enabled":true,
            "method":TradeMethod,
            "candleSize":60,
            "historySize":10},
        TradeMethod: TradeSetting[TradeMethod],
        "paperTrader":{
            "fee":0.25,
            "slippage":0.05,
            "simulationBalance":{
                "asset":1,
                "currency":100
            },
            "reportRoundtrips":true,
            "enabled":true
        },
        "candleWriter":{
            "enabled":true,
            "adapter":"sqlite"
        },
        "type":
        "paper trader",
        "performanceAnalyzer":{
            "riskFreeReturn":2,
            "enabled":true},
        "valid":true
    }

    RESULT = httpPost(URL,CONFIG)
    print(RESULT)

def createConfig(TradeSetting, Database, DateRange, candleSize=10, gekko_config=None):

    TradeMethod = list(TradeSetting.keys())[0]
    true = True
    false= False

    CONFIG = {
        "gekkoConfig": {
            "debug":false,
            "info":false,
            "watch": Database,
            "paperTrader": {
                "fee": 0.25, # declare deprecated 'fee' so keeps working w/ old gekko;
                "feeMaker": 0.15,
                "feeTaker": 0.25,
                "feeUsing": 'maker',
                "slippage":0.05,
                "simulationBalance": {
                    "asset":1,
                    "currency":100
                },
                "reportRoundtrips":true,
                "enabled":true
            },
            "tradingAdvisor": {
                "enabled":true,
                "method": TradeMethod,
                "candleSize": candleSize, # candleSize: smaller = heavier computation + better possible results;
                "historySize":10
            },
            TradeMethod: TradeSetting[TradeMethod],
             "backtest": {
                 "daterange": DateRange
             },
             "performanceAnalyzer": {
                 "riskFreeReturn":2,
                 "enabled":true
             },
            "valid":true
        },
            "data": {
                "candleProps": ["id", "start", "open", "high", "low", "close", "vwp", "volume", "trades"],
                "indicatorResults":true,
                "report":true,
                "roundtrips":false,
                "trades":true
            }
    }
    if gekko_config == None:
        gekko_config = CONFIG
    return gekko_config

def getCandles(DateRange, size=100):
    globalconf = getSettings('Global')
    base = random.choice(globalconf.GekkoURLs)

    URL = base + "/api/getCandles"
    CONFIG = {
        "watch": {
            "exchange": "poloniex",
            "currency": "BTC",
            "asset": "ETH"
            },
        "daterange": DateRange,
        "adapter": "sqlite",
        "sqlite": {
            "path": "plugins/sqlite",

            "dataDirectory": "history",
            "version": 0.1,

            "dependencies": [{
                "module": "sqlite3",
                "version": "3.1.4"
                }]
            },
        "candleSize": size
    }

    RESULT = httpPost(URL, CONFIG)
    return RESULT

def Evaluate(candleSize, Database, DateRange, phenotype, GekkoInstanceUrl):
    # IndividualToSettings(IND, STRAT) is a function that depends on GA algorithm,
    # so should be provided;

    result = [ runBacktest(GekkoInstanceUrl,
                           phenotype, Database,
                           DR,
                           candleSize=candleSize) for DR in DateRange ]

    RelativeProfits = [ R['relativeProfit']-R['market'] for R in result]
    avgTrades = sum( [R['trades'] for R in result] ) / len(DateRange)
    mRelativeProfit = sum(RelativeProfits)/len(RelativeProfits)

    avgSharpe = sum ( [R['sharpe'] for R in result if R['sharpe']]) / len(DateRange)

    return (mRelativeProfit, avgSharpe), avgTrades

def getDateRange(Limits, deltaDays=3):
    DateFormat="%Y-%m-%d %H:%M:%S"

    deltams=deltaDays * 24 * 60 * 60

    DateRange = {
        "from": "%s" % epochToString(Limits['to']-deltams),
        "to": "%s" % epochToString(Limits['to'])
    }
    return DateRange

def getRandomDateRange(Limits, deltaDays):

    FLms = Limits['from']
    TLms = Limits['to']
    deltams=deltaDays * 24 * 60 * 60

    if deltams > (TLms - FLms):
        print("Fatal: deltaDays on Settings.py set to a value bigger than current dataset.\n Edit Settings file to fit your chosen candlestick data.")
        exit()

    Starting= random.randint(FLms,TLms-deltams)
    DateRange = {
        "from": "%s" % epochToString(Starting),
        "to": "%s" % epochToString(Starting+deltams)
    }

    return DateRange

epochToString = lambda D: datetime.datetime.utcfromtimestamp(D).strftime("%Y-%m-%d %H:%M:%S")

def globalEvaluationDataset(DatasetLimits, deltaDays, NB):
    Dataset = []
    for W in range(NB):
        DateRange = getRandomDateRange(DatasetLimits, deltaDays)
        Dataset.append(DateRange)

    return Dataset

<commit_msg>dataset not found debug<commit_after>#!/bin/python

#from subprocess import run
from subprocess import Popen, PIPE
import random
from copy import deepcopy
import operator
from functools import reduce
from urllib import request, parse
import json
import requests
import datetime
import os

from Settings import getSettings

gekkoDIR = 'TBD'

def getURL(path):
    pass
#return random.choice(gekkoURLs)+path
epochToString = lambda D: datetime.datetime.utcfromtimestamp(D).strftime(DateFormat)

def initializeGekko(): # not used yet.
    CMD = ['node', gekkoDIR + '/gekko', '--ui']
    D = Popen(CMD, stdin=PIPE, stdout=PIPE, stderr=PIPE)

def httpPost(URL, data={}):
    try:

        Request = requests.post(URL, json=data)
        Response = json.loads(Request.text)

    except ConnectionRefusedError:
        print("Error: Gekko comm error! Check your local Gekko instance.")
        exit()
    except Exception as e:
        print("Error: config failure")
        print(URL)
        print(data)
        raise e

    return Response

def getAllScanset():
    globalconf = getSettings('Global')
    base = random.choice(globalconf.GekkoURLs)
    URL = base + '/api/scansets'

    RESP = httpPost(URL)

    return RESP['datasets']

def getAvailableDataset(exchange_source=None):

    DataSetPack = getAllScanset()

    scanset = []
    for s in DataSetPack:
        Valid = True
        for k in "exchange currency asset".split(" "):
            if exchange_source and s[k] != exchange_source[k]:
                Valid = False
                continue
        if Valid:
            scanset.append(s)

    if len(scanset) == 0:
        if exchange_source:
            raise RuntimeError("scanset not available: %s\n\tscanset found: %s" % (exchange_source, DataSetPack))
        else:
            raise RuntimeError("no scanset available! check Gekko candle database.")

    for EXCHANGE in scanset:
        ranges = EXCHANGE['ranges']
        range_spans = [x['to']-x['from'] for x in ranges]
        LONGEST = range_spans.index(max(range_spans))
        EXCHANGE['max_span'] = range_spans[LONGEST]
        EXCHANGE['max_span_index'] = LONGEST

    exchange_longest_spans = [ x['max_span'] for x in scanset ]
    best_exchange = exchange_longest_spans.index(max(exchange_longest_spans))

    chosenScansetRange = scanset[ best_exchange]['ranges'][scanset[best_exchange]['max_span_index'] ]

    specKeys = ['exchange', 'currency', 'asset']

    chosenScansetSpecifications = {K:scanset[best_exchange][K] for K in scanset[best_exchange] if K in specKeys}

    return chosenScansetSpecifications, chosenScansetRange

def loadHostsFile(HostsFilePath):
    remoteGekkos=[]
    if os.path.isfile(HostsFilePath):
        H = open(HostsFilePath).read().split('\n')
        for W in H:
            if W and not '=' in W and not '[' in W:
                remoteGekkos.append("http://%s:3000" % W)

    return remoteGekkos

def runBacktest(GekkoInstanceUrl, TradeSetting, Database, DateRange, candleSize=10, gekko_config=None):
    gekko_config = createConfig(TradeSetting, Database, DateRange, candleSize, gekko_config)
    url = GekkoInstanceUrl+'/api/backtest'
    result = httpPost(url, gekko_config)
    # sometime report is False(not dict)
    if type(result['report']) is bool:
        print("Warning: report not found, probable Gekko fail!")
        print(DateRange)

        # That fail is so rare that has no impact.. still happens randomly;
        return {'relativeProfit':0, 'market':0, 'trades':0, 'sharpe':0} # fake backtest report

    #rProfit = result['report']['relativeProfit']
    #nbTransactions = result['report']['trades']
    #market = result['report']['market']
    return result['report']

def firePaperTrader(GekkoInstanceUrl, TradeSetting, Exchange, Currency, Asset):

    TradeMethod = list(TradeSetting.keys())[0]
    true = True
    false= False

    CONFIG = {
        "market":{
            "type":"leech",
            "from":"2017-09-13T15:42:00Z" # TIME ATM;
        },
        "mode":"realtime",
        "watch":{
            "exchange": Exchange,
            "currency": Currency,
            "asset": Asset
        },
        "tradingAdvisor":{
            "enabled":true,
            "method":TradeMethod,
            "candleSize":60,
            "historySize":10},
        TradeMethod: TradeSetting[TradeMethod],
        "paperTrader":{
            "fee":0.25,
            "slippage":0.05,
            "simulationBalance":{
                "asset":1,
                "currency":100
            },
            "reportRoundtrips":true,
            "enabled":true
        },
        "candleWriter":{
            "enabled":true,
            "adapter":"sqlite"
        },
        "type":
        "paper trader",
        "performanceAnalyzer":{
            "riskFreeReturn":2,
            "enabled":true},
        "valid":true
    }

    RESULT = httpPost(URL,CONFIG)
    print(RESULT)

def createConfig(TradeSetting, Database, DateRange, candleSize=10, gekko_config=None):

    TradeMethod = list(TradeSetting.keys())[0]
    true = True
    false= False

    CONFIG = {
        "gekkoConfig": {
            "debug":false,
            "info":false,
            "watch": Database,
            "paperTrader": {
                "fee": 0.25, # declare deprecated 'fee' so keeps working w/ old gekko;
                "feeMaker": 0.15,
                "feeTaker": 0.25,
                "feeUsing": 'maker',
                "slippage":0.05,
                "simulationBalance": {
                    "asset":1,
                    "currency":100
                },
                "reportRoundtrips":true,
                "enabled":true
            },
            "tradingAdvisor": {
                "enabled":true,
                "method": TradeMethod,
                "candleSize": candleSize, # candleSize: smaller = heavier computation + better possible results;
                "historySize":10
            },
            TradeMethod: TradeSetting[TradeMethod],
             "backtest": {
                 "daterange": DateRange
             },
             "performanceAnalyzer": {
                 "riskFreeReturn":2,
                 "enabled":true
             },
            "valid":true
        },
            "data": {
                "candleProps": ["id", "start", "open", "high", "low", "close", "vwp", "volume", "trades"],
                "indicatorResults":true,
                "report":true,
                "roundtrips":false,
                "trades":true
            }
    }
    if gekko_config == None:
        gekko_config = CONFIG
    return gekko_config

def getCandles(DateRange, size=100):
    globalconf = getSettings('Global')
    base = random.choice(globalconf.GekkoURLs)

    URL = base + "/api/getCandles"
    CONFIG = {
        "watch": {
            "exchange": "poloniex",
            "currency": "BTC",
            "asset": "ETH"
            },
        "daterange": DateRange,
        "adapter": "sqlite",
        "sqlite": {
            "path": "plugins/sqlite",

            "dataDirectory": "history",
            "version": 0.1,

            "dependencies": [{
                "module": "sqlite3",
                "version": "3.1.4"
                }]
            },
        "candleSize": size
    }

    RESULT = httpPost(URL, CONFIG)
    return RESULT

def Evaluate(candleSize, Database, DateRange, phenotype, GekkoInstanceUrl):
    # IndividualToSettings(IND, STRAT) is a function that depends on GA algorithm,
    # so should be provided;

    result = [ runBacktest(GekkoInstanceUrl,
                           phenotype, Database,
                           DR,
                           candleSize=candleSize) for DR in DateRange ]

    RelativeProfits = [ R['relativeProfit']-R['market'] for R in result]
    avgTrades = sum( [R['trades'] for R in result] ) / len(DateRange)
    mRelativeProfit = sum(RelativeProfits)/len(RelativeProfits)

    avgSharpe = sum ( [R['sharpe'] for R in result if R['sharpe']]) / len(DateRange)

    return (mRelativeProfit, avgSharpe), avgTrades

def getDateRange(Limits, deltaDays=3):
    DateFormat="%Y-%m-%d %H:%M:%S"

    deltams=deltaDays * 24 * 60 * 60

    DateRange = {
        "from": "%s" % epochToString(Limits['to']-deltams),
        "to": "%s" % epochToString(Limits['to'])
    }
    return DateRange

def getRandomDateRange(Limits, deltaDays):

    FLms = Limits['from']
    TLms = Limits['to']
    deltams=deltaDays * 24 * 60 * 60

    if deltams > (TLms - FLms):
        print("Fatal: deltaDays on Settings.py set to a value bigger than current dataset.\n Edit Settings file to fit your chosen candlestick data.")
        exit()

    Starting= random.randint(FLms,TLms-deltams)
    DateRange = {
        "from": "%s" % epochToString(Starting),
        "to": "%s" % epochToString(Starting+deltams)
    }

    return DateRange

epochToString = lambda D: datetime.datetime.utcfromtimestamp(D).strftime("%Y-%m-%d %H:%M:%S")

def globalEvaluationDataset(DatasetLimits, deltaDays, NB):
    Dataset = []
    for W in range(NB):
        DateRange = getRandomDateRange(DatasetLimits, deltaDays)
        Dataset.append(DateRange)

    return Dataset

<|endoftext|>