<commit_before>#!/usr/bin/env python
'''
Qubinode - Quick Bitcoin Node Deploy

Usage:
  qubinode.py spawn (do|digitalocean) [--do-size=<slug>]
                    [--priv-key-path=<path>] [--pub-key-path=<path>]
  qubinode.py local [--release=<version>] [--prune=<MB>]
                    [--swapfile-size=<MB>] [--swapfile-path=<path>]
  qubinode.py list-versions
  qubinode.py list-providers

Options:
  -h --help                  Show this screen.
  --version                  Show version.
  -b --batch                 Non-interactive and choose all options by default.

Install options:
  -s --swapfile=<MB>         Create a swapfile
  --swapfile-path=<path>     Specify swapfile path [default: /swapfile]

Bitcoin options:
  -r --release=<version>     Bitcoin node release [default: XT/0.11D]
  -p --prune=<MB>            Blockchain pruning [default: 2048]

Spawn options:
  --priv-key-path=<path>     [default: ~/.ssh/qubinode]
  --pub-key-path=<path>      [default: ~/.ssh/qubinode.pub]

DigitalOcean options:
  --do-size=<slug>           Size of the provider's instance [default: 512MB]

'''
import os
import sys
import glob
import shutil
import string
import functools
import random
import hashlib
import subprocess
import textwrap

import requests
from docopt import docopt
from Crypto.PublicKey import RSA
import paramiko
import digitalocean as do

__version__ = '0.0.1'

NAMES = {
    'XT': 'BitcoinXT',
}

RELEASES = {
    'XT/0.11D': {
        'url': 'https://github.com/bitcoinxt/bitcoinxt/releases/download/v0.11D/bitcoin-xt-0.11.0-D-linux64.tar.gz',
        'dir': 'bitcoin-xt-0.11.0-D',
        'sha256': 'ba0e8d553271687bc8184a4a7070e5d350171036f13c838db49bb0aabe5c5e49',
        'install': 'copy_to_root',
    },
}

PROVIDERS = {
    'do': {
        'name': 'DigitalOcean',
        'class': 'DigitalOcean',
    }
}

class ChecksumException(Exception):
    pass

class ExtractionException(Exception):
    pass

class InstallationException(Exception):
    pass

class Qubinode:
    def run(self):
        self.config = Config()
        self.config.setup()

        if self.config.spawn:
            self.boot()

        if self.config.local:
            Installer(self.config).setup()

    def boot(self):
        '''
        Run the provider class that will start the VM.

        e.g. the "do" provider will instantiate the DigitalOcean class.
        '''
        fun = globals()[self.config.provider['class']]
        fun(self.config).setup()

class Config(object):
    def __init__(self):
        self.args = {}

    def __getattribute__(self, key):
        '''
        Allows attribute access to the docopt args dictionary.

        e.g. config.batch instead of config.args.batch
        '''
        if key != 'args' and key in self.args:
            return self.args[key]
        else:
            return object.__getattribute__(self, key)

    @property
    def provider_key(self):
        if self.do or self.digitalocean:
            return 'do'

    @property
    def provider(self):
        return PROVIDERS[self.provider_key]

    def parse_args(self):
        self.args = docopt(__doc__, version=__version__)

    def setup(self):
        self.parse_args()
        # self.ask()
        self.normalise()

    def normalise_path(self, key):
        self.args[key] = os.path.expanduser(self.args[key])
        self.args[key] = os.path.abspath(self.args[key])

    def normalise(self):
        self.normalise_path('--priv-key-path')
        self.normalise_path('--pub-key-path')

        # Allows attribute access using allow characters, e.g.:
        # --priv-key-path -> self.config.priv_key_path
        for k, v in self.args.items():
            self.args[k.replace('--', '').replace('-', '_')] = v

    def ask_version(self):
        #for version, release in VERSIONS.items():
        #    print('{}: {}'.format(version, NAMES[version.split('/')[0]]
        self.release = 'XT/0.11D'

class Provider:
    def __init__(self, config):
        self.config = config
        self.ensure_key_pairs()

    def ensure_key_pairs(self):
        if not os.path.exists(self.config.priv_key_path):
            self.generate_key_pairs()
        self.load_key_pairs()

    def ensure_key_paths(self):
        try:
            os.makedirs(os.path.dirname(self.config.priv_key_path))
        except OSError:
            pass

        try:
            os.makedirs(os.path.dirname(self.config.priv_key_path))
        except OSError:
            pass

    def generate_key_pairs(self):
        self.ensure_key_paths()

        key = RSA.generate(2048)

        with open(self.config.priv_key_path, 'w') as f:
            f.write(key.exportKey('PEM'))
        with open(self.config.pub_key_path, 'w') as f:
            f.write(key.publickey().exportKey('OpenSSH'))

        os.chmod(self.config.priv_key_path, 0600)

    def load_key_pairs(self):
        self.pub_key = open(self.config.pub_key_path).read()
        self.priv_key = open(self.config.priv_key_path).read()

    def connect(self):
	self.ip_address = '188.166.71.64'
	print('Connecting to {}...'.format(self.ip_address))
	pkey = paramiko.RSAKey.from_private_key_file(
	    self.config.priv_key_path
	)
        self.transport = paramiko.Transport((self.ip_address, 22))
        self.transport.connect(username='root', pkey=pkey)

    def remote_put(self, sftp, filename):
        try:
            if sftp.lstat(filename):
                sftp.unlink(filename)
        except IOError:
            pass

        sftp.put(filename, os.path.join('qubinode', filename))

    def deploy(self):
        print('Uploading Qubinode...')
        sftp = paramiko.SFTPClient.from_transport(self.transport)

        try:
            sftp.lstat('qubinode')
        except IOError:
            sftp.mkdir('qubinode')

        self.remote_put(sftp, 'bootstrap.sh')
        self.remote_put(sftp, 'qubinode.py')

    def run(self):
        print('Executing Qubinode on host...')
        self.channel = self.transport.open_channel('session')
        #self.channel.get_pty()
        self.channel.exec_command(
                'bash -c "cd qubinode && bash bootstrap.sh local '
                '--swapfile-size=2048 --prune=2048"'
            )

        while not self.channel.exit_status_ready():
            while self.channel.recv_ready():
                data = self.channel.recv(1024)
                sys.stdout.write(data)
            while self.channel.recv_stderr_ready():
                data = self.channel.recv_stderr(1024)
                sys.stdout.write(data)

        status = self.channel.recv_exit_status()
        print(status)

        print('Done!')

class DigitalOcean(Provider):
    def setup(self):
        #self.ask_token()
        #self.prepare_ssh()
        #self.get_regions()
        #self.choose_random_region()
        #self.create_droplet()
        #self.wait_for_droplet()
        self.connect()
        self.deploy()
        self.run()

    def ask_token(self):
        print(textwrap.dedent('''
            We will now:
             - Generate or reuse an SSH key pair in *TODO* some directory.
             - Create a $5/mo, 512MB, 1 CPU instance in a random data center.

            Once started:
             - A SSH key pair will be left in *TODO* some directory which can
               be used for SSHing into the droplet.
             - Your droplet will continue living forever until you stop it.
             - Your DigitalOcean access token will not be saved to disk and
               will be forgotten once the script has ended.

            Press Ctrl-C to abort!
        '''))

        self.token = raw_input('Enter a generated DigitalOcean access token: ')
        self.manager = functools.partial(do.Manager, token=self.token)
        self.droplet = functools.partial(do.Droplet, token=self.token)
        self.ssh_do = functools.partial(do.SSHKey, token=self.token)

    def prepare_ssh(self):
        self.ssh_id = self.ssh_do(name='qubinode', public_key=self.pub_key)
        if not self.ssh_id.load_by_pub_key(self.pub_key):
            self.ssh_id.create()

    def get_regions(self):
        self.regions = self.manager().get_all_regions()

    def choose_random_region(self):
        self.region = random.choice(self.regions).slug

    def create_droplet(self):
        self.instance = self.droplet(
            name='qubinode',
            region=self.region,
            image='ubuntu-14-04-x64',
            size_slug='512mb',
            ssh_keys=[self.pub_key],
        )
        self.instance.create()

    def wait_for_droplet(self):
        print('Waiting for Droplet (1-2 minutes)...')

        while True:
            print('...')
            actions = self.instance.get_actions()
            for action in actions:
                action.load()
                print(action.status)
                if action.status == 'completed':
                    return True

    def ip_address(self):
        return self.instance.ip_address

class Installer:
    def __init__(self, config):
        self.config = config
        self.release = RELEASES[self.config.release]
        self.filename = self.release['url'].split('/')[-1]
        self.data_dir = os.path.expanduser('~/.bitcoin')

    def setup(self):
        self.swap()
        self.fetch()
        self.checksum()
        self.extract()
        self.install()
        self.upstart()
        self.logrotate()
        self.configure()
        self.restart()

    def shell(self, cmd):
        return subprocess.call(cmd, shell=True)

    def random_string(self, length=40):
        return ''.join([
            random.choice(string.ascii_letters)
            for a in xrange(length)
        ])

    def swap(self, mb=2048):
        '''
        Creates and activates a swapfile, defaulting to 2GB.

        Also adds it to /etc/fstab
        '''
        if os.path.exists('/swapfile'):
            print('Swapfile already set up...')
            return

        print('Setting up swapfile')
        self.shell('fallocate -l {}M /swapfile'.format(mb))
        self.shell('chmod 600 /swapfile')
        self.shell('mkswap /swapfile')
        self.shell('swapon /swapfile')

        open('/etc/fstab', 'a').write('\n/swapfile none swap defaults 0 0\n')

    def fetch(self):
        if os.path.exists(self.filename):
            return
        url = self.release['url']
        print('Downloading {}'.format(url))
        r = requests.get(url, stream=True)
	with open(self.filename, 'wb') as f:
	    for chunk in r.iter_content(chunk_size=1024):
		if chunk:
		    f.write(chunk)
        print('Downloading {} complete...'.format(url))

    def downloaded_hash(self):
        return hashlib.sha256(open(self.filename, 'rb').read()).hexdigest()

    def checksum(self):
        print('Checking sha256 of package...')
        downloaded_hash = self.downloaded_hash()
        if downloaded_hash == self.release['sha256']:
            return

        raise ChecksumException(
            '{} from {} expected {} got {}'.format(
                self.filename,
                self.release['url'],
                self.release['sha256'],
                downloaded_hash,
            )
        )

    def extract(self):
        self.shell('tar xvf {}'.format(self.filename))

        if os.path.exists(self.release['dir']):
            return

        raise ExtractionException('Could not see extracted directory')

    def install(self):
        '''
        Dynamically get the method for installing files. e.g. "copy_to_root"
        would translate to self.copy_to_root().
        '''
        print('Installing files...')
        fun = getattr(self, self.release['install'])
        fun()

    def upstart(self):
        '''
        Create an upstart script so bitcoind can start on boot.
        '''
        print('Creating boot scripts...')
        with open('/etc/init/bitcoind.conf', 'w') as f:
            f.write(textwrap.dedent('''
            description "bitcoind"

            start on filesystem
            stop on runlevel [!2345]
            respawn
            respawn limit 10 180
            kill timeout 60

            script
            user=root
            home=/root/.bitcoin
            cmd=/usr/bin/bitcoind
            args="-disablewallet -printtoconsole"
            pidfile=$home/bitcoind.pid
            exec start-stop-daemon --start --make-pidfile -c $user --chdir $home --pidfile $pidfile --exec $cmd -- $args
            end script
            '''))

    def logrotate(self):
        print('Creating logrotate configuration...')
        with open('/etc/logrotate.d/bitcoin-debug-log', 'w') as f:
            f.write(textwrap.dedent('''
	    /root/.bitcoin/debug.log
	    {
		rotate 5
		copytruncate
		daily
		missingok
		notifempty
		compress
		delaycompress
		sharedscripts
	    }
            '''))

    def configure(self):
        print('Creating bitcoin.conf')
        if not os.path.exists(self.data_dir):
            os.mkdir(self.data_dir)

        with open(os.path.join(self.data_dir, 'bitcoin.conf'), 'w') as f:
            if self.config.prune:
                f.write('prune={}\n'.format(self.config.prune))
            f.write('rpcuser={}\n'.format(self.random_string()))
            f.write('rpcpassword={}\n'.format(self.random_string()))

    def restart(self):
        print('Starting daemon...')
        self.shell('stop bitcoind')
        self.shell('start bitcoind')

    def copy_to_root(self):
        '''
        Copies all directories and files from the extracted directory
        into /usr/.

        e.g. bitcoind-xt-0.11D/bin/bitcoind -> /usr/bin/bitcoind
        '''
        self.shell('cd {} && cp -vr * /usr/'.format(
            self.release['dir'],
        ))

        if os.path.exists('/usr/bin/bitcoind'):
            return

        raise InstallationException('/usr/bin/bitcoind does not exist')

if __name__ == '__main__':
    Qubinode().run()

<commit_msg>less hardcody and debuggy<commit_after>#!/usr/bin/env python
'''
Qubinode - Quick Bitcoin Node Deploy

Usage:
  qubinode.py spawn (do|digitalocean) [--do-size=<slug>]
                    [--priv-key-path=<path>] [--pub-key-path=<path>]
  qubinode.py local [--release=<version>] [--prune=<MB>]
                    [--swapfile-size=<MB>] [--swapfile-path=<path>]
  qubinode.py list-versions
  qubinode.py list-providers

Options:
  -h --help                  Show this screen.
  --version                  Show version.
  -b --batch                 Non-interactive and choose all options by default.

Install options:
  -s --swapfile=<MB>         Create a swapfile
  --swapfile-path=<path>     Specify swapfile path [default: /swapfile]

Bitcoin options:
  -r --release=<version>     Bitcoin node release [default: XT/0.11D]
  -p --prune=<MB>            Blockchain pruning [default: 2048]

Spawn options:
  --priv-key-path=<path>     [default: ~/.ssh/qubinode]
  --pub-key-path=<path>      [default: ~/.ssh/qubinode.pub]

DigitalOcean options:
  --do-size=<slug>           Size of the provider's instance [default: 512MB]

'''
import os
import sys
import glob
import shutil
import string
import functools
import random
import hashlib
import subprocess
import textwrap

import requests
from docopt import docopt
from Crypto.PublicKey import RSA
import paramiko
import digitalocean as do

__version__ = '0.0.1'

NAMES = {
    'XT': 'BitcoinXT',
}

RELEASES = {
    'XT/0.11D': {
        'url': 'https://github.com/bitcoinxt/bitcoinxt/releases/download/v0.11D/bitcoin-xt-0.11.0-D-linux64.tar.gz',
        'dir': 'bitcoin-xt-0.11.0-D',
        'sha256': 'ba0e8d553271687bc8184a4a7070e5d350171036f13c838db49bb0aabe5c5e49',
        'install': 'copy_to_root',
    },
}

PROVIDERS = {
    'do': {
        'name': 'DigitalOcean',
        'class': 'DigitalOcean',
    }
}

class ChecksumException(Exception):
    pass

class ExtractionException(Exception):
    pass

class InstallationException(Exception):
    pass

class Qubinode:
    def run(self):
        self.config = Config()
        self.config.setup()

        if self.config.spawn:
            self.boot()

        if self.config.local:
            Installer(self.config).setup()

    def boot(self):
        '''
        Run the provider class that will start the VM.

        e.g. the "do" provider will instantiate the DigitalOcean class.
        '''
        fun = globals()[self.config.provider['class']]
        fun(self.config).setup()

class Config(object):
    def __init__(self):
        self.args = {}

    def __getattribute__(self, key):
        '''
        Allows attribute access to the docopt args dictionary.

        e.g. config.batch instead of config.args.batch
        '''
        if key != 'args' and key in self.args:
            return self.args[key]
        else:
            return object.__getattribute__(self, key)

    @property
    def provider_key(self):
        if self.do or self.digitalocean:
            return 'do'

    @property
    def provider(self):
        return PROVIDERS[self.provider_key]

    def parse_args(self):
        self.args = docopt(__doc__, version=__version__)

    def setup(self):
        self.parse_args()
        # self.ask()
        self.normalise()

    def normalise_path(self, key):
        self.args[key] = os.path.expanduser(self.args[key])
        self.args[key] = os.path.abspath(self.args[key])

    def normalise(self):
        self.normalise_path('--priv-key-path')
        self.normalise_path('--pub-key-path')

        # Allows attribute access using allow characters, e.g.:
        # --priv-key-path -> self.config.priv_key_path
        for k, v in self.args.items():
            self.args[k.replace('--', '').replace('-', '_')] = v

    def ask_version(self):
        #for version, release in VERSIONS.items():
        #    print('{}: {}'.format(version, NAMES[version.split('/')[0]]
        self.release = 'XT/0.11D'

class Provider:
    def __init__(self, config):
        self.config = config
        self.ensure_key_pairs()

    def ensure_key_pairs(self):
        if not os.path.exists(self.config.priv_key_path):
            self.generate_key_pairs()
        self.load_key_pairs()

    def ensure_key_paths(self):
        try:
            os.makedirs(os.path.dirname(self.config.priv_key_path))
        except OSError:
            pass

        try:
            os.makedirs(os.path.dirname(self.config.priv_key_path))
        except OSError:
            pass

    def generate_key_pairs(self):
        self.ensure_key_paths()

        key = RSA.generate(2048)

        with open(self.config.priv_key_path, 'w') as f:
            f.write(key.exportKey('PEM'))
        with open(self.config.pub_key_path, 'w') as f:
            f.write(key.publickey().exportKey('OpenSSH'))

        os.chmod(self.config.priv_key_path, 0600)

    def load_key_pairs(self):
        self.pub_key = open(self.config.pub_key_path).read()
        self.priv_key = open(self.config.priv_key_path).read()

    def connect(self):
	print('Connecting to {}...'.format(self.ip_address))
	pkey = paramiko.RSAKey.from_private_key_file(
	    self.config.priv_key_path
	)
        self.transport = paramiko.Transport((self.ip_address, 22))
        self.transport.connect(username='root', pkey=pkey)

    def remote_put(self, sftp, filename):
        try:
            if sftp.lstat(filename):
                sftp.unlink(filename)
        except IOError:
            pass

        sftp.put(filename, os.path.join('qubinode', filename))

    def deploy(self):
        print('Uploading Qubinode...')
        sftp = paramiko.SFTPClient.from_transport(self.transport)

        try:
            sftp.lstat('qubinode')
        except IOError:
            sftp.mkdir('qubinode')

        self.remote_put(sftp, 'bootstrap.sh')
        self.remote_put(sftp, 'qubinode.py')

    def run(self):
        print('Executing Qubinode on host...')
        self.channel = self.transport.open_channel('session')
        #self.channel.get_pty()
        self.channel.exec_command(
                'bash -c "cd qubinode && bash bootstrap.sh local '
                '--swapfile-size=2048 --prune=2048"'
            )

        while not self.channel.exit_status_ready():
            while self.channel.recv_ready():
                data = self.channel.recv(1024)
                sys.stdout.write(data)
            while self.channel.recv_stderr_ready():
                data = self.channel.recv_stderr(1024)
                sys.stdout.write(data)

        status = self.channel.recv_exit_status()
        print(status)

        print('Done!')

class DigitalOcean(Provider):
    def setup(self):
        self.ask_token()
        self.prepare_ssh()
        self.get_regions()
        self.choose_random_region()
        self.create_droplet()
        self.wait_for_droplet()
        self.connect()
        self.deploy()
        self.run()

    def ask_token(self):
        print(textwrap.dedent('''
            We will now:
             - Generate or reuse an SSH key pair in *TODO* some directory.
             - Create a $5/mo, 512MB, 1 CPU instance in a random data center.

            Once started:
             - A SSH key pair will be left in *TODO* some directory which can
               be used for SSHing into the droplet.
             - Your droplet will continue living forever until you stop it.
             - Your DigitalOcean access token will not be saved to disk and
               will be forgotten once the script has ended.

            Press Ctrl-C to abort!
        '''))

        self.token = raw_input('Enter a generated DigitalOcean access token: ')
        self.manager = functools.partial(do.Manager, token=self.token)
        self.droplet = functools.partial(do.Droplet, token=self.token)
        self.ssh_do = functools.partial(do.SSHKey, token=self.token)

    def prepare_ssh(self):
        self.ssh_id = self.ssh_do(name='qubinode', public_key=self.pub_key)
        if not self.ssh_id.load_by_pub_key(self.pub_key):
            self.ssh_id.create()

    def get_regions(self):
        self.regions = self.manager().get_all_regions()

    def choose_random_region(self):
        self.region = random.choice(self.regions).slug

    def create_droplet(self):
        self.instance = self.droplet(
            name='qubinode',
            region=self.region,
            image='ubuntu-14-04-x64',
            size_slug='512mb',
            ssh_keys=[self.pub_key],
        )
        self.instance.create()

    def wait_for_droplet(self):
        print('Waiting for Droplet (1-2 minutes)...')

        while True:
            print('...')
            actions = self.instance.get_actions()
            for action in actions:
                action.load()
                print(action.status)
                if action.status == 'completed':
                    return True

    @property
    def ip_address(self):
        return self.instance.ip_address

class Installer:
    def __init__(self, config):
        self.config = config
        self.release = RELEASES[self.config.release]
        self.filename = self.release['url'].split('/')[-1]
        self.data_dir = os.path.expanduser('~/.bitcoin')

    def setup(self):
        self.swap()
        self.fetch()
        self.checksum()
        self.extract()
        self.install()
        self.upstart()
        self.logrotate()
        self.configure()
        self.restart()

    def shell(self, cmd):
        return subprocess.call(cmd, shell=True)

    def random_string(self, length=40):
        return ''.join([
            random.choice(string.ascii_letters)
            for a in xrange(length)
        ])

    def swap(self, mb=2048):
        '''
        Creates and activates a swapfile, defaulting to 2GB.

        Also adds it to /etc/fstab
        '''
        if os.path.exists('/swapfile'):
            print('Swapfile already set up...')
            return

        print('Setting up swapfile')
        self.shell('fallocate -l {}M /swapfile'.format(mb))
        self.shell('chmod 600 /swapfile')
        self.shell('mkswap /swapfile')
        self.shell('swapon /swapfile')

        open('/etc/fstab', 'a').write('\n/swapfile none swap defaults 0 0\n')

    def fetch(self):
        if os.path.exists(self.filename):
            return
        url = self.release['url']
        print('Downloading {}'.format(url))
        r = requests.get(url, stream=True)
	with open(self.filename, 'wb') as f:
	    for chunk in r.iter_content(chunk_size=1024):
		if chunk:
		    f.write(chunk)
        print('Downloading {} complete...'.format(url))

    def downloaded_hash(self):
        return hashlib.sha256(open(self.filename, 'rb').read()).hexdigest()

    def checksum(self):
        print('Checking sha256 of package...')
        downloaded_hash = self.downloaded_hash()
        if downloaded_hash == self.release['sha256']:
            return

        raise ChecksumException(
            '{} from {} expected {} got {}'.format(
                self.filename,
                self.release['url'],
                self.release['sha256'],
                downloaded_hash,
            )
        )

    def extract(self):
        self.shell('tar xvf {}'.format(self.filename))

        if os.path.exists(self.release['dir']):
            return

        raise ExtractionException('Could not see extracted directory')

    def install(self):
        '''
        Dynamically get the method for installing files. e.g. "copy_to_root"
        would translate to self.copy_to_root().
        '''
        print('Installing files...')
        fun = getattr(self, self.release['install'])
        fun()

    def upstart(self):
        '''
        Create an upstart script so bitcoind can start on boot.
        '''
        print('Creating boot scripts...')
        with open('/etc/init/bitcoind.conf', 'w') as f:
            f.write(textwrap.dedent('''
            description "bitcoind"

            start on filesystem
            stop on runlevel [!2345]
            respawn
            respawn limit 10 180
            kill timeout 60

            script
            user=root
            home=/root/.bitcoin
            cmd=/usr/bin/bitcoind
            args="-disablewallet -printtoconsole"
            pidfile=$home/bitcoind.pid
            exec start-stop-daemon --start --make-pidfile -c $user --chdir $home --pidfile $pidfile --exec $cmd -- $args
            end script
            '''))

    def logrotate(self):
        print('Creating logrotate configuration...')
        with open('/etc/logrotate.d/bitcoin-debug-log', 'w') as f:
            f.write(textwrap.dedent('''
	    /root/.bitcoin/debug.log
	    {
		rotate 5
		copytruncate
		daily
		missingok
		notifempty
		compress
		delaycompress
		sharedscripts
	    }
            '''))

    def configure(self):
        print('Creating bitcoin.conf')
        if not os.path.exists(self.data_dir):
            os.mkdir(self.data_dir)

        with open(os.path.join(self.data_dir, 'bitcoin.conf'), 'w') as f:
            if self.config.prune:
                f.write('prune={}\n'.format(self.config.prune))
            f.write('rpcuser={}\n'.format(self.random_string()))
            f.write('rpcpassword={}\n'.format(self.random_string()))

    def restart(self):
        print('Starting daemon...')
        self.shell('stop bitcoind')
        self.shell('start bitcoind')

    def copy_to_root(self):
        '''
        Copies all directories and files from the extracted directory
        into /usr/.

        e.g. bitcoind-xt-0.11D/bin/bitcoind -> /usr/bin/bitcoind
        '''
        self.shell('cd {} && cp -vr * /usr/'.format(
            self.release['dir'],
        ))

        if os.path.exists('/usr/bin/bitcoind'):
            return

        raise InstallationException('/usr/bin/bitcoind does not exist')

if __name__ == '__main__':
    Qubinode().run()

<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A server that runs FFmpeg on files in Google Cloud Storage.
"""

import os
import subprocess
from typing import Tuple

from flask import Flask
from flask import request
from flask import safe_join
from google.cloud import storage
from google.cloud.exceptions import NotFound

app = Flask(__name__)

@app.errorhandler(NotFound)
def _handle_gcloud_not_found(_):
    return 'Google Cloud storage object not found', 404

@app.route('/ffmpeg', methods=['POST'])
def process_ffmpeg_command():
    """Runs ffmpeg according to the request's specification.

    The POST request supports the following arguments:
        input_file: the input file in the Google Cloud Bucket specified
                    Example: bucket_name/path/to/file
        output_file: the output file that should be stored in the
                     Google Cloud Bucket specified
                     Example: bucket_name/path/to/file
    This route changes the video container format by running the following:
        ffmpeg -i {input_file} {output_file}

    Args:

    Returns:
        A string with the ffmpeg logs.
    """
    remote_input_file = request.form['input_file']
    local_input_file = _localize_filename(remote_input_file)
    download_file(local_input_file, remote_input_file)
    remote_output_file = request.form['output_file']
    local_output_file = _localize_filename(remote_output_file)

    ffmpeg_logs = subprocess.run(
        ["ffmpeg", "-i", local_input_file, local_output_file],
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        check=False).stdout
    os.remove(local_input_file)
    upload_file(local_output_file, remote_output_file)
    os.remove(local_output_file)
    return ffmpeg_logs

def download_file(local_filename: str, remote_filename: str) -> None:
    """Downloads the file from Google Cloud Storage

    This function downloads the file to the /tmp directory.

    Args:
        local_filename: the filepath of the downloaded file
        remote_filename: the filename preceded by the Google Cloud Bucket name
                         Example: bucket_name/path/to/file

    Returns:
        None
    """
    client = storage.Client()
    local_filename = _localize_filename(remote_filename)
    os.makedirs(os.path.dirname(local_filename), exist_ok=True)
    with open(local_filename, 'wb') as downloaded_file:
        client.download_blob_to_file('gs://' + remote_filename, downloaded_file)

def upload_file(local_filename: str, remote_filename: str) -> None:
    """Uploads the file from Google Cloud Storage.

    This function uploads the file to the Google Cloud Bucket specified.

    Args:
        local_filename: the filepath of the local file to upload
        remote_filename: the filename preceded by the Google Cloud Bucket name
                         Example: bucket_name/path/to/file

    Returns:
        None
    """
    client = storage.Client()
    bucket_name, blob_name = _split_remote_filename(remote_filename)
    bucket = client.bucket(bucket_name)
    blob = bucket.blob(blob_name)
    blob.upload_from_filename(local_filename)

def _split_remote_filename(remote_filename: str) -> Tuple[str, str]:
    """Splits a remote filename into a bucket and a blob name."""
    split_path = os.path.normpath(remote_filename).split('/', 1)
    bucket_name = split_path[0] if len(split_path) >= 1 else ""
    blob_name = split_path[1] if len(split_path) >= 2 else ""
    return bucket_name, blob_name

def _localize_filename(filename: str) -> str:
    """Takes a remote filename and returns its local counterpart.

    Throws an exception if the filename falls out of the directory.
    """
    return safe_join('/tmp/', filename)
<commit_msg>Reworded and fixed capitalization of comments<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A server that runs FFmpeg on files in Google Cloud Storage.
"""

import os
import subprocess
from typing import Tuple

from flask import Flask
from flask import request
from flask import safe_join
from google.cloud import storage
from google.cloud.exceptions import NotFound

app = Flask(__name__)

@app.errorhandler(NotFound)
def _handle_gcloud_not_found(_):
    return 'Google Cloud storage object not found', 404

@app.route('/ffmpeg', methods=['POST'])
def process_ffmpeg_command():
    """Runs ffmpeg according to the request's specification.

    The POST request supports the following arguments:
        input_file: A GCS input file path
                    Example: bucket_name/path/to/file
        output_file: A GCS output file path
                     Example: bucket_name/path/to/file
    This route changes the video container format by running the following:
        ffmpeg -i {input_file} {output_file}

    Returns:
        A string with the ffmpeg logs.
    """
    remote_input_file = request.form['input_file']
    local_input_file = _localize_filename(remote_input_file)
    download_file(local_input_file, remote_input_file)
    remote_output_file = request.form['output_file']
    local_output_file = _localize_filename(remote_output_file)

    ffmpeg_logs = subprocess.run(
        ["ffmpeg", "-i", local_input_file, local_output_file],
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        check=False).stdout
    os.remove(local_input_file)
    upload_file(local_output_file, remote_output_file)
    os.remove(local_output_file)
    return ffmpeg_logs

def download_file(local_filename: str, remote_filename: str) -> None:
    """Downloads the file from Google Cloud Storage

    This function downloads the file to the /tmp directory.

    Args:
        local_filename: the filepath of the downloaded file
        remote_filename: the filename preceded by the Google Cloud Bucket name
                         Example: bucket_name/path/to/file

    Returns:
        None
    """
    client = storage.Client()
    local_filename = _localize_filename(remote_filename)
    os.makedirs(os.path.dirname(local_filename), exist_ok=True)
    with open(local_filename, 'wb') as downloaded_file:
        client.download_blob_to_file('gs://' + remote_filename, downloaded_file)

def upload_file(local_filename: str, remote_filename: str) -> None:
    """Uploads the file from Google Cloud Storage.

    This function uploads the file to the Google Cloud Bucket specified.

    Args:
        local_filename: The filepath of the local file to upload
        remote_filename: A GCS output file path
                         Example: bucket_name/path/to/file

    Returns:
        None
    """
    client = storage.Client()
    bucket_name, blob_name = _split_remote_filename(remote_filename)
    bucket = client.bucket(bucket_name)
    blob = bucket.blob(blob_name)
    blob.upload_from_filename(local_filename)

def _split_remote_filename(remote_filename: str) -> Tuple[str, str]:
    """Splits a remote filename into a bucket and a blob name."""
    split_path = os.path.normpath(remote_filename).split('/', 1)
    bucket_name = split_path[0] if len(split_path) >= 1 else ""
    blob_name = split_path[1] if len(split_path) >= 2 else ""
    return bucket_name, blob_name

def _localize_filename(filename: str) -> str:
    """Takes a remote filename and returns its local counterpart.

    Throws an exception if the filename falls out of the directory.
    """
    return safe_join('/tmp/', filename)
<|endoftext|>


<commit_before>"""
===========
NewTabmagic
===========

Magics for opening new browser tabs.

============
Sample Usage
============

    Start magic:

        In [1]: %load_ext newtabmagic

        In [2]: %newtab --browser firefox

        In [3]: %newtab --port 8889

        In [4]: %newtab --server start
        Starting job # 0 in a separate thread.
        Server running at http://127.0.0.1:8889/

    Open help pages in new browser tabs:

        In [5]: %newtab IPython.core.debugger.Tracer

        In [6]: import IPython
        In [7]: Tracer = IPython.core.debugger.Tracer()
        In [8]: %newtab Tracer

    Show state:

        In [9]: # random \
        ...: %newtab --show
        browser: firefox
        content-type: html
        server pid: 3096
        server poll: None
        server running: True
        server port: 8889
        server root url: http://127.0.0.1:8889/

    Stop server:

        In [10]: %newtab --server stop
        Server process is terminated.

    Note: shutting down IPython stops the server.

"""
from __future__ import print_function

import inspect
import os
import pydoc
import sys
import subprocess
import time
import tornado.ioloop
import tornado.web

from IPython import get_ipython
from IPython.core.error import UsageError
from IPython.core.magic import (
    Magics,
    magics_class,
    line_magic)
from IPython.core.magic_arguments import (
    argument,
    magic_arguments,
    parse_argstring)

@magics_class
class NewTabMagics(Magics):
    """Magic class for opening new browser tabs."""

    def __init__(self, shell):
        super(NewTabMagics, self).__init__(shell)
        self._content_type = 'html'
        self._content_types = ['html', 'text']
        self._browser = None
        self._server = ServerProcess()
        self.new_tabs_enabled = True
        self._cmds = []

    @line_magic
    @magic_arguments()
    @argument(
        'names',
        help='Variable names and dotted paths, separated by spaces.',
        nargs='*'
    )
    @argument(
        '--browser',
        help='Set name of browser used to open new tabs.',
        nargs='+'
    )
    @argument(
        '--port',
        type=int,
        help='Set port used by the newtabmagic server.'
    )
    @argument(
        '--server',
        help='Control the newtabmagic server.',
        choices=['stop', 'start']
    )
    @argument(
        '--show',
        help='Show state of magic.',
        action='store_true'
    )
    @argument(
        '--content-type',
        choices=['text', 'html'],
        help='Set content type of files opened in new tabs.'
    )
    def newtab(self, line):
        """Line magic for opening new browser tabs."""

        args = parse_argstring(self.newtab, line)

        if args.port:
            self._server.port = args.port

        if args.server:
            if args.server == 'start':
                self._server.start()
            elif args.server == 'stop':
                self._server.stop()

        if args.browser:
            self.browser = args.browser

        if args.content_type:
            self._content_type = args.content_type

        if args.names:
            if self._browser:
                self._open_new_tabs(args.names)
            else:
                msg = "Browser not initialized\n"
                raise UsageError(msg)

        if args.show:
            self._show()

    def _open_new_tabs(self, names):
        """Open browser tabs for a list of variable names and paths."""
        self._cmds = []
        for name in names:
            url, msg = self._get_url(name)
            if not url:
                print(msg)
            else:
                cmd = [self._browser, url]
                self._cmds.append(cmd)
                self._open_new_tab(cmd)

    def _get_url(self, name):
        """Get url associated with name, returning None if not found"""
        return self._get_pydoc_url(name)

    def _get_pydoc_url(self, name):
        """Get pydoc url for name of variable or path."""
        msg = ''
        page = self._get_pydoc_page_name(name)
        if page:
            url = self._pydoc_url(page)
        else:
            url = None
            msg = 'Documentation not found: {}'.format(name)
        return url, msg

    def _get_pydoc_page_name(self, path):
        """Return name of pydoc page, or None if path is not valid."""
        obj = self._get_object(path)
        if obj is not None:
            page_name = fully_qualified_name(obj)
        else:
            page_name = None
        return page_name

    def _get_object(self, path):
        """Return object, or None if the object does not exist."""
        parts = [part for part in path.split('.') if part]
        if parts[0] in self.shell.user_ns:
            obj = self.shell.user_ns[parts[0]]
            if parts[1:]:
                obj = _getattr_path(obj, parts[1:])
        else:
            obj = pydoc.locate(path)
        return obj

    @property
    def command_lines(self):
        """Command lines used (most recently) to invoke subproccesses."""
        return self._cmds

    @property
    def browser(self):
        """Name of browser used to open new tabs."""
        return self._browser

    @browser.setter
    def browser(self, args):
        """Set browser by name or path."""
        path = ' '.join(args).strip('\'\"')
        self._browser = path

    def _open_new_tab(self, cmd):
        """Open a new browser tab by invoking a subprocess."""
        try:
            if self.new_tabs_enabled:
                # chromium browser writes to screen, so redirect
                subprocess.Popen(cmd, stdout=subprocess.PIPE)
        except OSError:
            msg = "Browser named {} failed to open new tab\n".format(
                self._browser)
            raise UsageError(msg)

    def _show(self):
        """Show state of magic."""
        msg = ''
        msg += 'browser: {}\n'.format(self._browser)
        msg += 'content-type: {}\n'.format(self._content_type)
        print(msg, end='')
        self._server.show()

    def base_url(self):
        """Base url for newtabmagic server."""
        return self._server.url()

    def _pydoc_url(self, page):
        """Return url for pydoc help page."""
        root = self._server.url()
        if self._content_type == 'html':
            url = root + page + '.html'
        else:
            url = root + page + '.txt'
        return url

class ServerProcess(object):
    """Class for the newtabmagic server process."""

    def __init__(self):
        self._process = None
        self._port = 8889

    def start(self):
        """Start server if not previously started."""
        msg = ''
        if not self.running():
            self._process = start_server_background(self._port)
        else:
            msg = 'Server already started\n'
        msg += 'Server running at {}'.format(self.url())
        print(msg)

    def stop(self):
        """Stop server process."""

        msg = ''
        if self._process:
            _stop_process(self._process, 'Server process')
        else:
            msg += 'Server not started.\n'
        if msg:
            print(msg, end='')

    def running(self):
        """If the server has been started, is it still running?"""
        return self._process is not None and self._process.poll() is None

    def show(self):
        """Show state."""
        msg = ''
        if self._process:
            msg += 'server pid: {}\n'.format(self._process.pid)
            msg += 'server poll: {}\n'.format(self._process.poll())
        msg += 'server running: {}\n'.format(self.running())
        msg += 'server port: {}\n'.format(self._port)
        msg += 'server root url: {}\n'.format(self.url())
        print(msg, end='')

    def url(self):
        """Base url. Includes protocol, host, and port number."""
        proto = 'http'
        ip = '127.0.0.1'
        return '{}://{}:{}/'.format(proto, ip, self._port)

    @property
    def port(self):
        """Port number server listens on."""
        return self._port

    @port.setter
    def port(self, port):
        """Set port number if server is not running."""
        if not self.running():
            self._port = port
        else:
            print('Server already running. Port number not changed')

def fully_qualified_name(obj):
    """Returns fully qualified name, including module name, except for the
    built-in module."""

    builtins = ['builtins', '__builtin__']

    if inspect.ismodule(obj):
        name = obj.__name__
    else:
        module_name = _get_module_name(obj)
        if not module_name or module_name in builtins:
            name = _qualname(obj, module_name)
        else:
            name = module_name + '.' + _qualname(obj, module_name)
    return name

def _qualname(obj, module_name):
    """Qualified name not including module name."""

    if sys.version_info >= (3, 3):
        return _qualname33(obj)
    elif sys.version_info >= (3, 0):
        return _qualname32(obj)
    else:
        return _qualname27(obj, module_name)

def _qualname33(obj):
    """Qualified name, not including module name, for Python 3.3+."""
    try:
        return obj.undecorated.__qualname__
    except AttributeError:
        try:
            return obj.__qualname__
        except AttributeError:
            return type(obj).__name__

def _qualname32(obj):
    """Qualified name, not including module name, for Python 3.0-3.2."""
    try:
        return obj.__objclass__.__name__ + "." + obj.__name__
    except AttributeError:
        try:
            if inspect.ismodule(obj.__self__):
                return obj.__name__
            else:
                return obj.__self__.__class__.__name__ + "." + obj.__name__
        except AttributeError:
            try:
                return obj.__name__
            except AttributeError:
                return type(obj).__name__

def _qualname27(obj, module_name):
    """Qualified name, not including module name, Python 2.7."""
    try:
        return obj.__objclass__.__name__ + "." + obj.__name__
    except AttributeError:
        try:
            attr = obj.__name__
            # loop through base classes looking for class where
            # attr is defined.
            for base_class in obj.im_class.mro():
                qual_name = base_class.__name__ + '.' + attr
                if pydoc.locate(module_name + '.' + qual_name):
                    return qual_name
        except AttributeError:  # no im_class
            try:
                if obj.__self__ is not None:
                    return obj.__self__.__class__.__name__ + "." + obj.__name__
            except AttributeError: # no __self__
                pass
            try:
                return obj.__name__
            except AttributeError:
                return type(obj).__name__

def _getattr_path(obj, attrs):
    """Get a named attribute from an object, returning None if the
    attribute does not exist.

    Keyword Arguments:
    attrs -- a list of attributes obtained from a path by applying
                String.split('.')
    """

    try:
        for attr in attrs:
            obj = getattr(obj, attr)
    except AttributeError:
        obj = None
    return obj

def _get_module_name(object_):
    """Return module name of object, or module name of object's class."""
    try:
        name = object_.__module__
    except AttributeError:
        name = object_.__class__.__module__
    return name

def _stop_process(p, name):
    """Stop process, by applying terminate and kill."""
    # Based on code in IPython.core.magics.script.ScriptMagics.shebang
    if p.poll() is not None:
        print("{} is already stopped.".format(name))
        return
    p.terminate()
    time.sleep(0.1)
    if p.poll() is not None:
        print("{} is terminated.".format(name))
        return
    p.kill()
    print("{} is killed.".format(name))

class HtmlHandler(tornado.web.RequestHandler):
    """Handler for html pages."""
    def get(self):
        page = get_html(self.request.uri)
        self.write(page)

class TextHandler(tornado.web.RequestHandler):
    """Handler for plain text pages."""
    def get(self):
        page = get_text(self.request.uri)
        self.set_header("Content-Type", "text/plain")
        self.write(page)

def get_html(url):
    """Get method for HTML pages."""
    if sys.version_info[0] >= 3:
        page = pydoc._url_handler(url)  # pylint: disable=W0212
    else:
        if url.endswith('.html'):
            page = pydoc_html_help(url[1:-5])
        elif url == '/':
            # index page is not available for Python 2
            page = "Index page not available"
    return page

def get_text(uri):
    """Get method for plain text pages."""
    path = uri[1:-4]  # remove leading '/' and trailing '.txt'
    page = pydoc_text_help(path)
    return page

def pydoc_html_help(path):
    """Returns a pydoc help page in html for the object referred
    to by path.

    Keyword Arguments:
    path -- an object name or a dotted path to an object

    Reference: pydoc.writedoc

    """
    try:
        obj, name = pydoc.resolve(path)
        html_doc = pydoc.html.document(obj, name)
        page = pydoc.html.page(pydoc.describe(obj), html_doc)
    except (ImportError, pydoc.ErrorDuringImport):
        raise tornado.web.HTTPError(404)
    return page

def pydoc_text_help(path):
    """Returns a pydoc help page in plain text for the object
    referred to by path.

    Keyword Arguments:
    path -- an object name or a dotted path to an object
    """
    try:
        if sys.version_info[0] == 2:
            page = pydoc.plain(pydoc.render_doc(path))
        else:
            page = pydoc.render_doc(path, renderer=pydoc.plaintext)
    except (ImportError, pydoc.ErrorDuringImport):
        raise tornado.web.HTTPError(404)
    return page.encode('utf-8')

def start_server(port):
    """Starts the newtab server on the given port."""
    application = tornado.web.Application([
        (r'/', HtmlHandler),
        (r'/[\w\.]*.html', HtmlHandler),
        (r'/[\w\.]*.txt', TextHandler),
    ])
    application.listen(port)

    tornado.ioloop.IOLoop.instance().start()

def start_server_background(port):
    """Start the newtab server server as a background process."""

    # Using script cell magic so that shutting down
    # IPython stops the server process.

    module = 'newtabmagic'
    path = repr(os.path.dirname(os.path.realpath(__file__)))

    lines = ('import sys\n'
             'sys.path.append({path})\n'
             'import {module}\n'
             '{module}.start_server({port})')
    cell = lines.format(path=path, module=module, port=port)

    line = "python --proc proc --bg"
    ip = get_ipython()
    ip.run_cell_magic("script", line, cell)

    return ip.user_ns['proc']

def load_ipython_extension(ip):
    """Load NewTabMagics extension."""
    ip.register_magics(NewTabMagics)
<commit_msg>Making fully qualified name function non-public<commit_after>"""
===========
NewTabmagic
===========

Magics for opening new browser tabs.

============
Sample Usage
============

    Start magic:

        In [1]: %load_ext newtabmagic

        In [2]: %newtab --browser firefox

        In [3]: %newtab --port 8889

        In [4]: %newtab --server start
        Starting job # 0 in a separate thread.
        Server running at http://127.0.0.1:8889/

    Open help pages in new browser tabs:

        In [5]: %newtab IPython.core.debugger.Tracer

        In [6]: import IPython
        In [7]: Tracer = IPython.core.debugger.Tracer()
        In [8]: %newtab Tracer

    Show state:

        In [9]: # random \
        ...: %newtab --show
        browser: firefox
        content-type: html
        server pid: 3096
        server poll: None
        server running: True
        server port: 8889
        server root url: http://127.0.0.1:8889/

    Stop server:

        In [10]: %newtab --server stop
        Server process is terminated.

    Note: shutting down IPython stops the server.

"""
from __future__ import print_function

import inspect
import os
import pydoc
import sys
import subprocess
import time
import tornado.ioloop
import tornado.web

from IPython import get_ipython
from IPython.core.error import UsageError
from IPython.core.magic import (
    Magics,
    magics_class,
    line_magic)
from IPython.core.magic_arguments import (
    argument,
    magic_arguments,
    parse_argstring)

@magics_class
class NewTabMagics(Magics):
    """Magic class for opening new browser tabs."""

    def __init__(self, shell):
        super(NewTabMagics, self).__init__(shell)
        self._content_type = 'html'
        self._content_types = ['html', 'text']
        self._browser = None
        self._server = ServerProcess()
        self.new_tabs_enabled = True
        self._cmds = []

    @line_magic
    @magic_arguments()
    @argument(
        'names',
        help='Variable names and dotted paths, separated by spaces.',
        nargs='*'
    )
    @argument(
        '--browser',
        help='Set name of browser used to open new tabs.',
        nargs='+'
    )
    @argument(
        '--port',
        type=int,
        help='Set port used by the newtabmagic server.'
    )
    @argument(
        '--server',
        help='Control the newtabmagic server.',
        choices=['stop', 'start']
    )
    @argument(
        '--show',
        help='Show state of magic.',
        action='store_true'
    )
    @argument(
        '--content-type',
        choices=['text', 'html'],
        help='Set content type of files opened in new tabs.'
    )
    def newtab(self, line):
        """Line magic for opening new browser tabs."""

        args = parse_argstring(self.newtab, line)

        if args.port:
            self._server.port = args.port

        if args.server:
            if args.server == 'start':
                self._server.start()
            elif args.server == 'stop':
                self._server.stop()

        if args.browser:
            self.browser = args.browser

        if args.content_type:
            self._content_type = args.content_type

        if args.names:
            if self._browser:
                self._open_new_tabs(args.names)
            else:
                msg = "Browser not initialized\n"
                raise UsageError(msg)

        if args.show:
            self._show()

    def _open_new_tabs(self, names):
        """Open browser tabs for a list of variable names and paths."""
        self._cmds = []
        for name in names:
            url, msg = self._get_url(name)
            if not url:
                print(msg)
            else:
                cmd = [self._browser, url]
                self._cmds.append(cmd)
                self._open_new_tab(cmd)

    def _get_url(self, name):
        """Get url associated with name, returning None if not found"""
        return self._get_pydoc_url(name)

    def _get_pydoc_url(self, name):
        """Get pydoc url for name of variable or path."""
        msg = ''
        page = self._get_pydoc_page_name(name)
        if page:
            url = self._pydoc_url(page)
        else:
            url = None
            msg = 'Documentation not found: {}'.format(name)
        return url, msg

    def _get_pydoc_page_name(self, path):
        """Return name of pydoc page, or None if path is not valid."""
        obj = self._get_object(path)
        if obj is not None:
            page_name = _fully_qualified_name(obj)
        else:
            page_name = None
        return page_name

    def _get_object(self, path):
        """Return object, or None if the object does not exist."""
        parts = [part for part in path.split('.') if part]
        if parts[0] in self.shell.user_ns:
            obj = self.shell.user_ns[parts[0]]
            if parts[1:]:
                obj = _getattr_path(obj, parts[1:])
        else:
            obj = pydoc.locate(path)
        return obj

    @property
    def command_lines(self):
        """Command lines used (most recently) to invoke subproccesses."""
        return self._cmds

    @property
    def browser(self):
        """Name of browser used to open new tabs."""
        return self._browser

    @browser.setter
    def browser(self, args):
        """Set browser by name or path."""
        path = ' '.join(args).strip('\'\"')
        self._browser = path

    def _open_new_tab(self, cmd):
        """Open a new browser tab by invoking a subprocess."""
        try:
            if self.new_tabs_enabled:
                # chromium browser writes to screen, so redirect
                subprocess.Popen(cmd, stdout=subprocess.PIPE)
        except OSError:
            msg = "Browser named {} failed to open new tab\n".format(
                self._browser)
            raise UsageError(msg)

    def _show(self):
        """Show state of magic."""
        msg = ''
        msg += 'browser: {}\n'.format(self._browser)
        msg += 'content-type: {}\n'.format(self._content_type)
        print(msg, end='')
        self._server.show()

    def base_url(self):
        """Base url for newtabmagic server."""
        return self._server.url()

    def _pydoc_url(self, page):
        """Return url for pydoc help page."""
        root = self._server.url()
        if self._content_type == 'html':
            url = root + page + '.html'
        else:
            url = root + page + '.txt'
        return url

class ServerProcess(object):
    """Class for the newtabmagic server process."""

    def __init__(self):
        self._process = None
        self._port = 8889

    def start(self):
        """Start server if not previously started."""
        msg = ''
        if not self.running():
            self._process = start_server_background(self._port)
        else:
            msg = 'Server already started\n'
        msg += 'Server running at {}'.format(self.url())
        print(msg)

    def stop(self):
        """Stop server process."""

        msg = ''
        if self._process:
            _stop_process(self._process, 'Server process')
        else:
            msg += 'Server not started.\n'
        if msg:
            print(msg, end='')

    def running(self):
        """If the server has been started, is it still running?"""
        return self._process is not None and self._process.poll() is None

    def show(self):
        """Show state."""
        msg = ''
        if self._process:
            msg += 'server pid: {}\n'.format(self._process.pid)
            msg += 'server poll: {}\n'.format(self._process.poll())
        msg += 'server running: {}\n'.format(self.running())
        msg += 'server port: {}\n'.format(self._port)
        msg += 'server root url: {}\n'.format(self.url())
        print(msg, end='')

    def url(self):
        """Base url. Includes protocol, host, and port number."""
        proto = 'http'
        ip = '127.0.0.1'
        return '{}://{}:{}/'.format(proto, ip, self._port)

    @property
    def port(self):
        """Port number server listens on."""
        return self._port

    @port.setter
    def port(self, port):
        """Set port number if server is not running."""
        if not self.running():
            self._port = port
        else:
            print('Server already running. Port number not changed')

def _fully_qualified_name(obj):
    """Returns fully qualified name, including module name, except for the
    built-in module."""

    builtins = ['builtins', '__builtin__']

    if inspect.ismodule(obj):
        name = obj.__name__
    else:
        module_name = _get_module_name(obj)
        if not module_name or module_name in builtins:
            name = _qualname(obj, module_name)
        else:
            name = module_name + '.' + _qualname(obj, module_name)
    return name

def _qualname(obj, module_name):
    """Qualified name not including module name."""

    if sys.version_info >= (3, 3):
        return _qualname33(obj)
    elif sys.version_info >= (3, 0):
        return _qualname32(obj)
    else:
        return _qualname27(obj, module_name)

def _qualname33(obj):
    """Qualified name, not including module name, for Python 3.3+."""
    try:
        return obj.undecorated.__qualname__
    except AttributeError:
        try:
            return obj.__qualname__
        except AttributeError:
            return type(obj).__name__

def _qualname32(obj):
    """Qualified name, not including module name, for Python 3.0-3.2."""
    try:
        return obj.__objclass__.__name__ + "." + obj.__name__
    except AttributeError:
        try:
            if inspect.ismodule(obj.__self__):
                return obj.__name__
            else:
                return obj.__self__.__class__.__name__ + "." + obj.__name__
        except AttributeError:
            try:
                return obj.__name__
            except AttributeError:
                return type(obj).__name__

def _qualname27(obj, module_name):
    """Qualified name, not including module name, Python 2.7."""
    try:
        return obj.__objclass__.__name__ + "." + obj.__name__
    except AttributeError:
        try:
            attr = obj.__name__
            # loop through base classes looking for class where
            # attr is defined.
            for base_class in obj.im_class.mro():
                qual_name = base_class.__name__ + '.' + attr
                if pydoc.locate(module_name + '.' + qual_name):
                    return qual_name
        except AttributeError:  # no im_class
            try:
                if obj.__self__ is not None:
                    return obj.__self__.__class__.__name__ + "." + obj.__name__
            except AttributeError: # no __self__
                pass
            try:
                return obj.__name__
            except AttributeError:
                return type(obj).__name__

def _getattr_path(obj, attrs):
    """Get a named attribute from an object, returning None if the
    attribute does not exist.

    Keyword Arguments:
    attrs -- a list of attributes obtained from a path by applying
                String.split('.')
    """

    try:
        for attr in attrs:
            obj = getattr(obj, attr)
    except AttributeError:
        obj = None
    return obj

def _get_module_name(object_):
    """Return module name of object, or module name of object's class."""
    try:
        name = object_.__module__
    except AttributeError:
        name = object_.__class__.__module__
    return name

def _stop_process(p, name):
    """Stop process, by applying terminate and kill."""
    # Based on code in IPython.core.magics.script.ScriptMagics.shebang
    if p.poll() is not None:
        print("{} is already stopped.".format(name))
        return
    p.terminate()
    time.sleep(0.1)
    if p.poll() is not None:
        print("{} is terminated.".format(name))
        return
    p.kill()
    print("{} is killed.".format(name))

class HtmlHandler(tornado.web.RequestHandler):
    """Handler for html pages."""
    def get(self):
        page = get_html(self.request.uri)
        self.write(page)

class TextHandler(tornado.web.RequestHandler):
    """Handler for plain text pages."""
    def get(self):
        page = get_text(self.request.uri)
        self.set_header("Content-Type", "text/plain")
        self.write(page)

def get_html(url):
    """Get method for HTML pages."""
    if sys.version_info[0] >= 3:
        page = pydoc._url_handler(url)  # pylint: disable=W0212
    else:
        if url.endswith('.html'):
            page = pydoc_html_help(url[1:-5])
        elif url == '/':
            # index page is not available for Python 2
            page = "Index page not available"
    return page

def get_text(uri):
    """Get method for plain text pages."""
    path = uri[1:-4]  # remove leading '/' and trailing '.txt'
    page = pydoc_text_help(path)
    return page

def pydoc_html_help(path):
    """Returns a pydoc help page in html for the object referred
    to by path.

    Keyword Arguments:
    path -- an object name or a dotted path to an object

    Reference: pydoc.writedoc

    """
    try:
        obj, name = pydoc.resolve(path)
        html_doc = pydoc.html.document(obj, name)
        page = pydoc.html.page(pydoc.describe(obj), html_doc)
    except (ImportError, pydoc.ErrorDuringImport):
        raise tornado.web.HTTPError(404)
    return page

def pydoc_text_help(path):
    """Returns a pydoc help page in plain text for the object
    referred to by path.

    Keyword Arguments:
    path -- an object name or a dotted path to an object
    """
    try:
        if sys.version_info[0] == 2:
            page = pydoc.plain(pydoc.render_doc(path))
        else:
            page = pydoc.render_doc(path, renderer=pydoc.plaintext)
    except (ImportError, pydoc.ErrorDuringImport):
        raise tornado.web.HTTPError(404)
    return page.encode('utf-8')

def start_server(port):
    """Starts the newtab server on the given port."""
    application = tornado.web.Application([
        (r'/', HtmlHandler),
        (r'/[\w\.]*.html', HtmlHandler),
        (r'/[\w\.]*.txt', TextHandler),
    ])
    application.listen(port)

    tornado.ioloop.IOLoop.instance().start()

def start_server_background(port):
    """Start the newtab server server as a background process."""

    # Using script cell magic so that shutting down
    # IPython stops the server process.

    module = 'newtabmagic'
    path = repr(os.path.dirname(os.path.realpath(__file__)))

    lines = ('import sys\n'
             'sys.path.append({path})\n'
             'import {module}\n'
             '{module}.start_server({port})')
    cell = lines.format(path=path, module=module, port=port)

    line = "python --proc proc --bg"
    ip = get_ipython()
    ip.run_cell_magic("script", line, cell)

    return ip.user_ns['proc']

def load_ipython_extension(ip):
    """Load NewTabMagics extension."""
    ip.register_magics(NewTabMagics)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import argparse
import json
import os

import paramiko

DEFAULT_GERRIT_HOST = 'review.openstack.org'
DEFAULT_GERRIT_PORT = 29418

def ssh_client(host, port, user=None, key=None):
    client = paramiko.SSHClient()
    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    client.load_system_host_keys()
    client.connect(host, port=port, key_filename=key, username=user)
    return client

def get_watched_reviews(client):
    reviews = []

    while True:
        query = [
            'gerrit', 'query', 'is:watched', 'is:open', 'limit:100',
            '--all-approvals', '--patch-sets', '--format=JSON']
        if reviews:
            query.append('resume_sortkey:%s' % reviews[-2]['sortKey'])
        stdin, stdout, stderr = client.exec_command(' '.join(query))

        for line in stdout:
            reviews.append(json.loads(line))
        if reviews[-1]['rowCount'] == 0:
            break

    return [x for x in reviews if 'id' in x]

def sort_reviews_by_last_updated(reviews):
    """Sort reviews in ascending order by last update date."""
    return sorted(reviews, key=lambda review: review['lastUpdated'])

def render_reviews(reviews, maximum=3):
    for review in reviews[:maximum]:
        print review['url'], review['subject'].strip()

def ignore_blocked_reviews(reviews):
    filtered_reviews = []
    for review in reviews:
        votes = [x['value'] for x in review['patchSets'][-1]['approvals']]
        if "-2" not in votes:
            filtered_reviews.append(review)
    return filtered_reviews

def require_jenkins_upvote(reviews):
    filtered_reviews = []
    for review in reviews:
        for vote in review['patchSets'][-1]['approvals']:
            if vote['by']['username'] == 'jenkins' and vote['value'] == '1':
                filtered_reviews.append(review)
    return filtered_reviews

def require_smokestack_upvote(reviews):
    filtered_reviews = []
    for review in reviews:
        for vote in review['patchSets'][-1]['approvals']:
            if vote['by']['username'] == 'smokestack' and vote['value'] == '1':
                filtered_reviews.append(review)
    return filtered_reviews

def ignore_my_reviews(reviews, username):
    """Ignore reviews created by me."""
    filtered_reviews = []
    for review in reviews:
        if review['owner']['username'] != username:
            filtered_reviews.append(review)
    return filtered_reviews

def ignore_wip(reviews):
    return [x for x in reviews if x['status'] != 'WORKINPROGRESS']

def main(args):
    client = ssh_client(
        host=args.host, port=args.port, user=args.username, key=args.key)

    reviews = get_watched_reviews(client)

    # filter out reviews that are not prime review targets
    reviews = ignore_wip(reviews)
    reviews = ignore_blocked_reviews(reviews)
    reviews = require_jenkins_upvote(reviews)
    reviews = require_smokestack_upvote(reviews)
    reviews = ignore_my_reviews(reviews, username=args.username)

    # review old stuff before it expires
    reviews = sort_reviews_by_last_updated(reviews)

    if reviews and args.no_action:
        render_reviews(reviews, maximum=100)
    elif reviews and not args.no_action:
        render_reviews(reviews, maximum=1)

        # open the oldest code review in a browser
        os.system('open %s' % reviews[0]['url'])
    elif not reviews:
        print 'Nothing to review!'

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        prog='next-review',
        description='Start your next gerrit code review without any hassle.')
    parser.add_argument(
        '--host', default=DEFAULT_GERRIT_HOST,
        help='SSH hostname for gerrit')
    parser.add_argument(
        '--port', type=int, default=DEFAULT_GERRIT_PORT,
        help='SSH port for gerrit')
    parser.add_argument(
        '--username', default=None,
        help='Your SSH username for gerrit (optional but HIGHLY recommended)')
    parser.add_argument(
        '--key', default=None,
        help='Path to your SSH public key for gerrit')
    parser.add_argument(
        '-N', '--no-action', action='store_true',
        help='Do not attempt to open the review')

    args = parser.parse_args()
    main(args)
<commit_msg>Username is technically optional<commit_after>#!/usr/bin/env python

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import argparse
import json
import os

import paramiko

DEFAULT_GERRIT_HOST = 'review.openstack.org'
DEFAULT_GERRIT_PORT = 29418

def ssh_client(host, port, user=None, key=None):
    client = paramiko.SSHClient()
    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    client.load_system_host_keys()
    client.connect(host, port=port, key_filename=key, username=user)
    return client

def get_watched_reviews(client):
    reviews = []

    while True:
        query = [
            'gerrit', 'query', 'is:watched', 'is:open', 'limit:100',
            '--all-approvals', '--patch-sets', '--format=JSON']
        if reviews:
            query.append('resume_sortkey:%s' % reviews[-2]['sortKey'])
        stdin, stdout, stderr = client.exec_command(' '.join(query))

        for line in stdout:
            reviews.append(json.loads(line))
        if reviews[-1]['rowCount'] == 0:
            break

    return [x for x in reviews if 'id' in x]

def sort_reviews_by_last_updated(reviews):
    """Sort reviews in ascending order by last update date."""
    return sorted(reviews, key=lambda review: review['lastUpdated'])

def render_reviews(reviews, maximum=3):
    for review in reviews[:maximum]:
        print review['url'], review['subject'].strip()

def ignore_blocked_reviews(reviews):
    filtered_reviews = []
    for review in reviews:
        votes = [x['value'] for x in review['patchSets'][-1]['approvals']]
        if "-2" not in votes:
            filtered_reviews.append(review)
    return filtered_reviews

def require_jenkins_upvote(reviews):
    filtered_reviews = []
    for review in reviews:
        for vote in review['patchSets'][-1]['approvals']:
            if vote['by']['username'] == 'jenkins' and vote['value'] == '1':
                filtered_reviews.append(review)
    return filtered_reviews

def require_smokestack_upvote(reviews):
    filtered_reviews = []
    for review in reviews:
        for vote in review['patchSets'][-1]['approvals']:
            if vote['by']['username'] == 'smokestack' and vote['value'] == '1':
                filtered_reviews.append(review)
    return filtered_reviews

def ignore_my_reviews(reviews, username=None):
    """Ignore reviews created by me."""
    filtered_reviews = []
    for review in reviews:
        if review['owner']['username'] != username:
            filtered_reviews.append(review)
    return filtered_reviews

def ignore_wip(reviews):
    return [x for x in reviews if x['status'] != 'WORKINPROGRESS']

def main(args):
    client = ssh_client(
        host=args.host, port=args.port, user=args.username, key=args.key)

    reviews = get_watched_reviews(client)

    # filter out reviews that are not prime review targets
    reviews = ignore_wip(reviews)
    reviews = ignore_blocked_reviews(reviews)
    reviews = require_jenkins_upvote(reviews)
    reviews = require_smokestack_upvote(reviews)
    reviews = ignore_my_reviews(reviews, username=args.username)

    # review old stuff before it expires
    reviews = sort_reviews_by_last_updated(reviews)

    if reviews and args.no_action:
        render_reviews(reviews, maximum=100)
    elif reviews and not args.no_action:
        render_reviews(reviews, maximum=1)

        # open the oldest code review in a browser
        os.system('open %s' % reviews[0]['url'])
    elif not reviews:
        print 'Nothing to review!'

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        prog='next-review',
        description='Start your next gerrit code review without any hassle.')
    parser.add_argument(
        '--host', default=DEFAULT_GERRIT_HOST,
        help='SSH hostname for gerrit')
    parser.add_argument(
        '--port', type=int, default=DEFAULT_GERRIT_PORT,
        help='SSH port for gerrit')
    parser.add_argument(
        '--username', default=None,
        help='Your SSH username for gerrit (optional but HIGHLY recommended)')
    parser.add_argument(
        '--key', default=None,
        help='Path to your SSH public key for gerrit')
    parser.add_argument(
        '-N', '--no-action', action='store_true',
        help='Do not attempt to open the review')

    args = parser.parse_args()
    main(args)
<|endoftext|>