<commit_before>#!/usr/bin/env python3
# Copyright (c) 2017-2019 The Particl Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

import time

from test_framework.test_particl import ParticlTestFramework, connect_nodes_bi
from test_framework.authproxy import JSONRPCException

class ForkTest(ParticlTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 6
        self.extra_args = [ ['-debug',] for i in range(self.num_nodes)]

    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()

    def setup_network(self, split=False):
        self.add_nodes(self.num_nodes, extra_args=self.extra_args)
        self.start_nodes()

        connect_nodes_bi(self.nodes, 0, 1)
        connect_nodes_bi(self.nodes, 0, 2)
        connect_nodes_bi(self.nodes, 1, 2)

        connect_nodes_bi(self.nodes, 3, 4)
        connect_nodes_bi(self.nodes, 3, 5)
        connect_nodes_bi(self.nodes, 4, 5)

        self.sync_all()

    def run_test(self):
        nodes = self.nodes

        # stop staking
        nodes[0].reservebalance(True, 10000000)
        nodes[3].reservebalance(True, 10000000)

        nodes[0].extkeyimportmaster('pact mammal barrel matrix local final lecture chunk wasp survey bid various book strong spread fall ozone daring like topple door fatigue limb olympic', '', 'true')
        nodes[0].getnewextaddress('lblExtTest')
        nodes[0].rescanblockchain()
        assert(nodes[0].getwalletinfo()['total_balance'] == 25000)

        nodes[3].extkeyimportmaster('abandon baby cabbage dad eager fabric gadget habit ice kangaroo lab absorb')
        assert(nodes[3].getwalletinfo()['total_balance'] == 100000)

        # start staking
        nBlocksShorterChain = 2
        nBlocksLongerChain = 5

        ro = nodes[0].walletsettings('stakelimit', {'height':nBlocksShorterChain})
        ro = nodes[3].walletsettings('stakelimit', {'height':nBlocksLongerChain})
        ro = nodes[0].reservebalance(False)
        ro = nodes[3].reservebalance(False)

        self.wait_for_height(nodes[0], nBlocksShorterChain, 1000)

        # stop group1 from staking
        ro = nodes[0].reservebalance(True, 10000000)

        self.wait_for_height(nodes[3], nBlocksLongerChain, 2000)

        # stop group2 from staking
        ro = nodes[3].reservebalance(True, 10000000)

        node0_chain = []
        for k in range(1, nBlocksLongerChain+1):
            try:
                ro = nodes[0].getblockhash(k)
            except JSONRPCException as e:
                assert('Block height out of range' in e.error['message'])
                ro = ''
            node0_chain.append(ro)
            print('node0 ',k, " - ", ro)

        node3_chain = []
        for k in range(1, 6):
            ro = nodes[3].getblockhash(k)
            node3_chain.append(ro)
            print('node3 ',k, ' - ', ro)

        # connect groups
        connect_nodes_bi(self.nodes, 0, 3)

        fPass = False
        for i in range(15):
            time.sleep(2)

            fPass = True
            for k in range(1, nBlocksLongerChain+1):
                try:
                    ro = nodes[0].getblockhash(k)
                except JSONRPCException as e:
                    assert('Block height out of range' in e.error['message'])
                    ro = ''
                if not ro == node3_chain[k]:
                    fPass = False
                    break
            if fPass:
                break
        #assert(fPass)

        node0_chain = []
        for k in range(1, nBlocksLongerChain+1):
            try:
                ro = nodes[0].getblockhash(k)
            except JSONRPCException as e:
                assert('Block height out of range' in e.error['message'])
                ro = ''
            node0_chain.append(ro)
            print('node0 ',k, ' - ', ro)

        ro = nodes[0].getblockchaininfo()
        assert(ro['blocks'] == 5)

        ro = nodes[3].getblockchaininfo()
        assert(ro['blocks'] == 5)

if __name__ == '__main__':
    ForkTest().main()
<commit_msg>test: Confirm failed stakes are orphaned.<commit_after>#!/usr/bin/env python3
# Copyright (c) 2017-2019 The Particl Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

import time

from test_framework.test_particl import ParticlTestFramework, connect_nodes_bi
from test_framework.authproxy import JSONRPCException

class ForkTest(ParticlTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 6
        self.extra_args = [ ['-debug',] for i in range(self.num_nodes)]

    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()

    def setup_network(self, split=False):
        self.add_nodes(self.num_nodes, extra_args=self.extra_args)
        self.start_nodes()

        connect_nodes_bi(self.nodes, 0, 1)
        connect_nodes_bi(self.nodes, 0, 2)
        connect_nodes_bi(self.nodes, 1, 2)

        connect_nodes_bi(self.nodes, 3, 4)
        connect_nodes_bi(self.nodes, 3, 5)
        connect_nodes_bi(self.nodes, 4, 5)

        self.sync_all()

    def run_test(self):
        nodes = self.nodes

        # stop staking
        nodes[0].reservebalance(True, 10000000)
        nodes[3].reservebalance(True, 10000000)

        nodes[0].extkeyimportmaster('pact mammal barrel matrix local final lecture chunk wasp survey bid various book strong spread fall ozone daring like topple door fatigue limb olympic', '', 'true')
        nodes[0].getnewextaddress('lblExtTest')
        nodes[0].rescanblockchain()
        assert(nodes[0].getwalletinfo()['total_balance'] == 25000)

        nodes[3].extkeyimportmaster('abandon baby cabbage dad eager fabric gadget habit ice kangaroo lab absorb')
        assert(nodes[3].getwalletinfo()['total_balance'] == 100000)

        n0_wi_before = nodes[0].getwalletinfo()

        # start staking
        nBlocksShorterChain = 2
        nBlocksLongerChain = 5

        ro = nodes[0].walletsettings('stakelimit', {'height':nBlocksShorterChain})
        ro = nodes[3].walletsettings('stakelimit', {'height':nBlocksLongerChain})
        ro = nodes[0].reservebalance(False)
        ro = nodes[3].reservebalance(False)

        self.wait_for_height(nodes[0], nBlocksShorterChain, 1000)

        # stop group1 from staking
        ro = nodes[0].reservebalance(True, 10000000)

        self.wait_for_height(nodes[3], nBlocksLongerChain, 2000)

        # stop group2 from staking
        ro = nodes[3].reservebalance(True, 10000000)

        node0_chain = []
        for k in range(1, nBlocksLongerChain+1):
            try:
                ro = nodes[0].getblockhash(k)
            except JSONRPCException as e:
                assert('Block height out of range' in e.error['message'])
                ro = ''
            node0_chain.append(ro)
            print('node0 ',k, " - ", ro)

        node3_chain = []
        for k in range(1, 6):
            ro = nodes[3].getblockhash(k)
            node3_chain.append(ro)
            print('node3 ',k, ' - ', ro)

        # connect groups
        connect_nodes_bi(self.nodes, 0, 3)

        fPass = False
        for i in range(15):
            time.sleep(2)

            fPass = True
            for k in range(1, nBlocksLongerChain+1):
                try:
                    ro = nodes[0].getblockhash(k)
                except JSONRPCException as e:
                    assert('Block height out of range' in e.error['message'])
                    ro = ''
                if not ro == node3_chain[k]:
                    fPass = False
                    break
            if fPass:
                break
        #assert(fPass)

        node0_chain = []
        for k in range(1, nBlocksLongerChain+1):
            try:
                ro = nodes[0].getblockhash(k)
            except JSONRPCException as e:
                assert('Block height out of range' in e.error['message'])
                ro = ''
            node0_chain.append(ro)
            print('node0 ',k, ' - ', ro)

        ro = nodes[0].getblockchaininfo()
        assert(ro['blocks'] == 5)
        ro = nodes[3].getblockchaininfo()
        assert(ro['blocks'] == 5)

        n0_wi_after = nodes[0].getwalletinfo()

        assert(n0_wi_after['total_balance'] == n0_wi_before['total_balance'])
        assert(n0_wi_before['txcount'] == 1)
        assert(n0_wi_after['txcount'] == 3)

        n0_ft = nodes[0].filtertransactions()
        assert(len(n0_ft) == 3)
        assert(n0_ft[0]['category'] == 'orphaned_stake')
        assert(n0_ft[1]['category'] == 'orphaned_stake')
        n0_lt = nodes[0].listtransactions()
        assert(n0_lt[-1]['category'] == 'orphaned_stake')
        assert(n0_lt[-2]['category'] == 'orphaned_stake')

if __name__ == '__main__':
    ForkTest().main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import time
import json
import sys
import os
import subprocess
from subprocess import PIPE, Popen
import xml.etree.ElementTree as ET
from datetime import datetime, timedelta
from dateutil import tz
from termcolor import colored
import argcomplete
import boto3
from botocore.exceptions import ClientError
import psutil
from .cf_utils import set_region, resolve_account, InstanceInfo
from .aws_infra_util import find_include

def letter_to_target_id(letter):
    return ord(letter)-ord("f")+5

def target_id_to_letter(target_id):
    return str(unichr(target_id-5+ord("f")))

def wmic_partition_get():
    return wmic_get("partition")

def wmic_diskdrive_get():
    return wmic_get("diskdrive")

def wmic_volume_get():
    return wmic_get("volume")

def wmic_get(command):
    ret = []
    proc = Popen(["wmic", command, "get",
                  "/format:rawxml"], stdout=PIPE, stderr=PIPE)
    output = proc.communicate()[0]
    tree = ET.fromstring(output)
    for elem in tree.iter("RESULTS"):
        for inst in elem.iter("INSTANCE"):
            disk = {}
            for prop in inst.iter("PROPERTY"):
                try:
                    disk[prop.attrib['NAME']] = int(prop.findtext("*"))
                except ValueError:
                    disk[prop.attrib['NAME']] = prop.findtext("*")
                except TypeError:
                    continue
            ret.append(disk)
    return ret

def wmic_disk_with_target_id(target_id):
    return [x for x in wmic_diskdrive_get() if x['SCSITargetId'] == target_id][0]

def wmic_max_target_id():
    return sorted([x['SCSITargetId'] for x in wmic_diskdrive_get()],
                  reverse=True)[0]

def disk_by_drive_letter(drive_letter):
    ret = {}
    proc = Popen(["powershell.exe", find_include("disk-by-drive-letter.ps1"),
                  drive_letter.upper() + ":"], stdout=PIPE, stderr=PIPE)
    output = proc.communicate()[0]
    tree = ET.fromstring(output)
    for obj in tree.iter("Object"):
        for prop in obj.iter("Property"):
            try:
                ret[prop.attrib['Name']] = int(prop.findtext("."))
            except ValueError:
                ret[prop.attrib['Name']] = prop.findtext(".")
            except TypeError:
                continue
    return ret

def latest_snapshot():
    """Get the latest snapshot with a given tag
    """
    parser = argparse.ArgumentParser(description=latest_snapshot.__doc__)
    parser.add_argument("tag", help="The tag to find snapshots with")
    argcomplete.autocomplete(parser)
    args = parser.parse_args()
    print first_free_device()
    snapshot = get_latest_snapshot(args.tag, args.tag)
    if snapshot:
        print snapshot.id
    else:
        sys.exit(1)

def volume_from_snapshot(tag_key, tag_value, mount_path, availability_zone=None,
                         size_gb=None):
    set_region()
    snapshot = get_latest_snapshot(tag_key, tag_value)
    if snapshot:
        print "Found snapshot " + snapshot.id
        volume = create_volume(snapshot.id, availability_zone=availability_zone,
                               size_gb=size_gb)
    else:
        if not size_gb:
            size_gb = 32
        print "Creating empty volyme of size " + str(size_gb)
        volume = create_empty_volume(size_gb,
                                     availability_zone=availability_zone)
    device = first_free_device()
    print "Attaching volume " + volume + " to " + device
    attach_volume(volume, device)
    if not snapshot:
        #empty device
        if sys.platform.startswith('win'):
            #Windows format
            disk = wmic_disk_with_target_id(letter_to_target_id(device[-1:]))
            drive_letter = mount_path[0].upper()
            disk_number = str(disk['Index'])
            subprocess.check_call(["powershell.exe", "Get-Disk", disk_number,
                                   "|", "Set-Disk", "-IsOffline", "$False"])
            subprocess.check_call(["powershell.exe", "Initialize-Disk",
                                   disk_number, "-PartitionStyle", "MBR"])
            subprocess.check_call(["powershell.exe", "New-Partition",
                                   "-DiskNumber", disk_number,
                                   "-UseMaximumSize", "-DriveLetter",
                                   drive_letter])
            print "Formatting " + device + "(" + drive_letter + ":)"
            subprocess.check_call(["powershell.exe", "Format-Volume",
                                   "-DriveLetter", drive_letter, "-FileSystem",
                                   "NTFS", "-Force", "-Confirm:$False"])
        else:
            #linux format
            print "Formatting " + device
            subprocess.check_call(["mkfs.ext4", device])
    else:
        if sys.platform.startswith('win'):
            target_id = letter_to_target_id(device[-1:])
            disk = wmic_disk_with_target_id(target_id)
            drive_letter = mount_path[0].upper()
            disk_number = str(disk['Index'])
            with open(os.devnull, 'w') as devnull:
                subprocess.call(["powershell.exe", "Initialize-Disk",
                                 disk_number, "-PartitionStyle", "MBR"],
                                stderr=devnull, stdout=devnull)
            subprocess.check_call(["powershell.exe", "Get-Disk", disk_number,
                                   "|", "Set-Disk", "-IsOffline", "$False"])
            with open(os.devnull, 'w') as devnull:
                subprocess.check_call(["powershell.exe", "Get-Partition",
                                       "-DiskNumber", disk_number,
                                       "-PartitionNumber", "1"
                                       "|", "Set-Partition", "-NewDriveLetter",
                                       drive_letter], stdout=devnull,
                                      stderr=devnull)
            #resize win partition if necessary
            if size_gb and not size_gb == snapshot.volume_size:
                proc = subprocess.Popen(["powershell.exe",
                                         "$((Get-PartitionSupportedSize -Dri" +\
                                         "veLetter " + drive_letter + \
                                         ").SizeMax)"],
                                        stdout=subprocess.PIPE)
                max_size = proc.communicate()[0]
                subprocess.check_call(["powershell.exe", "Resize-Partition",
                                       "-DriveLetter", drive_letter, "-Size",
                                       max_size])
        else:
            if size_gb and not size_gb == snapshot.volume_size:
                print "Resizing " + device + " from " + snapshot.volume_size +\
                      "GB to " + size_gb
                subprocess.check_call(["e2fsck", "-f", "-p", device])
                subprocess.check_call(["resize2fs", device])

def first_free_device():
    if sys.platform.startswith('win'):
        max_target = wmic_max_target_id()
        if max_target == 0:
            return "/dev/xvdf"
        else:
            return "/dev/xvd" + target_id_to_letter(max_target + 1)
    else:
        devices = [x.device for x in psutil.disk_partitions()]
        print devices
        for letter in "fghijklmnopqrstuvxyz":
            device = "/dev/xvd" + letter
            if device not in devices and not os.path.exists(device):
                return device
    return None

def get_latest_snapshot(tag_name, tag_value):
    """Get the latest snapshot with a given tag
    """
    set_region()
    ec2res = boto3.resource("ec2")
    snapshots = sorted(ec2res.snapshots.filter(
        Filters=[{'Name': 'tag:' + tag_name,
                  'Values': [tag_value]}]),
                       key=lambda k: k.start_time, reverse=True)
    if snapshots:
        return snapshots[0]
    else:
        return None

# Usage: create_volume snapshot-id [size_gb]
def create_volume(snapshot_id, availability_zone=None, size_gb=None):
    set_region()
    ec2 = boto3.client("ec2")
    args = {'SnapshotId': snapshot_id,
            'VolumeType': 'gp2'}
    if not availability_zone:
        availability_zone = InstanceInfo().availability_zone()
    args['AvailabilityZone'] = availability_zone
    if size_gb:
        args['Size'] = size_gb
    resp = ec2.create_volume(**args)
    wait_for_volume_status(resp['VolumeId'], "available")
    return resp['VolumeId']

# Usage: create_empty_volume size_gb
def create_empty_volume(size_gb, availability_zone=None):
    set_region()
    ec2 = boto3.client("ec2")
    args = {'Size': size_gb,
            'VolumeType': 'gp2'}
    if not availability_zone:
        availability_zone = InstanceInfo().availability_zone()
    args['AvailabilityZone'] = availability_zone
    resp = ec2.create_volume(**args)
    wait_for_volume_status(resp['VolumeId'], "available")
    return resp['VolumeId']

def wait_for_volume_status(volume_id, status, timeout_sec=300):
    set_region()
    start = time.time()
    ec2 = boto3.client("ec2")
    volume = None
    while not match_volume_state(volume, status):
        time.sleep(2)
        if time.time() - start > timeout_sec:
            raise Exception("Failed waiting for status '" + status + "' for " +\
                            volume_id + " (timeout: " + str(timeout_sec) + ")")
        resp = ec2.describe_volumes(VolumeIds=[volume_id])
        if "Volumes" in resp:
            volume = resp['Volumes'][0]

def match_volume_state(volume, status):
    if not volume:
        return False
    if status == "attached":
        return 'Attachments' in volume and len(volume['Attachments']) > 0 and \
               volume['Attachments'][0]['State'] == "attached"
    else:
        return volume['State'] == status

def wait_for_snapshot_complete(snapshot_id, timeout_sec=900):
    set_region()
    start = time.time()
    ec2 = boto3.client("ec2")
    snapshot = None
    while not is_snapshot_complete(snapshot):
        time.sleep(2)
        if time.time() - start > timeout_sec:
            raise Exception("Failed waiting for status 'completed' for " +\
                            snapshot_id + " (timeout: " + str(timeout_sec) + ")")
        resp = ec2.describe_snapshots(SnapshotIds=[snapshot_id])
        if "Snapshots" in resp:
            snapshot = resp['Snapshots'][0]

def is_snapshot_complete(snapshot):
    return snapshot is not None and 'State' in snapshot and \
           snapshot['State'] == 'completed'

# Usage: attach_volume volume-id device-path
def attach_volume(volume_id, device_path):
    set_region()
    ec2 = boto3.client("ec2")
    instance_id = InstanceInfo().instance_id()
    ec2.attach_volume(VolumeId=volume_id, InstanceId=instance_id,
                      Device=device_path)
    wait_for_volume_status(volume_id, "attached")

# Usage: delete_on_termination device-path
def delete_on_termination(device_path):
    set_region()
    ec2 = boto3.client("ec2")
    instance_id = InstanceInfo().instance_id()
    ec2.modify_instance_attribute(InstanceId=instance_id,
                                  BlockDeviceMappings=[{
                                      "DeviceName": device_path,
                                      "Ebs":{"DeleteOnTermination": True}}])

def detach_volume(mount_path):
    set_region()
    device = device_from_mount_path(mount_path)
    ec2 = boto3.client("ec2")
    instance_id = InstanceInfo().instance_id()
    volume = ec2.describe_volumes(Filters=[{"Name": "attachment.device",
                                            "Values": [device]},
                                           {"Name": "attachment.instance-id",
                                            "Values": [instance_id]}])
    volume_id = volume['Volumes'][0]['VolumeId']
    ec2.detach_volume(VolumeId=volume_id, InstanceId=instance_id)

# Usage: create_snapshot volume_id tag_key tag_value
def create_snapshot(tag_key, tag_value, mount_path, wait=False):
    set_region()
    device = device_from_mount_path(mount_path)
    with open(os.devnull, 'w') as devnull:
        subprocess.call(["sync", mount_path[0]], stdout=devnull,
                        stderr=devnull)
    ec2 = boto3.client("ec2")
    instance_id = InstanceInfo().instance_id()
    volume = ec2.describe_volumes(Filters=[{"Name": "attachment.device",
                                            "Values": [device]},
                                           {"Name": "attachment.instance-id",
                                            "Values": [instance_id]}])
    volume_id = volume['Volumes'][0]['VolumeId']
    snap = ec2.create_snapshot(VolumeId=volume_id)
    ec2.create_tags(Resources=[snap['SnapshotId']],
                    Tags=[{'Key': tag_key, 'Value': tag_value},
                          {'Key': 'Name', 'Value': tag_value}])
    if wait:
        wait_for_snapshot_complete(snap['SnapshotId'])
    return snap['SnapshotId']

def device_from_mount_path(mount_path):
    if sys.platform.startswith('win'):
        return "/dev/xvd" + \
                 target_id_to_letter(disk_by_drive_letter(
                     mount_path[0])['TargetId'])
    else:
        return [x for x in psutil.disk_partitions() \
                      if x.mountpoint == mount_path][0].device

def clean_snapshots(days, tags):
    set_region()
    ec2 = boto3.client("ec2")
    account_id = resolve_account()
    newest_timestamp = datetime.utcnow() - timedelta(days=days)
    newest_timestamp = newest_timestamp .replace(tzinfo=None)
    paginator = ec2.get_paginator('describe_snapshots')
    for page in paginator.paginate(OwnerIds=[account_id],
                                   Filters=[{'Name': 'tag-value',
                                             'Values': tags}],
                                   PaginationConfig={'PageSize': 1000}):
        for snapshot in page['Snapshots']:
            tags = {}
            for tag in snapshot['Tags']:
                tags[tag['Key']] = tag['Value']
            print_time = snapshot['StartTime'].replace(tzinfo=\
                                                       tz.tzlocal()).timetuple()
            compare_time = snapshot['StartTime'].replace(tzinfo=None)
            if compare_time < newest_timestamp:
                print colored("Deleting " + snapshot['SnapshotId'], "yellow") +\
                              " || " +\
                              time.strftime("%a, %d %b %Y %H:%M:%S",
                                            print_time) + \
                              " || " + json.dumps(tags)
                try:
                    ec2.delete_snapshot(SnapshotId=snapshot['SnapshotId'])
                    time.sleep(0.2)
                except ClientError as err:
                    print colored("Delete failed: " + \
                                  err.response['Error']['Message'], "red")
            else:
                print colored("Skipping " + snapshot['SnapshotId'], "cyan") +\
                              " || " + \
                              time.strftime("%a, %d %b %Y %H:%M:%S",
                                            print_time) +\
                              " || " + json.dumps(tags)
<commit_msg>Forgot mounting<commit_after>#!/usr/bin/env python

import argparse
import time
import json
import sys
import os
import subprocess
from subprocess import PIPE, Popen
import xml.etree.ElementTree as ET
from datetime import datetime, timedelta
from dateutil import tz
from termcolor import colored
import argcomplete
import boto3
from botocore.exceptions import ClientError
import psutil
from .cf_utils import set_region, resolve_account, InstanceInfo
from .aws_infra_util import find_include

def letter_to_target_id(letter):
    return ord(letter)-ord("f")+5

def target_id_to_letter(target_id):
    return str(unichr(target_id-5+ord("f")))

def wmic_partition_get():
    return wmic_get("partition")

def wmic_diskdrive_get():
    return wmic_get("diskdrive")

def wmic_volume_get():
    return wmic_get("volume")

def wmic_get(command):
    ret = []
    proc = Popen(["wmic", command, "get",
                  "/format:rawxml"], stdout=PIPE, stderr=PIPE)
    output = proc.communicate()[0]
    tree = ET.fromstring(output)
    for elem in tree.iter("RESULTS"):
        for inst in elem.iter("INSTANCE"):
            disk = {}
            for prop in inst.iter("PROPERTY"):
                try:
                    disk[prop.attrib['NAME']] = int(prop.findtext("*"))
                except ValueError:
                    disk[prop.attrib['NAME']] = prop.findtext("*")
                except TypeError:
                    continue
            ret.append(disk)
    return ret

def wmic_disk_with_target_id(target_id):
    return [x for x in wmic_diskdrive_get() if x['SCSITargetId'] == target_id][0]

def wmic_max_target_id():
    return sorted([x['SCSITargetId'] for x in wmic_diskdrive_get()],
                  reverse=True)[0]

def disk_by_drive_letter(drive_letter):
    ret = {}
    proc = Popen(["powershell.exe", find_include("disk-by-drive-letter.ps1"),
                  drive_letter.upper() + ":"], stdout=PIPE, stderr=PIPE)
    output = proc.communicate()[0]
    tree = ET.fromstring(output)
    for obj in tree.iter("Object"):
        for prop in obj.iter("Property"):
            try:
                ret[prop.attrib['Name']] = int(prop.findtext("."))
            except ValueError:
                ret[prop.attrib['Name']] = prop.findtext(".")
            except TypeError:
                continue
    return ret

def latest_snapshot():
    """Get the latest snapshot with a given tag
    """
    parser = argparse.ArgumentParser(description=latest_snapshot.__doc__)
    parser.add_argument("tag", help="The tag to find snapshots with")
    argcomplete.autocomplete(parser)
    args = parser.parse_args()
    print first_free_device()
    snapshot = get_latest_snapshot(args.tag, args.tag)
    if snapshot:
        print snapshot.id
    else:
        sys.exit(1)

def volume_from_snapshot(tag_key, tag_value, mount_path, availability_zone=None,
                         size_gb=None):
    set_region()
    snapshot = get_latest_snapshot(tag_key, tag_value)
    if snapshot:
        print "Found snapshot " + snapshot.id
        volume = create_volume(snapshot.id, availability_zone=availability_zone,
                               size_gb=size_gb)
    else:
        if not size_gb:
            size_gb = 32
        print "Creating empty volyme of size " + str(size_gb)
        volume = create_empty_volume(size_gb,
                                     availability_zone=availability_zone)
    device = first_free_device()
    print "Attaching volume " + volume + " to " + device
    attach_volume(volume, device)
    if not snapshot:
        #empty device
        if sys.platform.startswith('win'):
            #Windows format
            disk = wmic_disk_with_target_id(letter_to_target_id(device[-1:]))
            drive_letter = mount_path[0].upper()
            disk_number = str(disk['Index'])
            subprocess.check_call(["powershell.exe", "Get-Disk", disk_number,
                                   "|", "Set-Disk", "-IsOffline", "$False"])
            subprocess.check_call(["powershell.exe", "Initialize-Disk",
                                   disk_number, "-PartitionStyle", "MBR"])
            subprocess.check_call(["powershell.exe", "New-Partition",
                                   "-DiskNumber", disk_number,
                                   "-UseMaximumSize", "-DriveLetter",
                                   drive_letter])
            print "Formatting " + device + "(" + drive_letter + ":)"
            subprocess.check_call(["powershell.exe", "Format-Volume",
                                   "-DriveLetter", drive_letter, "-FileSystem",
                                   "NTFS", "-Force", "-Confirm:$False"])
        else:
            #linux format
            print "Formatting " + device
            subprocess.check_call(["mkfs.ext4", device])
    else:
        if sys.platform.startswith('win'):
            target_id = letter_to_target_id(device[-1:])
            disk = wmic_disk_with_target_id(target_id)
            drive_letter = mount_path[0].upper()
            disk_number = str(disk['Index'])
            with open(os.devnull, 'w') as devnull:
                subprocess.call(["powershell.exe", "Initialize-Disk",
                                 disk_number, "-PartitionStyle", "MBR"],
                                stderr=devnull, stdout=devnull)
            subprocess.check_call(["powershell.exe", "Get-Disk", disk_number,
                                   "|", "Set-Disk", "-IsOffline", "$False"])
            with open(os.devnull, 'w') as devnull:
                subprocess.check_call(["powershell.exe", "Get-Partition",
                                       "-DiskNumber", disk_number,
                                       "-PartitionNumber", "1"
                                       "|", "Set-Partition", "-NewDriveLetter",
                                       drive_letter], stdout=devnull,
                                      stderr=devnull)
            #resize win partition if necessary
            if size_gb and not size_gb == snapshot.volume_size:
                proc = subprocess.Popen(["powershell.exe",
                                         "$((Get-PartitionSupportedSize -Dri" +\
                                         "veLetter " + drive_letter + \
                                         ").SizeMax)"],
                                        stdout=subprocess.PIPE)
                max_size = proc.communicate()[0]
                subprocess.check_call(["powershell.exe", "Resize-Partition",
                                       "-DriveLetter", drive_letter, "-Size",
                                       max_size])
        else:
            if size_gb and not size_gb == snapshot.volume_size:
                print "Resizing " + device + " from " + snapshot.volume_size +\
                      "GB to " + size_gb
                subprocess.check_call(["e2fsck", "-f", "-p", device])
                subprocess.check_call(["resize2fs", device])
    if not sys.platform.startswith('win'):
        if not os.path.isdir(mount_path):
            os.makedirs(mount_path)
        subprocess.check_call(["mount", device, mount_path])

def first_free_device():
    if sys.platform.startswith('win'):
        max_target = wmic_max_target_id()
        if max_target == 0:
            return "/dev/xvdf"
        else:
            return "/dev/xvd" + target_id_to_letter(max_target + 1)
    else:
        devices = [x.device for x in psutil.disk_partitions()]
        print devices
        for letter in "fghijklmnopqrstuvxyz":
            device = "/dev/xvd" + letter
            if device not in devices and not os.path.exists(device):
                return device
    return None

def get_latest_snapshot(tag_name, tag_value):
    """Get the latest snapshot with a given tag
    """
    set_region()
    ec2res = boto3.resource("ec2")
    snapshots = sorted(ec2res.snapshots.filter(
        Filters=[{'Name': 'tag:' + tag_name,
                  'Values': [tag_value]}]),
                       key=lambda k: k.start_time, reverse=True)
    if snapshots:
        return snapshots[0]
    else:
        return None

# Usage: create_volume snapshot-id [size_gb]
def create_volume(snapshot_id, availability_zone=None, size_gb=None):
    set_region()
    ec2 = boto3.client("ec2")
    args = {'SnapshotId': snapshot_id,
            'VolumeType': 'gp2'}
    if not availability_zone:
        availability_zone = InstanceInfo().availability_zone()
    args['AvailabilityZone'] = availability_zone
    if size_gb:
        args['Size'] = size_gb
    resp = ec2.create_volume(**args)
    wait_for_volume_status(resp['VolumeId'], "available")
    return resp['VolumeId']

# Usage: create_empty_volume size_gb
def create_empty_volume(size_gb, availability_zone=None):
    set_region()
    ec2 = boto3.client("ec2")
    args = {'Size': size_gb,
            'VolumeType': 'gp2'}
    if not availability_zone:
        availability_zone = InstanceInfo().availability_zone()
    args['AvailabilityZone'] = availability_zone
    resp = ec2.create_volume(**args)
    wait_for_volume_status(resp['VolumeId'], "available")
    return resp['VolumeId']

def wait_for_volume_status(volume_id, status, timeout_sec=300):
    set_region()
    start = time.time()
    ec2 = boto3.client("ec2")
    volume = None
    while not match_volume_state(volume, status):
        time.sleep(2)
        if time.time() - start > timeout_sec:
            raise Exception("Failed waiting for status '" + status + "' for " +\
                            volume_id + " (timeout: " + str(timeout_sec) + ")")
        resp = ec2.describe_volumes(VolumeIds=[volume_id])
        if "Volumes" in resp:
            volume = resp['Volumes'][0]

def match_volume_state(volume, status):
    if not volume:
        return False
    if status == "attached":
        return 'Attachments' in volume and len(volume['Attachments']) > 0 and \
               volume['Attachments'][0]['State'] == "attached"
    else:
        return volume['State'] == status

def wait_for_snapshot_complete(snapshot_id, timeout_sec=900):
    set_region()
    start = time.time()
    ec2 = boto3.client("ec2")
    snapshot = None
    while not is_snapshot_complete(snapshot):
        time.sleep(2)
        if time.time() - start > timeout_sec:
            raise Exception("Failed waiting for status 'completed' for " +\
                            snapshot_id + " (timeout: " + str(timeout_sec) + ")")
        resp = ec2.describe_snapshots(SnapshotIds=[snapshot_id])
        if "Snapshots" in resp:
            snapshot = resp['Snapshots'][0]

def is_snapshot_complete(snapshot):
    return snapshot is not None and 'State' in snapshot and \
           snapshot['State'] == 'completed'

# Usage: attach_volume volume-id device-path
def attach_volume(volume_id, device_path):
    set_region()
    ec2 = boto3.client("ec2")
    instance_id = InstanceInfo().instance_id()
    ec2.attach_volume(VolumeId=volume_id, InstanceId=instance_id,
                      Device=device_path)
    wait_for_volume_status(volume_id, "attached")

# Usage: delete_on_termination device-path
def delete_on_termination(device_path):
    set_region()
    ec2 = boto3.client("ec2")
    instance_id = InstanceInfo().instance_id()
    ec2.modify_instance_attribute(InstanceId=instance_id,
                                  BlockDeviceMappings=[{
                                      "DeviceName": device_path,
                                      "Ebs":{"DeleteOnTermination": True}}])

def detach_volume(mount_path):
    set_region()
    device = device_from_mount_path(mount_path)
    ec2 = boto3.client("ec2")
    instance_id = InstanceInfo().instance_id()
    volume = ec2.describe_volumes(Filters=[{"Name": "attachment.device",
                                            "Values": [device]},
                                           {"Name": "attachment.instance-id",
                                            "Values": [instance_id]}])
    volume_id = volume['Volumes'][0]['VolumeId']
    ec2.detach_volume(VolumeId=volume_id, InstanceId=instance_id)

# Usage: create_snapshot volume_id tag_key tag_value
def create_snapshot(tag_key, tag_value, mount_path, wait=False):
    set_region()
    device = device_from_mount_path(mount_path)
    with open(os.devnull, 'w') as devnull:
        subprocess.call(["sync", mount_path[0]], stdout=devnull,
                        stderr=devnull)
    ec2 = boto3.client("ec2")
    instance_id = InstanceInfo().instance_id()
    volume = ec2.describe_volumes(Filters=[{"Name": "attachment.device",
                                            "Values": [device]},
                                           {"Name": "attachment.instance-id",
                                            "Values": [instance_id]}])
    volume_id = volume['Volumes'][0]['VolumeId']
    snap = ec2.create_snapshot(VolumeId=volume_id)
    ec2.create_tags(Resources=[snap['SnapshotId']],
                    Tags=[{'Key': tag_key, 'Value': tag_value},
                          {'Key': 'Name', 'Value': tag_value}])
    if wait:
        wait_for_snapshot_complete(snap['SnapshotId'])
    return snap['SnapshotId']

def device_from_mount_path(mount_path):
    if sys.platform.startswith('win'):
        return "/dev/xvd" + \
                 target_id_to_letter(disk_by_drive_letter(
                     mount_path[0])['TargetId'])
    else:
        return [x for x in psutil.disk_partitions() \
                      if x.mountpoint == mount_path][0].device

def clean_snapshots(days, tags):
    set_region()
    ec2 = boto3.client("ec2")
    account_id = resolve_account()
    newest_timestamp = datetime.utcnow() - timedelta(days=days)
    newest_timestamp = newest_timestamp .replace(tzinfo=None)
    paginator = ec2.get_paginator('describe_snapshots')
    for page in paginator.paginate(OwnerIds=[account_id],
                                   Filters=[{'Name': 'tag-value',
                                             'Values': tags}],
                                   PaginationConfig={'PageSize': 1000}):
        for snapshot in page['Snapshots']:
            tags = {}
            for tag in snapshot['Tags']:
                tags[tag['Key']] = tag['Value']
            print_time = snapshot['StartTime'].replace(tzinfo=\
                                                       tz.tzlocal()).timetuple()
            compare_time = snapshot['StartTime'].replace(tzinfo=None)
            if compare_time < newest_timestamp:
                print colored("Deleting " + snapshot['SnapshotId'], "yellow") +\
                              " || " +\
                              time.strftime("%a, %d %b %Y %H:%M:%S",
                                            print_time) + \
                              " || " + json.dumps(tags)
                try:
                    ec2.delete_snapshot(SnapshotId=snapshot['SnapshotId'])
                    time.sleep(0.2)
                except ClientError as err:
                    print colored("Delete failed: " + \
                                  err.response['Error']['Message'], "red")
            else:
                print colored("Skipping " + snapshot['SnapshotId'], "cyan") +\
                              " || " + \
                              time.strftime("%a, %d %b %Y %H:%M:%S",
                                            print_time) +\
                              " || " + json.dumps(tags)
<|endoftext|>


<commit_before>from pony import orm
from datetime import datetime
import feedparser
from flask import Flask, render_template, request, redirect, url_for
from urllib.parse import urlparse

db = orm.Database()
#orm.sql_debug(True)
db.bind('sqlite', 'fbdb.sqlite', create_db=True)

app = Flask(__name__)

class Feed(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    articles = orm.Set('Article')
    categories = orm.Set('Category')
    title = orm.Required(str)
    url = orm.Required(str, unique=True)
    etag = orm.Optional(str)
    modified = orm.Optional(str)

class Article(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    feed = orm.Required('Feed')
    author = orm.Optional('Author')
    title = orm.Required(str)
    url = orm.Required(str, unique=True)
    read = orm.Optional(bool, default=False)
    published = orm.Required(datetime)
    summary = orm.Optional(str)

class Category(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    title = orm.Required(str, unique=True)
    feeds = orm.Set('Feed')

class Author(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    name = orm.Required(str, unique=True)
    articles = orm.Set('Article')

db.generate_mapping(create_tables=True)

@orm.db_session
def add_category(title):
    exists = Category.get(title=title)
    if exists:
        return exists.id
    else:
        new_category = Category(title=title)
        orm.commit()
        return new_category.id

@orm.db_session
def del_category(id):
    Category[id].delete()

@orm.db_session
def add_author(name):
    exists=Author.get(name=name)
    if exists:
        return exists.id
    else:
        new_author = Author(name=name)
        orm.commit()
        return new_author.id

@orm.db_session
def del_author(id):
    Author[id].delete()

@app.template_filter('datetime')
def format_datetime(value, format='full'):
    formats = {'full': '%m/%d/%Y %I:%M%p', 'date': '%m/%d/%Y', 'time': '%I:%M%p'}
    return value.strftime(formats[format])

@app.route('/')
@orm.db_session
def home():
    uncategorized = list(Feed.select(lambda u: not u.categories))
    categories = list(Category.select())
    return render_template('home.html', uncategorized=uncategorized, categories=categories)

@app.route('/feed/<int:id>')
@orm.db_session
def feed(id):
    try:
        feed = Feed[id]
        articles = list(orm.select(a for a in Article if a.feed is feed).order_by(orm.desc(Article.published)))
        return render_template('feed.html', feed=feed, articles=articles)
    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Feed', id=id)

@app.route('/add_feed', methods=['POST'])
@orm.db_session
def add_feed():
    if request.form['url']:
        url = request.form['url']
        new_feed = Feed.get(url=url)
        if not new_feed:
            p = feedparser.parse(url)
            title = p.feed.title if 'title' in p.feed else url
            new_feed = Feed(title=title, url=url)

    return redirect(redirect_referrer())

@app.route('/del_feed/<int:id>')
@orm.db_session
def del_feed(id):
    try:
        Feed[id].delete()
        return redirect(url_for('home'))
    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Feed', id=id)

@app.route('/edit_feed/<int:id>', methods=['GET', 'POST'])
@orm.db_session
def edit_feed(id):
    try:
        feed = Feed[id]
        if request.method == 'POST' and request.form['submit'] == 'Save':
            feed.title = request.form['title']
            feed.url = request.form['url']
            old = set(c.id for c in feed.categories)
            new = set(request.form.getlist('category'))
            if  new != old:
                feed.categories.clear()
                [feed.categories.add(Category[c]) for c in new]
        elif request.method == 'POST' and request.form['submit'] == 'Delete':
            return redirect(url_for('del_feed', id=id))

        other_categories = list(orm.select(c for c in Category if c not in feed.categories))
        return render_template('edit_feed.html', feed=feed, other_categories=other_categories)

    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Feed', id=id)

@app.route('/category')
@app.route('/category/<int:id>')
@orm.db_session
def category(id=-1):
    if id < 0:
        articles = list(orm.select(a for a in Article if not a.feed.categories).order_by(orm.desc(Article.published)))
        return render_template('category.html', articles=articles)
    try:
        category = Category[id]
        articles = list(orm.select(a for a in Article if a.feed in category.feeds).order_by(orm.desc(Article.published)))
        return render_template('category.html', category=category, articles=articles)
    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Category', id=id)

@app.route('/add_category', methods=['POST'])
@orm.db_session
def add_category():
    if request.form['category']:
        title = request.form['category']
        new_category = Category.get(title=title)
        if not new_category:
            new_category = Category(title=title)

    return redirect(redirect_referrer())

@app.route('/del_category/<int:id>')
@orm.db_session
def del_category(id):
    try:
        Category[id].delete()
        return redirect(url_for('home'))
    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Category', id=id)

@app.route('/edit_category/<int:id>', methods=['GET', 'POST'])
@orm.db_session
def edit_category(id):
    try:
        category = Category[id]
        if request.method == 'POST' and request.form['submit'] == 'Save':
            category.title = request.form['title']
            old = set(f.id for f in category.feeds)
            new = set(request.form.getlist('feed'))
            if  new != old:
                category.feeds.clear()
                [category.feeds.add(Feed[f]) for f in new]
        elif request.method == 'POST' and request.form['submit'] == 'Delete':
            return redirect(url_for('del_category', id=id))

        other_feeds = list(orm.select(f for f in Feed if f not in category.feeds))
        return render_template('edit_category.html', category=category, other_feeds=other_feeds)
    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Category', id=id)

@app.route('/fetch/<int:id>')
@orm.db_session
def fetch(id):
    try:
        feed = Feed[id]
        p = feedparser.parse(feed.url, etag=feed.etag, modified=feed.modified)
        feed.etag = p.etag if 'etag' in p else ''
        feed.modified = p.modified if 'modified' in p else ''
        for e in p.entries if 'entries' in p else []:
            author = Author[add_author(e.author)] if 'author' in e and e.author else Author[add_author('None')]
            title = e.title if 'title' in e else ''
            url = e.link if 'link' in e else feed.url
            published = datetime(*e.published_parsed[:6]) if 'published_parsed' in e else datetime.utcnow()
            summary = e.summary if 'summary' in e else ''
            a = Article.get(url=url)
            if a:
                if a.published != published:
                    a.feed = feed
                    a.author = author
                    a.title = title
                    a.published = published
                    a.summary = summary
            else:
                new_article = Article(feed=feed, author=author, title=title, url=url, published=published, summary=summary)
        return redirect(url_for('feed', id=id))
    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Feed', id=id)

@app.route('/fetch_all')
@orm.db_session
def fetch_all():
    for feed_id in orm.select(f.id for f in Feed):
        fetch(feed_id)
    return redirect(url_for('home'))

@app.route('/fetch_category')
@app.route('/fetch_category/<int:id>')
@orm.db_session
def fetch_category(id=-1):
    if id < 0:
        feeds = orm.select(f for f in Feed if not f.categories)
        for feed in feeds:
            fetch(feed.id)
        return redirect(url_for('category'))
    else:
        try:
            category = Category[id]
            for feed in category.feeds:
                fetch(feed.id)
            return redirect(url_for('category', id=id))
        except orm.ObjectNotFound:
            return render_template('missing.html', entity='Category', id=id)

def redirect_referrer(default='home'):
    if urlparse(url_for(default, _external=True)).netloc == urlparse(request.referrer).netloc:
        return request.referrer
    else:
        return url_for(default)
<commit_msg>Summaries Stripped and Limited<commit_after>from pony import orm
from datetime import datetime
import feedparser
from flask import Flask, render_template, request, redirect, url_for
from urllib.parse import urlparse
from bs4 import BeautifulSoup, Comment

db = orm.Database()
#orm.sql_debug(True)
db.bind('sqlite', 'fbdb.sqlite', create_db=True)

app = Flask(__name__)

class Feed(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    articles = orm.Set('Article')
    categories = orm.Set('Category')
    title = orm.Required(str)
    url = orm.Required(str, unique=True)
    etag = orm.Optional(str)
    modified = orm.Optional(str)

class Article(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    feed = orm.Required('Feed')
    author = orm.Optional('Author')
    title = orm.Required(str)
    url = orm.Required(str, unique=True)
    read = orm.Optional(bool, default=False)
    published = orm.Required(datetime)
    summary = orm.Optional(str)

class Category(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    title = orm.Required(str, unique=True)
    feeds = orm.Set('Feed')

class Author(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    name = orm.Required(str, unique=True)
    articles = orm.Set('Article')

db.generate_mapping(create_tables=True)

@orm.db_session
def add_category(title):
    exists = Category.get(title=title)
    if exists:
        return exists.id
    else:
        new_category = Category(title=title)
        orm.commit()
        return new_category.id

@orm.db_session
def del_category(id):
    Category[id].delete()

@orm.db_session
def add_author(name):
    exists=Author.get(name=name)
    if exists:
        return exists.id
    else:
        new_author = Author(name=name)
        orm.commit()
        return new_author.id

@orm.db_session
def del_author(id):
    Author[id].delete()

@app.template_filter('datetime')
def format_datetime(value, format='full'):
    formats = {'full': '%m/%d/%Y %I:%M%p', 'date': '%m/%d/%Y', 'time': '%I:%M%p'}
    return value.strftime(formats[format])

@app.route('/')
@orm.db_session
def home():
    uncategorized = list(Feed.select(lambda u: not u.categories))
    categories = list(Category.select())
    return render_template('home.html', uncategorized=uncategorized, categories=categories)

@app.route('/feed/<int:id>')
@orm.db_session
def feed(id):
    try:
        feed = Feed[id]
        articles = list(orm.select(a for a in Article if a.feed is feed).order_by(orm.desc(Article.published)))
        return render_template('feed.html', feed=feed, articles=articles)
    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Feed', id=id)

@app.route('/add_feed', methods=['POST'])
@orm.db_session
def add_feed():
    if request.form['url']:
        url = request.form['url']
        new_feed = Feed.get(url=url)
        if not new_feed:
            p = feedparser.parse(url)
            title = p.feed.title if 'title' in p.feed else url
            new_feed = Feed(title=title, url=url)

    return redirect(redirect_referrer())

@app.route('/del_feed/<int:id>')
@orm.db_session
def del_feed(id):
    try:
        Feed[id].delete()
        return redirect(url_for('home'))
    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Feed', id=id)

@app.route('/edit_feed/<int:id>', methods=['GET', 'POST'])
@orm.db_session
def edit_feed(id):
    try:
        feed = Feed[id]
        if request.method == 'POST' and request.form['submit'] == 'Save':
            feed.title = request.form['title']
            feed.url = request.form['url']
            old = set(c.id for c in feed.categories)
            new = set(request.form.getlist('category'))
            if  new != old:
                feed.categories.clear()
                [feed.categories.add(Category[c]) for c in new]
        elif request.method == 'POST' and request.form['submit'] == 'Delete':
            return redirect(url_for('del_feed', id=id))

        other_categories = list(orm.select(c for c in Category if c not in feed.categories))
        return render_template('edit_feed.html', feed=feed, other_categories=other_categories)

    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Feed', id=id)

@app.route('/category')
@app.route('/category/<int:id>')
@orm.db_session
def category(id=-1):
    if id < 0:
        articles = list(orm.select(a for a in Article if not a.feed.categories).order_by(orm.desc(Article.published)))
        return render_template('category.html', articles=articles)
    try:
        category = Category[id]
        articles = list(orm.select(a for a in Article if a.feed in category.feeds).order_by(orm.desc(Article.published)))
        return render_template('category.html', category=category, articles=articles)
    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Category', id=id)

@app.route('/add_category', methods=['POST'])
@orm.db_session
def add_category():
    if request.form['category']:
        title = request.form['category']
        new_category = Category.get(title=title)
        if not new_category:
            new_category = Category(title=title)

    return redirect(redirect_referrer())

@app.route('/del_category/<int:id>')
@orm.db_session
def del_category(id):
    try:
        Category[id].delete()
        return redirect(url_for('home'))
    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Category', id=id)

@app.route('/edit_category/<int:id>', methods=['GET', 'POST'])
@orm.db_session
def edit_category(id):
    try:
        category = Category[id]
        if request.method == 'POST' and request.form['submit'] == 'Save':
            category.title = request.form['title']
            old = set(f.id for f in category.feeds)
            new = set(request.form.getlist('feed'))
            if  new != old:
                category.feeds.clear()
                [category.feeds.add(Feed[f]) for f in new]
        elif request.method == 'POST' and request.form['submit'] == 'Delete':
            return redirect(url_for('del_category', id=id))

        other_feeds = list(orm.select(f for f in Feed if f not in category.feeds))
        return render_template('edit_category.html', category=category, other_feeds=other_feeds)
    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Category', id=id)

@app.route('/fetch/<int:id>')
@orm.db_session
def fetch(id):
    try:
        feed = Feed[id]
        p = feedparser.parse(feed.url, etag=feed.etag, modified=feed.modified)
        feed.etag = p.etag if 'etag' in p else ''
        feed.modified = p.modified if 'modified' in p else ''
        for e in p.entries if 'entries' in p else []:
            author = Author[add_author(e.author)] if 'author' in e and e.author else Author[add_author('None')]
            title = e.title if 'title' in e else ''
            url = e.link if 'link' in e else feed.url
            published = datetime(*e.published_parsed[:6]) if 'published_parsed' in e else datetime.utcnow()
            summary = strip_summary(e.summary) if 'summary' in e else ''
            a = Article.get(url=url)
            if a:
                if a.published != published:
                    a.feed = feed
                    a.author = author
                    a.title = title
                    a.published = published
                    a.summary = summary
            else:
                new_article = Article(feed=feed, author=author, title=title, url=url, published=published, summary=summary)
        return redirect(url_for('feed', id=id))
    except orm.ObjectNotFound:
        return render_template('missing.html', entity='Feed', id=id)

@app.route('/fetch_all')
@orm.db_session
def fetch_all():
    for feed_id in orm.select(f.id for f in Feed):
        fetch(feed_id)
    return redirect(url_for('home'))

@app.route('/fetch_category')
@app.route('/fetch_category/<int:id>')
@orm.db_session
def fetch_category(id=-1):
    if id < 0:
        feeds = orm.select(f for f in Feed if not f.categories)
        for feed in feeds:
            fetch(feed.id)
        return redirect(url_for('category'))
    else:
        try:
            category = Category[id]
            for feed in category.feeds:
                fetch(feed.id)
            return redirect(url_for('category', id=id))
        except orm.ObjectNotFound:
            return render_template('missing.html', entity='Category', id=id)

def redirect_referrer(default='home'):
    if urlparse(url_for(default, _external=True)).netloc == urlparse(request.referrer).netloc:
        return request.referrer
    else:
        return url_for(default)

def strip_summary(summary):
    soup = BeautifulSoup(summary, 'html.parser')
    text = soup.find_all(text=True)
    return ' '.join(list(filter(visible, text)))[:300] + '...'

def visible(element):
    if element.parent.name in ['style', 'script', '[document]', 'head', 'title', 'meta']:
        return False
    elif isinstance(element, Comment):
        return False
    return True
<|endoftext|>


<commit_before>"""
Here are a collection of examples of how this module can be used.
See the README or the natsort homepage for more details.

    >>> a = ['a2', 'a8', 'a7', 'a5', 'a9', 'a1', 'a4', 'a10', 'a3', 'a6']
    >>> sorted(a)
    ['a1', 'a10', 'a2', 'a3', 'a4', 'a5', 'a6', 'a7', 'a8', 'a9']
    >>> natsorted(a)
    ['a1', 'a2', 'a3', 'a4', 'a5', 'a6', 'a7', 'a8', 'a9', 'a10']

    >>> a = ['a50', 'a51.', 'a50.4', 'a5.034e1', 'a50.300']
    >>> sorted(a)
    ['a5.034e1', 'a50', 'a50.300', 'a50.4', 'a51.']
    >>> natsorted(a)
    ['a50', 'a50.300', 'a5.034e1', 'a50.4', 'a51.']
    >>> natsorted(a, number_type=None)
    ['a5.034e1', 'a50', 'a50.4', 'a50.300', 'a51.']

    >>> a = ['1.9.9a', '1.11', '1.9.9b', '1.11.4', '1.10.1']
    >>> sorted(a)
    ['1.10.1', '1.11', '1.11.4', '1.9.9a', '1.9.9b']
    >>> natsorted(a)
    ['1.10.1', '1.11', '1.11.4', '1.9.9a', '1.9.9b']
    >>> natsorted(a, number_type=None)
    ['1.9.9a', '1.9.9b', '1.10.1', '1.11', '1.11.4']

    >>> a = ['name.1', 'name.101', 'name.01', 'name.200', 'name.21']
    >>> sorted(a)
    ['name.01', 'name.1', 'name.101', 'name.200', 'name.21']
    >>> natsorted(a)
    ['name.01', 'name.1', 'name.101', 'name.200', 'name.21']
    >>> natsorted(a, number_type=None)
    ['name.1', 'name.01', 'name.21', 'name.101', 'name.200']

    >>> a = ['version-2', 'version-20', 'version-4', 'version-1']
    >>> sorted(a)
    ['version-1', 'version-2', 'version-20', 'version-4']
    >>> natsorted(a)
    ['version-20', 'version-4', 'version-2', 'version-1']
    >>> natsorted(a, number_type=int)
    ['version-20', 'version-4', 'version-2', 'version-1']
    >>> natsorted(a, number_type=None)
    ['version-1', 'version-2', 'version-4', 'version-20']

    >>> a = [6, 4.5, '7', u'2.5']
    >>> sorted(a)
    [4.5, 6, u'2.5', '7']
    >>> natsorted(a)
    [u'2.5', 4.5, 6, '7']

"""

import re
# The regex that locates version numbers and floats
float_re = re.compile(r'([-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)')
# The regex of the exponent
exp_re = re.compile(r'^[eE][-+]?[0-9]+$')
# A basic digit splitter
digit_re = re.compile(r'(\d+)')
# Integer regex
int_re = re.compile(r'([-+]?[0-9]+)')

def remove_empty(s):
    """\
    Remove empty strings from a list.

        >>> a = ['a', 2, '', 'b']
        >>> remove_empty(a)
        ['a', 2, 'b']

    """
    while True:
        try:
            s.remove('')
        except ValueError:
            break
    return s

def find_floats(s):
    """\
    Locate all the floats in a string, and return a tuple of
    strings and floats.

        >>> find_floats('name3.5')
        ['name', 3.5]
        >>> find_floats('a5.034e1')
        ['a', 50.34]
        >>> find_floats('b-40.2')
        ['b', -40.2]

    """

    # Split.  If there are no splits, return now
    s = float_re.split(s)
    if len(s) == 1:
        return tuple(s)

    # Remove all the None elements and exponentials from the list.
    # This results from the way split works when there are parenthesis
    # in the regular expression
    ix = [i for i, x in enumerate(s) if x is None or exp_re.match(x)]
    for i in reversed(ix):
        s.pop(i)

    # Now convert floats to floats, and leave strings as strings
    s = remove_empty(s)
    for i in xrange(len(s)):
        try:
            s[i] = float(s[i])
        except ValueError:
            pass

    return s

def _simple_finder(s, regex):
    """Helper to split ints"""

    # Split.  If there are no splits, return now
    s = regex.split(s)
    if len(s) == 1:
        return tuple(s)

    # Now convert ints to ints, and leave strings as strings
    s = remove_empty(s)
    for i in xrange(len(s)):
        try:
            s[i] = int(s[i])
        except ValueError:
            pass

    return s

def find_ints(s):
    """\
    Locate all the ints in a string, and return a tuple of
    strings and ints.

        >>> find_ints('name3.5')
        ['name', 3, '.', 5]
        >>> find_ints('a5.034e1')
        ['a', 5, '.', 34, 'e', 1]
        >>> find_ints('b-40.2')
        ['b', -40, '.', 2]

    """
    return _simple_finder(s, int_re)

def find_digits(s):
    """\
    Locate all the digits in a string, and return a tuple of
    strings and ints.

        >>> find_digits('name3.5')
        ['name', 3, '.', 5]
        >>> find_digits('a5.034e1')
        ['a', 5, '.', 34, 'e', 1]
        >>> find_digits('b-40.2')
        ['b-', 40, '.', 2]

    """
    return _simple_finder(s, digit_re)

def natsort_key(s, number_type=float):
    """\
    Key to sort strings and numbers naturally, not by ASCII.
    It also has basic support for version numbers.
    For use in passing to the :py:func:`sorted` builtin or
    :py:meth:`sort` attribute of lists.

        >>> a = ['num3', 'num5', 'num2']
        >>> a.sort(key=natsort_key)
        >>> a
        ['num2', 'num3', 'num5']
        >>> class Foo:
        ...    def __init__(self, bar):
        ...        self.bar = bar
        ...    def __repr__(self):
        ...        return "Foo('{0}')".format(self.bar)
        >>> b = [Foo('num3'), Foo('num5'), Foo('num2')]
        >>> b.sort(key=lambda x: natsort_key(x.bar))
        >>> b
        [Foo('num2'), Foo('num3'), Foo('num5')]
        >>> from operator import attrgetter
        >>> c = [Foo('num3'), Foo('num5'), Foo('num2')]
        >>> f = attrgetter('bar')
        >>> c.sort(key=lambda x: natsort_key(f(x)))
        >>> c
        [Foo('num2'), Foo('num3'), Foo('num5')]

    """

    # If we are dealing with non-strings, return now
    if not isinstance(s, basestring):
        return (s,)

    # Convert to the proper tuple and return
    find_method = {float: find_floats, int: find_ints, None: find_digits}
    try:
        return tuple(find_method[number_type](s))
    except KeyError:
        raise ValueError("natsort_key: 'search' parameter {0} invalid".format(str(number_type)))

def natsorted(seq, key=lambda x: x, number_type=float):
    """\
    Sorts a sequence naturally (alphabetically and numerically),
    not by ASCII.

        >>> a = ['num3', 'num5', 'num2']
        >>> natsorted(a)
        ['num2', 'num3', 'num5']
        >>> class Foo:
        ...    def __init__(self, bar):
        ...        self.bar = bar
        ...    def __repr__(self):
        ...        return "Foo('{0}')".format(self.bar)
        >>> b = [Foo('num3'), Foo('num5'), Foo('num2')]
        >>> from operator import attrgetter
        >>> natsorted(b, key=attrgetter('bar'))
        [Foo('num2'), Foo('num3'), Foo('num5')]

    :argument seq:
        The sequence to be sorted.
    :type seq: sequence-like
    :rtype: list
    """
    return sorted(seq, key=lambda x: natsort_key(key(x), number_type=number_type))

def index_natsorted(seq, key=lambda x: x, number_type=float):
    """\
    Sorts a sequence naturally, but returns a list of sorted the
    indeces and not the sorted list.

        >>> a = ['num3', 'num5', 'num2']
        >>> b = ['foo', 'bar', 'baz']
        >>> index = index_natsorted(a)
        >>> index
        [2, 0, 1]
        >>> # Sort both lists by the sort order of a
        >>> [a[i] for i in index]
        ['num2', 'num3', 'num5']
        >>> [b[i] for i in index]
        ['baz', 'foo', 'bar']
        >>> class Foo:
        ...    def __init__(self, bar):
        ...        self.bar = bar
        ...    def __repr__(self):
        ...        return "Foo('{0}')".format(self.bar)
        >>> c = [Foo('num3'), Foo('num5'), Foo('num2')]
        >>> from operator import attrgetter
        >>> index_natsorted(c, key=attrgetter('bar'))
        [2, 0, 1]

    :argument seq:
        The sequence that you want the sorted index of.
    :type seq: sequence-like
    :rtype: list
    """
    from operator import itemgetter
    item1 = itemgetter(1)
    # Pair the index and sequence together, then sort by
    index_seq_pair = [[x, key(y)] for x, y in zip(xrange(len(seq)), seq)]
    index_seq_pair.sort(key=lambda x: natsort_key(item1(x), number_type=number_type))
    return [x[0] for x in index_seq_pair]

def test():
    from doctest import DocTestSuite
    return DocTestSuite()

# Test this module
if __name__ == '__main__':
    import doctest
    doctest.testmod()
<commit_msg>Made float, int and digit commands use same function base. This involved changing the float regex to use non-capturing groups.<commit_after>"""
Here are a collection of examples of how this module can be used.
See the README or the natsort homepage for more details.

    >>> a = ['a2', 'a8', 'a7', 'a5', 'a9', 'a1', 'a4', 'a10', 'a3', 'a6']
    >>> sorted(a)
    ['a1', 'a10', 'a2', 'a3', 'a4', 'a5', 'a6', 'a7', 'a8', 'a9']
    >>> natsorted(a)
    ['a1', 'a2', 'a3', 'a4', 'a5', 'a6', 'a7', 'a8', 'a9', 'a10']

    >>> a = ['a50', 'a51.', 'a50.4', 'a5.034e1', 'a50.300']
    >>> sorted(a)
    ['a5.034e1', 'a50', 'a50.300', 'a50.4', 'a51.']
    >>> natsorted(a)
    ['a50', 'a50.300', 'a5.034e1', 'a50.4', 'a51.']
    >>> natsorted(a, number_type=None)
    ['a5.034e1', 'a50', 'a50.4', 'a50.300', 'a51.']

    >>> a = ['1.9.9a', '1.11', '1.9.9b', '1.11.4', '1.10.1']
    >>> sorted(a)
    ['1.10.1', '1.11', '1.11.4', '1.9.9a', '1.9.9b']
    >>> natsorted(a)
    ['1.10.1', '1.11', '1.11.4', '1.9.9a', '1.9.9b']
    >>> natsorted(a, number_type=None)
    ['1.9.9a', '1.9.9b', '1.10.1', '1.11', '1.11.4']

    >>> a = ['name.1', 'name.101', 'name.01', 'name.200', 'name.21']
    >>> sorted(a)
    ['name.01', 'name.1', 'name.101', 'name.200', 'name.21']
    >>> natsorted(a)
    ['name.01', 'name.1', 'name.101', 'name.200', 'name.21']
    >>> natsorted(a, number_type=None)
    ['name.1', 'name.01', 'name.21', 'name.101', 'name.200']

    >>> a = ['version-2', 'version-20', 'version-4', 'version-1']
    >>> sorted(a)
    ['version-1', 'version-2', 'version-20', 'version-4']
    >>> natsorted(a)
    ['version-20', 'version-4', 'version-2', 'version-1']
    >>> natsorted(a, number_type=int)
    ['version-20', 'version-4', 'version-2', 'version-1']
    >>> natsorted(a, number_type=None)
    ['version-1', 'version-2', 'version-4', 'version-20']

    >>> a = [6, 4.5, '7', u'2.5']
    >>> sorted(a)
    [4.5, 6, u'2.5', '7']
    >>> natsorted(a)
    [u'2.5', 4.5, 6, '7']

"""

import re
# The regex that locates floats
float_re = re.compile(r'([-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?)')
# A basic digit splitter
digit_re = re.compile(r'(\d+)')
# Integer regex
int_re = re.compile(r'([-+]?[0-9]+)')

def remove_empty(s):
    """\
    Remove empty strings from a list.

        >>> a = ['a', 2, '', 'b']
        >>> remove_empty(a)
        ['a', 2, 'b']

    """
    while True:
        try:
            s.remove('')
        except ValueError:
            break
    return s

def _number_finder(s, regex, numconv):
    """Helper to split numbers"""

    # Split.  If there are no splits, return now
    s = regex.split(s)
    if len(s) == 1:
        return tuple(s)

    # Now convert the numbers to numbers, and leave strings as strings
    s = remove_empty(s)
    for i in xrange(len(s)):
        try:
            s[i] = numconv(s[i])
        except ValueError:
            pass

    return s

def find_floats(s):
    """\
    Locate all the floats in a string, and return a tuple of
    strings and floats.

        >>> find_floats('name3.5')
        ['name', 3.5]
        >>> find_floats('a5.034e1')
        ['a', 50.34]
        >>> find_floats('b-40.2')
        ['b', -40.2]

    """
    return _number_finder(s, float_re, float)

def find_ints(s):
    """\
    Locate all the ints in a string, and return a tuple of
    strings and ints.

        >>> find_ints('name3.5')
        ['name', 3, '.', 5]
        >>> find_ints('a5.034e1')
        ['a', 5, '.', 34, 'e', 1]
        >>> find_ints('b-40.2')
        ['b', -40, '.', 2]

    """
    return _number_finder(s, int_re, int)

def find_digits(s):
    """\
    Locate all the digits in a string, and return a tuple of
    strings and ints.

        >>> find_digits('name3.5')
        ['name', 3, '.', 5]
        >>> find_digits('a5.034e1')
        ['a', 5, '.', 34, 'e', 1]
        >>> find_digits('b-40.2')
        ['b-', 40, '.', 2]

    """
    return _number_finder(s, digit_re, int)

def natsort_key(s, number_type=float):
    """\
    Key to sort strings and numbers naturally, not by ASCII.
    It also has basic support for version numbers.
    For use in passing to the :py:func:`sorted` builtin or
    :py:meth:`sort` attribute of lists.

        >>> a = ['num3', 'num5', 'num2']
        >>> a.sort(key=natsort_key)
        >>> a
        ['num2', 'num3', 'num5']
        >>> class Foo:
        ...    def __init__(self, bar):
        ...        self.bar = bar
        ...    def __repr__(self):
        ...        return "Foo('{0}')".format(self.bar)
        >>> b = [Foo('num3'), Foo('num5'), Foo('num2')]
        >>> b.sort(key=lambda x: natsort_key(x.bar))
        >>> b
        [Foo('num2'), Foo('num3'), Foo('num5')]
        >>> from operator import attrgetter
        >>> c = [Foo('num3'), Foo('num5'), Foo('num2')]
        >>> f = attrgetter('bar')
        >>> c.sort(key=lambda x: natsort_key(f(x)))
        >>> c
        [Foo('num2'), Foo('num3'), Foo('num5')]

    """

    # If we are dealing with non-strings, return now
    if not isinstance(s, basestring):
        return (s,)

    # Convert to the proper tuple and return
    find_method = {float: find_floats, int: find_ints, None: find_digits}
    try:
        return tuple(find_method[number_type](s))
    except KeyError:
        raise ValueError("natsort_key: 'search' parameter {0} invalid".format(str(number_type)))

def natsorted(seq, key=lambda x: x, number_type=float):
    """\
    Sorts a sequence naturally (alphabetically and numerically),
    not by ASCII.

        >>> a = ['num3', 'num5', 'num2']
        >>> natsorted(a)
        ['num2', 'num3', 'num5']
        >>> class Foo:
        ...    def __init__(self, bar):
        ...        self.bar = bar
        ...    def __repr__(self):
        ...        return "Foo('{0}')".format(self.bar)
        >>> b = [Foo('num3'), Foo('num5'), Foo('num2')]
        >>> from operator import attrgetter
        >>> natsorted(b, key=attrgetter('bar'))
        [Foo('num2'), Foo('num3'), Foo('num5')]

    :argument seq:
        The sequence to be sorted.
    :type seq: sequence-like
    :rtype: list
    """
    return sorted(seq, key=lambda x: natsort_key(key(x), number_type=number_type))

def index_natsorted(seq, key=lambda x: x, number_type=float):
    """\
    Sorts a sequence naturally, but returns a list of sorted the
    indeces and not the sorted list.

        >>> a = ['num3', 'num5', 'num2']
        >>> b = ['foo', 'bar', 'baz']
        >>> index = index_natsorted(a)
        >>> index
        [2, 0, 1]
        >>> # Sort both lists by the sort order of a
        >>> [a[i] for i in index]
        ['num2', 'num3', 'num5']
        >>> [b[i] for i in index]
        ['baz', 'foo', 'bar']
        >>> class Foo:
        ...    def __init__(self, bar):
        ...        self.bar = bar
        ...    def __repr__(self):
        ...        return "Foo('{0}')".format(self.bar)
        >>> c = [Foo('num3'), Foo('num5'), Foo('num2')]
        >>> from operator import attrgetter
        >>> index_natsorted(c, key=attrgetter('bar'))
        [2, 0, 1]

    :argument seq:
        The sequence that you want the sorted index of.
    :type seq: sequence-like
    :rtype: list
    """
    from operator import itemgetter
    item1 = itemgetter(1)
    # Pair the index and sequence together, then sort by
    index_seq_pair = [[x, key(y)] for x, y in zip(xrange(len(seq)), seq)]
    index_seq_pair.sort(key=lambda x: natsort_key(item1(x), number_type=number_type))
    return [x[0] for x in index_seq_pair]

def test():
    from doctest import DocTestSuite
    return DocTestSuite()

# Test this module
if __name__ == '__main__':
    import doctest
    doctest.testmod()
<|endoftext|>


<commit_before># Obzidi4n was here
from bs4 import BeautifulSoup
import cloudscraper, csv, re, requests, shutil

pluginFile = open('config/pluginurls.csv')
pluginReader = csv.reader(pluginFile)
pluginData = list(pluginReader)
pluginNum = pluginReader.line_num
manualPluginList = 'Manual Plugins:\n'

# command: update all
for i in range(1, pluginNum):

	pluginName = pluginData[i][0]
	pluginUrl = pluginData[i][2]

	print("Processing",i,"of",pluginNum,":",pluginName)

	# check if spigot
	if pluginData[i][1] == 'spigot':

		# invoke cloudscraper
		print ('Source: Spigot')
		scraper = cloudscraper.create_scraper()

		# get spigot page
		pluginApi = pluginUrl + '/history'
		r_page = scraper.get(pluginApi)

		# parse download link with BeautifulSoup
		soup = BeautifulSoup(r_page.text, 'html.parser')
		soup2 = soup.find('a', 'secondaryContent')
		target = 'http://www.spigotmc.org/' + soup2['href']

		# get response object
		r = scraper.get(target, stream=True, verify="/etc/ssl/certs/ca-certificates.crt")

		# get filename from header, or use pluginName
		try:
			d = response.headers['content-disposition']
			f = re.search('filename=\"(.+)\"', d)
			fileName = f.group(1)
		except KeyError:
			fileName = pluginName + '.jar'  # this presumes it's a jar, of course..

		# report
		print('Target:', target)
		print('File: ', fileName, '\n')

		# stream / save file
		with open('../common-files/plugins/%s' % fileName, 'wb') as fd:
			for chunk in r.iter_content(chunk_size=128):
				fd.write(chunk)
		del r

		print('Saved \n\n')

	# check if jenkins
	elif pluginData[i][1] == 'jenkins':

		pluginApi = pluginUrl + '/lastStableBuild/api/xml'
		r_page = requests.get(pluginApi, verify=False)
		soup = BeautifulSoup(r_page.text, 'html.parser')

		# loop through all relativepaths
		for tag in soup.find_all('artifact'):

			# download
			fileName = tag.filename.string
			target = pluginUrl + '/lastStableBuild/artifact/' + tag.relativepath.string

			r = requests.get(target, stream=True, verify="/etc/ssl/certs/ca-certificates.crt")

			# report
			print('Plugin:', pluginName)
			print('Target:', target)
			print('File: ', fileName, '\n')

			 # stream / save file
			with open('../common-files/plugins/%s' % fileName, 'wb') as fd:
				for chunk in r.iter_content(chunk_size=128):
					fd.write(chunk)
			del r

			print('Saved \n\n')

	# check if enginehub
	elif pluginData[i][1] == 'enginehub':

		# get page
		pluginApi = pluginUrl
		r_page = requests.get(pluginApi)

		# parse download link with BeautifulSoup
		soup = BeautifulSoup(r_page.text, 'html.parser')
		soup2 = soup.find(class_="col-md-8")
		soup3 = soup2.find('a')
		target = soup3['href']

		r = requests.get(target, stream=True, verify="/etc/ssl/certs/ca-certificates.crt")
		fileName = pluginName + '.jar'

		# report
		print('Plugin:', pluginName)
		print('Target:', target)
		print('File: ', fileName, '\n')

		# stream / save file
		with open('../common-files/plugins/%s' % fileName, 'wb') as fd:
			for chunk in r.iter_content(chunk_size=128):
				fd.write(chunk)
		del r

		print('Saved \n\n')

	# check if static
	elif pluginData[i][1] == 'static':

		target = pluginUrl
		fileName = pluginName + '.jar'
		r = requests.get(target, stream=True, verify="/etc/ssl/certs/ca-certificates.crt")

		# report
		print('Plugin:', pluginName)
		print('Target:', target)
		print('File: ', fileName, '\n')

		# stream / save file
		with open('../common-files/plugins/%s' % fileName, 'wb') as fd:
			for chunk in r.iter_content(chunk_size=128):
				fd.write(chunk)
		del r

		print('Saved \n\n')

	# check if manual
	elif pluginData[i][1] == 'manual':

		manualPluginList += pluginName
		manualPluginList += '\n'

		# report
		print('Plugin:', pluginName)
		print('Skipped for manual installation \n')

	# else error
	else:
		print('error updating', pluginName, ', please check the CSV file.')

# print list of manual plugins
print(manualPluginList)
<commit_msg>moar update<commit_after># Obzidi4n was here
from bs4 import BeautifulSoup
import cloudscraper, csv, re, requests, shutil

pluginFile = open('config/pluginurls.csv')
pluginReader = csv.reader(pluginFile)
pluginData = list(pluginReader)
pluginNum = pluginReader.line_num
manualPluginList = 'Manual Plugins:\n'

# command: update all
for i in range(1, pluginNum):

	pluginName = pluginData[i][0]
	pluginUrl = pluginData[i][2]

	print("Processing",i,"of",pluginNum,":",pluginName)

	# check if spigot
	if pluginData[i][1] == 'spigot':

		# invoke cloudscraper
		print ('Source: Spigot')
		scraper = cloudscraper.create_scraper()

		# get spigot page
		pluginApi = pluginUrl + '/history'
		r_page = scraper.get(pluginApi)

		# parse download link with BeautifulSoup
		soup = BeautifulSoup(r_page.text, 'html.parser')
		soup2 = soup.find('a', 'secondaryContent')
		target = 'http://www.spigotmc.org/' + soup2['href']

		# get response object
		r = scraper.get(target, stream=True, verify="/etc/ssl/certs/ca-certificates.crt")

		# get filename from header, or use pluginName
		try:
			d = r.headers['content-disposition']
			f = re.search('filename=\"(.+)\"', d)
			fileName = f.group(1)
		except KeyError:
			fileName = pluginName + '.jar'  # this presumes it's a jar, of course..

		# report
		print('Target:', target)
		print('File: ', fileName, '\n')

		# stream / save file
		with open('../common-files/plugins/%s' % fileName, 'wb') as fd:
			for chunk in r.iter_content(chunk_size=128):
				fd.write(chunk)
		del r

		print('Saved \n\n')

	# check if jenkins
	elif pluginData[i][1] == 'jenkins':

		pluginApi = pluginUrl + '/lastStableBuild/api/xml'
		r_page = requests.get(pluginApi, verify=False)
		soup = BeautifulSoup(r_page.text, 'html.parser')

		# loop through all relativepaths
		for tag in soup.find_all('artifact'):

			# download
			fileName = tag.filename.string
			target = pluginUrl + '/lastStableBuild/artifact/' + tag.relativepath.string

			r = requests.get(target, stream=True, verify="/etc/ssl/certs/ca-certificates.crt")

			# report
			print('Plugin:', pluginName)
			print('Target:', target)
			print('File: ', fileName, '\n')

			 # stream / save file
			with open('../common-files/plugins/%s' % fileName, 'wb') as fd:
				for chunk in r.iter_content(chunk_size=128):
					fd.write(chunk)
			del r

			print('Saved \n\n')

	# check if enginehub
	elif pluginData[i][1] == 'enginehub':

		# get page
		pluginApi = pluginUrl
		r_page = requests.get(pluginApi)

		# parse download link with BeautifulSoup
		soup = BeautifulSoup(r_page.text, 'html.parser')
		soup2 = soup.find(class_="col-md-8")
		soup3 = soup2.find('a')
		target = soup3['href']

		r = requests.get(target, stream=True, verify="/etc/ssl/certs/ca-certificates.crt")
		fileName = pluginName + '.jar'

		# report
		print('Plugin:', pluginName)
		print('Target:', target)
		print('File: ', fileName, '\n')

		# stream / save file
		with open('../common-files/plugins/%s' % fileName, 'wb') as fd:
			for chunk in r.iter_content(chunk_size=128):
				fd.write(chunk)
		del r

		print('Saved \n\n')

	# check if static
	elif pluginData[i][1] == 'static':

		target = pluginUrl
		fileName = pluginName + '.jar'
		r = requests.get(target, stream=True, verify="/etc/ssl/certs/ca-certificates.crt")

		# report
		print('Plugin:', pluginName)
		print('Target:', target)
		print('File: ', fileName, '\n')

		# stream / save file
		with open('../common-files/plugins/%s' % fileName, 'wb') as fd:
			for chunk in r.iter_content(chunk_size=128):
				fd.write(chunk)
		del r

		print('Saved \n\n')

	# check if manual
	elif pluginData[i][1] == 'manual':

		manualPluginList += pluginName
		manualPluginList += '\n'

		# report
		print('Plugin:', pluginName)
		print('Skipped for manual installation \n')

	# else error
	else:
		print('error updating', pluginName, ', please check the CSV file.')

# print list of manual plugins
print(manualPluginList)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Build the gh-pages

from collections import OrderedDict
import re
import sys
import json

from lintlib import parse_all, log

lint_subheadline = re.compile(r'''^\*\*([\w\s]+?)[:?.!]?\*\*(.*)''')
rust_code_block = re.compile(r'''```rust.+?```''', flags=re.DOTALL)

CONF_TEMPLATE = """\
This lint has the following configuration variables:

* `%s: %s`: %s (defaults to `%s`)."""

def parse_code_block(match):
    lines = []

    for line in match.group(0).split('\n'):
        if not line.startswith('# '):
            lines.append(line)

    return '\n'.join(lines)

def parse_lint_def(lint):
    lint_dict = {}
    lint_dict['id'] = lint.name
    lint_dict['group'] = lint.group
    lint_dict['level'] = lint.level
    lint_dict['docs'] = OrderedDict()

    last_section = None

    for line in lint.doc:
        match = re.match(lint_subheadline, line)
        if match:
            last_section = match.groups()[0]
            text = match.groups()[1]
        else:
            text = line

        if not last_section:
            log.warning("Skipping comment line as it was not preceded by a heading")
            log.debug("in lint `%s`, line `%s`", lint.name, line)

        if last_section not in lint_dict['docs']:
            lint_dict['docs'][last_section] = ""

        lint_dict['docs'][last_section] += text + "\n"

    for section in lint_dict['docs']:
        lint_dict['docs'][section] = re.sub(rust_code_block, parse_code_block, lint_dict['docs'][section].strip())

    return lint_dict

def main():
    lintlist, configs = parse_all()
    lints = {}
    for lint in lintlist:
        lints[lint.name] = parse_lint_def(lint)
        if lint.name in configs:
            lints[lint.name]['docs']['Configuration'] = \
                CONF_TEMPLATE % configs[lint.name]

    outfile = sys.argv[1] if len(sys.argv) > 1 else "util/gh-pages/lints.json"
    with open(outfile, "w") as fp:
        json.dump(list(lints.values()), fp, indent=2)
        log.info("wrote JSON for great justice")

if __name__ == "__main__":
    main()
<commit_msg>Sort output of export.py script<commit_after>#!/usr/bin/env python

# Build the gh-pages

from collections import OrderedDict
import re
import sys
import json

from lintlib import parse_all, log

lint_subheadline = re.compile(r'''^\*\*([\w\s]+?)[:?.!]?\*\*(.*)''')
rust_code_block = re.compile(r'''```rust.+?```''', flags=re.DOTALL)

CONF_TEMPLATE = """\
This lint has the following configuration variables:

* `%s: %s`: %s (defaults to `%s`)."""

def parse_code_block(match):
    lines = []

    for line in match.group(0).split('\n'):
        if not line.startswith('# '):
            lines.append(line)

    return '\n'.join(lines)

def parse_lint_def(lint):
    lint_dict = {}
    lint_dict['id'] = lint.name
    lint_dict['group'] = lint.group
    lint_dict['level'] = lint.level
    lint_dict['docs'] = OrderedDict()

    last_section = None

    for line in lint.doc:
        match = re.match(lint_subheadline, line)
        if match:
            last_section = match.groups()[0]
            text = match.groups()[1]
        else:
            text = line

        if not last_section:
            log.warning("Skipping comment line as it was not preceded by a heading")
            log.debug("in lint `%s`, line `%s`", lint.name, line)

        if last_section not in lint_dict['docs']:
            lint_dict['docs'][last_section] = ""

        lint_dict['docs'][last_section] += text + "\n"

    for section in lint_dict['docs']:
        lint_dict['docs'][section] = re.sub(rust_code_block, parse_code_block, lint_dict['docs'][section].strip())

    return lint_dict

def main():
    lintlist, configs = parse_all()
    lints = {}
    for lint in lintlist:
        lints[lint.name] = parse_lint_def(lint)
        if lint.name in configs:
            lints[lint.name]['docs']['Configuration'] = \
                CONF_TEMPLATE % configs[lint.name]

    outfile = sys.argv[1] if len(sys.argv) > 1 else "util/gh-pages/lints.json"
    with open(outfile, "w") as fp:
        lints = list(lints.values())
        lints.sort(key=lambda x: x['id'])
        json.dump(lints, fp, indent=2)
        log.info("wrote JSON for great justice")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#-----------------------------------------------------------------------------
#  Copyright (C) 2013 The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

import os

from tornado.httpclient import AsyncHTTPClient
from tornado.httputil import url_concat
from tornado.log import app_log

from .utils import url_path_join, quote

#-----------------------------------------------------------------------------
# Async GitHub Client
#-----------------------------------------------------------------------------

class AsyncGitHubClient(object):
    """AsyncHTTPClient wrapper with methods for common requests"""
    github_api_url = 'https://api.github.com/'
    auth = None

    def __init__(self, client=None):
        self.client = client or AsyncHTTPClient()
        self.authenticate()

    def authenticate(self):
        self.auth = {
            'client_id': os.environ.get('GITHUB_OAUTH_KEY', ''),
            'client_secret': os.environ.get('GITHUB_OAUTH_SECRET', ''),
            'token' : os.environ.get('GITHUB_API_TOKEN', ''),
        }
        self.auth = {k:v for k,v in self.auth.items() if v}

    def github_api_request(self, path, callback=None, params=None, **kwargs):
        """Make a GitHub API request to URL

        URL is constructed from url and params, if specified.
        callback and **kwargs are passed to client.fetch unmodified.
        """
        url = url_path_join(self.github_api_url, path)

        params = {} if params is None else params
        kwargs.setdefault('user_agent', 'Tornado-Async-GitHub-Client')
        if self.auth:
            params.update(self.auth)
        url = url_concat(url, params)
        future = self.client.fetch(url, callback, **kwargs)
        return future

    def get_gist(self, gist_id, callback=None, **kwargs):
        """Get a gist"""
        path = u'gists/{}'.format(gist_id)
        return self.github_api_request(path, callback, **kwargs)

    def get_contents(self, user, repo, path, callback=None, ref=None, **kwargs):
        """Make contents API request - either file contents or directory listing"""
        path = quote(u'repos/{user}/{repo}/contents/{path}'.format(
            **locals()
        ))
        if ref is not None:
            params = kwargs.setdefault('params', {})
            params['ref'] = ref
        return self.github_api_request(path, callback, **kwargs)

    def get_repos(self, user, callback=None, **kwargs):
        """List a user's repos"""
        path = u"users/{user}/repos".format(user=user)
        return self.github_api_request(path, callback ,**kwargs)

    def get_gists(self, user, callback=None, **kwargs):
        """List a user's gists"""
        path = u"users/{user}/gists".format(user=user)
        return self.github_api_request(path, callback ,**kwargs)

<commit_msg>changes required to run sharing server on mac os<commit_after>#-----------------------------------------------------------------------------
#  Copyright (C) 2013 The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

import os
import certifi

from tornado.httpclient import AsyncHTTPClient
from tornado.httputil import url_concat
from tornado.log import app_log

from .utils import url_path_join, quote

#-----------------------------------------------------------------------------
# Async GitHub Client
#-----------------------------------------------------------------------------

class AsyncGitHubClient(object):
    """AsyncHTTPClient wrapper with methods for common requests"""
    github_api_url = 'https://api.github.com/'
    auth = None

    def __init__(self, client=None):
        self.client = client or AsyncHTTPClient()
        self.authenticate()

    def authenticate(self):
        self.auth = {
            'client_id': os.environ.get('GITHUB_OAUTH_KEY', ''),
            'client_secret': os.environ.get('GITHUB_OAUTH_SECRET', ''),
            'token' : os.environ.get('GITHUB_API_TOKEN', ''),
        }
        self.auth = {k:v for k,v in self.auth.items() if v}

    def github_api_request(self, path, callback=None, params=None, **kwargs):
        """Make a GitHub API request to URL

        URL is constructed from url and params, if specified.
        callback and **kwargs are passed to client.fetch unmodified.
        """
        url = url_path_join(self.github_api_url, path)

        params = {} if params is None else params
        kwargs.setdefault('user_agent', 'Tornado-Async-GitHub-Client')
        kwargs.setdefault('ca_certs', certifi.where());
        if self.auth:
            params.update(self.auth)
        url = url_concat(url, params)
        future = self.client.fetch(url, callback, **kwargs)
        return future

    def get_gist(self, gist_id, callback=None, **kwargs):
        """Get a gist"""
        path = u'gists/{}'.format(gist_id)
        return self.github_api_request(path, callback, **kwargs)

    def get_contents(self, user, repo, path, callback=None, ref=None, **kwargs):
        """Make contents API request - either file contents or directory listing"""
        path = quote(u'repos/{user}/{repo}/contents/{path}'.format(
            **locals()
        ))
        if ref is not None:
            params = kwargs.setdefault('params', {})
            params['ref'] = ref
        return self.github_api_request(path, callback, **kwargs)

    def get_repos(self, user, callback=None, **kwargs):
        """List a user's repos"""
        path = u"users/{user}/repos".format(user=user)
        return self.github_api_request(path, callback ,**kwargs)

    def get_gists(self, user, callback=None, **kwargs):
        """List a user's gists"""
        path = u"users/{user}/gists".format(user=user)
        return self.github_api_request(path, callback ,**kwargs)

<|endoftext|>