<commit_before>#
#

# Copyright (C) 2010 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

"""Module for a simple query language"""

from ganeti import errors

# Logic operators
OP_OR = "|"
OP_AND = "&"

# Unary operators
OP_NOT = "!"
OP_TRUE = "?"

# Binary operators
OP_EQUAL = "="
OP_NOT_EQUAL = "!="
OP_GLOB = "=*"
OP_REGEXP = "=~"
OP_CONTAINS = "=[]"

def ReadSimpleFilter(namefield, filter_):
  """Function extracting wanted names from restricted filter.

  This should only be used until proper filtering is implemented. The filter
  must either be empty or of the format C{["|", ["=", field, "name1"], ["=",
  field, "name2"], ...]}.

  """
  if filter_ is None:
    return []

  if not isinstance(filter_, list):
    raise errors.ParameterError("Filter should be list")

  if not filter_ or filter_[0] != OP_OR:
    raise errors.ParameterError("Filter should start with OR operator")

  if len(filter_) < 2:
    raise errors.ParameterError("Invalid filter, OR operator should have"
                                " operands")

  result = []

  for idx, item in enumerate(filter_[1:]):
    if not isinstance(item, list):
      raise errors.ParameterError("Invalid OR operator, operand %s not a"
                                  " list" % idx)

    if len(item) != 3 or item[0] != OP_EQUAL:
      raise errors.ParameterError("Invalid OR operator, operand %s is not an"
                                  " equality filter" % idx)

    (_, name, value) = item

    if not isinstance(value, basestring):
      raise errors.ParameterError("Operand %s for OR should compare against a"
                                  " string" % idx)

    if name != namefield:
      raise errors.ParameterError("Operand %s for OR should filter field '%s',"
                                  " not '%s'" % (idx, namefield, name))

    result.append(value)

  return result

def MakeSimpleFilter(namefield, values):
  """Builds a filter for use with L{ReadSimpleFilter}.

  @param namefield: Name of field containing item name
  @param values: List of names

  """
  if values:
    return [OP_OR] + [[OP_EQUAL, namefield, i] for i in values]

  return None
<commit_msg>qlang: Add some more documentation for filters<commit_after>#
#

# Copyright (C) 2010, 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

"""Module for a simple query language

A query filter is always a list. The first item in the list is the operator
(e.g. C{[OP_AND, ...]}), while the other items depend on the operator. For
logic operators (e.g. L{OP_AND}, L{OP_OR}), they are subfilters whose results
are combined. Unary operators take exactly one other item (e.g. a subfilter for
L{OP_NOT} and a field name for L{OP_TRUE}). Binary operators take exactly two
operands, usually a field name and a value to compare against. Filters are
converted to callable functions by L{query._CompileFilter}.

"""

from ganeti import errors

# Logic operators with one or more operands, each of which is a filter on its
# own
OP_OR = "|"
OP_AND = "&"

# Unary operators with exactly one operand
OP_NOT = "!"
OP_TRUE = "?"

# Binary operators with exactly two operands, the field name and an
# operator-specific value
OP_EQUAL = "="
OP_NOT_EQUAL = "!="
OP_GLOB = "=*"
OP_REGEXP = "=~"
OP_CONTAINS = "=[]"

def ReadSimpleFilter(namefield, filter_):
  """Function extracting wanted names from restricted filter.

  This should only be used until proper filtering is implemented. The filter
  must either be empty or of the format C{["|", ["=", field, "name1"], ["=",
  field, "name2"], ...]}.

  """
  if filter_ is None:
    return []

  if not isinstance(filter_, list):
    raise errors.ParameterError("Filter should be list")

  if not filter_ or filter_[0] != OP_OR:
    raise errors.ParameterError("Filter should start with OR operator")

  if len(filter_) < 2:
    raise errors.ParameterError("Invalid filter, OR operator should have"
                                " operands")

  result = []

  for idx, item in enumerate(filter_[1:]):
    if not isinstance(item, list):
      raise errors.ParameterError("Invalid OR operator, operand %s not a"
                                  " list" % idx)

    if len(item) != 3 or item[0] != OP_EQUAL:
      raise errors.ParameterError("Invalid OR operator, operand %s is not an"
                                  " equality filter" % idx)

    (_, name, value) = item

    if not isinstance(value, basestring):
      raise errors.ParameterError("Operand %s for OR should compare against a"
                                  " string" % idx)

    if name != namefield:
      raise errors.ParameterError("Operand %s for OR should filter field '%s',"
                                  " not '%s'" % (idx, namefield, name))

    result.append(value)

  return result

def MakeSimpleFilter(namefield, values):
  """Builds a filter for use with L{ReadSimpleFilter}.

  @param namefield: Name of field containing item name
  @param values: List of names

  """
  if values:
    return [OP_OR] + [[OP_EQUAL, namefield, i] for i in values]

  return None
<|endoftext|>


<commit_before>"""
mpltex
======

"""

golden_ratio = (5**.5-1.0)/2.0

def pt_to_in(pt):
    return pt / 72.27

def pt_to_mm(pt):
    raise NotImplementedError

def get_rcParams(fig_width_pt = 232.0, scale = 1.0, dpi = 600):
    """TODO: write a description

    """
    fig_width = pt_to_in(fig_width_pt) # width in inches
    fig_height = fig_width * golden_ratio # height in inches
    fig_size =  [fig_width * scale, fig_height * scale]
    params = {  'axes.labelsize': 10,
                'text.fontsize': 8,
                'legend.fontsize': 8,
                'xtick.labelsize': 8,
                'ytick.labelsize': 8,
                'font.family': 'serif',
                'font.serif': 'Times',
                'text.usetex': True,
                'figure.figsize': fig_size,
                'figure.dpi': dpi}
    return params
<commit_msg>added more info to a docstring<commit_after>"""
mpltex
======

"""

golden_ratio = (5**.5-1.0)/2.0

def pt_to_in(pt):
    return pt / 72.27

def pt_to_mm(pt):
    raise NotImplementedError

def get_rcParams(fig_width_pt = 232.0, scale = 1.0, dpi = 600):
    """TODO: write a description

    Parameters
    ----------
    fig_width_pt: float (232.0)
        Width of the figure in points.
        Use LaTeX \showthe\linewidth to figure out what number to use.
            232.0 for two-column IEEEtran articles
            YYYY for MIT/WHOI thesis figures

    """
    fig_width = pt_to_in(fig_width_pt) # width in inches
    fig_height = fig_width * golden_ratio # height in inches
    fig_size =  [fig_width * scale, fig_height * scale]
    params = {  'axes.labelsize': 10,
                'text.fontsize': 8,
                'legend.fontsize': 8,
                'xtick.labelsize': 8,
                'ytick.labelsize': 8,
                'font.family': 'serif',
                'font.serif': 'Times',
                'text.usetex': True,
                'figure.figsize': fig_size,
                'figure.dpi': dpi}
    return params
<|endoftext|>


<commit_before>#!/usr/bin/env python2

import requests
import json
import time
import sys

class Xmr():

    def __init__(self):
        self.headers = {"Content-Type": "application/json"}
        self.xmrapi = "https://xmr.to/api/v1/xmr2btc/"
        self.walletapi = "http://localhost:18082/json_rpc"

    def gentrans(self, amount, destination):
        params = {"btc_amount": amount, "btc_dest_address": destination}
        api = self.xmrapi + "order_create/"
        _r = self.post(params, api)
        return _r.json()["uuid"]

    def getstatus(self, uuid):
        params = {"uuid": uuid}
        api = self.xmrapi + "order_status_query/"
        _r = self.post(params, api)
        return _r.json()

    def post(self, params, api):
        _r = requests.post(api, headers=self.headers,
                           data=json.dumps(params))
        return _r

    def sendxmr(self, _r):
        amount = self.toxmrvalue(_r["xmr_required_amount"])
        dest = [{'address': _r["xmr_receiving_address"],
                'amount': amount}]
        params = {"method": "transfer",
                  "params": {"destinations": dest,
                             "payment_id": _r["xmr_required_payment_id"],
                             "fee": 0,
                             "mixin": 3,
                             "unlock_time": 0}}
        params.update({"id": 0, "jsonrpc": "2.0"})

        self.post(params, self.walletapi)

    def toxmrvalue(self, amount):
        amount = str(amount)
        padding = 12
        leadpadding = 0
        if amount.count("."):
            first, last = amount.split(".")
            first = first.lstrip("0")
            last = last.rstrip("0")
            amount = first + last
            padding = padding - len(last)
            if first == "":
                leadpadding += 1
        rr = "0"*leadpadding + amount + "0"*padding
        return int(rr)

if __name__ == "__main__":
    if len(sys.argv) != 3:
        print "%s [amount in btc] [destination address]" % sys.argv[0]
        sys.exit(1)
    amount = sys.argv[1]
    destination = sys.argv[2]
    _t = Xmr()
    uuid = _t.gentrans(amount, destination)
    _r = _t.getstatus(uuid)
    while _r["state"] == "TO_BE_CREATED":
        print("Waiting for transaction to be created")
        time.sleep(2)
        _r = _t.getstatus(uuid)
    _t.sendxmr(_r)
    _r = _t.getstatus(uuid)
    while _r["state"] != "BTC_SENT":
        print("Waiting for transaction to be paid")
        time.sleep(15)
        _r = _t.getstatus(uuid)
    else:
        print("Transaction sent!")
<commit_msg>use argparser<commit_after>#!/usr/bin/env python2

import requests
import json
import time
import sys
import argparse

class Xmr():

    def __init__(self):
        self.headers = {"Content-Type": "application/json"}
        self.xmrapi = "https://xmr.to/api/v1/xmr2btc/"
        self.walletapi = "http://localhost:18082/json_rpc"

    def gentrans(self, amount, destination):
        params = {"btc_amount": amount, "btc_dest_address": destination}
        api = self.xmrapi + "order_create/"
        _r = self.post(params, api)
        return _r.json()["uuid"]

    def getstatus(self, uuid):
        params = {"uuid": uuid}
        api = self.xmrapi + "order_status_query/"
        _r = self.post(params, api)
        return _r.json()

    def post(self, params, api):
        _r = requests.post(api, headers=self.headers,
                           data=json.dumps(params))
        return _r

    def sendxmr(self, _r):
        amount = self.toxmrvalue(_r["xmr_required_amount"])
        dest = [{'address': _r["xmr_receiving_address"],
                'amount': amount}]
        params = {"method": "transfer",
                  "params": {"destinations": dest,
                             "payment_id": _r["xmr_required_payment_id"],
                             "fee": 0,
                             "mixin": 3,
                             "unlock_time": 0}}
        params.update({"id": 0, "jsonrpc": "2.0"})

        self.post(params, self.walletapi)

    def toxmrvalue(self, amount):
        amount = str(amount)
        padding = 12
        leadpadding = 0
        if amount.count("."):
            first, last = amount.split(".")
            first = first.lstrip("0")
            last = last.rstrip("0")
            amount = first + last
            padding = padding - len(last)
            if first == "":
                leadpadding += 1
        rr = "0"*leadpadding + amount + "0"*padding
        return int(rr)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Send btc via xmr.to")
    parser.add_argument("amount")
    parser.add_argument("address")
    args = parser.parse_args()
    amount = args.amount
    destination = args.address

    _t = Xmr()
    uuid = _t.gentrans(amount, destination)
    _r = _t.getstatus(uuid)
    while _r["state"] == "TO_BE_CREATED":
        print("Waiting for transaction to be created")
        time.sleep(2)
        _r = _t.getstatus(uuid)
    _t.sendxmr(_r)
    _r = _t.getstatus(uuid)
    while _r["state"] != "BTC_SENT":
        print("Waiting for transaction to be paid")
        time.sleep(15)
        _r = _t.getstatus(uuid)
    else:
        print("Transaction sent!")
<|endoftext|>


<commit_before>"""
=====================================================
Overplotting HEK feature/event polygons on SunPy maps
=====================================================

This example shows how to overplot HEK outlines on SunPy maps.
"""

##############################################################################
# Start by importing the necessary modules.
import numpy as np

import matplotlib.pyplot as plt

import astropy.units as u
from astropy.coordinates import SkyCoord
from astropy.time import TimeDelta

import sunpy.map
import sunpy.data.sample
from sunpy.net import hek
from sunpy.time import parse_time
from sunpy.coordinates import frames
from sunpy.physics.differential_rotation import solar_rotate_coordinate

##############################################################################
# Load in an AIA map:

aia_map = sunpy.map.Map(sunpy.data.sample.AIA_171_IMAGE)

##############################################################################
# Let's look for sunspots in the HEK close to the time of the AIA map. First
# create a client:

hek_client = hek.HEKClient()

##############################################################################
# Look for coronal holes detected using the SPoCA feature recognition method:

start_time = aia_map.date - TimeDelta(2*u.hour)
end_time = aia_map.date + TimeDelta(2*u.hour)
responses = hek_client.search(hek.attrs.Time(start_time, end_time),
                              hek.attrs.CH, hek.attrs.FRM.Name == 'SPoCA')

##############################################################################
# Let's find the biggest coronal hole within 80 degrees north/south of the
# equator:

area = 0.0
for i, response in enumerate(responses):
    if response['area_atdiskcenter'] > area and np.abs(response['hgc_y']) < 80.0:
        area = response['area_atdiskcenter']
        response_index = i

##############################################################################
# Now let's get the boundary of the coronal hole

ch = responses[response_index]
p1 = ch["hpc_boundcc"][9:-2]
p2 = p1.split(',')
p3 = [v.split(" ") for v in p2]
ch_date = parse_time(ch['event_starttime'])

##############################################################################
# The coronal hole was detected at a certain time.  To plot it on a map, we
# need to rotate it to the map observation time.

ch_boundary = SkyCoord(
    [(float(v[0]), float(v[1])) * u.arcsec for v in p3],
    obstime=ch_date,
    frame=frames.Helioprojective)
rotated_ch_boundary = solar_rotate_coordinate(ch_boundary, time=aia_map.date)

##############################################################################
# Now let's plot the rotated coronal hole boundary on the AIA map, and fill
# it with some matplotlib hatching.

fig = plt.figure()
ax = plt.subplot(projection=aia_map)
aia_map.plot(axes=ax)
ax.plot_coord(rotated_ch_boundary, color='c')
ax.set_title('{:s}\n{:s}'.format(aia_map.name, ch['frm_specificid']))
plt.colorbar()
plt.show()
<commit_msg>Update overplot_hek_polygon.py<commit_after>"""
================================================
Overplotting HEK feature/event polygons on a map
================================================

How to overplot HEK outlines on a map.
"""
import numpy as np

import matplotlib.pyplot as plt

import astropy.units as u
from astropy.coordinates import SkyCoord
from astropy.time import TimeDelta

import sunpy.map
import sunpy.data.sample
from sunpy.net import hek
from sunpy.time import parse_time
from sunpy.coordinates import frames
from sunpy.physics.differential_rotation import solar_rotate_coordinate

###############################################################################
# We start with the sample data
aia_map = sunpy.map.Map(sunpy.data.sample.AIA_171_IMAGE)

##############################################################################
# Look for coronal holes detected using the SPoCA feature recognition method:
hek_client = hek.HEKClient()
start_time = aia_map.date - TimeDelta(2*u.hour)
end_time = aia_map.date + TimeDelta(2*u.hour)
responses = hek_client.search(hek.attrs.Time(start_time, end_time),
                              hek.attrs.CH, hek.attrs.FRM.Name == 'SPoCA')

##############################################################################
# Let's find the biggest coronal hole within 80 degrees north/south of the
# equator:
area = 0.0
for i, response in enumerate(responses):
    if response['area_atdiskcenter'] > area and np.abs(response['hgc_y']) < 80.0:
        area = response['area_atdiskcenter']
        response_index = i

##############################################################################
# Next let's get the boundary of the coronal hole
ch = responses[response_index]
p1 = ch["hpc_boundcc"][9:-2]
p2 = p1.split(',')
p3 = [v.split(" ") for v in p2]
ch_date = parse_time(ch['event_starttime'])

##############################################################################
# The coronal hole was detected at different time than the AIA image was
# taken so we need to rotate it to the map observation time.
ch_boundary = SkyCoord(
    [(float(v[0]), float(v[1])) * u.arcsec for v in p3],
    obstime=ch_date,
    frame=frames.Helioprojective)
rotated_ch_boundary = solar_rotate_coordinate(ch_boundary, time=aia_map.date)

##############################################################################
# Now let's plot the rotated coronal hole boundary on the AIA map, and fill
# it with hatching.
fig = plt.figure()
ax = plt.subplot(projection=aia_map)
aia_map.plot(axes=ax)
ax.plot_coord(rotated_ch_boundary, color='c')
ax.set_title('{:s}\n{:s}'.format(aia_map.name, ch['frm_specificid']))
plt.colorbar()
plt.show()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Copyright (c) 2015 Andrew Azarov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from datetime import datetime
from subprocess import Popen, PIPE
import yaml
import json

def assrt(check, error=None):
    if check:
        return 1
    else:
        if error:
            raise AssertionError(error)
        raise AssertionError

def do(line):
    return Popen(line, shell=True, bufsize=4096, stdout=PIPE)

class gnt_ext_backup(object):

    """gnt_ext_backup object

    Keyword arguments:

    unique_id           -- Unique identifier for backups (default set to current date and hour)
    retention_period    -- Backup retention period (default set to 1 day)
    backup_user_server  -- login@server.hostname credentials for SSH (default not set). Key auth for SSH should be setup beforehand,
                            I suggest chroot on target (or even jail) with only lz4, sh and find commands
    lv_backup_extension -- Extension for LV name of backup snapshot, without dot (default is bak)
    backup_folder       -- Remote server folder to place backups (default is ./upload/)
    backup_extension    -- Extension for resulting backup files (default is raw)
    compression         -- Dictionary of ingress and egress (default lz4 commands, do not remove the pipes!)
    debug               -- Do not perform actions if set, just print them

    """

    def __init__(self, **kwargs):
        # Set instance defaults
        # For simplicity set to timestamp
        self.unique_id = datetime.now().strftime("%Y-%m-%d-%H")
        self.retention_period = 7
        self.backup_user_server = None
        self.lv_backup_extension = 'bak'
        self.backup_folder = './upload/'
        self.backup_extension = 'raw'
        self.compression = {'egress': '| lz4 -1c |', 'ingress': 'lz4 -dc >'}
        self.debug = 0
        self.instances_names = None
        self.dd_buffer = '128M'
        self.lv_size = '1G'
        for i in ['unique_id', 'retention_period', 'backup_user_server',
                  'lv_backup_extension', 'backup_extension', 'backup_folder',
                  'compression', 'debug', 'instances_names', 'dd_buffer', 'lv_size']:
            if i in kwargs and kwargs[i]:
                setattr(self, i, kwargs[i])
            assrt(self.__dict__[i] is not None, "%s is not set" % i)

        if not self.instances_names:
            instances_raw_info = do('gnt-instance info --all')
        else:
            instances_raw_info = do(
                'gnt-instance info ' + ' '.join(self.instances_names))
        self.instances = yaml.load(instances_raw_info.stdout.read())

        self.ssh_cmd = 'ssh -oStrictHostKeyChecking=no ' + \
            self.backup_user_server
        assrt(isinstance(self.retention_period, int), "%r is not int" %
              self.retention_period)
        assrt([isinstance(ids, str) for ids in self.compression.values()],
              "%r is not correct" % self.compression)
        assrt(isinstance(self.lv_backup_extension, str), "%r is not str" %
              self.lv_backup_extension)
        assrt(isinstance(self.backup_extension, str), "%r is not str" %
              self.backup_extension)
        assrt(isinstance(self.backup_folder, str), "%r is not str" %
              self.backup_folder)
        assrt(isinstance(self.backup_user_server, str), "%r is not str" %
              self.backup_user_server)
        assrt(len(self.backup_user_server.split('@')) == 2, "%r is incorrect" %
              self.backup_user_server)

    def perform_backup(self):
        for instance in self.instances:
            name = instance['Instance name']
            primary_node = [i['primary']
                            for i in instance['Nodes'] if 'primary' in i][0]
            disks = [(i['logical_id'], i['on primary'].split()[0])
                     for i in instance['Disks']]
            command = "gnt-cluster command -n " + primary_node
            for disk in disks:
                drive = {}
                drive['vg'], drive['lv'] = disk[0].split('/')

                print(
                    '{}: Backing up {} {}'.format(self.unique_id, name, disk[0]))
                cmd_list = [
                    [
                        command,
                        "\"lvcreate -L" + self.lv_size + " -s -n",
                        '.'.join(
                            [drive['lv'], self.unique_id, self.lv_backup_extension]),
                        disk[1],
                        "\""
                    ],
                    [
                        command,
                        "\"dd if=" +
                        '.'.join(
                            [disk[1], self.unique_id, self.lv_backup_extension]),
                        "bs=" + self.dd_buffer,
                        self.compression['egress'],
                        self.ssh_cmd,
                        "'" + self.compression['ingress'],
                        self.backup_folder +
                        '.'.join(
                            [self.unique_id, drive['lv'], name, primary_node, self.backup_extension]),
                        "'\""
                    ],
                    [
                        command,
                        "\"lvremove -f",
                        '.'.join(
                            [disk[0], self.unique_id, self.lv_backup_extension]),
                        "\""
                    ]
                ]
                for cmd in cmd_list:
                    if self.debug:
                        print(' '.join(cmd))
                    else:
                        do(' '.join(cmd)).wait()
                print('{}: Done {} {}'.format(self.unique_id, name, disk[0]))
                print('-' * 100)
        cmd = [
            self.ssh_cmd,
            "\"",
            'find',
            self.backup_folder,
            '-name \'' + '.'.join(
                ['*', '*', '*', '*', self.backup_extension]) + '\'',
            '-ctime +' + str(self.retention_period),
            '-delete',
            "\""
        ]
        if self.debug:
            print(' '.join(cmd))
        else:
            do(' '.join(cmd)).wait()
        print('Done cleaning old backups')
        print('-' * 100)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("-i",
                        "--id",
                        dest='unique_id',
                        type=str,
                        default=None,
                        help="Unique id to identify backups, default is date with hour",
                        required=False)
    parser.add_argument("-n",
                        "--instances_names",
                        dest='instances_names',
                        type=str,
                        nargs='+',
                        help="Space separated list of instances to backup",
                        required=False)
    parser.add_argument("-r",
                        "--retention_period",
                        dest='retention_period',
                        type=int,
                        default=None,
                        help="Backup retention period in days, default is 1",
                        required=False)
    parser.add_argument("-b",
                        "--backup_user_server",
                        dest='backup_user_server',
                        type=str,
                        default=None,
                        help="Backup login and server ssh style: login@backup.server",
                        required=True)
    parser.add_argument("-l",
                        "--lv_backup_extension",
                        type=str,
                        dest='lv_backup_extension',
                        default=None,
                        help="LV backup snapshot extension to identify",
                        required=False)
    parser.add_argument("-e",
                        "--backup_extension",
                        type=str,
                        dest='backup_extension',
                        default=None,
                        help="Backup extension for files on the backup target",
                        required=False)
    parser.add_argument("-f",
                        "--backup_folder",
                        type=str,
                        dest='backup_folder',
                        default=None,
                        help="Backup folder on remote server",
                        required=False)
    parser.add_argument("-c",
                        "--compression",
                        type=json.loads,
                        dest='compression',
                        default=None,
                        help="JSON array like {'egress': '| lz4 -1c |', 'ingress': 'lz4 -dc >'}",
                        required=False)
    parser.add_argument("-d",
                        "--debug",
                        type=int,
                        dest='debug',
                        default=None,
                        help="If debug is 1 - disable performing actions and just print them out",
                        required=False)
    parser.add_argument("--lv_size",
                        type=str,
                        dest='lv_size',
                        default=None,
                        help="LV snapshot size - default 1G, check man lvcreate for snapshot sizes",
                        required=False)
    parser.add_argument("--dd_buffer",
                        type=str,
                        dest='dd_buffer',
                        default=None,
                        help="dd buffer size, default 128MB for fast gbit transfers, adjust to your memory and network size",
                        required=False)
    a = parser.parse_args()
    arguments = {}
    for i in ['unique_id', 'retention_period', 'backup_user_server',
              'lv_backup_extension', 'backup_extension', 'backup_folder',
              'compression', 'debug', 'instances_names', 'lv_size', 'dd_buffer']:
        if hasattr(a, i) and getattr(a, i):
            arguments[i] = getattr(a, i)
    backup_job = gnt_ext_backup(**arguments)
    backup_job.perform_backup()
<commit_msg>Small update<commit_after>#!/usr/bin/env python
"""
Copyright (c) 2015 Andrew Azarov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from datetime import datetime
from subprocess import Popen, PIPE
import yaml
import json

def assrt(check, error=None):
    if check:
        return 1
    else:
        if error:
            raise AssertionError(error)
        raise AssertionError

def do(line):
    return Popen(line, shell=True, bufsize=4096, stdout=PIPE)

class gnt_ext_backup(object):

    """gnt_ext_backup object

    Keyword arguments:

    unique_id           -- Unique identifier for backups (default set to current date and hour)
    retention_period    -- Backup retention period (default set to 1 day)
    backup_user_server  -- login@server.hostname credentials for SSH (default not set). Key auth for SSH should be setup beforehand,
                            I suggest chroot on target (or even jail) with only lz4, sh and find commands
    lv_backup_extension -- Extension for LV name of backup snapshot, without dot (default is bak)
    backup_folder       -- Remote server folder to place backups (default is ./upload/)
    backup_extension    -- Extension for resulting backup files (default is raw)
    compression         -- Dictionary of ingress and egress (default lz4 commands, do not remove the pipes!)
    debug               -- Do not perform actions if set, just print them

    """

    def __init__(self, **kwargs):
        # Set instance defaults
        # For simplicity set to timestamp
        self.unique_id = datetime.now().strftime("%Y-%m-%d-%H")
        self.retention_period = 7
        self.backup_user_server = None
        self.lv_backup_extension = 'bak'
        self.backup_folder = './upload/'
        self.backup_extension = 'raw'
        self.compression = {'egress': '| lz4 -1c |', 'ingress': 'lz4 -dc >'}
        self.debug = 0
        self.instances_names = None
        self.dd_buffer = '128M'
        self.lv_size = '1G'
        for i in ['unique_id', 'retention_period', 'backup_user_server',
                  'lv_backup_extension', 'backup_extension', 'backup_folder',
                  'compression', 'debug', 'instances_names', 'dd_buffer', 'lv_size']:
            if i in kwargs and kwargs[i]:
                setattr(self, i, kwargs[i])
            if i != 'instances_names':  # It can be None
                assrt(self.__dict__[i] is not None, "%s is not set" % i)

        if not self.instances_names:
            instances_raw_info = do('gnt-instance info --all')
        else:
            instances_raw_info = do(
                'gnt-instance info ' + ' '.join(self.instances_names))
        self.instances = yaml.load(instances_raw_info.stdout.read())

        self.ssh_cmd = 'ssh -oStrictHostKeyChecking=no ' + \
            self.backup_user_server
        assrt(isinstance(self.retention_period, int), "%r is not int" %
              self.retention_period)
        assrt([isinstance(ids, str) for ids in self.compression.values()],
              "%r is not correct" % self.compression)
        assrt(isinstance(self.lv_backup_extension, str), "%r is not str" %
              self.lv_backup_extension)
        assrt(isinstance(self.backup_extension, str), "%r is not str" %
              self.backup_extension)
        assrt(isinstance(self.backup_folder, str), "%r is not str" %
              self.backup_folder)
        assrt(isinstance(self.backup_user_server, str), "%r is not str" %
              self.backup_user_server)
        assrt(len(self.backup_user_server.split('@')) == 2, "%r is incorrect" %
              self.backup_user_server)

    def perform_backup(self):
        for instance in self.instances:
            name = instance['Instance name']
            primary_node = [i['primary']
                            for i in instance['Nodes'] if 'primary' in i][0]
            disks = [(i['logical_id'], i['on primary'].split()[0])
                     for i in instance['Disks']]
            command = "gnt-cluster command -n " + primary_node
            for disk in disks:
                drive = {}
                drive['vg'], drive['lv'] = disk[0].split('/')

                print(
                    '{}: Backing up {} {}'.format(self.unique_id, name, disk[0]))
                cmd_list = [
                    [
                        command,
                        "\"lvcreate -L" + self.lv_size + " -s -n",
                        '.'.join(
                            [drive['lv'], self.unique_id, self.lv_backup_extension]),
                        disk[1],
                        "\""
                    ],
                    [
                        command,
                        "\"dd if=" +
                        '.'.join(
                            [disk[1], self.unique_id, self.lv_backup_extension]),
                        "bs=" + self.dd_buffer,
                        self.compression['egress'],
                        self.ssh_cmd,
                        "'" + self.compression['ingress'],
                        self.backup_folder +
                        '.'.join(
                            [self.unique_id, drive['lv'], name, primary_node, self.backup_extension]),
                        "'\""
                    ],
                    [
                        command,
                        "\"lvremove -f",
                        '.'.join(
                            [disk[0], self.unique_id, self.lv_backup_extension]),
                        "\""
                    ]
                ]
                for cmd in cmd_list:
                    if self.debug:
                        print(' '.join(cmd))
                    else:
                        do(' '.join(cmd)).wait()
                print('{}: Done {} {}'.format(self.unique_id, name, disk[0]))
                print('-' * 100)
        cmd = [
            self.ssh_cmd,
            "\"",
            'find',
            self.backup_folder,
            '-name \'' + '.'.join(
                ['*', '*', '*', '*', self.backup_extension]) + '\'',
            '-ctime +' + str(self.retention_period),
            '-delete',
            "\""
        ]
        if self.debug:
            print(' '.join(cmd))
        else:
            do(' '.join(cmd)).wait()
        print('Done cleaning old backups')
        print('-' * 100)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("-i",
                        "--id",
                        dest='unique_id',
                        type=str,
                        default=None,
                        help="Unique id to identify backups, default is date with hour",
                        required=False)
    parser.add_argument("-n",
                        "--instances_names",
                        dest='instances_names',
                        type=str,
                        nargs='+',
                        help="Space separated list of instances to backup",
                        required=False)
    parser.add_argument("-r",
                        "--retention_period",
                        dest='retention_period',
                        type=int,
                        default=None,
                        help="Backup retention period in days, default is 1",
                        required=False)
    parser.add_argument("-b",
                        "--backup_user_server",
                        dest='backup_user_server',
                        type=str,
                        default=None,
                        help="Backup login and server ssh style: login@backup.server",
                        required=True)
    parser.add_argument("-l",
                        "--lv_backup_extension",
                        type=str,
                        dest='lv_backup_extension',
                        default=None,
                        help="LV backup snapshot extension to identify",
                        required=False)
    parser.add_argument("-e",
                        "--backup_extension",
                        type=str,
                        dest='backup_extension',
                        default=None,
                        help="Backup extension for files on the backup target",
                        required=False)
    parser.add_argument("-f",
                        "--backup_folder",
                        type=str,
                        dest='backup_folder',
                        default=None,
                        help="Backup folder on remote server",
                        required=False)
    parser.add_argument("-c",
                        "--compression",
                        type=json.loads,
                        dest='compression',
                        default=None,
                        help="JSON array like {'egress': '| lz4 -1c |', 'ingress': 'lz4 -dc >'}",
                        required=False)
    parser.add_argument("-d",
                        "--debug",
                        type=int,
                        dest='debug',
                        default=None,
                        help="If debug is 1 - disable performing actions and just print them out",
                        required=False)
    parser.add_argument("--lv_size",
                        type=str,
                        dest='lv_size',
                        default=None,
                        help="LV snapshot size - default 1G, check man lvcreate for snapshot sizes",
                        required=False)
    parser.add_argument("--dd_buffer",
                        type=str,
                        dest='dd_buffer',
                        default=None,
                        help="dd buffer size, default 128MB for fast gbit transfers, adjust to your memory and network size",
                        required=False)
    a = parser.parse_args()
    arguments = {}
    for i in ['unique_id', 'retention_period', 'backup_user_server',
              'lv_backup_extension', 'backup_extension', 'backup_folder',
              'compression', 'debug', 'instances_names', 'lv_size', 'dd_buffer']:
        if hasattr(a, i) and getattr(a, i):
            arguments[i] = getattr(a, i)
    backup_job = gnt_ext_backup(**arguments)
    backup_job.perform_backup()
<|endoftext|>


<commit_before>#!/usr/bin/env python

#import boto3
import os
import sys
import json

#def assume_role():
#    invoker_role_arn = os.environ["INVOKER_ROLE_ARN"]
#    sts_client = boto3.client('sts')
#    return sts_client.assume_role(
#        RoleArn=invoker_role_arn,
#        RoleSessionName='integration_test')['Credentials']

def format_(credentials):
    key_map = {"SecretAccessKey": "AWS_SECRET_ACCESS_KEY",
               "SessionToken": "AWS_SESSION_TOKEN",
               "AccessKeyId": "AWS_ACCESS_KEY_ID"}

    for key, value in key_map.items():
        print "export {0}={1}".format(value, credentials[key])

if __name__ == "__main__":
    format_(json.loads(sys.stdin.read())['Credentials'])
<commit_msg>print the AWS_SECURITY_TOKEN too<commit_after>#!/usr/bin/env python

#import boto3
import os
import sys
import json

#def assume_role():
#    invoker_role_arn = os.environ["INVOKER_ROLE_ARN"]
#    sts_client = boto3.client('sts')
#    return sts_client.assume_role(
#        RoleArn=invoker_role_arn,
#        RoleSessionName='integration_test')['Credentials']

def format_(credentials):
    key_map = {"SecretAccessKey": "AWS_SECRET_ACCESS_KEY",
               "SessionToken": "AWS_SESSION_TOKEN",
               "AccessKeyId": "AWS_ACCESS_KEY_ID"}

    for key, value in key_map.items():
        print "export {0}={1}".format(value, credentials[key])
    print "export AWS_SECURITY_TOKEN=".format(credentials['SessionToken'])

if __name__ == "__main__":
    format_(json.loads(sys.stdin.read())['Credentials'])
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
import ssl
import socket
import time
import datetime
import threading
import asn1crypto.x509
import requests

from .manual import *

DETECT_CIPHERS = True
DEFAULT_TIMEOUT = 5.0

def make_ssocket(protocol, ciphers=ssl._DEFAULT_CIPHERS):
    """Creates a TCP socket wrapped in SSL."""
    return ssl.wrap_socket( socket.socket(),
                            ssl_version=protocol,
                            ciphers=ciphers)

def get_supported_ciphers(dest, protocol):
    """Returns a list of ciphers supported by a host.

    Arguments:
        dest -- the (IP, port) to connect to
        protocol -- the SSL/TLS version to test; it is also
                     the index into the ssl._PROTOCOL_NAMES array

    The returned list contains cipher names.
    """
    chosen_ciphers = []
    # start by enabling all ciphers
    cipher_string = "ALL:aNULL:eNULL"
    while True:
        try:
            ssock = make_ssocket(protocol, cipher_string)
            ssock.connect(dest)
            # the connection will be made with the highest-priority cipher
            chosen_ciphers.append( ssock.cipher()[0] )
            ssock.close()
            # having detected the cipher, disable it
            cipher_string += ":!"+chosen_ciphers[-1]
        except OSError:
            # if no connection can be made with the remaining ciphers,
            #  it means we've enumerated all the supported ones
            break
    return chosen_ciphers

def get_certificate(dest):
    """Returns a DER-encoded certificate (as a bytes object).

    Arguments:
            dest -- the (IP, port) to connect to
    """
    try:
        ssock = make_ssocket(ssl.PROTOCOL_SSLv23)  # connects to SSLv3+
        ssock.connect(dest)
        der_cert = ssock.getpeercert(binary_form=True)
        ssock.close()
        return der_cert
    except OSError:
        return None
    # c = asn1crypto.x509.Certificate.load(der_cert)

########################################################################
########################################################################
########################################################################

def get_certificate_chain(url):
    """Returns a list of DER-encoded certificate (as bytes).

    Arguments: url -- the site to connect to"""
    try:
        sock = socket.socket()
        sock.connect((url, 443))
    except OSError:
        try:
            time.sleep(5)
            sock.settimeout(10)
            sock.connect((url, 443))
        except:
            return []

    hello = create_client_hello(SSLVersion.SSLv3,
                                [cs for cs in CipherSuite],
                                sni_url = url,
                                max_ssl_version = SSLVersion.TLSv1_2)
    try:
        sock.send(hello)
        time.sleep(2)             # to make sure all data has arrived
        resp = sock.recv(100000)
    except ConnectionError:
        return []

    # find a Certificate record
    while resp:
        try:    record = TLSRecord(resp)
        except: return []

        if record.type == RecordType.handshake and record.data[0:1] == HandshakeType.certificate.value:
            return extract_certificate_chain(record)
        resp = resp[5+len(record):]

    return []

def get_HTTP_header_fields(url, fieldnames):
    """Returns the contents of the specified HTTP response header fields.

    Arguments:
        url -- the website to check (in the form '[xyz://]www.example.com/')
        fieldnames -- a list of header field names; can be a single string,
                      in which case it is treated like a list of length one

    Returns:
        field_contents -- a mapping (dict) from fieldnames to their values;
                          empty in case of error or if there are no headers
    """
    url = "https://" + url.split("//", maxsplit=1)[-1]
    if type(fieldnames) is str:
        fieldnames = [fieldnames]

    field_contents = {}

    try:
        resp = requests.head(url, allow_redirects=True)
    except requests.RequestException as e:
        return {}

    return {k:v for (k,v) in resp.headers.items() if k in fieldnames}

class Record:
    """A class for storing the results of a single scan."""
    def __init__(self, host):
        self.hostname, self.IP = host
        self.protocols = []
        self.cipher_suites = []
        self.timestamp = int(time.time())
        self.certificate = None
        self.certificate_chain = []
        self.extensions = []

    def add_protocol(self, protocol):
        if protocol not in self.protocols:
            self.protocols.append( protocol )

    def add_cipher_suite(self, csuite):
        if csuite not in self.cipher_suites:
            self.cipher_suites.append( csuite )

    def add_to_DB(self, db_cursor):
        SQL_GET_INDEX = "SELECT COALESCE(MAX(indeks),0) FROM certificates;"
        SQL_GET_CERT  = "SELECT indeks FROM certificates WHERE certificate = ?;"
        SQL_INSERT_CERT = "INSERT INTO certificates VALUES (?,?);"
        SQL_INSERT_MAIN = "INSERT INTO scan_results VALUES (?,?,?,?,?,?,?,?);"

        certs = []
        for cert in self.certificate_chain:   # insert any new certificates first

            # get the index of next cert
            db_cursor.execute(SQL_GET_INDEX)
            next_index = db_cursor.fetchone()[0] + 1

            # check whether cert already exists
            db_cursor.execute(SQL_GET_CERT, (cert,));
            index = db_cursor.fetchone()

            if not index:
                certs += [next_index]
                db_cursor.execute(SQL_INSERT_CERT, (next_index, cert))
                next_index += 1
            else:
                certs += [index[0]]

        # Here, `certs` has the certificate chain as indices;
        #  now we add the record into the main table.

        protos_bytes  = SSLVersion.as_bytes(self.protocols)
        csuite_bytes  = CipherSuite.as_bytes(self.cipher_suites)
        ext_bytes     = ExtensionType.as_bytes(self.extensions)
        cchain_string = "$".join(str(i) for i in certs)
        HTTP_fields = "_$_".join(k+":"+v for k,v in self.HTTP_header_fields.items())
        if not protos_bytes:  protos_bytes  = None
        if not csuite_bytes:  csuite_bytes  = None
        if not ext_bytes:     ext_bytes     = None
        if not cchain_string: cchain_string = None
        if not HTTP_fields:   HTTP_fields   = None

        db_cursor.execute(SQL_INSERT_MAIN,
                           (self.hostname, self.IP, self.timestamp,
                            protos_bytes, csuite_bytes, cchain_string,
                            ext_bytes, HTTP_fields))

    def __str__(self):
        s = "host: " + self.hostname + "\n"
        s += "IP: " + self.IP + "\n"
        s += "timestamp: " + datetime.datetime.fromtimestamp(self.timestamp)\
                                              .isoformat(" ") + "\n"
        s += "supports:\n"
        for proto in self.protocols:
            s += "  "+proto.name+"\n"

        for cipher in proto.cipher_suites:
            s += "  "+cipher+"\n"

        return s

def init_db_tables(db_cursor):
    """Create the necessary database tables, if they do not already exist."""
    db_cursor.execute(
        "CREATE TABLE IF NOT EXISTS scan_results(\n"
        "    hostname           TEXT,\n"
        "    ip                 TEXT,\n"
        "    timestamp          INTEGER,\n"
        "    protocols          BLOB,\n"
        "    ciphersuites       BLOB,\n"
        "    certificate_chain  TEXT,\n"
        "    extensions         BLOB,\n"
        "    HTTP_header_fields TEXT,\n"
        "    PRIMARY KEY (hostname, timestamp));\n"
    )

    db_cursor.execute("CREATE TABLE IF NOT EXISTS certificates(\n"
                      "    indeks      INTEGER,\n"
                      "    certificate BLOB);\n"
    )

def process(host):
    hostname, IP = host
    if (hostname=="?"):
        dest = (IP, 443)
    else:
        dest = (hostname, 443)
    url = hostname

    # set default socket timeout
    socket.setdefaulttimeout(DEFAULT_TIMEOUT)
    # a dictionary to record the supported protocols
    supported = {}

    record = Record(host)

    # detect supported protocol versions
    record.protocols = [v for v in SSLVersion if try_protocol(v, url)]

    # detect supported cipher suites
    # cs_ssl3   = scan_all_ciphers(SSLVersion.SSLv3,   url)
    # cs_tls1   = scan_all_ciphers(SSLVersion.TLSv1,   url)
    # cs_tls1_1 = scan_all_ciphers(SSLVersion.TLSv1_1, url)
    cs_tls1_2 = scan_all_ciphers(SSLVersion.TLSv1_2, url)

    # record.cipher_suites = list(set(cs_ssl3 + cs_tls1 + cs_tls1_1 + cs_tls1_2))
    record.cipher_suites = sorted(cs_tls1_2)

    # record the certificate chain
    record.certificate_chain = get_certificate_chain(url)

    # detect supported extensions
    extensions = scan_all_extensions(SSLVersion.TLSv1_2, url)
    record.extensions = sorted(extensions)

    # record certain response header fields
    fields = get_HTTP_header_fields(hostname,
                                    ["Server",
                                     "Strict-Transport-Security",
                                     "Public-Key-Pins",
                                     "Public-Key-Pins-Report-Only"])
    record.HTTP_header_fields = fields;

    return record

class SSLTester(threading.Thread):

    def __init__(self, target_feed, results_feed):
        threading.Thread.__init__(self)
        self.target_feed = target_feed
        self.results_feed = results_feed

    def run(self):
        for target in self.target_feed:
            res = self._scan(target)
            self.results_feed.put(res)

    def _scan(self, target):
        return process(target)
<commit_msg>Speed up scan, fix two small bugs.<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
import ssl
import socket
import time
import datetime
import threading
import asn1crypto.x509
import requests

from .manual import *

DETECT_CIPHERS = True
DEFAULT_TIMEOUT = 5.0

def make_ssocket(protocol, ciphers=ssl._DEFAULT_CIPHERS):
    """Creates a TCP socket wrapped in SSL."""
    return ssl.wrap_socket( socket.socket(),
                            ssl_version=protocol,
                            ciphers=ciphers)

def get_supported_ciphers(dest, protocol):
    """Returns a list of ciphers supported by a host.

    Arguments:
        dest -- the (IP, port) to connect to
        protocol -- the SSL/TLS version to test; it is also
                     the index into the ssl._PROTOCOL_NAMES array

    The returned list contains cipher names.
    """
    chosen_ciphers = []
    # start by enabling all ciphers
    cipher_string = "ALL:aNULL:eNULL"
    while True:
        try:
            ssock = make_ssocket(protocol, cipher_string)
            ssock.connect(dest)
            # the connection will be made with the highest-priority cipher
            chosen_ciphers.append( ssock.cipher()[0] )
            ssock.close()
            # having detected the cipher, disable it
            cipher_string += ":!"+chosen_ciphers[-1]
        except OSError:
            # if no connection can be made with the remaining ciphers,
            #  it means we've enumerated all the supported ones
            break
    return chosen_ciphers

def get_certificate(dest):
    """Returns a DER-encoded certificate (as a bytes object).

    Arguments:
            dest -- the (IP, port) to connect to
    """
    try:
        ssock = make_ssocket(ssl.PROTOCOL_SSLv23)  # connects to SSLv3+
        ssock.connect(dest)
        der_cert = ssock.getpeercert(binary_form=True)
        ssock.close()
        return der_cert
    except OSError:
        return None
    # c = asn1crypto.x509.Certificate.load(der_cert)

########################################################################
########################################################################
########################################################################

def get_certificate_chain(url):
    """Returns a list of DER-encoded certificate (as bytes).

    Arguments: url -- the site to connect to"""
    try:
        sock = socket.socket()
        sock.connect((url, 443))
    except OSError:
        try:
            time.sleep(5)
            sock.settimeout(10)
            sock.connect((url, 443))
        except:
            return []

    hello = create_client_hello(SSLVersion.SSLv3,
                                [cs for cs in CipherSuite],
                                sni_url = url,
                                max_ssl_version = SSLVersion.TLSv1_2)
    try:
        sock.send(hello)
        time.sleep(2)             # to make sure all data has arrived
        resp = sock.recv(100000)
    except OSError:
        return []

    # find a Certificate record
    while resp:
        try:    record = TLSRecord(resp)
        except: return []

        if record.type == RecordType.handshake and record.data[0:1] == HandshakeType.certificate.value:
            return extract_certificate_chain(record)
        resp = resp[5+len(record):]

    return []

def get_HTTP_header_fields(url, fieldnames):
    """Returns the contents of the specified HTTP response header fields.

    Arguments:
        url -- the website to check (in the form '[xyz://]www.example.com/')
        fieldnames -- a list of header field names; can be a single string,
                      in which case it is treated like a list of length one

    Returns:
        field_contents -- a mapping (dict) from fieldnames to their values;
                          empty in case of error or if there are no headers
    """
    url = "https://" + url.split("//", maxsplit=1)[-1]
    if type(fieldnames) is str:
        fieldnames = [fieldnames]

    field_contents = {}

    try:
        resp = requests.head(url, allow_redirects=True)
    except requests.RequestException as e:
        return {}

    return {k:v for (k,v) in resp.headers.items() if k in fieldnames}

class Record:
    """A class for storing the results of a single scan."""
    def __init__(self, host):
        self.hostname, self.IP = host
        self.protocols = []
        self.cipher_suites = []
        self.timestamp = int(time.time())
        self.certificate = None
        self.certificate_chain = []
        self.extensions = []
        self.HTTP_header_fields = {}

    def add_protocol(self, protocol):
        if protocol not in self.protocols:
            self.protocols.append( protocol )

    def add_cipher_suite(self, csuite):
        if csuite not in self.cipher_suites:
            self.cipher_suites.append( csuite )

    def add_to_DB(self, db_cursor):
        SQL_GET_INDEX = "SELECT COALESCE(MAX(indeks),0) FROM certificates;"
        SQL_GET_CERT  = "SELECT indeks FROM certificates WHERE certificate = ?;"
        SQL_INSERT_CERT = "INSERT INTO certificates VALUES (?,?);"
        SQL_INSERT_MAIN = "INSERT INTO scan_results VALUES (?,?,?,?,?,?,?,?);"

        certs = []
        for cert in self.certificate_chain:   # insert any new certificates first

            # get the index of next cert
            db_cursor.execute(SQL_GET_INDEX)
            next_index = db_cursor.fetchone()[0] + 1

            # check whether cert already exists
            db_cursor.execute(SQL_GET_CERT, (cert,));
            index = db_cursor.fetchone()

            if not index:
                certs += [next_index]
                db_cursor.execute(SQL_INSERT_CERT, (next_index, cert))
                next_index += 1
            else:
                certs += [index[0]]

        # Here, `certs` has the certificate chain as indices;
        #  now we add the record into the main table.

        protos_bytes  = SSLVersion.as_bytes(self.protocols)
        csuite_bytes  = CipherSuite.as_bytes(self.cipher_suites)
        ext_bytes     = ExtensionType.as_bytes(self.extensions)
        cchain_string = "$".join(str(i) for i in certs)
        HTTP_fields = "_$_".join(k+":"+v for k,v in self.HTTP_header_fields.items())
        if not protos_bytes:  protos_bytes  = None
        if not csuite_bytes:  csuite_bytes  = None
        if not ext_bytes:     ext_bytes     = None
        if not cchain_string: cchain_string = None
        if not HTTP_fields:   HTTP_fields   = None

        db_cursor.execute(SQL_INSERT_MAIN,
                           (self.hostname, self.IP, self.timestamp,
                            protos_bytes, csuite_bytes, cchain_string,
                            ext_bytes, HTTP_fields))

    def __str__(self):
        s = "host: " + self.hostname + "\n"
        s += "IP: " + self.IP + "\n"
        s += "timestamp: " + datetime.datetime.fromtimestamp(self.timestamp)\
                                              .isoformat(" ") + "\n"
        s += "supports:\n"
        for proto in self.protocols:
            s += "  "+proto.name+"\n"

        for cipher in proto.cipher_suites:
            s += "  "+cipher+"\n"

        return s

def init_db_tables(db_cursor):
    """Create the necessary database tables, if they do not already exist."""
    db_cursor.execute(
        "CREATE TABLE IF NOT EXISTS scan_results(\n"
        "    hostname           TEXT,\n"
        "    ip                 TEXT,\n"
        "    timestamp          INTEGER,\n"
        "    protocols          BLOB,\n"
        "    ciphersuites       BLOB,\n"
        "    certificate_chain  TEXT,\n"
        "    extensions         BLOB,\n"
        "    HTTP_header_fields TEXT,\n"
        "    PRIMARY KEY (hostname, timestamp));\n"
    )

    db_cursor.execute("CREATE TABLE IF NOT EXISTS certificates(\n"
                      "    indeks      INTEGER,\n"
                      "    certificate BLOB);\n"
    )

def process(host):
    hostname, IP = host
    if (hostname=="?"):
        dest = (IP, 443)
    else:
        dest = (hostname, 443)
    url = hostname

    # set default socket timeout
    socket.setdefaulttimeout(DEFAULT_TIMEOUT)
    # a dictionary to record the supported protocols
    supported = {}

    record = Record(host)

    # detect supported protocol versions
    record.protocols = [v for v in SSLVersion if try_protocol(v, url)]

    if record.protocols:
        # detect supported cipher suites
        #  (don't duplicate, try just on the highest-supported protocol)
        if SSLVersion.TLSv1_2 in record.protocols:
            cs = scan_all_ciphers(SSLVersion.TLSv1_2, url)
        elif SSLVersion.TLSv1_1 in record.protocols:
            cs = scan_all_ciphers(SSLVersion.TLSv1_1, url)
        elif SSLVersion.TLSv1 in record.protocols:
            cs = scan_all_ciphers(SSLVersion.TLSv1,   url)
        else:
            cs = scan_all_ciphers(SSLVersion.SSLv3,   url)

        # record.cipher_suites = list(set(cs_ssl3 + cs_tls1 + cs_tls1_1 + cs_tls1_2))
        record.cipher_suites = sorted(cs)

        # record the certificate chain
        record.certificate_chain = get_certificate_chain(url)

        # detect supported extensions
        extensions = scan_all_extensions(SSLVersion.TLSv1_2, url)
        record.extensions = sorted(extensions)

        # record certain response header fields
        fields = get_HTTP_header_fields(hostname,
                                        ["Server",
                                        "Strict-Transport-Security",
                                        "Public-Key-Pins",
                                        "Public-Key-Pins-Report-Only"])
        record.HTTP_header_fields = fields;

    return record

class SSLTester(threading.Thread):

    def __init__(self, target_feed, results_feed):
        threading.Thread.__init__(self)
        self.target_feed = target_feed
        self.results_feed = results_feed

    def run(self):
        for target in self.target_feed:
            res = self._scan(target)
            self.results_feed.put(res)

    def _scan(self, target):
        return process(target)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# import * to get render_template
from flask import *

import random
import time

from thing import PiThing

"""main.py uses Flask web server for network communications
and PiThing to read and write to raspberry pi gpio pins.
"""

# instantiate flask app
app = Flask(__name__)

# instantiate pi_thing as a global variable
pi_thing = PiThing()

# default route raspberry pi /
@app.route("/")
def index():
    # get current switch value
    switch_value = pi_thing.read_switch()
    # render index.html, passing in switch_value
    # user must manually refresh browser, make new get request to see any changes to web page.
    # app is using request/response.
    # Later can change to websocket and javascript that runs in browser.
    return render_template('index.html', switch=switch_value)

# use url last component to pass argument led_state to the server
# name of defined function doesn't matter
@app.route("/led/<int:led_state>", methods=['POST'])
def led(led_state):
    if led_state == 0:
        pi_thing.set_led(False)
    elif led_state == 1:
        pi_thing.set_led(True)
    else:
        # http status code 400 bad request error
        return ('Unknown LED state!', 400)
    return ('', 204)

# endpoint /switch
# Firefox asks if want to save or open with a program
# Chrome works
@app.route("/switch")
def switch():
    # http://flask.pocoo.org/docs/0.11/patterns/streaming/
    def get_switch_values():
        while True:
            # server sent event specifies format:
            # data: <value>\n\n
            # http://www.html5rocks.com/en/tutorials/eventsource/basics/
            yield('data: {0}\n\n'.format(random.randrange(0,100)))
            time.sleep(1.0)
    return Response(get_switch_values(), mimetype='text/event-stream')

@app.route("/foo")
def achoo():
    return "Achoo Foo!"

if __name__ == "__main__":
    # listen for connections from any machine on network
    # in browser enter url http://0.0.0.0:5000/
    # set threaed true so flask can use multiple threads
    # this keeps /switch infinite loop "while True" from hogging all execution time
    # also enables app to handle multiple requests
    app.run(host='0.0.0.0', debug=True, threaded=True)
<commit_msg>In main.py data send switch_value<commit_after>#!/usr/bin/env python3

# import * to get render_template
from flask import *

import random
import time

from thing import PiThing

"""main.py uses Flask web server for network communications
and PiThing to read and write to raspberry pi gpio pins.
"""

# instantiate flask app
app = Flask(__name__)

# instantiate pi_thing as a global variable
pi_thing = PiThing()

# default route raspberry pi /
@app.route("/")
def index():
    # get current switch value
    switch_value = pi_thing.read_switch()
    # render index.html, passing in switch_value
    return render_template('index.html', switch=switch_value)

# use url last component to pass argument led_state to the server
# name of defined function doesn't matter
@app.route("/led/<int:led_state>", methods=['POST'])
def led(led_state):
    if led_state == 0:
        pi_thing.set_led(False)
    elif led_state == 1:
        pi_thing.set_led(True)
    else:
        # http status code 400 bad request error
        return ('Unknown LED state!', 400)
    return ('', 204)

# endpoint /switch
# Firefox asks if want to save or open with a program
# Chrome works
@app.route("/switch")
def switch():
    # http://flask.pocoo.org/docs/0.11/patterns/streaming/
    def get_switch_values():
        while True:
            # get current switch value 0 or 1
            switch_value = pi_thing.read_switch()

            # server sent event specifies format:
            # data: <value>\n\n
            # http://www.html5rocks.com/en/tutorials/eventsource/basics/
            yield('data: {0}\n\n'.format(switch_value))

            time.sleep(1.0)
    return Response(get_switch_values(), mimetype='text/event-stream')

@app.route("/foo")
def achoo():
    return "Achoo Foo!"

if __name__ == "__main__":
    # listen for connections from any machine on network
    # in browser enter url http://0.0.0.0:5000/
    # set threaed true so flask can use multiple threads
    # this keeps /switch infinite loop "while True" from hogging all execution time
    # also enables app to handle multiple requests
    app.run(host='0.0.0.0', debug=True, threaded=True)
<|endoftext|>


<commit_before>from google.protobuf.descriptor import FieldDescriptor as FD
import opts_pb2

# pb2dict and dict2pb are methods to convert pb to/from dict.
# Inspired by:
#   protobuf-to-dict - https://github.com/benhodgson/protobuf-to-dict
#   protobuf-json    - https://code.google.com/p/protobuf-json/
#   protobuf source  - https://code.google.com/p/protobuf/
# Both protobuf-to-dict/json do not fit here because of several reasons,
# here are some of them:
#   - both have a common bug in treating optional field with empty
#     repeated inside.
#   - protobuf-to-json is not avalible in pip or in any other python
#     repo, so it is hard to distribute and we can't rely on it.
#   - both do not treat enums in a way we would like to. They convert
#     protobuf enum to int, but we need a string here, because it is
#     much more informative. BTW, protobuf text_format converts pb
#     enums to string value too. (i.e. "march : x86_64" is better then
#     "march : 1").

_basic_cast = {
	FD.TYPE_DOUBLE		: float,
	FD.TYPE_FLOAT		: float,
	FD.TYPE_FIXED64		: float,
	FD.TYPE_FIXED32		: float,
	FD.TYPE_SFIXED64	: float,
	FD.TYPE_SFIXED32	: float,

	FD.TYPE_INT64		: long,
	FD.TYPE_UINT64		: long,
	FD.TYPE_SINT64		: long,

	FD.TYPE_INT32		: int,
	FD.TYPE_UINT32		: int,
	FD.TYPE_SINT32		: int,

	FD.TYPE_BOOL		: bool,

	FD.TYPE_STRING		: unicode
}

def _marked_as_hex(field):
	return field.GetOptions().Extensions[opts_pb2.criu].hex

def _pb2dict_cast(field, value, fmt = None, is_hex = False):
	if not is_hex:
		is_hex = _marked_as_hex(field)
	if fmt:
		print_hex = 'hex' in fmt
	else:
		print_hex = False

	if field.type == FD.TYPE_MESSAGE:
		return pb2dict(value, fmt, is_hex)
	elif field.type == FD.TYPE_BYTES:
		return value.encode('base64')
	elif field.type == FD.TYPE_ENUM:
		return field.enum_type.values_by_number.get(value, None).name
	elif field.type in _basic_cast:
		cast = _basic_cast[field.type]
		if (cast == int or cast == long) and is_hex and print_hex:
			# Fields that have (criu).hex = true option set
			# should be stored in hex string format.
			return "0x%x" % value
		else:
			return cast(value)
	else:
		raise Exception("Field(%s) has unsupported type %d" % (field.name, field.type))

def pb2dict(pb, fmt = None, is_hex = False):
	"""
	Convert protobuf msg to dictionary.
	Takes a protobuf message and returns a dict.
	"""
	d = {}
	for field, value in pb.ListFields():
		if field.label == FD.LABEL_REPEATED:
			d_val = []
			for v in value:
				d_val.append(_pb2dict_cast(field, v, fmt, is_hex))
		else:
			d_val = _pb2dict_cast(field, value, fmt, is_hex)

		d[field.name] = d_val
	return d

def _dict2pb_cast(field, value):
	# Not considering TYPE_MESSAGE here, as repeated
	# and non-repeated messages need special treatment
	# in this case, and are hadled separately.
	if field.type == FD.TYPE_BYTES:
		return value.decode('base64')
	elif field.type == FD.TYPE_ENUM:
		return field.enum_type.values_by_name.get(value, None).number
	else:
		cast = _basic_cast[field.type]
		if (cast == int or cast == long) and isinstance(value, unicode):
			# Some int or long fields might be stored as hex
			# strings. See _pb2dict_cast.
			return cast(value, 0)
		else:
			return cast(value)

def dict2pb(d, pb):
	"""
	Convert dictionary to protobuf msg.
	Takes dict and protobuf message to be merged into.
	"""
	for field in pb.DESCRIPTOR.fields:
		if field.name not in d:
			continue
		value = d[field.name]
		if field.label == FD.LABEL_REPEATED:
			pb_val = getattr(pb, field.name, None)
			for v in value:
				if field.type == FD.TYPE_MESSAGE:
					dict2pb(v, pb_val.add())
				else:
					pb_val.append(_dict2pb_cast(field, v))
		else:
			if field.type == FD.TYPE_MESSAGE:
				# SetInParent method acts just like has_* = true in C,
				# and helps to properly treat cases when we have optional
				# field with empty repeated inside.
				getattr(pb, field.name).SetInParent()

				dict2pb(value, getattr(pb, field.name, None))
			else:
				setattr(pb, field.name, _dict2pb_cast(field, value))
	return pb
<commit_msg>pycriu: images: pb2dict: don't forget to throw an exception, if field has unsupported type<commit_after>from google.protobuf.descriptor import FieldDescriptor as FD
import opts_pb2

# pb2dict and dict2pb are methods to convert pb to/from dict.
# Inspired by:
#   protobuf-to-dict - https://github.com/benhodgson/protobuf-to-dict
#   protobuf-json    - https://code.google.com/p/protobuf-json/
#   protobuf source  - https://code.google.com/p/protobuf/
# Both protobuf-to-dict/json do not fit here because of several reasons,
# here are some of them:
#   - both have a common bug in treating optional field with empty
#     repeated inside.
#   - protobuf-to-json is not avalible in pip or in any other python
#     repo, so it is hard to distribute and we can't rely on it.
#   - both do not treat enums in a way we would like to. They convert
#     protobuf enum to int, but we need a string here, because it is
#     much more informative. BTW, protobuf text_format converts pb
#     enums to string value too. (i.e. "march : x86_64" is better then
#     "march : 1").

_basic_cast = {
	FD.TYPE_DOUBLE		: float,
	FD.TYPE_FLOAT		: float,
	FD.TYPE_FIXED64		: float,
	FD.TYPE_FIXED32		: float,
	FD.TYPE_SFIXED64	: float,
	FD.TYPE_SFIXED32	: float,

	FD.TYPE_INT64		: long,
	FD.TYPE_UINT64		: long,
	FD.TYPE_SINT64		: long,

	FD.TYPE_INT32		: int,
	FD.TYPE_UINT32		: int,
	FD.TYPE_SINT32		: int,

	FD.TYPE_BOOL		: bool,

	FD.TYPE_STRING		: unicode
}

def _marked_as_hex(field):
	return field.GetOptions().Extensions[opts_pb2.criu].hex

def _pb2dict_cast(field, value, fmt = None, is_hex = False):
	if not is_hex:
		is_hex = _marked_as_hex(field)
	if fmt:
		print_hex = 'hex' in fmt
	else:
		print_hex = False

	if field.type == FD.TYPE_MESSAGE:
		return pb2dict(value, fmt, is_hex)
	elif field.type == FD.TYPE_BYTES:
		return value.encode('base64')
	elif field.type == FD.TYPE_ENUM:
		return field.enum_type.values_by_number.get(value, None).name
	elif field.type in _basic_cast:
		cast = _basic_cast[field.type]
		if (cast == int or cast == long) and is_hex and print_hex:
			# Fields that have (criu).hex = true option set
			# should be stored in hex string format.
			return "0x%x" % value
		else:
			return cast(value)
	else:
		raise Exception("Field(%s) has unsupported type %d" % (field.name, field.type))

def pb2dict(pb, fmt = None, is_hex = False):
	"""
	Convert protobuf msg to dictionary.
	Takes a protobuf message and returns a dict.
	"""
	d = {}
	for field, value in pb.ListFields():
		if field.label == FD.LABEL_REPEATED:
			d_val = []
			for v in value:
				d_val.append(_pb2dict_cast(field, v, fmt, is_hex))
		else:
			d_val = _pb2dict_cast(field, value, fmt, is_hex)

		d[field.name] = d_val
	return d

def _dict2pb_cast(field, value):
	# Not considering TYPE_MESSAGE here, as repeated
	# and non-repeated messages need special treatment
	# in this case, and are hadled separately.
	if field.type == FD.TYPE_BYTES:
		return value.decode('base64')
	elif field.type == FD.TYPE_ENUM:
		return field.enum_type.values_by_name.get(value, None).number
	elif field.type in _basic_cast:
		cast = _basic_cast[field.type]
		if (cast == int or cast == long) and isinstance(value, unicode):
			# Some int or long fields might be stored as hex
			# strings. See _pb2dict_cast.
			return cast(value, 0)
		else:
			return cast(value)
	else:
		raise Exception("Field(%s) has unsupported type %d" % (field.name, field.type))

def dict2pb(d, pb):
	"""
	Convert dictionary to protobuf msg.
	Takes dict and protobuf message to be merged into.
	"""
	for field in pb.DESCRIPTOR.fields:
		if field.name not in d:
			continue
		value = d[field.name]
		if field.label == FD.LABEL_REPEATED:
			pb_val = getattr(pb, field.name, None)
			for v in value:
				if field.type == FD.TYPE_MESSAGE:
					dict2pb(v, pb_val.add())
				else:
					pb_val.append(_dict2pb_cast(field, v))
		else:
			if field.type == FD.TYPE_MESSAGE:
				# SetInParent method acts just like has_* = true in C,
				# and helps to properly treat cases when we have optional
				# field with empty repeated inside.
				getattr(pb, field.name).SetInParent()

				dict2pb(value, getattr(pb, field.name, None))
			else:
				setattr(pb, field.name, _dict2pb_cast(field, value))
	return pb
<|endoftext|>


<commit_before>"""
The MIT License (MIT)

Copyright (c) 2016-2017 Louis-Philippe Querel l_querel@encs.concordia.ca

Permission is hereby granted, free of charge, to any person obtaining a copy of this software
and associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
from datetime import datetime

import config
from postgres import Postgres

from utility.Logging import logger

class Service_DB:

    def __init__(self, reprocess_failures_hours):
        self.db = Postgres(config.get_local_settings())
        self.REPROCESS_FAILURES_HOURS = reprocess_failures_hours
        self.COMMIT_LOG_TOOL_COLUMNS = frozenset(["artifacts_archived"])

    def get_unprocessed_commits(self):

        cursor = self.db.get_cursor()

        if config.REPO_TO_ANALYSE:

            # determine if it is a string or a list
            if isinstance(config.REPO_TO_ANALYSE, (list, tuple)) and not isinstance(config.REPO_TO_ANALYSE, (str, basestring)):
                equality = "in"
                value = tuple(config.REPO_TO_ANALYSE)
            else:
                equality = "="
                value = config.REPO_TO_ANALYSE

            query = """
                    SELECT repository_id, commit_hash, author_date
                    FROM COMMITS
                    WHERE COMMITS.COMMIT_HASH NOT IN (SELECT COMMIT FROM STATIC_COMMIT_PROCESSED AS PROCESSED)
                    AND repository_id {equality} %s
                    ORDER BY author_date_unix_timestamp DESC
                    LIMIT 10;
                    """.format(equality=equality)

            cursor.execute(query, (value, ))
        else:
            query = """
                    SELECT repository_id, commit_hash
                    FROM COMMITS
                    WHERE COMMITS.COMMIT_HASH NOT IN (SELECT COMMIT FROM STATIC_COMMIT_PROCESSED AS PROCESSED)
                    ORDER BY author_date_unix_timestamp DESC
                    LIMIT 10;
                    """

            cursor.execute(query)
        rows = cursor.fetchall()

        REPO = 0
        COMMIT = 1
        AUTHOR_DATE = 2

        commits = []

        #datetime.strptime('Tue Jun 28 23:29:52 2016 -0700'[], '%a %b %d %H:%M:%S %Y %z')
        for row in rows:
            raw_author_date = row[AUTHOR_DATE]
            # Strip off the timezone not to have to deal with it
            author_datetime = datetime.strptime(raw_author_date[:len(raw_author_date)-6], '%a %b %d %H:%M:%S %Y')
            commits.append({"repo": row[REPO], "commit": row[COMMIT], "author_date": author_datetime})

        self.queued_commit(commits)

        return commits

    """
    Delete any records of previous runs that have not finished running since a specified interval
    """
    def truncate_commit_processing(self):
        cursor = self.db.get_cursor()
        cursor.execute(
            """
            DELETE FROM STATIC_COMMIT_PROCESSED
            WHERE STATUS <> 'PROCESSED' AND modified < NOW() - INTERVAL '%s hour';
            """ % self.REPROCESS_FAILURES_HOURS
        )
        self.db.db.commit()

    def queued_commit(self, commits):
        cursor = self.db.get_cursor()

        cursor.executemany("""
            INSERT INTO STATIC_COMMIT_PROCESSED
            (REPO, COMMIT, STATUS)
            VALUES
            (%(repo)s, %(commit)s, 'QUEUED');
            """, commits)
        self.db.db.commit()

    def processing_commit(self, repo, commit):
        cursor = self.db.get_cursor()

        cursor.execute("""
            UPDATE STATIC_COMMIT_PROCESSED
             SET STATUS = 'PROCESSING', MODIFIED = NOW()
             WHERE REPO = %s AND COMMIT = %s;
            """, (repo, commit))
        self.db.db.commit()

    def processed_commit(self, repo, commit, build, log=""):
        cursor = self.db.get_cursor()

        cursor.execute("""
            UPDATE STATIC_COMMIT_PROCESSED
             SET STATUS = 'PROCESSED', BUILD = %s, BUILD_LOG = %s, MODIFIED = NOW()
             WHERE REPO = %s AND COMMIT = %s;
            """, (build, log, repo, commit))
        self.db.db.commit()

    def processing_commit_sql_failed(self, repo, commit, error_message):
        self.processed_commit(repo, commit, "SQL_ERROR", error_message)

    def commit_log_tool(self, repo, commit, column, value):

        if column not in self.COMMIT_LOG_TOOL_COLUMNS:
            logger.error("Column %s is not a log tool column" % column)
        else:
            cursor = self.db.get_cursor()

            cursor.execute("""
                    UPDATE STATIC_COMMIT_PROCESSED
                     SET {column} = %s
                     WHERE REPO = %s AND COMMIT = %s;
                    """.format(column=column), (value, repo, commit))

    def add_commit_warning_lines(self, warnings):

        cursor = self.db.get_cursor()
        cursor.executemany("""
        INSERT INTO STATIC_COMMIT_LINE_WARNING
        (REPO, COMMIT, RESOURCE, LINE, SFP, CWE, GENERATOR_TOOL, WEAKNESS)
        VALUES
        (%(repo_id)s, %(commit_id)s, %(resource)s, %(line_number)s, %(SFP)s, %(CWE)s, %(generator_tool)s, %(description)s)
        """, warnings)

    def add_commit_warning_blames(self, blames):

        cursor = self.db.get_cursor()
        cursor.executemany("""
        INSERT INTO static_commit_line_blame
        (REPO, COMMIT, RESOURCE, LINE, ORIGIN_COMMIT, ORIGIN_RESOURCE, ORIGIN_LINE, IS_NEW_LINE)
        VALUES
        (%(repo_id)s, %(commit_id)s, %(resource)s, %(line)s, %(origin_commit)s, %(origin_resource)s, %(origin_line)s, %(is_new_line)s)
        """, blames)

    def add_commit_history_graph(self, relations):
        cursor = self.db.get_cursor()
        cursor.executemany("""
        INSERT INTO commit_history_graph
        (REPO, COMMIT, PARENT_COMMIT)
        VALUES
        (%(repo_id)s, %(commit_id)s, %(parent_commit)s)
        """, relations)

    def clear_commit_data(self, repo, commit):
        cursor = self.db.get_cursor()

        cursor.execute("""DELETE FROM STATIC_COMMIT_LINE_WARNING WHERE REPO=%s AND COMMIT=%s;""", (repo, commit))
        cursor.execute("""DELETE FROM STATIC_COMMIT_LINE_BLAME WHERE REPO=%s AND COMMIT=%s;""", (repo, commit))
        cursor.execute("""DELETE FROM COMMIT_HISTORY_GRAPH WHERE REPO=%s AND COMMIT=%s;""", (repo, commit))

def get_service_db():
    return Service_DB(config.REPROCESS_FAILURES_HOURS).db<commit_msg>#58 Reseting the connection after a db error to continue with the analysis of subsequent commits<commit_after>"""
The MIT License (MIT)

Copyright (c) 2016-2017 Louis-Philippe Querel l_querel@encs.concordia.ca

Permission is hereby granted, free of charge, to any person obtaining a copy of this software
and associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
from datetime import datetime

import config
from postgres import Postgres

from utility.Logging import logger

class Service_DB:

    def __init__(self, reprocess_failures_hours):
        self.db = Postgres(config.get_local_settings())
        self.REPROCESS_FAILURES_HOURS = reprocess_failures_hours
        self.COMMIT_LOG_TOOL_COLUMNS = frozenset(["artifacts_archived"])

    def get_unprocessed_commits(self):

        cursor = self.db.get_cursor()

        if config.REPO_TO_ANALYSE:

            # determine if it is a string or a list
            if isinstance(config.REPO_TO_ANALYSE, (list, tuple)) and not isinstance(config.REPO_TO_ANALYSE, (str, basestring)):
                equality = "in"
                value = tuple(config.REPO_TO_ANALYSE)
            else:
                equality = "="
                value = config.REPO_TO_ANALYSE

            query = """
                    SELECT repository_id, commit_hash, author_date
                    FROM COMMITS
                    WHERE COMMITS.COMMIT_HASH NOT IN (SELECT COMMIT FROM STATIC_COMMIT_PROCESSED AS PROCESSED)
                    AND repository_id {equality} %s
                    ORDER BY author_date_unix_timestamp DESC
                    LIMIT 10;
                    """.format(equality=equality)

            cursor.execute(query, (value, ))
        else:
            query = """
                    SELECT repository_id, commit_hash
                    FROM COMMITS
                    WHERE COMMITS.COMMIT_HASH NOT IN (SELECT COMMIT FROM STATIC_COMMIT_PROCESSED AS PROCESSED)
                    ORDER BY author_date_unix_timestamp DESC
                    LIMIT 10;
                    """

            cursor.execute(query)
        rows = cursor.fetchall()

        REPO = 0
        COMMIT = 1
        AUTHOR_DATE = 2

        commits = []

        #datetime.strptime('Tue Jun 28 23:29:52 2016 -0700'[], '%a %b %d %H:%M:%S %Y %z')
        for row in rows:
            raw_author_date = row[AUTHOR_DATE]
            # Strip off the timezone not to have to deal with it
            author_datetime = datetime.strptime(raw_author_date[:len(raw_author_date)-6], '%a %b %d %H:%M:%S %Y')
            commits.append({"repo": row[REPO], "commit": row[COMMIT], "author_date": author_datetime})

        self.queued_commit(commits)

        return commits

    """
    Delete any records of previous runs that have not finished running since a specified interval
    """
    def truncate_commit_processing(self):
        cursor = self.db.get_cursor()
        cursor.execute(
            """
            DELETE FROM STATIC_COMMIT_PROCESSED
            WHERE STATUS <> 'PROCESSED' AND modified < NOW() - INTERVAL '%s hour';
            """ % self.REPROCESS_FAILURES_HOURS
        )
        self.db.db.commit()

    def queued_commit(self, commits):
        cursor = self.db.get_cursor()

        cursor.executemany("""
            INSERT INTO STATIC_COMMIT_PROCESSED
            (REPO, COMMIT, STATUS)
            VALUES
            (%(repo)s, %(commit)s, 'QUEUED');
            """, commits)
        self.db.db.commit()

    def processing_commit(self, repo, commit):
        cursor = self.db.get_cursor()

        cursor.execute("""
            UPDATE STATIC_COMMIT_PROCESSED
             SET STATUS = 'PROCESSING', MODIFIED = NOW()
             WHERE REPO = %s AND COMMIT = %s;
            """, (repo, commit))
        self.db.db.commit()

    def processed_commit(self, repo, commit, build, log=""):
        cursor = self.db.get_cursor()

        cursor.execute("""
            UPDATE STATIC_COMMIT_PROCESSED
             SET STATUS = 'PROCESSED', BUILD = %s, BUILD_LOG = %s, MODIFIED = NOW()
             WHERE REPO = %s AND COMMIT = %s;
            """, (build, log, repo, commit))
        self.db.db.commit()

    def processing_commit_sql_failed(self, repo, commit, error_message):
        # Cancel the current transaction
        self.db.db.reset()

        # Log the error in the database
        self.processed_commit(repo, commit, "SQL_ERROR", error_message)

    def commit_log_tool(self, repo, commit, column, value):

        if column not in self.COMMIT_LOG_TOOL_COLUMNS:
            logger.error("Column %s is not a log tool column" % column)
        else:
            cursor = self.db.get_cursor()

            cursor.execute("""
                    UPDATE STATIC_COMMIT_PROCESSED
                     SET {column} = %s
                     WHERE REPO = %s AND COMMIT = %s;
                    """.format(column=column), (value, repo, commit))

    def add_commit_warning_lines(self, warnings):

        cursor = self.db.get_cursor()
        cursor.executemany("""
        INSERT INTO STATIC_COMMIT_LINE_WARNING
        (REPO, COMMIT, RESOURCE, LINE, SFP, CWE, GENERATOR_TOOL, WEAKNESS)
        VALUES
        (%(repo_id)s, %(commit_id)s, %(resource)s, %(line_number)s, %(SFP)s, %(CWE)s, %(generator_tool)s, %(description)s)
        """, warnings)

    def add_commit_warning_blames(self, blames):

        cursor = self.db.get_cursor()
        cursor.executemany("""
        INSERT INTO static_commit_line_blame
        (REPO, COMMIT, RESOURCE, LINE, ORIGIN_COMMIT, ORIGIN_RESOURCE, ORIGIN_LINE, IS_NEW_LINE)
        VALUES
        (%(repo_id)s, %(commit_id)s, %(resource)s, %(line)s, %(origin_commit)s, %(origin_resource)s, %(origin_line)s, %(is_new_line)s)
        """, blames)

    def add_commit_history_graph(self, relations):
        cursor = self.db.get_cursor()
        cursor.executemany("""
        INSERT INTO commit_history_graph
        (REPO, COMMIT, PARENT_COMMIT)
        VALUES
        (%(repo_id)s, %(commit_id)s, %(parent_commit)s)
        """, relations)

    def clear_commit_data(self, repo, commit):
        cursor = self.db.get_cursor()

        cursor.execute("""DELETE FROM STATIC_COMMIT_LINE_WARNING WHERE REPO=%s AND COMMIT=%s;""", (repo, commit))
        cursor.execute("""DELETE FROM STATIC_COMMIT_LINE_BLAME WHERE REPO=%s AND COMMIT=%s;""", (repo, commit))
        cursor.execute("""DELETE FROM COMMIT_HISTORY_GRAPH WHERE REPO=%s AND COMMIT=%s;""", (repo, commit))

def get_service_db():
    return Service_DB(config.REPROCESS_FAILURES_HOURS).db<|endoftext|>


<commit_before>"""
mpltex
======

"""

golden_ratio = (5**.5-1.0)/2.0

def pt_to_in(pt):
    return pt / 72.27

def pt_to_mm(pt):
    raise NotImplementedError

def get_rcParams(fig_width_pt = 232.0, scale = 1.0, dpi = 600):
    """TODO: write a description

    """
    fig_width = pt_to_in(fig_width_pt) # width in inches
    fig_height = fig_width * golden_ratio # height in inches
    fig_size =  [fig_width * scale, fig_height * scale]
    params = {  'axes.labelsize': 10,
                'text.fontsize': 8,
                'legend.fontsize': 8,
                'xtick.labelsize': 8,
                'ytick.labelsize': 8,
                'font.family': 'serif',
                'font.serif': 'Times',
                'text.usetex': True,
                'figure.figsize': fig_size,
                'figure.dpi': dpi}
    return params
<commit_msg>amsmath and bm packages in LaTeX preamble<commit_after>"""
mpltex
======

"""

golden_ratio = (5**.5-1.0)/2.0
packages = ['amsmath', 'amssymb', 'amsfonts', 'amsbsy', 'bm']
usepackages = [r'\usepackage{{{0}}}'.format(pkg) for pkg in packages]

def pt_to_in(pt):
    return pt / 72.27

def pt_to_mm(pt):
    raise NotImplementedError

def get_rcParams(fig_width_pt = 232.0, scale = 1.0, dpi = 600):
    """TODO: write a description

    """
    fig_width = pt_to_in(fig_width_pt) # width in inches
    fig_height = fig_width * golden_ratio # height in inches
    fig_size =  [fig_width * scale, fig_height * scale]
    params = {  'axes.labelsize': 10,
                'text.fontsize': 8,
                'legend.fontsize': 8,
                'xtick.labelsize': 8,
                'ytick.labelsize': 8,
                'font.family': 'serif',
                'font.serif': 'Times',
                'text.usetex': True,
                'text.latex.preamble': usepackages,
                'figure.figsize': fig_size,
                'figure.dpi': dpi}
    return params
<|endoftext|>