<commit_before>from base64 import b64encode
import urllib
import json
from os.path import exists, basename

import requests
from requests.exceptions import ConnectionError

from domain import Figure, Image, Dataset, Activity

def check_image(fn):
    def wrapped(*args, **kwargs):
        # if len(args) < 1 or not isinstance(args[0], Image):
        #     raise Exception('Invalid Image')
        if args[1].identifier in (None, ''):
            raise Exception('Invalid identifier', args[0].identifier)
        return fn(*args, **kwargs)

    return wrapped

def exists(fn):
    def wrapped(*args, **kwargs):
        resp = fn(*args, **kwargs)
        if resp.status_code == 200:
            return True
        elif resp.status_code == 404:
            return False
        else:
            raise Exception(resp.text)
    return wrapped

def http_resp(fn):
    def wrapped(*args, **kwargs):
        resp = fn(*args, **kwargs)
        if resp.status_code == 200:
            return resp
        else:
            raise Exception(resp.text)
    return wrapped

class AssociationException(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class GcisClient(object):
    def __init__(self, url, username, password):
        self.headers = {
            'Accept': 'application/json'
        }

        self.base_url = url
        self.headers['Authorization'] = 'Basic ' + b64encode(username + ':' + password)

    @http_resp
    def create_figure(self, report_id, chapter_id, figure, skip_images=False):
        if figure.identifier in (None, ''):
            raise Exception('Invalid figure identifier', figure.identifier)

        #Is GCIS not inferring this from the url parameter?
        if figure.chapter_identifier in (None, ''):
            figure.chapter_identifier = chapter_id

        url = '{b}/report/{rpt}/chapter/{chp}/figure/'.format(
            b=self.base_url, rpt=report_id, chp=chapter_id
        )

        resp = requests.post(url, data=figure.as_json(), headers=self.headers, verify=False)

        if skip_images is False:
            for image in figure.images:
                self.create_image(image),
                self.associate_image_with_figure(image.identifier, report_id, figure.identifier)

        return resp

    @http_resp
    def update_figure(self, report_id, chapter_id, figure, skip_images=False):
        if figure.identifier in (None, ''):
            raise Exception('Invalid identifier', figure.identifier)

        #Is GCIS not inferring this from the url parameter?
        if figure.chapter_identifier in (None, ''):
            figure.chapter_identifier = chapter_id

        update_url = '{b}/report/{rpt}/chapter/{chp}/figure/{fig}'.format(
            b=self.base_url, rpt=report_id, chp=chapter_id, fig=figure.identifier
        )

        resp = requests.post(update_url, data=figure.as_json(), headers=self.headers, verify=False)

        if skip_images is False:
            for image in figure.images:
                self.update_image(image)

        return resp

    @http_resp
    def delete_figure(self, report_id, figure_id):
        url = '{b}/report/{rpt}/figure/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)
        return requests.delete(url, headers=self.headers, verify=False)

    @check_image
    def create_image(self, image, report_id=None, figure_id=None):
        url = '{b}/image/'.format(b=self.base_url)
        resp = requests.post(url, data=image.as_json(), headers=self.headers, verify=False)
        try:
            if image.local_path is not None:
                self.upload_image_file(image.identifier, image.local_path)
            if figure_id and report_id:
                self.associate_image_with_figure(image.identifier, report_id, figure_id)
            for dataset in image.datasets:
                self.create_or_update_dataset(dataset)
                self.create_or_update_activity(dataset.activity)
                self.associate_dataset_with_image(dataset.identifier, image.identifier,
                                                  activity_id=dataset.activity.identifier)
        except ConnectionError, e:
            print e
        return resp

    @check_image
    def update_image(self, image):
        update_url = '{b}/image/{img}'.format(b=self.base_url, img=image.identifier)
        for dataset in image.datasets:
            self.update_activity(dataset.activity)
            self.associate_dataset_with_image(dataset.identifier, image.identifier,
                                              activity_id=dataset.activity.identifier)

        return requests.post(update_url, data=image.as_json(), headers=self.headers, verify=False)

    @check_image
    def delete_image(self, image):
        delete_url = '{b}/image/{img}'.format(b=self.base_url, img=image.identifier)
        return requests.delete(delete_url, headers=self.headers, verify=False)

    def associate_image_with_figure(self, image_id, report_id, figure_id):
        url = '{b}/report/{rpt}/figure/rel/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)
        return requests.post(url, data=json.dumps({'add_image_identifier': image_id}), headers=self.headers, verify=False)

    def upload_image_file(self, image_id, local_path):
        url = '{b}/image/files/{id}/{fn}'.format(b=self.base_url, id=image_id, fn=basename(local_path))
        # For future multi-part encoding support
        # return requests.put(url, headers=headers, files={'file': (filename, open(filepath, 'rb'))})
        if not exists(local_path):
            raise Exception('File not found: ' + local_path)

        return requests.put(url, data=open(local_path, 'rb'), headers=self.headers, verify=False)

    #Full listing
    def get_figure_listing(self, report_id, chapter_id=None):
        chapter_filter = '/chapter/' + chapter_id if chapter_id else ''

        url = '{b}/report/{rpt}{chap}/figure?{p}'.format(
            b=self.base_url, rpt=report_id, chap=chapter_filter, p=urllib.urlencode({'all': '1'})
        )
        resp = requests.get(url, headers=self.headers, verify=False)

        try:
            return [Figure(figure) for figure in resp.json()]
        except ValueError:
            raise Exception('Add a better exception string here')

    def get_figure(self, report_id, figure_id, chapter_id=None):
        chapter_filter = '/chapter/' + chapter_id if chapter_id else ''

        url = '{b}/report/{rpt}{chap}/figure/{fig}?{p}'.format(
            b=self.base_url, rpt=report_id, chap=chapter_filter, fig=figure_id, p=urllib.urlencode({'all': '1'})
        )
        resp = requests.get(url, headers=self.headers, verify=False)

        try:
            return Figure(resp.json())
        except ValueError:
            raise Exception(resp.text)

    @exists
    def figure_exists(self, report_id, figure_id, chapter_id=None):
        chapter_filter = '/chapter/' + chapter_id if chapter_id else ''

        url = '{b}/report/{rpt}{chap}/figure/{fig}?{p}'.format(
            b=self.base_url, rpt=report_id, chap=chapter_filter, fig=figure_id, p=urllib.urlencode({'all': '1'})
        )
        return requests.head(url, headers=self.headers, verify=False)

    def get_image(self, image_id):
        url = '{b}/image/{img}'.format(b=self.base_url, img=image_id)
        resp = requests.get(url, headers=self.headers, verify=False)

        try:
            return Image(resp.json())
        except ValueError:
            raise Exception(resp.text)

    @exists
    def image_exists(self, image_id):
        url = '{b}/image/{img}'.format(b=self.base_url, img=image_id)
        return requests.head(url, headers=self.headers, verify=False)

    def has_all_associated_images(self, report_id, figure_id, target_image_ids):
        try:
            figure_image_ids = [i.identifier for i in self.get_figure(report_id, figure_id).images]
        except Exception, e:
            print e.message
            return False, set()

        target_set = set(target_image_ids)
        gcis_set = set(figure_image_ids)
        deltas = target_set - gcis_set

        if target_set.issubset(gcis_set):
            return True, deltas
        else:
            return False, deltas

    def test_login(self):
        url = '{b}/login.json'.format(b=self.base_url)
        resp = requests.get(url, headers=self.headers, verify=False)
        return resp.status_code, resp.text

    def get_keyword_listing(self):
        url = '{b}/gcmd_keyword?{p}'.format(b=self.base_url, p=urllib.urlencode({'all': '1'}))
        resp = requests.get(url, headers=self.headers, verify=False)

        return resp.json()

    def get_keyword(self, key_id):
        url = '{b}/gcmd_keyword/{k}'.format(b=self.base_url, k=key_id)
        return requests.get(url, headers=self.headers, verify=False).json()

    def associate_keyword_with_figure(self, keyword_id, report_id, figure_id):
        url = '{b}/report/{rpt}/figure/keywords/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)
        return requests.post(url, data=json.dumps({'identifier': keyword_id}), headers=self.headers, verify=False)

    def get_dataset(self, dataset_id):
        url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset_id)
        resp = requests.get(url, headers=self.headers, verify=False)
        try:
            return Dataset(resp.json())
        except ValueError:
            raise Exception(resp.text())

    @exists
    def dataset_exists(self, dataset_id):
        url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset_id)
        return requests.head(url, headers=self.headers, verify=False)

    def create_dataset(self, dataset):
        url = '{b}/dataset/'.format(b=self.base_url)
        return requests.post(url, data=dataset.as_json(), headers=self.headers, verify=False)

    def update_dataset(self, dataset):
        url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset.identifier)
        return requests.post(url, data=dataset.as_json(), headers=self.headers, verify=False)

    def delete_dataset(self, dataset):
        url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset.identifier)
        return requests.delete(url, headers=self.headers, verify=False)

    def associate_dataset_with_image(self, dataset_id, image_id, activity_id=None):
        url = '{b}/image/prov/{img}'.format(b=self.base_url, img=image_id)

        data = {
            'parent_uri': '/dataset/' + dataset_id,
            'parent_rel': 'prov:wasDerivedFrom'
        }
        if activity_id:
            data['activity'] = activity_id

        try:
            self.delete_dataset_image_assoc(dataset_id, image_id)
        except AssociationException as e:
            print e.value

        resp = requests.post(url, data=json.dumps(data), headers=self.headers, verify=False)

        if resp.status_code == 200:
            return resp
        else:
            raise Exception('Dataset association failed:\n{url}\n{resp}'.format(url=url, resp=resp.text))

    def delete_dataset_image_assoc(self, dataset_id, image_id):
        url = '{b}/image/prov/{img}'.format(b=self.base_url, img=image_id)

        data = {
            'delete': {
                'parent_uri': '/dataset/' + dataset_id,
                'parent_rel': 'prov:wasDerivedFrom'
            }
        }
        resp = requests.post(url, data=json.dumps(data), headers=self.headers, verify=False)

        if resp.status_code == 200:
            return resp
        else:
            raise AssociationException(
                'Dataset dissociation failed:\n{url}\n{resp}\n{d}'.format(url=url, resp=resp.text, d=data))

    def create_or_update_dataset(self, dataset):
        if self.dataset_exists(dataset.identifier):
            print 'Updating dataset: ' + dataset.identifier
            print self.update_dataset(dataset)
        else:
            print 'Creating dataset: ' + dataset.identifier
            print self.create_dataset(dataset)

    # @exists
    def activity_exists(self, activity_id):
        url = '{b}/activity/{act}'.format(b=self.base_url, act=activity_id)
        resp = requests.head(url, headers=self.headers, verify=False)
        if resp.status_code == 200:
            return True
        else:
            return False

    def get_activity(self, activity_id):
        url = '{b}/activity/{act}'.format(b=self.base_url, act=activity_id)
        resp = requests.get(url, headers=self.headers, verify=False)
        try:
            return Activity(resp.json())
        except ValueError:
            raise Exception(resp.text())

    @http_resp
    def create_activity(self, activity):
        url = '{b}/activity/'.format(b=self.base_url)
        return requests.post(url, data=activity.as_json(), headers=self.headers, verify=False)

    @http_resp
    def update_activity(self, activity):
        url = '{b}/activity/{act}'.format(b=self.base_url, act=activity.identifier)
        return requests.post(url, data=activity.as_json(), headers=self.headers, verify=False)

    @http_resp
    def delete_activity(self, activity):
        url = '{b}/activity/{act}'.format(b=self.base_url, act=activity.identifier)
        return requests.delete(url, headers=self.headers, verify=False)

    def create_or_update_activity(self, activity):
        if self.activity_exists(activity.identifier):
            self.update_activity(activity)
        else:
            self.create_activity(activity)
<commit_msg>Got rid of extraneous print statements<commit_after>from base64 import b64encode
import urllib
import json
from os.path import exists, basename

import requests
from requests.exceptions import ConnectionError

from domain import Figure, Image, Dataset, Activity

def check_image(fn):
    def wrapped(*args, **kwargs):
        # if len(args) < 1 or not isinstance(args[0], Image):
        #     raise Exception('Invalid Image')
        if args[1].identifier in (None, ''):
            raise Exception('Invalid identifier', args[0].identifier)
        return fn(*args, **kwargs)

    return wrapped

def exists(fn):
    def wrapped(*args, **kwargs):
        resp = fn(*args, **kwargs)
        if resp.status_code == 200:
            return True
        elif resp.status_code == 404:
            return False
        else:
            raise Exception(resp.text)
    return wrapped

def http_resp(fn):
    def wrapped(*args, **kwargs):
        resp = fn(*args, **kwargs)
        if resp.status_code == 200:
            return resp
        else:
            raise Exception(resp.text)
    return wrapped

class AssociationException(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class GcisClient(object):
    def __init__(self, url, username, password):
        self.headers = {
            'Accept': 'application/json'
        }

        self.base_url = url
        self.headers['Authorization'] = 'Basic ' + b64encode(username + ':' + password)

    @http_resp
    def create_figure(self, report_id, chapter_id, figure, skip_images=False):
        if figure.identifier in (None, ''):
            raise Exception('Invalid figure identifier', figure.identifier)

        #Is GCIS not inferring this from the url parameter?
        if figure.chapter_identifier in (None, ''):
            figure.chapter_identifier = chapter_id

        url = '{b}/report/{rpt}/chapter/{chp}/figure/'.format(
            b=self.base_url, rpt=report_id, chp=chapter_id
        )

        resp = requests.post(url, data=figure.as_json(), headers=self.headers, verify=False)

        if skip_images is False:
            for image in figure.images:
                self.create_image(image),
                self.associate_image_with_figure(image.identifier, report_id, figure.identifier)

        return resp

    @http_resp
    def update_figure(self, report_id, chapter_id, figure, skip_images=False):
        if figure.identifier in (None, ''):
            raise Exception('Invalid identifier', figure.identifier)

        #Is GCIS not inferring this from the url parameter?
        if figure.chapter_identifier in (None, ''):
            figure.chapter_identifier = chapter_id

        update_url = '{b}/report/{rpt}/chapter/{chp}/figure/{fig}'.format(
            b=self.base_url, rpt=report_id, chp=chapter_id, fig=figure.identifier
        )

        resp = requests.post(update_url, data=figure.as_json(), headers=self.headers, verify=False)

        if skip_images is False:
            for image in figure.images:
                self.update_image(image)

        return resp

    @http_resp
    def delete_figure(self, report_id, figure_id):
        url = '{b}/report/{rpt}/figure/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)
        return requests.delete(url, headers=self.headers, verify=False)

    @check_image
    def create_image(self, image, report_id=None, figure_id=None):
        url = '{b}/image/'.format(b=self.base_url)
        resp = requests.post(url, data=image.as_json(), headers=self.headers, verify=False)

        if image.local_path is not None:
            self.upload_image_file(image.identifier, image.local_path)
        if figure_id and report_id:
            self.associate_image_with_figure(image.identifier, report_id, figure_id)
        for dataset in image.datasets:
            self.create_or_update_dataset(dataset)
            self.create_or_update_activity(dataset.activity)
            self.associate_dataset_with_image(dataset.identifier, image.identifier,
                                              activity_id=dataset.activity.identifier)
        return resp

    @check_image
    def update_image(self, image):
        update_url = '{b}/image/{img}'.format(b=self.base_url, img=image.identifier)
        for dataset in image.datasets:
            self.update_activity(dataset.activity)
            self.associate_dataset_with_image(dataset.identifier, image.identifier,
                                              activity_id=dataset.activity.identifier)

        return requests.post(update_url, data=image.as_json(), headers=self.headers, verify=False)

    @check_image
    def delete_image(self, image):
        delete_url = '{b}/image/{img}'.format(b=self.base_url, img=image.identifier)
        return requests.delete(delete_url, headers=self.headers, verify=False)

    def associate_image_with_figure(self, image_id, report_id, figure_id):
        url = '{b}/report/{rpt}/figure/rel/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)
        return requests.post(url, data=json.dumps({'add_image_identifier': image_id}), headers=self.headers, verify=False)

    def upload_image_file(self, image_id, local_path):
        url = '{b}/image/files/{id}/{fn}'.format(b=self.base_url, id=image_id, fn=basename(local_path))
        # For future multi-part encoding support
        # return requests.put(url, headers=headers, files={'file': (filename, open(filepath, 'rb'))})
        if not exists(local_path):
            raise Exception('File not found: ' + local_path)

        return requests.put(url, data=open(local_path, 'rb'), headers=self.headers, verify=False)

    #Full listing
    def get_figure_listing(self, report_id, chapter_id=None):
        chapter_filter = '/chapter/' + chapter_id if chapter_id else ''

        url = '{b}/report/{rpt}{chap}/figure?{p}'.format(
            b=self.base_url, rpt=report_id, chap=chapter_filter, p=urllib.urlencode({'all': '1'})
        )
        resp = requests.get(url, headers=self.headers, verify=False)

        try:
            return [Figure(figure) for figure in resp.json()]
        except ValueError:
            raise Exception('Add a better exception string here')

    def get_figure(self, report_id, figure_id, chapter_id=None):
        chapter_filter = '/chapter/' + chapter_id if chapter_id else ''

        url = '{b}/report/{rpt}{chap}/figure/{fig}?{p}'.format(
            b=self.base_url, rpt=report_id, chap=chapter_filter, fig=figure_id, p=urllib.urlencode({'all': '1'})
        )
        resp = requests.get(url, headers=self.headers, verify=False)

        try:
            return Figure(resp.json())
        except ValueError:
            raise Exception(resp.text)

    @exists
    def figure_exists(self, report_id, figure_id, chapter_id=None):
        chapter_filter = '/chapter/' + chapter_id if chapter_id else ''

        url = '{b}/report/{rpt}{chap}/figure/{fig}?{p}'.format(
            b=self.base_url, rpt=report_id, chap=chapter_filter, fig=figure_id, p=urllib.urlencode({'all': '1'})
        )
        return requests.head(url, headers=self.headers, verify=False)

    def get_image(self, image_id):
        url = '{b}/image/{img}'.format(b=self.base_url, img=image_id)
        resp = requests.get(url, headers=self.headers, verify=False)

        try:
            return Image(resp.json())
        except ValueError:
            raise Exception(resp.text)

    @exists
    def image_exists(self, image_id):
        url = '{b}/image/{img}'.format(b=self.base_url, img=image_id)
        return requests.head(url, headers=self.headers, verify=False)

    def has_all_associated_images(self, report_id, figure_id, target_image_ids):
        try:
            figure_image_ids = [i.identifier for i in self.get_figure(report_id, figure_id).images]
        except Exception, e:
            print e.message
            return False, set()

        target_set = set(target_image_ids)
        gcis_set = set(figure_image_ids)
        deltas = target_set - gcis_set

        if target_set.issubset(gcis_set):
            return True, deltas
        else:
            return False, deltas

    def test_login(self):
        url = '{b}/login.json'.format(b=self.base_url)
        resp = requests.get(url, headers=self.headers, verify=False)
        return resp.status_code, resp.text

    def get_keyword_listing(self):
        url = '{b}/gcmd_keyword?{p}'.format(b=self.base_url, p=urllib.urlencode({'all': '1'}))
        resp = requests.get(url, headers=self.headers, verify=False)

        return resp.json()

    def get_keyword(self, key_id):
        url = '{b}/gcmd_keyword/{k}'.format(b=self.base_url, k=key_id)
        return requests.get(url, headers=self.headers, verify=False).json()

    def associate_keyword_with_figure(self, keyword_id, report_id, figure_id):
        url = '{b}/report/{rpt}/figure/keywords/{fig}'.format(b=self.base_url, rpt=report_id, fig=figure_id)
        return requests.post(url, data=json.dumps({'identifier': keyword_id}), headers=self.headers, verify=False)

    def get_dataset(self, dataset_id):
        url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset_id)
        resp = requests.get(url, headers=self.headers, verify=False)
        try:
            return Dataset(resp.json())
        except ValueError:
            raise Exception(resp.text())

    @exists
    def dataset_exists(self, dataset_id):
        url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset_id)
        return requests.head(url, headers=self.headers, verify=False)

    def create_dataset(self, dataset):
        url = '{b}/dataset/'.format(b=self.base_url)
        return requests.post(url, data=dataset.as_json(), headers=self.headers, verify=False)

    def update_dataset(self, dataset):
        url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset.identifier)
        return requests.post(url, data=dataset.as_json(), headers=self.headers, verify=False)

    def delete_dataset(self, dataset):
        url = '{b}/dataset/{ds}'.format(b=self.base_url, ds=dataset.identifier)
        return requests.delete(url, headers=self.headers, verify=False)

    def associate_dataset_with_image(self, dataset_id, image_id, activity_id=None):
        url = '{b}/image/prov/{img}'.format(b=self.base_url, img=image_id)

        data = {
            'parent_uri': '/dataset/' + dataset_id,
            'parent_rel': 'prov:wasDerivedFrom'
        }
        if activity_id:
            data['activity'] = activity_id

        try:
            self.delete_dataset_image_assoc(dataset_id, image_id)
        except AssociationException as e:
            print e.value

        resp = requests.post(url, data=json.dumps(data), headers=self.headers, verify=False)

        if resp.status_code == 200:
            return resp
        else:
            raise Exception('Dataset association failed:\n{url}\n{resp}'.format(url=url, resp=resp.text))

    def delete_dataset_image_assoc(self, dataset_id, image_id):
        url = '{b}/image/prov/{img}'.format(b=self.base_url, img=image_id)

        data = {
            'delete': {
                'parent_uri': '/dataset/' + dataset_id,
                'parent_rel': 'prov:wasDerivedFrom'
            }
        }
        resp = requests.post(url, data=json.dumps(data), headers=self.headers, verify=False)

        if resp.status_code == 200:
            return resp
        else:
            raise AssociationException(
                'Dataset dissociation failed:\n{url}\n{resp}\n{d}'.format(url=url, resp=resp.text, d=data))

    def create_or_update_dataset(self, dataset):
        if self.dataset_exists(dataset.identifier):
            print 'Updating dataset: ' + dataset.identifier
            self.update_dataset(dataset)
        else:
            print 'Creating dataset: ' + dataset.identifier
            self.create_dataset(dataset)

    # @exists
    def activity_exists(self, activity_id):
        url = '{b}/activity/{act}'.format(b=self.base_url, act=activity_id)
        resp = requests.head(url, headers=self.headers, verify=False)
        if resp.status_code == 200:
            return True
        else:
            return False

    def get_activity(self, activity_id):
        url = '{b}/activity/{act}'.format(b=self.base_url, act=activity_id)
        resp = requests.get(url, headers=self.headers, verify=False)
        try:
            return Activity(resp.json())
        except ValueError:
            raise Exception(resp.text())

    @http_resp
    def create_activity(self, activity):
        url = '{b}/activity/'.format(b=self.base_url)
        return requests.post(url, data=activity.as_json(), headers=self.headers, verify=False)

    @http_resp
    def update_activity(self, activity):
        url = '{b}/activity/{act}'.format(b=self.base_url, act=activity.identifier)
        return requests.post(url, data=activity.as_json(), headers=self.headers, verify=False)

    @http_resp
    def delete_activity(self, activity):
        url = '{b}/activity/{act}'.format(b=self.base_url, act=activity.identifier)
        return requests.delete(url, headers=self.headers, verify=False)

    def create_or_update_activity(self, activity):
        if self.activity_exists(activity.identifier):
            self.update_activity(activity)
        else:
            self.create_activity(activity)
<|endoftext|>


<commit_before>import subprocess

import pytest

from . import StorageTests
from vdirsyncer.storage.filesystem import FilesystemStorage
from vdirsyncer.vobject import Item

class TestFilesystemStorage(StorageTests):
    storage_class = FilesystemStorage

    @pytest.fixture
    def get_storage_args(self, tmpdir):
        def inner(collection="test"):
            rv = {"path": str(tmpdir), "fileext": ".txt", "collection": collection}
            if collection is not None:
                rv = self.storage_class.create_collection(**rv)
            return rv

        return inner

    def test_is_not_directory(self, tmpdir):
        with pytest.raises(OSError):
            f = tmpdir.join("hue")
            f.write("stub")
            self.storage_class(str(tmpdir) + "/hue", ".txt")

    def test_broken_data(self, tmpdir):
        s = self.storage_class(str(tmpdir), ".txt")

        class BrokenItem:
            raw = "Ц, Ш, Л, ж, Д, З, Ю".encode()
            uid = "jeezus"
            ident = uid

        with pytest.raises(TypeError):
            s.upload(BrokenItem)
        assert not tmpdir.listdir()

    def test_ident_with_slash(self, tmpdir):
        s = self.storage_class(str(tmpdir), ".txt")
        s.upload(Item("UID:a/b/c"))
        (item_file,) = tmpdir.listdir()
        assert "/" not in item_file.basename and item_file.isfile()

    def test_too_long_uid(self, tmpdir):
        s = self.storage_class(str(tmpdir), ".txt")
        item = Item("UID:" + "hue" * 600)
        href, etag = s.upload(item)
        assert item.uid not in href

    def test_post_hook_inactive(self, tmpdir, monkeypatch):
        def check_call_mock(*args, **kwargs):
            raise AssertionError()

        monkeypatch.setattr(subprocess, "call", check_call_mock)

        s = self.storage_class(str(tmpdir), ".txt", post_hook=None)
        s.upload(Item("UID:a/b/c"))

    def test_post_hook_active(self, tmpdir, monkeypatch):

        calls = []
        exe = "foo"

        def check_call_mock(call, *args, **kwargs):
            calls.append(True)
            assert len(call) == 2
            assert call[0] == exe

        monkeypatch.setattr(subprocess, "call", check_call_mock)

        s = self.storage_class(str(tmpdir), ".txt", post_hook=exe)
        s.upload(Item("UID:a/b/c"))
        assert calls

    def test_ignore_git_dirs(self, tmpdir):
        tmpdir.mkdir(".git").mkdir("foo")
        tmpdir.mkdir("a")
        tmpdir.mkdir("b")
        assert {c["collection"] for c in self.storage_class.discover(str(tmpdir))} == {
            "a",
            "b",
        }
<commit_msg>Add tests to filesystem storage for file ignorance<commit_after>import subprocess

import pytest

from . import StorageTests
from vdirsyncer.storage.filesystem import FilesystemStorage
from vdirsyncer.vobject import Item

class TestFilesystemStorage(StorageTests):
    storage_class = FilesystemStorage

    @pytest.fixture
    def get_storage_args(self, tmpdir):
        def inner(collection="test"):
            rv = {"path": str(tmpdir), "fileext": ".txt", "collection": collection}
            if collection is not None:
                rv = self.storage_class.create_collection(**rv)
            return rv

        return inner

    def test_is_not_directory(self, tmpdir):
        with pytest.raises(OSError):
            f = tmpdir.join("hue")
            f.write("stub")
            self.storage_class(str(tmpdir) + "/hue", ".txt")

    def test_broken_data(self, tmpdir):
        s = self.storage_class(str(tmpdir), ".txt")

        class BrokenItem:
            raw = "Ц, Ш, Л, ж, Д, З, Ю".encode()
            uid = "jeezus"
            ident = uid

        with pytest.raises(TypeError):
            s.upload(BrokenItem)
        assert not tmpdir.listdir()

    def test_ident_with_slash(self, tmpdir):
        s = self.storage_class(str(tmpdir), ".txt")
        s.upload(Item("UID:a/b/c"))
        (item_file,) = tmpdir.listdir()
        assert "/" not in item_file.basename and item_file.isfile()

    def test_ignore_tmp_files(self, tmpdir):
        """Test that files with .tmp suffix beside .ics files are ignored."""
        s = self.storage_class(str(tmpdir), '.ics')
        s.upload(Item('UID:xyzxyz'))
        item_file, = tmpdir.listdir()
        item_file.copy(item_file.new(ext='tmp'))
        assert len(tmpdir.listdir()) == 2
        assert len(list(s.list())) == 1

    def test_ignore_tmp_files_empty_fileext(self, tmpdir):
        """Test that files with .tmp suffix are ignored with empty fileext."""
        s = self.storage_class(str(tmpdir), '')
        s.upload(Item('UID:xyzxyz'))
        item_file, = tmpdir.listdir()
        item_file.copy(item_file.new(ext='tmp'))
        assert len(tmpdir.listdir()) == 2
        # assert False, tmpdir.listdir() # enable to see the created filename
        assert len(list(s.list())) == 1

    def test_too_long_uid(self, tmpdir):
        s = self.storage_class(str(tmpdir), ".txt")
        item = Item("UID:" + "hue" * 600)
        href, etag = s.upload(item)
        assert item.uid not in href

    def test_post_hook_inactive(self, tmpdir, monkeypatch):
        def check_call_mock(*args, **kwargs):
            raise AssertionError()

        monkeypatch.setattr(subprocess, "call", check_call_mock)

        s = self.storage_class(str(tmpdir), ".txt", post_hook=None)
        s.upload(Item("UID:a/b/c"))

    def test_post_hook_active(self, tmpdir, monkeypatch):

        calls = []
        exe = "foo"

        def check_call_mock(call, *args, **kwargs):
            calls.append(True)
            assert len(call) == 2
            assert call[0] == exe

        monkeypatch.setattr(subprocess, "call", check_call_mock)

        s = self.storage_class(str(tmpdir), ".txt", post_hook=exe)
        s.upload(Item("UID:a/b/c"))
        assert calls

    def test_ignore_git_dirs(self, tmpdir):
        tmpdir.mkdir(".git").mkdir("foo")
        tmpdir.mkdir("a")
        tmpdir.mkdir("b")
        assert {c["collection"] for c in self.storage_class.discover(str(tmpdir))} == {
            "a",
            "b",
        }
<|endoftext|>


<commit_before>#!/usr/bin/python
import ConfigParser
import getopt
import sys
import ipaddress
import random
import tor
import nmap
import time
import pycurl
import io
__author__ = 'jbollin'

hostlist = []
num_targets = 0
num_hosts = 0
targets_scanned = 0
first_run = True
sleep_time = 10

config = ConfigParser.ConfigParser()
config.read('scanner.cfg')
source_port = int(config.get("Scanner", "source_port"))

# helper functions
def refine_targetlist(targets):
    """
    Extract host ips from network blocks and randomize host scan order
    """
    global num_hosts
    outputlist = []
    for target_line in targets:
        target_line = target_line.rstrip()
        if "/" in target_line:
            try:
                for address in (ipaddress.ip_network(unicode(target_line)).hosts()):
                    outputlist.append(str(address))
            except ValueError:
                sys.exit("Invalid address or netmask: " + target_line)
        else:
            outputlist.append(target_line)
    random.shuffle(outputlist)
    # Use only a certain number of random hosts if user specified -n
    if num_hosts > 0:
        outputlist = outputlist[0:num_hosts]
    return outputlist

def query(url):
    """
    Uses pycurl to fetch a site using the proxy on the SOCKS_PORT.
    """
    socks_port = 9050
    output = io.BytesIO()

    curl_query = pycurl.Curl()
    curl_query.setopt(pycurl.URL, url)
    curl_query.setopt(pycurl.PROXY, 'localhost')
    curl_query.setopt(pycurl.PROXYPORT, socks_port)
    curl_query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
    curl_query.setopt(pycurl.WRITEFUNCTION, output.write)

    try:
        curl_query.perform()
        return output.getvalue()
    except pycurl.error as exc:
        print "[*] Exiting check tor service to see if it is started...\n"
        sys.exit("Unable to reach %s (%s)" % (url, exc))

def printhelp():
    print'    usage: ./nmap-tor.py <options>'
    print'    options:'
    print'      -h, --help        Display this message'
    print'      -t, --target      Specify single IP address or network of target'
    print'      -f, --targetlist  Specify file of IP addresses and/or networks to use as target'
    print'      -p, --portlist    Specify file of ports to be used on target'
    print'      -s, --sleep       Specify time in seconds to sleep between Nmap requests (default:10)'
    print'      -n, --numhosts    Specify number of hosts to be scanned from the provided list'

# System arguments for input and output files
try:
    opts, args = getopt.getopt(sys.argv[1:], "hf:t:p:s:", ["help", "target=", "targetlist=",
                                                           "portlist=", "sleep=", "numhosts="])
except getopt.GetoptError:
    print printhelp()
    sys.exit(2)
if len(opts) == 0:
    printhelp()
    sys.exit("\nError: There were no arguments specified\n")
for opt, arg in opts:
    if opt in ("-h", "--help"):
        print printhelp()
        sys.exit(2)
    elif opt in ("-f", "--targetlist"):
        inputfile = arg
        try:
            with open(inputfile) as hostfile:
                hostlist = []
                for host in hostfile:
                    hostlist.append(host)
        except:
            sys.exit("Input file for hosts is not valid")
    elif opt in ("-p", "--portlist"):
        inputfile = arg
        try:
            with open(inputfile) as portfile:
                targetports = []
                for port in portfile:
                    targetports.append(port.strip('\r\n'))
                num_ports = len(targetports)
        except:
            sys.exit("Input file for ports is not valid")
    elif opt in ("-t", "--target"):
        try:
            arg_ucode = unicode(arg)
            target_subnet = ipaddress.IPv4Network(arg_ucode)
            iplist = list(ipaddress.ip_network(target_subnet).hosts())
            for ip in iplist:
                hostlist.append(str(ip.compressed))
        except ipaddress.AddressValueError:
            sys.exit('Invalid IP address')
        except ipaddress.NetmaskValueError:
            sys.exit('Invalid subnet mask')
    elif opt in ("-n", "--numhosts"):
        num_hosts = int(arg)
    elif opt in ("-s", "--sleep"):
        sleep_time = float(arg)

print "[+] Nmap-Tor-Scanner starting up...\n"
targetlist = refine_targetlist(hostlist)

for target in targetlist:
    for dest_port in targetports:
        if not first_run:
            print "[+] Sleeping for " + str(sleep_time) + " seconds..."
            time.sleep(sleep_time)
            tor.changeIP()
        else:
            first_run = False
        print(query("https://www.atagar.com/echo.php"))
        print "trying {0:s} on {1:s}".format(target, dest_port)
        nmap.print_scan(nmap.do_scan(target, '-sT -p ' + str(dest_port)))
        targets_scanned += 1
        print "\n[+] (" + str(targets_scanned) + "/" + str(len(targetlist)) + ") " + \
              str(round((targets_scanned/float(len(targetlist)))*100, 1)) + "% targets scanned"
<commit_msg>fixed the percentage broken issued<commit_after>#!/usr/bin/python
import ConfigParser
import getopt
import sys
import ipaddress
import random
import tor
import nmap
import time
import pycurl
import io
__author__ = 'jbollin'

hostlist = []
num_targets = 0
num_hosts = 0
total_targets_and_hosts = 0
targets_scanned = 0
first_run = True
sleep_time = 10

config = ConfigParser.ConfigParser()
config.read('scanner.cfg')
source_port = int(config.get("Scanner", "source_port"))

# helper functions
def refine_targetlist(targets):
    """
    Extract host ips from network blocks and randomize host scan order
    """
    global num_hosts
    outputlist = []
    for target_line in targets:
        target_line = target_line.rstrip()
        if "/" in target_line:
            try:
                for address in (ipaddress.ip_network(unicode(target_line)).hosts()):
                    outputlist.append(str(address))
            except ValueError:
                sys.exit("Invalid address or netmask: " + target_line)
        else:
            outputlist.append(target_line)
    random.shuffle(outputlist)
    # Use only a certain number of random hosts if user specified -n
    if num_hosts > 0:
        outputlist = outputlist[0:num_hosts]
    return outputlist

def query(url):
    """
    Uses pycurl to fetch a site using the proxy on the SOCKS_PORT.
    """
    socks_port = 9050
    output = io.BytesIO()

    curl_query = pycurl.Curl()
    curl_query.setopt(pycurl.URL, url)
    curl_query.setopt(pycurl.PROXY, 'localhost')
    curl_query.setopt(pycurl.PROXYPORT, socks_port)
    curl_query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
    curl_query.setopt(pycurl.WRITEFUNCTION, output.write)

    try:
        curl_query.perform()
        return output.getvalue()
    except pycurl.error as exc:
        print "[*] Exiting check tor service to see if it is started...\n"
        sys.exit("Unable to reach %s (%s)" % (url, exc))

def printhelp():
    print'    usage: ./nmap-tor.py <options>'
    print'    options:'
    print'      -h, --help        Display this message'
    print'      -t, --target      Specify single IP address or network of target'
    print'      -f, --targetlist  Specify file of IP addresses and/or networks to use as target'
    print'      -p, --portlist    Specify file of ports to be used on target'
    print'      -s, --sleep       Specify time in seconds to sleep between Nmap requests (default:10)'
    print'      -n, --numhosts    Specify number of hosts to be scanned from the provided list'

# System arguments for input and output files
try:
    opts, args = getopt.getopt(sys.argv[1:], "hf:t:p:s:", ["help", "target=", "targetlist=",
                                                           "portlist=", "sleep=", "numhosts="])
except getopt.GetoptError:
    print printhelp()
    sys.exit(2)
if len(opts) == 0:
    printhelp()
    sys.exit("\nError: There were no arguments specified\n")
for opt, arg in opts:
    if opt in ("-h", "--help"):
        print printhelp()
        sys.exit(2)
    elif opt in ("-f", "--targetlist"):
        inputfile = arg
        try:
            with open(inputfile) as hostfile:
                hostlist = []
                for host in hostfile:
                    hostlist.append(host)
        except:
            sys.exit("Input file for hosts is not valid")
    elif opt in ("-p", "--portlist"):
        inputfile = arg
        try:
            with open(inputfile) as portfile:
                targetports = []
                for port in portfile:
                    targetports.append(port.strip('\r\n'))
                num_ports = len(targetports)
        except:
            sys.exit("Input file for ports is not valid")
    elif opt in ("-t", "--target"):
        try:
            arg_ucode = unicode(arg)
            target_subnet = ipaddress.IPv4Network(arg_ucode)
            iplist = list(ipaddress.ip_network(target_subnet).hosts())
            for ip in iplist:
                hostlist.append(str(ip.compressed))
        except ipaddress.AddressValueError:
            sys.exit('Invalid IP address')
        except ipaddress.NetmaskValueError:
            sys.exit('Invalid subnet mask')
    elif opt in ("-n", "--numhosts"):
        num_hosts = int(arg)
    elif opt in ("-s", "--sleep"):
        sleep_time = float(arg)

print "[+] Nmap-Tor-Scanner starting up...\n"
targetlist = refine_targetlist(hostlist)
total_targets_and_hosts = len(targetlist) * len(targetports)

for target in targetlist:
    for dest_port in targetports:
        if not first_run:
            print "[+] Sleeping for " + str(sleep_time) + " seconds..."
            time.sleep(sleep_time)
            tor.changeIP()
        else:
            first_run = False
        print(query("https://www.atagar.com/echo.php"))
        print "trying {0:s} on {1:s}".format(target, dest_port)
        nmap.print_scan(nmap.do_scan(target, '-sT -p ' + str(dest_port)))
        targets_scanned += 1
        print "\n[+] (" + str(targets_scanned) + "/" + str(total_targets_and_hosts) + ") " + \
              str(round((targets_scanned/float(total_targets_and_hosts))*100, 1)) + "% targets scanned"
<|endoftext|>


<commit_before># Copyright (c) 2015, Ralf Jung <post@ralfj.de>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#==============================================================================
import sys, os, os.path, subprocess
import configparser, itertools, re
import hmac, hashlib
import email.mime.text, email.utils, smtplib

mail_sender = "null@localhost"
config_file = os.path.join(os.path.dirname(__file__), 'git-mirror.conf')

class GitCommand:
    def __getattr__(self, name):
        def call(*args, capture_stderr = False, check = True):
            '''If <capture_stderr>, return stderr merged with stdout. Otherwise, return stdout and forward stderr to our own.
               If <check> is true, throw an exception of the process fails with non-zero exit code. Otherwise, do not.
               In any case, return a pair of the captured output and the exit code.'''
            cmd = ["git", name.replace('_', '-')] + list(args)
            with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT if capture_stderr else sys.stderr) as p:
                (stdout, stderr) = p.communicate()
                assert stderr is None
                code = p.returncode
                if check and code:
                    raise Exception("Error running {}: Non-zero exit code".format(cmd))
            return (stdout.decode('utf-8').strip('\n'), code)
        return call

git = GitCommand()
git_nullsha = 40*"0"

def git_is_forced_update(oldsha, newsha):
    out, code = git.merge_base("--is-ancestor", oldsha, newsha, check = False) # "Check if the first <commit> is an ancestor of the second <commit>"
    assert not out
    assert code in (0, 1)
    return False if code == 0 else True # if oldsha is an ancestor of newsha, then this was a "good" (non-forced) update

def read_config(defSection = 'DEFAULT'):
    '''Reads a config file that may have options outside of any section.'''
    config = configparser.ConfigParser()
    with open(config_file) as file:
        stream = itertools.chain(("["+defSection+"]\n",), file)
        config.read_file(stream)
    return config

def send_mail(subject, text, recipients, sender, replyTo = None):
    assert isinstance(recipients, list)
    if not len(recipients): return # nothing to do
    # construct content
    msg = email.mime.text.MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8')
    msg['Subject'] = subject
    msg['Date'] = email.utils.formatdate(localtime=True)
    msg['From'] = sender
    msg['To'] = ', '.join(recipients)
    if replyTo is not None:
        msg['Reply-To'] = replyTo
    # put into envelope and send
    s = smtplib.SMTP('localhost')
    s.sendmail(sender, recipients, msg.as_string())
    s.quit()

class Repo:
    def __init__(self, name, conf):
        '''Creates a repository from a section of the git-mirror configuration file'''
        self.name = name
        self.local = conf['local']
        self.owner = conf['owner'] # email address to notify in case of problems
        self.hmac_secret = conf['hmac-secret'].encode('utf-8') if 'hmac-secret' in conf else None
        self.deploy_key = conf['deploy-key'] # the SSH ky used for authenticating against remote hosts
        self.mirrors = {} # maps mirrors to their URLs
        mirror_prefix = 'mirror-'
        for name in filter(lambda s: s.startswith(mirror_prefix), conf.keys()):
            mirror = name[len(mirror_prefix):]
            self.mirrors[mirror] = conf[name]

    def mail_owner(self, msg):
        global mail_sender
        send_mail("git-mirror {}".format(self.name), msg, recipients = [self.owner], sender = mail_sender)

    def compute_hmac(self, data):
        assert self.hmac_secret is not None
        h = hmac.new(self.hmac_secret, digestmod = hashlib.sha1)
        h.update(data)
        return h.hexdigest()

    def find_mirror_by_url(self, match_urls):
        for mirror, url in self.mirrors.items():
            if url in match_urls:
                return mirror
        return None

    def setup_env(self):
        '''Setup the environment to work with this repository'''
        os.chdir(self.local)
        ssh_set_ident = os.path.join(os.path.dirname(__file__), 'ssh-set-ident.sh')
        os.putenv('GIT_SSH', ssh_set_ident)
        ssh_ident = os.path.join(os.path.expanduser('~/.ssh'), self.deploy_key)
        os.putenv('GIT_MIRROR_SSH_IDENT', ssh_ident)

    def update_mirrors(self, ref, oldsha, newsha):
        '''Update the <ref> from <oldsha> to <newsha> on all mirrors. The update must already have happened locally.'''
        assert len(oldsha) == 40 and len(newsha) == 40, "These are not valid SHAs."
        source_mirror = os.getenv("GIT_MIRROR_SOURCE") # in case of a self-call via the hooks, we can skip one of the mirrors
        self.setup_env()
        # check for a forced update
        is_forced = newsha != git_nullsha and oldsha != git_nullsha and git_is_forced_update(oldsha, newsha)
        # tell all the mirrors
        for mirror in self.mirrors:
            if mirror == source_mirror:
                continue
            sys.stdout.write("Updating mirror {}\n".format(mirror)); sys.stdout.flush()
            # update this mirror
            if is_forced:
                # forcibly update ref remotely (someone already did a force push and hence accepted data loss)
                git.push('--force', self.mirrors[mirror], newsha+":"+ref)
            else:
                # nicely update ref remotely (this avoids data loss due to race conditions)
                git.push(self.mirrors[mirror], newsha+":"+ref)

    def update_ref_from_mirror(self, ref, oldsha, newsha, mirror, suppress_stderr = False):
        '''Update the local version of this <ref> to what's currently on the given <mirror>. <oldsha> and <newsha> are checked. Then update all the other mirrors.'''
        self.setup_env()
        url = self.mirrors[mirror]
        # first check whether the remote really is at newsha
        remote_state, code = git.ls_remote(url, ref)
        if remote_state:
            remote_sha = remote_state.split()[0]
        else:
            remote_sha = git_nullsha
        assert newsha == remote_sha, "Someone lied about the new SHA, which should be {}.".format(newsha)
        # locally, we have to be at oldsha or newsha (the latter can happen if we already got this update, e.g. if it originated from us)
        local_state, code = git.show_ref(ref, check=False)
        if code == 0:
            local_sha = local_state.split()[0]
        else:
            if len(local_state):
                raise Exception("Something went wrong getting the local state of {}.".format(ref))
            local_sha = git_nullsha
        # some sanity checking, but deal gracefully with new branches appearing
        assert local_sha in (git_nullsha, oldsha, newsha), "Someone lied about the old SHA: Local ({}) is neither old ({}) nor new ({})".format(local_sha, oldsha, newsha)
        # if we are already at newsha locally, we also ran the local hooks, so we do not have to do anything
        if local_sha == newsha:
            return "Local repository is already up-to-date."
        # update local state from local_sha to newsha.
        if newsha != git_nullsha:
            # We *could* now fetch the remote ref and immediately update the local one. However, then we would have to
            # decide whether we want to allow a force-update or not. Also, the ref could already have changed remotely,
            # so that may update to some other commit.
            # Instead, we just fetch without updating any local ref. If the remote side changed in such a way that
            # <newsha> is not actually fetched, that's a race and will be noticed when updating the local ref.
            git.fetch(url, ref, capture_stderr = suppress_stderr)
            # now update the ref, checking the old value is still local_oldsha.
            git.update_ref(ref, newsha, 40*"0" if local_sha is None else local_sha)
        else:
            # ref does not exist anymore. delete it.
            assert local_sha != git_nullsha, "Why didn't we bail out earlier if there is nothing to do...?"
            git.update_ref("-d", ref, local_sha) # this checks that the old value is still local_sha
        # Now run the post-receive hooks. This will *also* push the changes to all mirrors, as we
        # are one of these hooks!
        os.putenv("GIT_MIRROR_SOURCE", mirror) # tell ourselves which repo we do *not* have to update
        with subprocess.Popen(['/bin/sh', 'hooks/post-receive'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as p:
            (stdout, stderr) = p.communicate("{} {} {}\n".format(oldsha, newsha, ref).encode('utf-8'))
            stdout = stdout.decode('utf-8')
            if p.returncode:
                raise Exception("post-receive git hook terminated with non-zero exit code {}:\n{}".format(p.returncode, stdout))
        return stdout

def find_repo_by_directory(repos, dir):
    for (name, repo) in repos.items():
        if dir == repo.local:
            return name
    return None

def load_repos():
    global mail_sender
    conf = read_config()
    mail_sender = conf['DEFAULT']['mail-sender']

    repos = {}
    for name, section in conf.items():
        if name != 'DEFAULT':
            repos[name] = Repo(name, section)
    return repos

<commit_msg>the post-receive hook really should be executable; run it directly<commit_after># Copyright (c) 2015, Ralf Jung <post@ralfj.de>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#==============================================================================
import sys, os, os.path, subprocess
import configparser, itertools, re
import hmac, hashlib
import email.mime.text, email.utils, smtplib

mail_sender = "null@localhost"
config_file = os.path.join(os.path.dirname(__file__), 'git-mirror.conf')

class GitCommand:
    def __getattr__(self, name):
        def call(*args, capture_stderr = False, check = True):
            '''If <capture_stderr>, return stderr merged with stdout. Otherwise, return stdout and forward stderr to our own.
               If <check> is true, throw an exception of the process fails with non-zero exit code. Otherwise, do not.
               In any case, return a pair of the captured output and the exit code.'''
            cmd = ["git", name.replace('_', '-')] + list(args)
            with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT if capture_stderr else sys.stderr) as p:
                (stdout, stderr) = p.communicate()
                assert stderr is None
                code = p.returncode
                if check and code:
                    raise Exception("Error running {}: Non-zero exit code".format(cmd))
            return (stdout.decode('utf-8').strip('\n'), code)
        return call

git = GitCommand()
git_nullsha = 40*"0"

def git_is_forced_update(oldsha, newsha):
    out, code = git.merge_base("--is-ancestor", oldsha, newsha, check = False) # "Check if the first <commit> is an ancestor of the second <commit>"
    assert not out
    assert code in (0, 1)
    return False if code == 0 else True # if oldsha is an ancestor of newsha, then this was a "good" (non-forced) update

def read_config(defSection = 'DEFAULT'):
    '''Reads a config file that may have options outside of any section.'''
    config = configparser.ConfigParser()
    with open(config_file) as file:
        stream = itertools.chain(("["+defSection+"]\n",), file)
        config.read_file(stream)
    return config

def send_mail(subject, text, recipients, sender, replyTo = None):
    assert isinstance(recipients, list)
    if not len(recipients): return # nothing to do
    # construct content
    msg = email.mime.text.MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8')
    msg['Subject'] = subject
    msg['Date'] = email.utils.formatdate(localtime=True)
    msg['From'] = sender
    msg['To'] = ', '.join(recipients)
    if replyTo is not None:
        msg['Reply-To'] = replyTo
    # put into envelope and send
    s = smtplib.SMTP('localhost')
    s.sendmail(sender, recipients, msg.as_string())
    s.quit()

class Repo:
    def __init__(self, name, conf):
        '''Creates a repository from a section of the git-mirror configuration file'''
        self.name = name
        self.local = conf['local']
        self.owner = conf['owner'] # email address to notify in case of problems
        self.hmac_secret = conf['hmac-secret'].encode('utf-8') if 'hmac-secret' in conf else None
        self.deploy_key = conf['deploy-key'] # the SSH ky used for authenticating against remote hosts
        self.mirrors = {} # maps mirrors to their URLs
        mirror_prefix = 'mirror-'
        for name in filter(lambda s: s.startswith(mirror_prefix), conf.keys()):
            mirror = name[len(mirror_prefix):]
            self.mirrors[mirror] = conf[name]

    def mail_owner(self, msg):
        global mail_sender
        send_mail("git-mirror {}".format(self.name), msg, recipients = [self.owner], sender = mail_sender)

    def compute_hmac(self, data):
        assert self.hmac_secret is not None
        h = hmac.new(self.hmac_secret, digestmod = hashlib.sha1)
        h.update(data)
        return h.hexdigest()

    def find_mirror_by_url(self, match_urls):
        for mirror, url in self.mirrors.items():
            if url in match_urls:
                return mirror
        return None

    def setup_env(self):
        '''Setup the environment to work with this repository'''
        os.chdir(self.local)
        ssh_set_ident = os.path.join(os.path.dirname(__file__), 'ssh-set-ident.sh')
        os.putenv('GIT_SSH', ssh_set_ident)
        ssh_ident = os.path.join(os.path.expanduser('~/.ssh'), self.deploy_key)
        os.putenv('GIT_MIRROR_SSH_IDENT', ssh_ident)

    def update_mirrors(self, ref, oldsha, newsha):
        '''Update the <ref> from <oldsha> to <newsha> on all mirrors. The update must already have happened locally.'''
        assert len(oldsha) == 40 and len(newsha) == 40, "These are not valid SHAs."
        source_mirror = os.getenv("GIT_MIRROR_SOURCE") # in case of a self-call via the hooks, we can skip one of the mirrors
        self.setup_env()
        # check for a forced update
        is_forced = newsha != git_nullsha and oldsha != git_nullsha and git_is_forced_update(oldsha, newsha)
        # tell all the mirrors
        for mirror in self.mirrors:
            if mirror == source_mirror:
                continue
            sys.stdout.write("Updating mirror {}\n".format(mirror)); sys.stdout.flush()
            # update this mirror
            if is_forced:
                # forcibly update ref remotely (someone already did a force push and hence accepted data loss)
                git.push('--force', self.mirrors[mirror], newsha+":"+ref)
            else:
                # nicely update ref remotely (this avoids data loss due to race conditions)
                git.push(self.mirrors[mirror], newsha+":"+ref)

    def update_ref_from_mirror(self, ref, oldsha, newsha, mirror, suppress_stderr = False):
        '''Update the local version of this <ref> to what's currently on the given <mirror>. <oldsha> and <newsha> are checked. Then update all the other mirrors.'''
        self.setup_env()
        url = self.mirrors[mirror]
        # first check whether the remote really is at newsha
        remote_state, code = git.ls_remote(url, ref)
        if remote_state:
            remote_sha = remote_state.split()[0]
        else:
            remote_sha = git_nullsha
        assert newsha == remote_sha, "Someone lied about the new SHA, which should be {}.".format(newsha)
        # locally, we have to be at oldsha or newsha (the latter can happen if we already got this update, e.g. if it originated from us)
        local_state, code = git.show_ref(ref, check=False)
        if code == 0:
            local_sha = local_state.split()[0]
        else:
            if len(local_state):
                raise Exception("Something went wrong getting the local state of {}.".format(ref))
            local_sha = git_nullsha
        # some sanity checking, but deal gracefully with new branches appearing
        assert local_sha in (git_nullsha, oldsha, newsha), "Someone lied about the old SHA: Local ({}) is neither old ({}) nor new ({})".format(local_sha, oldsha, newsha)
        # if we are already at newsha locally, we also ran the local hooks, so we do not have to do anything
        if local_sha == newsha:
            return "Local repository is already up-to-date."
        # update local state from local_sha to newsha.
        if newsha != git_nullsha:
            # We *could* now fetch the remote ref and immediately update the local one. However, then we would have to
            # decide whether we want to allow a force-update or not. Also, the ref could already have changed remotely,
            # so that may update to some other commit.
            # Instead, we just fetch without updating any local ref. If the remote side changed in such a way that
            # <newsha> is not actually fetched, that's a race and will be noticed when updating the local ref.
            git.fetch(url, ref, capture_stderr = suppress_stderr)
            # now update the ref, checking the old value is still local_oldsha.
            git.update_ref(ref, newsha, 40*"0" if local_sha is None else local_sha)
        else:
            # ref does not exist anymore. delete it.
            assert local_sha != git_nullsha, "Why didn't we bail out earlier if there is nothing to do...?"
            git.update_ref("-d", ref, local_sha) # this checks that the old value is still local_sha
        # Now run the post-receive hooks. This will *also* push the changes to all mirrors, as we
        # are one of these hooks!
        os.putenv("GIT_MIRROR_SOURCE", mirror) # tell ourselves which repo we do *not* have to update
        with subprocess.Popen(['hooks/post-receive'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as p:
            (stdout, stderr) = p.communicate("{} {} {}\n".format(oldsha, newsha, ref).encode('utf-8'))
            stdout = stdout.decode('utf-8')
            if p.returncode:
                raise Exception("post-receive git hook terminated with non-zero exit code {}:\n{}".format(p.returncode, stdout))
        return stdout

def find_repo_by_directory(repos, dir):
    for (name, repo) in repos.items():
        if dir == repo.local:
            return name
    return None

def load_repos():
    global mail_sender
    conf = read_config()
    mail_sender = conf['DEFAULT']['mail-sender']

    repos = {}
    for name, section in conf.items():
        if name != 'DEFAULT':
            repos[name] = Repo(name, section)
    return repos

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from mock import Mock

from twisted.internet import defer

from synapse.api.constants import EventTypes, Membership
from synapse.api.room_versions import RoomVersions
from synapse.types import RoomID, UserID

from tests import unittest
from tests.utils import create_room, setup_test_homeserver

class RoomMemberStoreTestCase(unittest.TestCase):
    @defer.inlineCallbacks
    def setUp(self):
        hs = yield setup_test_homeserver(
            self.addCleanup, resource_for_federation=Mock(), http_client=None
        )
        # We can't test the RoomMemberStore on its own without the other event
        # storage logic
        self.store = hs.get_datastore()
        self.event_builder_factory = hs.get_event_builder_factory()
        self.event_creation_handler = hs.get_event_creation_handler()

        self.u_alice = UserID.from_string("@alice:test")
        self.u_bob = UserID.from_string("@bob:test")

        # User elsewhere on another host
        self.u_charlie = UserID.from_string("@charlie:elsewhere")

        self.room = RoomID.from_string("!abc123:test")

        yield create_room(hs, self.room.to_string(), self.u_alice.to_string())

    @defer.inlineCallbacks
    def inject_room_member(self, room, user, membership, replaces_state=None):
        builder = self.event_builder_factory.for_room_version(
            RoomVersions.V1,
            {
                "type": EventTypes.Member,
                "sender": user.to_string(),
                "state_key": user.to_string(),
                "room_id": room.to_string(),
                "content": {"membership": membership},
            },
        )

        event, context = yield self.event_creation_handler.create_new_client_event(
            builder
        )

        yield self.store.persist_event(event, context)

        defer.returnValue(event)

    @defer.inlineCallbacks
    def test_one_member(self):
        yield self.inject_room_member(self.room, self.u_alice, Membership.JOIN)

        self.assertEquals(
            [self.room.to_string()],
            [
                m.room_id
                for m in (
                    yield self.store.get_rooms_for_user_where_membership_is(
                        self.u_alice.to_string(), [Membership.JOIN]
                    )
                )
            ],
        )
<commit_msg>Add unit test for current state membership bg update<commit_after># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from mock import Mock

from twisted.internet import defer

from synapse.api.constants import EventTypes, Membership
from synapse.api.room_versions import RoomVersions
from synapse.types import Requester, RoomID, UserID

from tests import unittest
from tests.utils import create_room, setup_test_homeserver

class RoomMemberStoreTestCase(unittest.TestCase):
    @defer.inlineCallbacks
    def setUp(self):
        hs = yield setup_test_homeserver(
            self.addCleanup, resource_for_federation=Mock(), http_client=None
        )
        # We can't test the RoomMemberStore on its own without the other event
        # storage logic
        self.store = hs.get_datastore()
        self.event_builder_factory = hs.get_event_builder_factory()
        self.event_creation_handler = hs.get_event_creation_handler()

        self.u_alice = UserID.from_string("@alice:test")
        self.u_bob = UserID.from_string("@bob:test")

        # User elsewhere on another host
        self.u_charlie = UserID.from_string("@charlie:elsewhere")

        self.room = RoomID.from_string("!abc123:test")

        yield create_room(hs, self.room.to_string(), self.u_alice.to_string())

    @defer.inlineCallbacks
    def inject_room_member(self, room, user, membership, replaces_state=None):
        builder = self.event_builder_factory.for_room_version(
            RoomVersions.V1,
            {
                "type": EventTypes.Member,
                "sender": user.to_string(),
                "state_key": user.to_string(),
                "room_id": room.to_string(),
                "content": {"membership": membership},
            },
        )

        event, context = yield self.event_creation_handler.create_new_client_event(
            builder
        )

        yield self.store.persist_event(event, context)

        defer.returnValue(event)

    @defer.inlineCallbacks
    def test_one_member(self):
        yield self.inject_room_member(self.room, self.u_alice, Membership.JOIN)

        self.assertEquals(
            [self.room.to_string()],
            [
                m.room_id
                for m in (
                    yield self.store.get_rooms_for_user_where_membership_is(
                        self.u_alice.to_string(), [Membership.JOIN]
                    )
                )
            ],
        )

class CurrentStateMembershipUpdateTestCase(unittest.HomeserverTestCase):
    def prepare(self, reactor, clock, homeserver):
        self.store = homeserver.get_datastore()
        self.room_creator = homeserver.get_room_creation_handler()

    def test_can_rerun_update(self):
        # First make sure we have completed all updates.
        while not self.get_success(self.store.has_completed_background_updates()):
            self.get_success(self.store.do_next_background_update(100), by=0.1)

        # Now let's create a room, which will insert a membership
        user = UserID("alice", "test")
        requester = Requester(user, None, False, None, None)
        self.get_success(self.room_creator.create_room(requester, {}))

        # Register the background update to run again.
        self.get_success(
            self.store._simple_insert(
                table="background_updates",
                values={
                    "update_name": "current_state_events_membership",
                    "progress_json": "{}",
                    "depends_on": None,
                },
            )
        )

        # ... and tell the DataStore that it hasn't finished all updates yet
        self.store._all_done = False

        # Now let's actually drive the updates to completion
        while not self.get_success(self.store.has_completed_background_updates()):
            self.get_success(self.store.do_next_background_update(100), by=0.1)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf8

import time
#import threading
import psutil
import socket
from datetime import timedelta
import RPi.GPIO as GPIO
import screenmanager
from fonts import font_clean
from display import disp

def lowBat(channel):
    print "Low battery!"

GPIO.setmode(GPIO.BCM)
GPIO.setup(4, GPIO.IN) #, pull_up_down=GPIO.PUD_DOWN
GPIO.add_event_detect(4, GPIO.FALLING, callback=lowBat, bouncetime=200)
#

class Status:
    def run(self):
        while True:
            #disp.clear()
            #draw = disp.draw()
            uptime = str(timedelta(seconds = float(open('/proc/uptime', 'r').readline().split()[0]))).rsplit('.', 1)[0]
            cpu_usage = str(psutil.cpu_percent())
            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            s.connect(('192.168.178.1', 1027))
            ip = s.getsockname()[0]
            s.close()
            screenmanager.draw_text(disp.buffer, "Uptime: ", (0, 250), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, uptime, (0, 200), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, "CPU Load: ", (20, 250), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, cpu_usage + " %", (20, 200), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, "IP: ", (40, 250), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, ip, (40, 50), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, "BT: ", (60, 250), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, str(GPIO.input(4)), (60, 200), 90, font_clean, fill=(255,0,0))
            #disp.display()
            time.sleep(1)
<commit_msg>Debug<commit_after>#!/usr/bin/env python
# coding: utf8

import time
#import threading
import psutil
import socket
from datetime import timedelta
import RPi.GPIO as GPIO
import screenmanager
from fonts import font_clean
from display import disp

def lowBat(channel):
    print "Low battery!"

GPIO.setmode(GPIO.BCM)
GPIO.setup(4, GPIO.IN) #, pull_up_down=GPIO.PUD_DOWN
GPIO.add_event_detect(4, GPIO.FALLING, callback=lowBat, bouncetime=200)
#

class Status:
    def run(self):
        while True:
            disp.reset()
            disp.begin()
            #disp.clear()
            draw = disp.draw()
            uptime = str(timedelta(seconds = float(open('/proc/uptime', 'r').readline().split()[0]))).rsplit('.', 1)[0]
            cpu_usage = str(psutil.cpu_percent())
            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            s.connect(('192.168.178.1', 1027))
            ip = s.getsockname()[0]
            s.close()
            screenmanager.draw_text(disp.buffer, "Uptime: ", (0, 250), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, uptime, (0, 200), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, "CPU Load: ", (20, 250), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, cpu_usage + " %", (20, 200), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, "IP: ", (40, 250), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, ip, (40, 50), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, "BT: ", (60, 250), 90, font_clean, fill=(255,255,255))
            screenmanager.draw_text(disp.buffer, str(GPIO.input(4)), (60, 200), 90, font_clean, fill=(255,0,0))
            disp.display()
            time.sleep(1)
<|endoftext|>


<commit_before>from app import db

class Message(db.Model):
    '''
    Read/Write the content of the message sent/received (type) from system.
    '''
    __tablename__ = 'messages'

    # Initially, I was also going to store an ID for each conversation, to allow
    # analysis of each conversation. That is outside the current project scope.
    id = db.Column(db.Integer, nullable=False, unique=True, primary_key=True)
    status = db.Column(db.String, nullable=False)
    content = db.Column(db.String, nullable=False)

    def __init__(self, _status, _content):
        self.status = _status  # Used to differentiate between sent/received.
        self.content = _content

    def __repr__(self):
        return 'Message & type: %r, -> %r.' % (self.content, self.status)

class Question(db.Model):
    '''
    Read/Write the question (OEQ, RS) sent to user, including modifying rating.
    '''
    __tablename__ = 'questions'

    id = db.Column(db.Integer, nullable=False, unique=True, primary_key=True)
    question = db.Column(db.String, nullable=False)
    rating = db.Column(db.Integer, nullable=False)

    def __init__(self, _question, _rating):
        self.question = _question
        self.rating = _rating

    def __repr__(self):
        return 'Question & rating: %r -> %r' % (self.question, self.rating)
<commit_msg>Added concept to questions table<commit_after>from app import db

class Message(db.Model):
    '''
    Read/Write the content of the message sent/received (type) from system.
    '''
    __tablename__ = 'messages'

    # Initially, I was also going to store an ID for each conversation, to allow
    # analysis of each conversation. That is outside the current project scope.
    id = db.Column(db.Integer, nullable=False, unique=True, primary_key=True)
    status = db.Column(db.String, nullable=False)
    content = db.Column(db.String, nullable=False)

    def __init__(self, _status, _content):
        self.status = _status  # Used to differentiate between sent/received.
        self.content = _content

    def __repr__(self):
        return 'Message & type: %r, -> %r.' % (self.content, self.status)

class Question(db.Model):
    '''
    Read/Write the question (OEQ, RS) sent to user, including modifying rating.
    '''
    __tablename__ = 'questions'

    id = db.Column(db.Integer, nullable=False, unique=True, primary_key=True)
    question = db.Column(db.String, nullable=False)
    concept = db.Column(db.String, nullable=False)
    rating = db.Column(db.Integer, nullable=False)

    def __init__(self, _question, _concept, _rating):
        self.question = _question
        self.concept = _concept
        self.rating = _rating

    def __repr__(self):
        return 'Concept: question -> rating: %r: %r : %r' \
            % (self.concept, self.question, self.rating)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import unittest

def find_profit(prices, window):
    """Given a certain window size and a list of prices, find the highest
    profit possible if exactly one share is bought then sold within that
    perid. Returns this profit."""

    # back_prices keeps track of previous prices
    # this is a copy so we don't have to access prices directly
    back_prices = []

    # pivot is the lowest price in the window
    pivot = None

    # next_pivot is the lowest price in the window after pivot
    # this is where pivot is moved if it falls out of the window
    next_pivot = None

    # accumulated maximum profit
    profit = 0

    # this is the only direct access of prices, and only assumes that an
    # __iter__ function is available
    for i, price in enumerate(prices):
        # add the current price to back_prices
        back_prices.append(price)

        # trim the back prices list to only be the window length
        while len(back_prices) > window + 1:
            back_prices.pop(0)

        # test to see if we've found a lower pivot
        if pivot is None or price < back_prices[pivot - i - 1]:
            # set the pivot and the pivot price
            pivot = i
            # bump the next_pivot if we've passed it
            next_pivot = max(next_pivot, pivot + 1)

        # test to see if we've found a lower next_pivot
        if next_pivot is None or (next_pivot <= i and price <
                                  back_prices[next_pivot - i - 1]):
            # set it and the next_price
            next_pivot = i

        # test to see if the pivot has fallen out of the window
        if i - pivot == window:
            # move the pivot to the next position
            pivot = next_pivot
            # set the next_pivot to one after the new pivot
            next_pivot = pivot + 1

        # update the profit accumulator
        profit = max(profit, price - back_prices[pivot - i - 1])

    # return the accumulated profit once done
    return profit

class StockProfitTests(unittest.TestCase):
    def testIncrease(self):
        self.assertEqual(find_profit([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], 2), 1.0)
        self.assertEqual(find_profit([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], 3), 2.0)
        self.assertEqual(find_profit([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], 4), 3.0)

    def testSimple(self):
        self.assertEqual(find_profit([1.0, 2.0, 3.0, 1.0, 3.0, 4.0], 5), 3.0)

        self.assertEqual(find_profit([7.0, 5.0, 6.0, 4.0, 5.0, 3.0,
                                      4.0, 2.0, 3.0, 1.0], 5), 1.0)

        self.assertEqual(find_profit([4.0, 3.0, 2.0, 4.0, 3.0, 1.0,
                                      1.1, 1.2, 1.3, 1.4], 5), 2.0)

if __name__ == "__main__":
    unittest.main()
<commit_msg>Add some more tests, appease pylint a bit<commit_after>#!/usr/bin/env python
"""find_profit is O(n) over a list, given a window, to find the maximum profit
possible given a single pair of trades taking place in that window"""
import unittest

def find_profit(prices, window):
    """Given a certain window size and a list of prices, find the highest
    profit possible if exactly one share is bought then sold within that
    perid. Returns this profit."""

    # back_prices keeps track of previous prices
    # this is a copy so we don't have to access prices directly
    back_prices = []

    # pivot is the lowest price in the window
    pivot = None

    # next_pivot is the lowest price in the window after pivot
    # this is where pivot is moved if it falls out of the window
    next_pivot = None

    # accumulated maximum profit
    profit = 0

    # this is the only direct access of prices, and only assumes that an
    # __iter__ function is available
    for i, price in enumerate(prices):
        # add the current price to back_prices
        back_prices.append(price)

        # trim the back prices list to only be the window length
        while len(back_prices) > window + 1:
            back_prices.pop(0)

        # test to see if we've found a lower pivot
        if pivot is None or price < back_prices[pivot - i - 1]:
            # set the pivot and the pivot price
            pivot = i
            # bump the next_pivot if we've passed it
            next_pivot = max(next_pivot, pivot + 1)

        # test to see if we've found a lower next_pivot
        if next_pivot is None or (next_pivot <= i and price <
                                  back_prices[next_pivot - i - 1]):
            # set it and the next_price
            next_pivot = i

        # test to see if the pivot has fallen out of the window
        if i - pivot == window:
            # move the pivot to the next position
            pivot = next_pivot
            # set the next_pivot to one after the new pivot
            next_pivot = pivot + 1

        # update the profit accumulator
        profit = max(profit, price - back_prices[pivot - i - 1])

    # return the accumulated profit once done
    return profit

# pylint: disable=R0904
class StockProfitTests(unittest.TestCase):
    """Unit tests for the find_profit function"""

    def test_increase(self):
        """Test an increasing window size with a simple list"""
        self.assertEqual(find_profit([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], 2), 1.0)
        self.assertEqual(find_profit([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], 3), 2.0)
        self.assertEqual(find_profit([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], 4), 3.0)

    def test_window_sizes(self):
        """Test various difficult lists with window sizes larger than the best
        size possible"""
        self.assertEqual(find_profit([1.0, 2.0, 3.0, 1.0, 3.0, 4.0], 5), 3.0)

        self.assertEqual(find_profit([7.0, 5.0, 6.0, 4.0, 5.0, 3.0,
                                      4.0, 2.0, 3.0, 1.0], 5), 1.0)

        self.assertEqual(find_profit([4.0, 3.0, 2.0, 4.0, 3.0, 1.0,
                                      1.1, 1.2, 1.3, 1.4], 5), 2.0)

    def test_shifting(self):
        """Test a growing window, where each increase makes for a different
        profit"""
        self.assertEqual(find_profit([2.0, 3.0, 1.0, 2.0, 4.0, 5.0, 7.0,
                                      8.0], 2), 2.0)
        self.assertEqual(find_profit([2.0, 3.0, 1.0, 2.0, 4.0, 5.0, 7.0,
                                      8.0], 3), 3.0)
        self.assertEqual(find_profit([2.0, 3.0, 1.0, 2.0, 4.0, 5.0, 7.0,
                                      8.0], 4), 5.0)
        self.assertEqual(find_profit([2.0, 3.0, 1.0, 2.0, 4.0, 5.0, 7.0,
                                      8.0], 5), 6.0)
        self.assertEqual(find_profit([2.0, 3.0, 1.0, 2.0, 4.0, 5.0, 7.0,
                                      8.0], 6), 7.0)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>import app
from app import db
from datetime import datetime, date

class Message(db.Model):
    __tablename__ = "messages"
    id = db.Column(db.Integer, primary_key=True)
    sent_timestamp = db.Column(db.DateTime, default=datetime.utcnow(), index=True)
    to_user_id = db.Column(db.Integer, nullable=False)
    from_user_id = db.Column(db.Integer, nullable=False)
    title = db.Column(db.String, index=True)
    body = db.Column(db.String, nullable=False, index=True)
    is_read = db.Column(db.Boolean, default=False)
    __table_args__ = (
        db.CheckConstraint("to_user_id != from_user_id"),
    )

    def from_user(self):
        user = User.query.filter_by(id=self.from_user_id).first()
        if user is not None:
            return user
        else:
            return None

    def to_user(self):
        user = User.query.filter_by(id=self.to_user_id).first()
        if user is not None:
            return user
        else:
            return None

    def __repr__(self):
        return "<Message: From: %d, to: %d, title: %r>" % (self.from_user_id, self.to_user_id, self.title)

class Task(db.Model):
    __tablename__ = "task"
    id = db.Column(db.Integer, primary_key=True)
    created = db.Column(db.DateTime, default=datetime.utcnow(), index=True)
    end_timestamp = db.Column(db.DateTime, default=datetime.utcnow(), index=True)
    description = db.Column(db.String, nullable=False, index=True)
    requester_id = db.Column(db.Integer, nullable=False)
    doer_id = db.Column(db.Integer, nullable=False)
    is_active = db.Column(db.Boolean, default=True)
    __table_args__ = (
        db.CheckConstraint("requester_id != doer_id"),
    )

    def requester(self):
        return db.session.query(User, Task).filter(self.requester_id == User.id).first()

    def doer(self):
        return db.session.query(User, Task).filter(self.doer_id == User.id).first()

    def __repr__(self):
        return "<Task: Requester: %d, doer: %d, '%r'>" % (self.requester_id, self.doer_id, self.description)

class User(db.Model):
    __tablename__ = "users"
    id = db.Column(db.Integer, primary_key=True)
    username = db.Column(db.String, index=True, unique=True)
    pw_hash = db.Column(db.String, index=True)
    age = db.Column(db.Integer)
    gender = db.Column(db.String(24))
    role = db.Column(db.String(24))
    orientation = db.Column(db.String(24))
    bio = db.Column(db.String)
    email = db.Column(db.String, nullable=False, unique=True)
    created = db.Column(db.DateTime, default=datetime.utcnow())
    user_role = db.Column(db.Integer, default=0)
    is_validated = db.Column(db.Boolean, default=False)
    password_reset_token = db.Column(db.String)
    password_reset_token_expiration = db.Column(db.DateTime)
    __table_args__ = (
        db.UniqueConstraint('username'),
        db.UniqueConstraint('email'),
        db.CheckConstraint('age is null or (age >= 18 and age <= 100)')
    )

    def get_all_inbox_messages(self):
        return Message.query.filter_by(to_user_id=self.id).order_by(Message.sent_timestamp.desc()).all()

    def get_all_outbox_messages(self):
        return Message.query.filter_by(from_user_id=self.id).order_by(Message.sent_timestamp.desc()).all()

    def num_unread_messages(self):
        return Message.query.filter_by(to_user_id=self.id).filter(~Message.is_read).count()

    def age_gender_role(self):
        return " ".join([str(x) if x is not None else "" for x in [self.age, self.gender, self.role]])

    def tasks_todo(self, active=True):
        return [x[1] for x in
                db.session.query(User, Task).filter(User.id == Task.doer_id and Task.is_active == bool(active)).
                    filter(User.id == self.id).all()
                ]

    def tasks_assigned(self, active=True):
        return [x[1] for x in
                    db.session.query(User, Task).filter(User.id == Task.requester_id and Task.is_active == bool(active)).
                        filter(User.id == self.id).all()
                ]

    def is_authenticated(self):
        return True

    def is_active(self):
        return True

    def is_anonymous(self):
        return False

    def get_id(self):
        return unicode(self.id)

    def __repr__(self):
        return "<User '%r'>" % (self.name)
<commit_msg>Added friends and friend_requests tables to models<commit_after>import app
from app import db
from datetime import datetime, date

friend_requests = db.Table('friend_requests',
    db.Column('id', db.Integer, primary_key=True),
    db.Column('user_id', db.Integer, db.ForeignKey('users.id')),
    db.Column('friend_id', db.Integer, db.ForeignKey('users.id')),
    db.Column('created', db.DateTime, default=datetime.utcnow)
)

friends = db.Table('friends',
    db.Column('id', db.Integer, primary_key=True),
    db.Column('user_id', db.Integer, db.ForeignKey('users.id')),
    db.Column('friend_id', db.Integer, db.ForeignKey('users.id')),
    db.Column('created', db.DateTime, default=datetime.utcnow)
)

class Message(db.Model):
    __tablename__ = "messages"
    id = db.Column(db.Integer, primary_key=True)
    sent_timestamp = db.Column(db.DateTime, default=datetime.utcnow(), index=True)
    to_user_id = db.Column(db.Integer, nullable=False)
    from_user_id = db.Column(db.Integer, nullable=False)
    title = db.Column(db.String, index=True)
    body = db.Column(db.String, nullable=False, index=True)
    is_read = db.Column(db.Boolean, default=False)
    __table_args__ = (
        db.CheckConstraint("to_user_id != from_user_id"),
    )

    def from_user(self):
        user = User.query.filter_by(id=self.from_user_id).first()
        if user is not None:
            return user
        else:
            return None

    def to_user(self):
        user = User.query.filter_by(id=self.to_user_id).first()
        if user is not None:
            return user
        else:
            return None

    def __repr__(self):
        return "<Message: From: %d, to: %d, title: %r>" % (self.from_user_id, self.to_user_id, self.title)

class Task(db.Model):
    __tablename__ = "task"
    id = db.Column(db.Integer, primary_key=True)
    created = db.Column(db.DateTime, default=datetime.utcnow(), index=True)
    end_timestamp = db.Column(db.DateTime, default=datetime.utcnow(), index=True)
    description = db.Column(db.String, nullable=False, index=True)
    requester_id = db.Column(db.Integer, nullable=False)
    doer_id = db.Column(db.Integer, nullable=False)
    is_active = db.Column(db.Boolean, default=True)
    __table_args__ = (
        db.CheckConstraint("requester_id != doer_id"),
    )

    def requester(self):
        return db.session.query(User, Task).filter(self.requester_id == User.id).first()

    def doer(self):
        return db.session.query(User, Task).filter(self.doer_id == User.id).first()

    def __repr__(self):
        return "<Task: Requester: %d, doer: %d, '%r'>" % (self.requester_id, self.doer_id, self.description)

class User(db.Model):
    __tablename__ = "users"
    id = db.Column(db.Integer, primary_key=True)
    username = db.Column(db.String, index=True, unique=True)
    pw_hash = db.Column(db.String, index=True)
    age = db.Column(db.Integer)
    gender = db.Column(db.String(24))
    role = db.Column(db.String(24))
    orientation = db.Column(db.String(24))
    bio = db.Column(db.String)
    email = db.Column(db.String, nullable=False, unique=True)
    created = db.Column(db.DateTime, default=datetime.utcnow())
    user_role = db.Column(db.Integer, default=0)
    is_validated = db.Column(db.Boolean, default=False)
    password_reset_token = db.Column(db.String)
    password_reset_token_expiration = db.Column(db.DateTime)
    __table_args__ = (
        db.UniqueConstraint('username'),
        db.UniqueConstraint('email'),
        db.CheckConstraint('age is null or (age >= 18 and age <= 100)')
    )

    def get_all_inbox_messages(self):
        return Message.query.filter_by(to_user_id=self.id).order_by(Message.sent_timestamp.desc()).all()

    def get_all_outbox_messages(self):
        return Message.query.filter_by(from_user_id=self.id).order_by(Message.sent_timestamp.desc()).all()

    def num_unread_messages(self):
        return Message.query.filter_by(to_user_id=self.id).filter(~Message.is_read).count()

    def age_gender_role(self):
        return " ".join([str(x) if x is not None else "" for x in [self.age, self.gender, self.role]])

    def tasks_todo(self, active=True):
        return [x[1] for x in
                db.session.query(User, Task).filter(User.id == Task.doer_id and Task.is_active == bool(active)).
                    filter(User.id == self.id).all()
                ]

    def tasks_assigned(self, active=True):
        return [x[1] for x in
                    db.session.query(User, Task).filter(User.id == Task.requester_id and Task.is_active == bool(active)).
                        filter(User.id == self.id).all()
                ]

    def is_authenticated(self):
        return True

    def is_active(self):
        return True

    def is_anonymous(self):
        return False

    def get_id(self):
        return unicode(self.id)

    def __repr__(self):
        return "<User '%r'>" % (self.name)
<|endoftext|>