<commit_before># Copyright (c) 2007 The Hewlett-Packard Development Company
# All rights reserved.
#
# Redistribution and use of this software in source and binary forms,
# with or without modification, are permitted provided that the
# following conditions are met:
#
# The software must be used only for Non-Commercial Use which means any
# use which is NOT directed to receiving any direct monetary
# compensation for, or commercial advantage from such use.  Illustrative
# examples of non-commercial use are academic research, personal study,
# teaching, education and corporate research & development.
# Illustrative examples of commercial use are distributing products for
# commercial advantage and providing services using the software for
# commercial advantage.
#
# If you wish to use this software or functionality therein that may be
# covered by patents for commercial use, please contact:
#     Director of Intellectual Property Licensing
#     Office of Strategy and Technology
#     Hewlett-Packard Company
#     1501 Page Mill Road
#     Palo Alto, California  94304
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.  Redistributions
# in binary form must reproduce the above copyright notice, this list of
# conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.  Neither the name of
# the COPYRIGHT HOLDER(s), HEWLETT-PACKARD COMPANY, nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.  No right of
# sublicense is granted herewith.  Derivatives of the software and
# output created using the software may be prepared, but only for
# Non-Commercial Uses.  Derivatives of the software may be shared with
# others provided: (i) the others agree to abide by the list of
# conditions herein which includes the Non-Commercial Use restrictions;
# and (ii) such Derivatives of the software include the above copyright
# notice to acknowledge the contribution from this software where
# applicable, this list of conditions and the disclaimer below.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Gabe Black

microcode = '''
def macroop SAL_R_I
{
    slli reg, reg, imm
};

def macroop SAL_M_I
{
    ld t1, ds, [scale, index, base], disp
    slli t1, t1, imm
    st t1, ds, [scale, index, base], disp
};

def macroop SAL_P_I
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    slli t1, t1, imm
    st t1, ds, [0, t0, t7], disp
};

def macroop SAL_R_R
{
    slli reg, reg, regm
};

def macroop SAL_M_R
{
    ld t1, ds, [scale, index, base], disp
    slli t1, t1, reg
    st t1, ds, [scale, index, base], disp
};

def macroop SAL_P_R
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    slli t1, t1, reg
    st t1, ds, [0, t0, t7], disp
};

def macroop SHR_R_I
{
    srli reg, reg, imm
};

def macroop SHR_M_I
{
    ld t1, ds, [scale, index, base], disp
    srli t1, t1, imm
    st t1, ds, [scale, index, base], disp
};

def macroop SHR_P_I
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    srli t1, t1, imm
    st t1, ds, [0, t0, t7], disp
};

def macroop SHR_R_R
{
    srli reg, reg, regm
};

def macroop SHR_M_R
{
    ld t1, ds, [scale, index, base], disp
    srli t1, t1, reg
    st t1, ds, [scale, index, base], disp
};

def macroop SHR_P_R
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    srli t1, t1, reg
    st t1, ds, [0, t0, t7], disp
};

def macroop SAR_R_I
{
    srai reg, reg, imm
};

def macroop SAR_M_I
{
    ld t1, ds, [scale, index, base], disp
    srai t1, t1, imm
    st t1, ds, [scale, index, base], disp
};

def macroop SAR_P_I
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    srai t1, t1, imm
    st t1, ds, [0, t0, t7], disp
};

def macroop SAR_R_R
{
    srai reg, reg, regm
};

def macroop SAR_M_R
{
    ld t1, ds, [scale, index, base], disp
    srai t1, t1, reg
    st t1, ds, [scale, index, base], disp
};

def macroop SAR_P_R
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    srai t1, t1, reg
    st t1, ds, [0, t0, t7], disp
};
'''
#let {{
#    class SAL(Inst):
#	"GenFault ${new UnimpInstFault}"
#    class SAR(Inst):
#	"GenFault ${new UnimpInstFault}"
#    class SHL(Inst):
#	"GenFault ${new UnimpInstFault}"
#    class SHR(Inst):
#	"GenFault ${new UnimpInstFault}"
#    class SHLD(Inst):
#	"GenFault ${new UnimpInstFault}"
#    class SHRD(Inst):
#	"GenFault ${new UnimpInstFault}"
#}};
<commit_msg>X86: Implement shift-by-one instructions, and make register shifts use registers.<commit_after># Copyright (c) 2007 The Hewlett-Packard Development Company
# All rights reserved.
#
# Redistribution and use of this software in source and binary forms,
# with or without modification, are permitted provided that the
# following conditions are met:
#
# The software must be used only for Non-Commercial Use which means any
# use which is NOT directed to receiving any direct monetary
# compensation for, or commercial advantage from such use.  Illustrative
# examples of non-commercial use are academic research, personal study,
# teaching, education and corporate research & development.
# Illustrative examples of commercial use are distributing products for
# commercial advantage and providing services using the software for
# commercial advantage.
#
# If you wish to use this software or functionality therein that may be
# covered by patents for commercial use, please contact:
#     Director of Intellectual Property Licensing
#     Office of Strategy and Technology
#     Hewlett-Packard Company
#     1501 Page Mill Road
#     Palo Alto, California  94304
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.  Redistributions
# in binary form must reproduce the above copyright notice, this list of
# conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.  Neither the name of
# the COPYRIGHT HOLDER(s), HEWLETT-PACKARD COMPANY, nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.  No right of
# sublicense is granted herewith.  Derivatives of the software and
# output created using the software may be prepared, but only for
# Non-Commercial Uses.  Derivatives of the software may be shared with
# others provided: (i) the others agree to abide by the list of
# conditions herein which includes the Non-Commercial Use restrictions;
# and (ii) such Derivatives of the software include the above copyright
# notice to acknowledge the contribution from this software where
# applicable, this list of conditions and the disclaimer below.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Gabe Black

microcode = '''
def macroop SAL_R_I
{
    slli reg, reg, imm
};

def macroop SAL_M_I
{
    ld t1, ds, [scale, index, base], disp
    slli t1, t1, imm
    st t1, ds, [scale, index, base], disp
};

def macroop SAL_P_I
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    slli t1, t1, imm
    st t1, ds, [0, t0, t7], disp
};

def macroop SAL_1_R
{
    slli reg, reg, 1
};

def macroop SAL_1_M
{
    ld t1, ds, [scale, index, base], disp
    slli t1, t1, 1
    st t1, ds, [scale, index, base], disp
};

def macroop SAL_1_P
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    slli t1, t1, 1
    st t1, ds, [0, t0, t7], disp
};

def macroop SAL_R_R
{
    sll reg, reg, regm
};

def macroop SAL_M_R
{
    ld t1, ds, [scale, index, base], disp
    sll t1, t1, reg
    st t1, ds, [scale, index, base], disp
};

def macroop SAL_P_R
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    sll t1, t1, reg
    st t1, ds, [0, t0, t7], disp
};

def macroop SHR_R_I
{
    srli reg, reg, imm
};

def macroop SHR_M_I
{
    ld t1, ds, [scale, index, base], disp
    srli t1, t1, imm
    st t1, ds, [scale, index, base], disp
};

def macroop SHR_P_I
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    srli t1, t1, imm
    st t1, ds, [0, t0, t7], disp
};

def macroop SHR_1_R
{
    srli reg, reg, 1
};

def macroop SHR_1_M
{
    ld t1, ds, [scale, index, base], disp
    srli t1, t1, 1
    st t1, ds, [scale, index, base], disp
};

def macroop SHR_1_P
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    srli t1, t1, 1
    st t1, ds, [0, t0, t7], disp
};

def macroop SHR_R_R
{
    srl reg, reg, regm
};

def macroop SHR_M_R
{
    ld t1, ds, [scale, index, base], disp
    srl t1, t1, reg
    st t1, ds, [scale, index, base], disp
};

def macroop SHR_P_R
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    srl t1, t1, reg
    st t1, ds, [0, t0, t7], disp
};

def macroop SAR_R_I
{
    srai reg, reg, imm
};

def macroop SAR_M_I
{
    ld t1, ds, [scale, index, base], disp
    srai t1, t1, imm
    st t1, ds, [scale, index, base], disp
};

def macroop SAR_P_I
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    srai t1, t1, imm
    st t1, ds, [0, t0, t7], disp
};

def macroop SAR_1_R
{
    srai reg, reg, 1
};

def macroop SAR_1_M
{
    ld t1, ds, [scale, index, base], disp
    srai t1, t1, 1
    st t1, ds, [scale, index, base], disp
};

def macroop SAR_1_P
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    srai t1, t1, 1
    st t1, ds, [0, t0, t7], disp
};

def macroop SAR_R_R
{
    sra reg, reg, regm
};

def macroop SAR_M_R
{
    ld t1, ds, [scale, index, base], disp
    sra t1, t1, reg
    st t1, ds, [scale, index, base], disp
};

def macroop SAR_P_R
{
    rdip t7
    ld t1, ds, [0, t0, t7], disp
    sra t1, t1, reg
    st t1, ds, [0, t0, t7], disp
};
'''
<|endoftext|>


<commit_before>#!/usr/bin/python
# coding: utf-7

import sys
import json
import requests
import pprint
import unittest
import string
import random

pp = pprint.PrettyPrinter(depth=6)

def random_title(length):
    return ''.join(random.choice(string.lowercase) for i in range(length))

def add_n(n):
    url = 'http://localhost:5000/api/v1/object'
    data = dict()
    data["submitter"] = "John Doe <john.doe@example.com>"
    data["object-item"] = dict()
    data["object-item"]['title'] = "Foo Bar Title"
    data["object-item"]['categories'] = [ "team:orange", "topic:ip", "subtopic:route-cache" ]
    data["object-item"]['version'] = 0
    data["attachment"] = dict()
    data["achievements"] = list()

    headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
    for i in range(n):
        data['object-item']['title'] = "{}".format(random_title(80))
        #pp.pprint(data)
        dj = json.dumps(data, sort_keys=True, separators=(',', ': '))
        r = requests.post(url, data=dj, headers=headers)
        #print("\nStatus Code:")
        #print(r.status_code)
        #print("\nRet Data:")
        ret_data = r.json()
        #pp.pprint(ret_data)
        assert len(ret_data['data']['id']) > 0
        processing_time = ret_data['processing-time']
        sys.stderr.write("\rEntry: {}, HTTPStatusCode: {} ServerProcTime {}s".format(i, r.status_code, processing_time))

    print("\r\n\n")

    url = 'http://localhost:5000/api/v1/objects'
    data = '''
    {
        "limit": 1000,
        "ordering": "by-submitting-date-reverse",
        "maturity-level": "all"
    }
    '''

    headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
    r = requests.get(url, data=data, headers=headers)
    print("\nStatus Code:")
    print(r.status_code)
    print("\nRet Data:")
    data = r.json()
    pp.pprint(data)

if __name__ == '__main__':
    add_n(5000)
<commit_msg>generate better title names<commit_after>#!/usr/bin/python
# coding: utf-7

import sys
import json
import requests
import pprint
import unittest
import string
import random

pp = pprint.PrettyPrinter(depth=6)

def random_title(length):
    words = ['Foo', 'Bar', 'Linux', 'Something', 'Yeah', 'Nope', 'Random']
    s = ' '.join(random.choice(words) for _ in range(5))
    return s

def add_n(n):
    url = 'http://localhost:5000/api/v1/object'
    data = dict()
    data["submitter"] = "John Doe <john.doe@example.com>"
    data["object-item"] = dict()
    data["object-item"]['categories'] = [ "team:orange", "topic:ip", "subtopic:route-cache" ]
    data["object-item"]['version'] = 0
    data["attachment"] = dict()
    data["achievements"] = list()

    headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
    for i in range(n):
        data['object-item']['title'] = "{}".format(random_title(80))
        #pp.pprint(data)
        dj = json.dumps(data, sort_keys=True, separators=(',', ': '))
        r = requests.post(url, data=dj, headers=headers)
        #print("\nStatus Code:")
        #print(r.status_code)
        #print("\nRet Data:")
        ret_data = r.json()
        #pp.pprint(ret_data)
        assert len(ret_data['data']['id']) > 0
        processing_time = ret_data['processing-time']
        sys.stderr.write("\rEntry: {}, HTTPStatusCode: {} ServerProcTime {}s".format(i, r.status_code, processing_time))

    print("\r\n\n")

    url = 'http://localhost:5000/api/v1/objects'
    data = '''
    {
        "limit": 0,
        "ordering": "by-submitting-date-reverse",
        "maturity-level": "all"
    }
    '''

    headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
    r = requests.get(url, data=data, headers=headers)
    print("\nStatus Code:")
    print(r.status_code)
    print("\nRet Data:")
    data = r.json()
    pp.pprint(data)

if __name__ == '__main__':
    add_n(100)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2015 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import msgpack
from message_coder import MessageCoderBase

# set of functions to encode/decode data tokens to/from a json description
class MessageCoder(MessageCoderBase):

    def encode(self, data):
        print "Encoding: "
        print data
        return msgpack.packb(data)

    def decode(self, data):
        data = msgpack.unpackb(data)
        print "Decoded: "
        print data
        return data
<commit_msg>coder/msgpack: Removed prints<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2015 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import msgpack
from message_coder import MessageCoderBase

# set of functions to encode/decode data tokens to/from a json description
class MessageCoder(MessageCoderBase):

    def encode(self, data):
        return msgpack.packb(data)

    def decode(self, data):
        data = msgpack.unpackb(data)
        return data
<|endoftext|>


<commit_before>from bitmessageaddress import BitmessageAddress
from settings_local import (
    BITMESSAGE_USERNAME,
    BITMESSAGE_PASSWORD,
    BITMESSAGE_HOST,
    BITMESSAGE_PORT,
    DEFAULT_ADDRESS_LABEL)

import xmlrpclib
import json
import base64

class BitmessageClient:

  def __init__(self):
    self.connect()
    self.get_addresses()
    self.update_address_if_empty()

  def connect(self):
    self.api = xmlrpclib.ServerProxy("http://{0}:{1}@{2}:{3}".format(
        BITMESSAGE_USERNAME,
        BITMESSAGE_PASSWORD,
        BITMESSAGE_HOST,
        BITMESSAGE_PORT))

  def create_random_address(self, label="main"):
    label_base64 = base64.encodestring(label)
    self.api.createRandomAddress(label_base64)

  def get_addresses(self):
    addresses_json = self.api.listAddresses2()
    address_list = json.loads(addresses_json)['addresses']

    self.addresses = \
        [BitmessageAddress(address_dict) for address_dict in address_list]

    if len(self.addresses) > 0:
      self.default_address = self.addresses[0]

    for address in self.addresses:
      if address.label == DEFAULT_ADDRESS_LABEL:
        self.default_address = address
        break

  def update_address_if_empty(self):
    if len(self.addresses) > 0:
      return
    self.create_random_address()
    self.get_addresses()

  def send_message(self, address, subject, message):
    message_id = self.api.sendMessage(
                    address,
                    self.default_address.address,
                    base64.encodestring(subject),
                    base64.encodestring(message))
    return message_id

  def send_broadcast(self, subject, message):
    message_id = self.api.sendBroadcast(
                    self.default_address.address,
                    base64.encodestring(subject),
                    base64.encodestring(message))
    return message_id

  def delete_address(self, address):
    self.api.deleteAddress(address)
<commit_msg>message_id -> ackdata<commit_after>from bitmessageaddress import BitmessageAddress
from settings_local import (
    BITMESSAGE_USERNAME,
    BITMESSAGE_PASSWORD,
    BITMESSAGE_HOST,
    BITMESSAGE_PORT,
    DEFAULT_ADDRESS_LABEL)

import xmlrpclib
import json
import base64

class BitmessageClient:

  def __init__(self):
    self.connect()
    self.get_addresses()
    self.update_address_if_empty()

  def connect(self):
    self.api = xmlrpclib.ServerProxy("http://{0}:{1}@{2}:{3}".format(
        BITMESSAGE_USERNAME,
        BITMESSAGE_PASSWORD,
        BITMESSAGE_HOST,
        BITMESSAGE_PORT))

  def create_random_address(self, label="main"):
    label_base64 = base64.encodestring(label)
    self.api.createRandomAddress(label_base64)

  def get_addresses(self):
    addresses_json = self.api.listAddresses2()
    address_list = json.loads(addresses_json)['addresses']

    self.addresses = \
        [BitmessageAddress(address_dict) for address_dict in address_list]

    if len(self.addresses) > 0:
      self.default_address = self.addresses[0]

    for address in self.addresses:
      if address.label == DEFAULT_ADDRESS_LABEL:
        self.default_address = address
        break

  def update_address_if_empty(self):
    if len(self.addresses) > 0:
      return
    self.create_random_address()
    self.get_addresses()

  def send_message(self, address, subject, message):
    ack_data = self.api.sendMessage(
                  address,
                  self.default_address.address,
                  base64.encodestring(subject),
                  base64.encodestring(message))
    return message_id

  def send_broadcast(self, subject, message):
    ack_data = self.api.sendBroadcast(
                  self.default_address.address,
                  base64.encodestring(subject),
                  base64.encodestring(message))
    return message_id

  def delete_address(self, address):
    self.api.deleteAddress(address)
<|endoftext|>


<commit_before>from datetime import datetime, timedelta
from unittest import TestCase

from mock import patch

from regulations.generator.versions import *

class VersionsTest(TestCase):
    @patch('regulations.generator.versions.api_reader')
    def test_fetch_grouped_history(self, reader):
        client = reader.ApiReader.return_value

        future = datetime.today() + timedelta(days=5)
        future = future.strftime('%Y-%m-%d')
        client.regversions.return_value = {'versions': [
            {'by_date': '2001-01-01', 'version': 'v1'},
            {'versions': 'v2'},
            {'by_date': '2003-03-03', 'version': 'v3'},
            {'versions': 'v4'},
            {'versions': 'v5'},
            {'by_date': future, 'version': 'v6'},
        ]}

        n1 = {'effective_on': '2001-01-01', 'publication_date': '2000-10-10'}
        n2 = {'effective_on': '2003-03-03', 'publication_date': '2001-01-01'}
        n3 = {'effective_on': '2003-03-03', 'publication_date': '2002-02-02'}
        n4 = {'effective_on': future, 'publication_date': '2005-05-05'}
        n5 = {'effective_on': future, 'publication_date': '2005-06-05'}
        n6 = {'effective_on': future, 'publication_date': '2005-07-05'}
        client.notices.return_value = {'results': [n1, n2, n3, n5, n4, n6]}

        history = fetch_grouped_history('111')
        self.assertEqual(3, len(history))
        v1, v2, v3 = history

        self.assertEqual('future', v1['timeline'])
        self.assertEqual('current', v2['timeline'])
        self.assertEqual('past', v3['timeline'])

        self.assertEqual(convert_to_python([n6, n5, n4]), v1['notices'])
        self.assertEqual(convert_to_python([n3, n2]), v2['notices'])
        self.assertEqual(convert_to_python([n1]), v3['notices'])
<commit_msg>Fix test for versions generator<commit_after>from datetime import datetime, timedelta
from unittest import TestCase

from mock import patch

from regulations.generator.versions import *

class VersionsTest(TestCase):
    @patch('regulations.generator.versions.api_reader')
    def test_fetch_grouped_history(self, reader):
        client = reader.ApiReader.return_value

        future = datetime.today() + timedelta(days=5)
        future = future.strftime('%Y-%m-%d')
        client.regversions.return_value = {'versions': [
            {'by_date': '2001-01-01', 'version': 'v1'},
            {'versions': 'v2'},
            {'by_date': '2003-03-03', 'version': 'v3'},
            {'versions': 'v4'},
            {'versions': 'v5'},
            {'by_date': future, 'version': 'v6'},
        ]}

        n1 = {'effective_on': '2001-01-01',
                'publication_date': '2000-10-10',
                'fr_url': 'http://n1'}
        n2 = {'effective_on': '2003-03-03',
                'publication_date': '2001-01-01',
                'fr_url': 'http://n2'}
        n3 = {'effective_on': '2003-03-03',
                'publication_date': '2002-02-02',
                'fr_url': 'http://n3'}
        n4 = {'effective_on': future,
                'publication_date': '2005-05-05',
                'fr_url': 'http://n4'}
        n5 = {'effective_on': future,
                'publication_date': '2005-06-05',
                'fr_url': 'http://n5'}
        n6 = {'effective_on': future,
                'publication_date': '2005-07-05',
                'fr_url': 'http://n6'}
        n6_1 = {'effective_on': future,
                'publication_date': '2005-07-05',
                'fr_url': 'http://n6'}
        client.notices.return_value = {
                'results': [n1, n2, n3, n5, n4, n6, n6_1]}

        history = fetch_grouped_history('111')
        self.assertEqual(3, len(history))
        v1, v2, v3 = history

        self.assertEqual('future', v1['timeline'])
        self.assertEqual('current', v2['timeline'])
        self.assertEqual('past', v3['timeline'])

        # n6_1 should NOT be present in this list
        self.assertEqual(convert_to_python([n6, n5, n4]), v1['notices'])
        self.assertEqual(convert_to_python([n3, n2]), v2['notices'])
        self.assertEqual(convert_to_python([n1]), v3['notices'])
<|endoftext|>


<commit_before>import uuid
import time

from collections import defaultdict

from ..catalog.variable import Variable
from ..catalog.dataset import Dataset
from ..catalog.geography import Geography
from ...clients import bigquery_client
from ....auth import get_default_credentials
from ....exceptions import EnrichmentException
from ....core.cartodataframe import CartoDataFrame
from ....core.logger import log
from ....utils.geom_utils import to_geojson
from ....utils.utils import timelogger

_ENRICHMENT_ID = 'enrichment_id'
_GEOM_COLUMN = '__geojson_geom'

AGGREGATION_DEFAULT = 'default'
AGGREGATION_NONE = 'none'

class VariableFilter(object):
    """This class can be used for filtering the results of
    :py:class:`Enrichment <cartoframes.data.observatory.Enrichment>` functions. It works by appending the
    `VariableFilter` SQL operators to the `WHERE` clause of the resulting enrichment SQL with the `AND` operator.

        Args:
            variable (str or :obj:`Variable`):
                The variable name or :obj:`Variable` instance

            query (str):
                The SQL query filter to be appended to the enrichment SQL query.

        Examples:

            - Equal to number: `VariableFilter(variable, '= 3')`
            - Equal to string: `VariableFilter(variable, "= 'the string'")`
            - Greater that 3: `VariableFilter(variable, '> 3')`

            The next example uses a `VariableFilter` instance to calculate the `SUM` of car-free households
            :obj:`Variable` of the :obj:`Catalog` for each polygon of `my_local_dataframe` pandas `DataFrame` only for
            areas with more than 100 car-free households,

            .. code::

                from cartoframes.data.observatory import Enrichment, Variable, VariableFilter

                variable = Variable.get('no_cars_d19dfd10')
                enriched_dataset_cdf = Enrichment().enrich_polygons(
                    my_local_dataframe,
                    variables=[variable],
                    aggregation=[VariableAggregation(variable, 'SUM')]
                    filters=[VariableFilter(variable, '> 100')]
                )
    """
    def __init__(self, variable, query):
        self.variable = _prepare_variable(variable)
        self.query = query

class EnrichmentService(object):
    """Base class for the Enrichment utility with commons auxiliary methods"""

    def __init__(self, credentials=None):
        self.credentials = credentials = credentials or get_default_credentials()
        self.bq_client = bigquery_client.BigQueryClient(credentials)
        self.bq_dataset = self.bq_client.bq_dataset
        self.bq_project = self.bq_client.bq_project
        self.bq_public_project = self.bq_client.bq_public_project
        self.enrichment_id = _ENRICHMENT_ID
        self.geom_column = _GEOM_COLUMN

    @timelogger
    def _execute_enrichment(self, queries, cartodataframe):

        dfs_enriched = list()
        awaiting_jobs = set()
        errors = list()

        def callback(job):
            if not job.errors:
                dfs_enriched.append(self.bq_client.download_to_dataframe(job))
            else:
                errors.extend(job.errors)

            awaiting_jobs.discard(job.job_id)

        for query in queries:
            job = self.bq_client.query(query)
            awaiting_jobs.add(job.job_id)
            job.add_done_callback(callback)

        while awaiting_jobs:
            time.sleep(0.5)

        if len(errors) > 0:
            raise Exception(errors)

        for df in dfs_enriched:
            cartodataframe = cartodataframe.merge(df, on=self.enrichment_id, how='left')

        # Remove extra columns
        cartodataframe.drop(self.enrichment_id, axis=1, inplace=True)
        cartodataframe.drop(self.geom_column, axis=1, inplace=True)

        return cartodataframe

    @timelogger
    def _prepare_data(self, dataframe, geom_col):
        cartodataframe = CartoDataFrame(dataframe, copy=True)

        if geom_col:
            cartodataframe.set_geometry(geom_col, inplace=True)

        if not cartodataframe.has_geometry():
            raise EnrichmentException('No valid geometry found. Please provide an input source with ' +
                                      'a valid geometry or specify the "geom_col" param with a geometry column.')

        # Add extra columns for the enrichment
        cartodataframe[self.enrichment_id] = range(cartodataframe.shape[0])
        cartodataframe[self.geom_column] = cartodataframe.geometry.apply(to_geojson)

        return cartodataframe

    def _get_temp_table_name(self):
        id_tablename = uuid.uuid4().hex
        return 'temp_{id}'.format(id=id_tablename)

    def _upload_data(self, tablename, cartodataframe):
        bq_dataframe = cartodataframe[[self.enrichment_id, self.geom_column]]
        schema = {self.enrichment_id: 'INTEGER', self.geom_column: 'GEOGRAPHY'}

        self.bq_client.upload_dataframe(
            dataframe=bq_dataframe,
            schema=schema,
            tablename=tablename
        )

    def _get_tables_metadata(self, variables):
        tables_metadata = defaultdict(lambda: defaultdict(list))

        for variable in variables:
            table_name = self.__get_enrichment_table(variable)
            tables_metadata[table_name]['variables'].append(variable)

            if 'dataset' not in tables_metadata[table_name].keys():
                tables_metadata[table_name]['dataset'] = self.__get_dataset(variable, table_name)

            if 'geo_table' not in tables_metadata[table_name].keys():
                tables_metadata[table_name]['geo_table'] = self.__get_geo_table(variable)

            if 'project' not in tables_metadata[table_name].keys():
                tables_metadata[table_name]['project'] = self.__get_project(variable)

        return tables_metadata

    def __get_enrichment_table(self, variable):
        if variable.project_name != self.bq_public_project:
            return 'view_{dataset}_{table}'.format(
                dataset=variable.schema_name,
                table=variable.dataset_name
            )
        else:
            return variable.dataset_name

    def __get_dataset(self, variable, table_name):
        if variable.project_name != self.bq_public_project:
            return '{project}.{dataset}.{table_name}'.format(
                project=self.bq_project,
                dataset=self.bq_dataset,
                table_name=table_name
            )
        else:
            return variable.dataset

    def __get_geo_table(self, variable):
        geography_id = Dataset.get(variable.dataset).geography
        geography = Geography.get(geography_id)
        _, dataset_geo_table, geo_table = geography_id.split('.')

        if not geography.is_public_data:
            return '{project}.{dataset}.view_{dataset_geo_table}_{geo_table}'.format(
                project=self.bq_project,
                dataset=self.bq_dataset,
                dataset_geo_table=dataset_geo_table,
                geo_table=geo_table
            )
        else:
            return '{project}.{dataset}.{geo_table}'.format(
                project=self.bq_public_project,
                dataset=dataset_geo_table,
                geo_table=geo_table
            )

    def __get_project(self, variable):
        project = self.bq_public_project

        if variable.project_name != self.bq_public_project:
            project = self.bq_project

        return project

    def _get_points_enrichment_sql(self, temp_table_name, variables, filters):
        tables_metadata = self._get_tables_metadata(variables).items()

        return [self._build_points_query(metadata, temp_table_name, filters)
                for _, metadata in tables_metadata]

    def _build_points_query(self, metadata, temp_table_name, filters):
        variables = ['enrichment_table.{}'.format(variable.column_name) for variable in metadata['variables']]
        enrichment_dataset = metadata['dataset']
        enrichment_geo_table = metadata['geo_table']
        data_table = '{project}.{user_dataset}.{temp_table_name}'.format(
            project=self.bq_project,
            user_dataset=self.bq_dataset,
            temp_table_name=temp_table_name
        )

        return '''
            SELECT data_table.{enrichment_id}, {variables},
                ST_Area(enrichment_geo_table.geom) AS do_geom_area
            FROM `{enrichment_dataset}` enrichment_table
                JOIN `{enrichment_geo_table}` enrichment_geo_table
                    ON enrichment_table.geoid = enrichment_geo_table.geoid
                JOIN `{data_table}` data_table
                    ON ST_Within(data_table.{geom_column}, enrichment_geo_table.geom)
            {where};
        '''.format(
            variables=', '.join(variables),
            geom_column=self.geom_column,
            enrichment_dataset=enrichment_dataset,
            enrichment_geo_table=enrichment_geo_table,
            enrichment_id=self.enrichment_id,
            where=_build_where_clausule(filters),
            data_table=data_table
        )

    def _get_polygon_enrichment_sql(self, temp_table_name, variables, filters, aggregation):
        tables_metadata = self._get_tables_metadata(variables).items()

        return [self._build_polygons_query(metadata, temp_table_name, filters, aggregation)
                for _, metadata in tables_metadata]

    def _build_polygons_query(self, metadata, temp_table_name, filters, aggregation):
        variables = metadata['variables']
        enrichment_dataset = metadata['dataset']
        enrichment_geo_table = metadata['geo_table']
        data_table = '{project}.{user_dataset}.{temp_table_name}'.format(
            project=self.bq_project,
            user_dataset=self.bq_dataset,
            temp_table_name=temp_table_name
        )

        if aggregation == AGGREGATION_NONE:
            grouper = ''
            columns = _build_polygons_query_variables_without_aggregation(self.geom_column, variables)
        else:
            grouper = 'group by data_table.{enrichment_id}'.format(enrichment_id=self.enrichment_id)
            columns = _build_polygons_query_variables_with_aggregation(self.geom_column, variables, aggregation)

        return '''
            SELECT data_table.{enrichment_id}, {columns}
            FROM `{enrichment_dataset}` enrichment_table
                JOIN `{enrichment_geo_table}` enrichment_geo_table
                    ON enrichment_table.geoid = enrichment_geo_table.geoid
                JOIN `{data_table}` data_table
                    ON ST_Intersects(data_table.{geom_column}, enrichment_geo_table.geom)
            {where}
            {grouper};
        '''.format(
                geom_column=self.geom_column,
                enrichment_dataset=enrichment_dataset,
                enrichment_geo_table=enrichment_geo_table,
                enrichment_id=self.enrichment_id,
                where=_build_where_clausule(filters),
                data_table=data_table,
                grouper=grouper or '',
                columns=columns
            )

def _build_polygons_query_variables_with_aggregation(geom_column, variables, aggregation):
    return ', '.join([
        _build_polygons_query_variable_with_aggregation(
            geom_column,
            variable,
            aggregation
        ) for variable in variables])

def _build_polygons_query_variable_with_aggregation(geom_column, variable, aggregation):
    variable_agg = _get_aggregation(variable, aggregation)

    if (variable_agg == 'sum'):
        return """
            {aggregation}(
                enrichment_table.{column} * (
                    ST_Area(ST_Intersection(enrichment_geo_table.geom, data_table.{geo_column}))
                    /
                    ST_area(data_table.{geo_column})
                )
            ) AS {aggregation}_{column}
            """.format(
                column=variable.column_name,
                geo_column=geom_column,
                aggregation=variable_agg)
    else:
        return """
            {aggregation}(enrichment_table.{column}) AS {aggregation}_{column}
            """.format(
                column=variable.column_name,
                aggregation=variable_agg)

def _build_polygons_query_variables_without_aggregation(geom_column, variables):
    variables = ['enrichment_table.{}'.format(variable.column_name) for variable in variables]

    return """
        {variables},
        ST_Area(ST_Intersection(enrichment_geo_table.geom, data_table.{geom_column})) AS intersected_area,
        ST_area(enrichment_geo_table.geom) AS do_area,
        ST_area(data_table.{geom_column}) AS user_area,
        enrichment_geo_table.geoid as do_geoid
        """.format(
            variables=', '.join(variables),
            geom_column=geom_column)

def _build_where_clausule(filters):
    where = ''
    if len(filters) > 0:
        where_clausules = ["enrichment_table.{} {}".format(f.variable.column_name, f.query) for f in filters]
        where = 'WHERE {}'.format('AND '.join(where_clausules))

    return where

@timelogger
def prepare_variables(variables, credentials, aggregation=None):
    if isinstance(variables, list):
        variables = [_prepare_variable(var, aggregation) for var in variables]
    else:
        variables = [_prepare_variable(variables, aggregation)]

    variables = list(filter(None, variables))

    _validate_bq_operations(variables, credentials)

    return variables

def _prepare_variable(variable, aggregation=None):
    if isinstance(variable, str):
        variable = Variable.get(variable)

    if not isinstance(variable, Variable):
        raise EnrichmentException("""
            variable should be a `<cartoframes.data.observatory> Variable` instance,
            Variable `id` property or Variable `slug` property
        """)

    if aggregation is not None:
        variable_agg = _get_aggregation(variable, aggregation)
        if not variable_agg and aggregation is not AGGREGATION_NONE:
            log.warning('%s skipped because it does not have aggregation method', variable.id)
            return None

    return variable

def _validate_bq_operations(variables, credentials):
    dataset_ids = list(set([variable.dataset for variable in variables]))

    for dataset_id in dataset_ids:
        dataset = Dataset.get(dataset_id)
        geography = Geography.get(dataset.geography)

        _is_subscribed(dataset, geography, credentials)
        _is_available_in_bq(dataset, geography)

def _is_available_in_bq(dataset, geography):
    if not dataset._is_available_in('bq'):
        raise EnrichmentException("""
            The Dataset '{}' is not ready for Enrichment. Please, contact us for more information.
        """.format(dataset))

    if not geography._is_available_in('bq'):
        raise EnrichmentException("""
            The Geography '{}' is not ready for Enrichment. Please, contact us for more information.
        """.format(geography))

def _is_subscribed(dataset, geography, credentials):
    if not dataset._is_subscribed(credentials):
        raise EnrichmentException("""
            You are not subscribed to the Dataset '{}' yet. Please, use the subscribe method first.
        """.format(dataset.id))

    if not geography._is_subscribed(credentials):
        raise EnrichmentException("""
            You are not subscribed to the Geography '{}' yet. Please, use the subscribe method first.
        """.format(geography.id))

def _get_aggregation(variable, aggregation):
    if aggregation in [None, AGGREGATION_NONE]:
        aggregation_method = None
    elif aggregation == AGGREGATION_DEFAULT:
        aggregation_method = variable.agg_method
    elif isinstance(aggregation, str):
        aggregation_method = aggregation
    elif isinstance(aggregation, dict):
        aggregation_method = aggregation.get(variable.id, variable.agg_method)
    else:
        raise ValueError('The `aggregation` parameter is invalid.')

    if aggregation_method is not None:
        return aggregation_method.lower()
<commit_msg>rm needless enrichment_id and geom_column props from service<commit_after>import uuid
import time

from collections import defaultdict

from ..catalog.variable import Variable
from ..catalog.dataset import Dataset
from ..catalog.geography import Geography
from ...clients import bigquery_client
from ....auth import get_default_credentials
from ....exceptions import EnrichmentException
from ....core.cartodataframe import CartoDataFrame
from ....core.logger import log
from ....utils.geom_utils import to_geojson
from ....utils.utils import timelogger

_ENRICHMENT_ID = 'enrichment_id'
_GEOM_COLUMN = '__geojson_geom'

AGGREGATION_DEFAULT = 'default'
AGGREGATION_NONE = 'none'

class VariableFilter(object):
    """This class can be used for filtering the results of
    :py:class:`Enrichment <cartoframes.data.observatory.Enrichment>` functions. It works by appending the
    `VariableFilter` SQL operators to the `WHERE` clause of the resulting enrichment SQL with the `AND` operator.

        Args:
            variable (str or :obj:`Variable`):
                The variable name or :obj:`Variable` instance

            query (str):
                The SQL query filter to be appended to the enrichment SQL query.

        Examples:

            - Equal to number: `VariableFilter(variable, '= 3')`
            - Equal to string: `VariableFilter(variable, "= 'the string'")`
            - Greater that 3: `VariableFilter(variable, '> 3')`

            The next example uses a `VariableFilter` instance to calculate the `SUM` of car-free households
            :obj:`Variable` of the :obj:`Catalog` for each polygon of `my_local_dataframe` pandas `DataFrame` only for
            areas with more than 100 car-free households,

            .. code::

                from cartoframes.data.observatory import Enrichment, Variable, VariableFilter

                variable = Variable.get('no_cars_d19dfd10')
                enriched_dataset_cdf = Enrichment().enrich_polygons(
                    my_local_dataframe,
                    variables=[variable],
                    aggregation=[VariableAggregation(variable, 'SUM')]
                    filters=[VariableFilter(variable, '> 100')]
                )
    """
    def __init__(self, variable, query):
        self.variable = _prepare_variable(variable)
        self.query = query

class EnrichmentService(object):
    """Base class for the Enrichment utility with commons auxiliary methods"""

    def __init__(self, credentials=None):
        self.credentials = credentials = credentials or get_default_credentials()
        self.bq_client = bigquery_client.BigQueryClient(credentials)
        self.bq_dataset = self.bq_client.bq_dataset
        self.bq_project = self.bq_client.bq_project
        self.bq_public_project = self.bq_client.bq_public_project

    @timelogger
    def _execute_enrichment(self, queries, cartodataframe):

        dfs_enriched = list()
        awaiting_jobs = set()
        errors = list()

        def callback(job):
            if not job.errors:
                dfs_enriched.append(self.bq_client.download_to_dataframe(job))
            else:
                errors.extend(job.errors)

            awaiting_jobs.discard(job.job_id)

        for query in queries:
            job = self.bq_client.query(query)
            awaiting_jobs.add(job.job_id)
            job.add_done_callback(callback)

        while awaiting_jobs:
            time.sleep(0.5)

        if len(errors) > 0:
            raise Exception(errors)

        for df in dfs_enriched:
            cartodataframe = cartodataframe.merge(df, on=_ENRICHMENT_ID, how='left')

        # Remove extra columns
        cartodataframe.drop(_ENRICHMENT_ID, axis=1, inplace=True)
        cartodataframe.drop(_GEOM_COLUMN, axis=1, inplace=True)

        return cartodataframe

    @timelogger
    def _prepare_data(self, dataframe, geom_col):
        cartodataframe = CartoDataFrame(dataframe, copy=True)

        if geom_col:
            cartodataframe.set_geometry(geom_col, inplace=True)

        if not cartodataframe.has_geometry():
            raise EnrichmentException('No valid geometry found. Please provide an input source with ' +
                                      'a valid geometry or specify the "geom_col" param with a geometry column.')

        # Add extra columns for the enrichment
        cartodataframe[_ENRICHMENT_ID] = range(cartodataframe.shape[0])
        cartodataframe[_GEOM_COLUMN] = cartodataframe.geometry.apply(to_geojson)

        return cartodataframe

    def _get_temp_table_name(self):
        id_tablename = uuid.uuid4().hex
        return 'temp_{id}'.format(id=id_tablename)

    def _upload_data(self, tablename, cartodataframe):
        bq_dataframe = cartodataframe[[_ENRICHMENT_ID, _GEOM_COLUMN]]
        schema = {_ENRICHMENT_ID: 'INTEGER', _GEOM_COLUMN: 'GEOGRAPHY'}

        self.bq_client.upload_dataframe(
            dataframe=bq_dataframe,
            schema=schema,
            tablename=tablename
        )

    def _get_tables_metadata(self, variables):
        tables_metadata = defaultdict(lambda: defaultdict(list))

        for variable in variables:
            table_name = self.__get_enrichment_table(variable)
            tables_metadata[table_name]['variables'].append(variable)

            if 'dataset' not in tables_metadata[table_name].keys():
                tables_metadata[table_name]['dataset'] = self.__get_dataset(variable, table_name)

            if 'geo_table' not in tables_metadata[table_name].keys():
                tables_metadata[table_name]['geo_table'] = self.__get_geo_table(variable)

            if 'project' not in tables_metadata[table_name].keys():
                tables_metadata[table_name]['project'] = self.__get_project(variable)

        return tables_metadata

    def __get_enrichment_table(self, variable):
        if variable.project_name != self.bq_public_project:
            return 'view_{dataset}_{table}'.format(
                dataset=variable.schema_name,
                table=variable.dataset_name
            )
        else:
            return variable.dataset_name

    def __get_dataset(self, variable, table_name):
        if variable.project_name != self.bq_public_project:
            return '{project}.{dataset}.{table_name}'.format(
                project=self.bq_project,
                dataset=self.bq_dataset,
                table_name=table_name
            )
        else:
            return variable.dataset

    def __get_geo_table(self, variable):
        geography_id = Dataset.get(variable.dataset).geography
        geography = Geography.get(geography_id)
        _, dataset_geo_table, geo_table = geography_id.split('.')

        if not geography.is_public_data:
            return '{project}.{dataset}.view_{dataset_geo_table}_{geo_table}'.format(
                project=self.bq_project,
                dataset=self.bq_dataset,
                dataset_geo_table=dataset_geo_table,
                geo_table=geo_table
            )
        else:
            return '{project}.{dataset}.{geo_table}'.format(
                project=self.bq_public_project,
                dataset=dataset_geo_table,
                geo_table=geo_table
            )

    def __get_project(self, variable):
        project = self.bq_public_project

        if variable.project_name != self.bq_public_project:
            project = self.bq_project

        return project

    def _get_points_enrichment_sql(self, temp_table_name, variables, filters):
        tables_metadata = self._get_tables_metadata(variables).items()

        return [self._build_points_query(metadata, temp_table_name, filters)
                for _, metadata in tables_metadata]

    def _build_points_query(self, metadata, temp_table_name, filters):
        variables = ['enrichment_table.{}'.format(variable.column_name) for variable in metadata['variables']]
        enrichment_dataset = metadata['dataset']
        enrichment_geo_table = metadata['geo_table']
        data_table = '{project}.{user_dataset}.{temp_table_name}'.format(
            project=self.bq_project,
            user_dataset=self.bq_dataset,
            temp_table_name=temp_table_name
        )

        return '''
            SELECT data_table.{enrichment_id}, {variables},
                ST_Area(enrichment_geo_table.geom) AS do_geom_area
            FROM `{enrichment_dataset}` enrichment_table
                JOIN `{enrichment_geo_table}` enrichment_geo_table
                    ON enrichment_table.geoid = enrichment_geo_table.geoid
                JOIN `{data_table}` data_table
                    ON ST_Within(data_table.{geom_column}, enrichment_geo_table.geom)
            {where};
        '''.format(
            variables=', '.join(variables),
            geom_column=_GEOM_COLUMN,
            enrichment_dataset=enrichment_dataset,
            enrichment_geo_table=enrichment_geo_table,
            enrichment_id=_ENRICHMENT_ID,
            where=_build_where_clausule(filters),
            data_table=data_table
        )

    def _get_polygon_enrichment_sql(self, temp_table_name, variables, filters, aggregation):
        tables_metadata = self._get_tables_metadata(variables).items()

        return [self._build_polygons_query(metadata, temp_table_name, filters, aggregation)
                for _, metadata in tables_metadata]

    def _build_polygons_query(self, metadata, temp_table_name, filters, aggregation):
        variables = metadata['variables']
        enrichment_dataset = metadata['dataset']
        enrichment_geo_table = metadata['geo_table']
        data_table = '{project}.{user_dataset}.{temp_table_name}'.format(
            project=self.bq_project,
            user_dataset=self.bq_dataset,
            temp_table_name=temp_table_name
        )

        if aggregation == AGGREGATION_NONE:
            grouper = ''
            columns = _build_polygons_query_variables_without_aggregation(variables)
        else:
            grouper = 'group by data_table.{enrichment_id}'.format(enrichment_id=_ENRICHMENT_ID)
            columns = _build_polygons_query_variables_with_aggregation(variables, aggregation)

        return '''
            SELECT data_table.{enrichment_id}, {columns}
            FROM `{enrichment_dataset}` enrichment_table
                JOIN `{enrichment_geo_table}` enrichment_geo_table
                    ON enrichment_table.geoid = enrichment_geo_table.geoid
                JOIN `{data_table}` data_table
                    ON ST_Intersects(data_table.{geom_column}, enrichment_geo_table.geom)
            {where}
            {grouper};
        '''.format(
                geom_column=_GEOM_COLUMN,
                enrichment_dataset=enrichment_dataset,
                enrichment_geo_table=enrichment_geo_table,
                enrichment_id=_ENRICHMENT_ID,
                where=_build_where_clausule(filters),
                data_table=data_table,
                grouper=grouper or '',
                columns=columns
            )

def _build_polygons_query_variables_with_aggregation(variables, aggregation):
    return ', '.join([
        _build_polygons_query_variable_with_aggregation(
            variable,
            aggregation
        ) for variable in variables])

def _build_polygons_query_variable_with_aggregation(variable, aggregation):
    variable_agg = _get_aggregation(variable, aggregation)

    if (variable_agg == 'sum'):
        return """
            {aggregation}(
                enrichment_table.{column} * (
                    ST_Area(ST_Intersection(enrichment_geo_table.geom, data_table.{geo_column}))
                    /
                    ST_area(data_table.{geo_column})
                )
            ) AS {aggregation}_{column}
            """.format(
                column=variable.column_name,
                geo_column=_GEOM_COLUMN,
                aggregation=variable_agg)
    else:
        return """
            {aggregation}(enrichment_table.{column}) AS {aggregation}_{column}
            """.format(
                column=variable.column_name,
                aggregation=variable_agg)

def _build_polygons_query_variables_without_aggregation(variables):
    variables = ['enrichment_table.{}'.format(variable.column_name) for variable in variables]

    return """
        {variables},
        ST_Area(ST_Intersection(enrichment_geo_table.geom, data_table.{geom_column})) AS intersected_area,
        ST_area(enrichment_geo_table.geom) AS do_area,
        ST_area(data_table.{geom_column}) AS user_area,
        enrichment_geo_table.geoid as do_geoid
        """.format(
            variables=', '.join(variables),
            geom_column=_GEOM_COLUMN)

def _build_where_clausule(filters):
    where = ''
    if len(filters) > 0:
        where_clausules = ["enrichment_table.{} {}".format(f.variable.column_name, f.query) for f in filters]
        where = 'WHERE {}'.format('AND '.join(where_clausules))

    return where

@timelogger
def prepare_variables(variables, credentials, aggregation=None):
    if isinstance(variables, list):
        variables = [_prepare_variable(var, aggregation) for var in variables]
    else:
        variables = [_prepare_variable(variables, aggregation)]

    variables = list(filter(None, variables))

    _validate_bq_operations(variables, credentials)

    return variables

def _prepare_variable(variable, aggregation=None):
    if isinstance(variable, str):
        variable = Variable.get(variable)

    if not isinstance(variable, Variable):
        raise EnrichmentException("""
            variable should be a `<cartoframes.data.observatory> Variable` instance,
            Variable `id` property or Variable `slug` property
        """)

    if aggregation is not None:
        variable_agg = _get_aggregation(variable, aggregation)
        if not variable_agg and aggregation is not AGGREGATION_NONE:
            log.warning('%s skipped because it does not have aggregation method', variable.id)
            return None

    return variable

def _validate_bq_operations(variables, credentials):
    dataset_ids = list(set([variable.dataset for variable in variables]))

    for dataset_id in dataset_ids:
        dataset = Dataset.get(dataset_id)
        geography = Geography.get(dataset.geography)

        _is_subscribed(dataset, geography, credentials)
        _is_available_in_bq(dataset, geography)

def _is_available_in_bq(dataset, geography):
    if not dataset._is_available_in('bq'):
        raise EnrichmentException("""
            The Dataset '{}' is not ready for Enrichment. Please, contact us for more information.
        """.format(dataset))

    if not geography._is_available_in('bq'):
        raise EnrichmentException("""
            The Geography '{}' is not ready for Enrichment. Please, contact us for more information.
        """.format(geography))

def _is_subscribed(dataset, geography, credentials):
    if not dataset._is_subscribed(credentials):
        raise EnrichmentException("""
            You are not subscribed to the Dataset '{}' yet. Please, use the subscribe method first.
        """.format(dataset.id))

    if not geography._is_subscribed(credentials):
        raise EnrichmentException("""
            You are not subscribed to the Geography '{}' yet. Please, use the subscribe method first.
        """.format(geography.id))

def _get_aggregation(variable, aggregation):
    if aggregation in [None, AGGREGATION_NONE]:
        aggregation_method = None
    elif aggregation == AGGREGATION_DEFAULT:
        aggregation_method = variable.agg_method
    elif isinstance(aggregation, str):
        aggregation_method = aggregation
    elif isinstance(aggregation, dict):
        aggregation_method = aggregation.get(variable.id, variable.agg_method)
    else:
        raise ValueError('The `aggregation` parameter is invalid.')

    if aggregation_method is not None:
        return aggregation_method.lower()
<|endoftext|>


<commit_before>#     Copyright 2018, Kay Hayen, mailto:kay.hayen@gmail.com
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
""" Scons interface.

Interaction with scons. Find the binary, and run it with a set of given
options.

"""

import contextlib
import os
import subprocess
import sys

from nuitka import Options, Tracing
from nuitka.__past__ import unicode  # pylint: disable=I0021,redefined-builtin
from nuitka.PythonVersions import getTargetPythonDLLPath, python_version
from nuitka.utils import Execution, Utils

def getSconsDataPath():
    """ Return path to where data for scons lives, e.g. static C source files.

    """

    return os.path.dirname(__file__)

def _getSconsInlinePath():
    """ Return path to inline copy of scons. """

    return os.path.join(
        getSconsDataPath(),
        "inline_copy"
    )

def _getSconsBinaryCall():
    """ Return a way to execute Scons.

    Using potentially in-line copy if no system Scons is available
    or if we are on Windows, there it is mandatory.
    """
    if Utils.getOS() != "Windows":
        scons_path = Execution.getExecutablePath("scons")

        if scons_path is not None:
            return [
                scons_path
            ]

    return [
        _getPythonForSconsExePath(),
        os.path.join(
            _getSconsInlinePath(),
            "bin",
            "scons.py"
        )
    ]

def _getPythonSconsExePathWindows():
    """ Find Python2 on Windows.

    First try a few guesses, the look into registry for user or system wide
    installations of Python2. Both Python 2.6 and 2.7 will do.
    """

    # Shortcuts for the default installation directories, to avoid going to
    # registry at all unless necessary. Any Python2 will do for Scons, so it
    # might be avoided entirely.

    if os.path.isfile(r"c:\Python27\python.exe"):
        return r"c:\Python27\python.exe"
    elif os.path.isfile(r"c:\Python26\python.exe"):
        return r"c:\Python26\python.exe"
    elif os.path.isfile(r"c:\Python35\python.exe"):
        return r"c:\Python35\python.exe"
    elif os.path.isfile(r"c:\Python36\python.exe"):
        return r"c:\Python36\python.exe"

    # Windows only code, pylint: disable=I0021,import-error,undefined-variable
    if python_version < 300:
        import _winreg as winreg # @UnresolvedImport pylint: disable=I0021,import-error,no-name-in-module
    else:
        import winreg  # @Reimport @UnresolvedImport pylint: disable=I0021,import-error,no-name-in-module

    for search in ("2.7", "2.6", "3.5", "3.6"):
        for hkey_branch in (winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER):
            for arch_key in 0, winreg.KEY_WOW64_32KEY, winreg.KEY_WOW64_64KEY:
                try:
                    key = winreg.OpenKey(
                        hkey_branch,
                        r"SOFTWARE\Python\PythonCore\%s\InstallPath" % search,
                        0,
                        winreg.KEY_READ | arch_key
                    )

                    return os.path.join(
                        winreg.QueryValue(key, ""),
                        "python.exe"
                    )
                except WindowsError:  # @UndefinedVariable
                    pass

def _getPythonForSconsExePath():
    """ Find a way to call any Python2.

    Scons needs it as it doesn't support Python3.
    """
    python_exe = Options.getPythonPathForScons()

    if python_exe is not None:
        return python_exe

    if python_version < 300 or python_version >= 350:
        return sys.executable
    elif Utils.getOS() == "Windows":
        python_exe = _getPythonSconsExePathWindows()

        if python_exe is not None:
            return python_exe
        else:
            sys.exit("""\
Error, while Nuitka works with Python 3.2 to 3.4, scons does not, and Nuitka
needs to find a Python executable 2.6/2.7 or 3.5 or higher. Simply under the
C:\\PythonXY, e.g. C:\\Python27 to execute the scons utility which is used
to build the C files to binary.

You may provide it using option "--python-for-scons=path_to_python.exe"
in case it is not visible in registry, e.g. due to using uninstalled
AnaConda Python.
""")

    for version_candidate in ("2.7", "2.6", "3.5", "3.6"):
        candidate = Execution.getExecutablePath("python" + version_candidate)

        if candidate is not None:
            return candidate

    # Lets be optimistic, this is most often going to be new enough or a
    # Python2 variant.
    return "python"

@contextlib.contextmanager
def _setupSconsEnvironment():
    """ Setup the scons execution environment.

    For the scons inline copy on Windows needs to find the library, using
    the "SCONS_LIB_DIR" environment variable "NUITKA_SCONS". And for the
    target Python we provide "NUITKA_PYTHON_DLL_PATH" to see where the
    Python DLL lives, in case it needs to be copied, and then the
    "NUITKA_PYTHON_EXE_PATH" to find the Python binary itself.
    """

    if Utils.getOS() == "Windows":
        # On Windows this Scons variable must be set by us.
        os.environ["SCONS_LIB_DIR"] = os.path.join(
            _getSconsInlinePath(),
            "lib",
            "scons-2.3.2"
        )

        # On Windows, we use the Python.DLL path for some things. We pass it
        # via environment variable
        os.environ["NUITKA_PYTHON_DLL_PATH"] = getTargetPythonDLLPath()

    os.environ["NUITKA_PYTHON_EXE_PATH"] = sys.executable

    # Remove environment variables that can only harm if we have to switch
    # major Python versions, these cannot help Python2 to execute scons, this
    # is a bit of noise, but helpful.
    if python_version >= 300:
        if "PYTHONPATH" in os.environ:
            old_pythonpath = os.environ["PYTHONPATH"]
            del os.environ["PYTHONPATH"]
        else:
            old_pythonpath = None

        if "PYTHONHOME" in os.environ:
            old_pythonhome = os.environ["PYTHONHOME"]
            del os.environ["PYTHONHOME"]
        else:
            old_pythonhome = None

    yield

    if python_version >= 300:
        if old_pythonpath is not None:
            os.environ["PYTHONPATH"] = old_pythonpath

        if old_pythonhome is not None:
            os.environ["PYTHONHOME"] = old_pythonhome

    if Utils.getOS() == "Windows":
        del os.environ["NUITKA_PYTHON_DLL_PATH"]
        del os.environ["NUITKA_PYTHON_EXE_PATH"]

def _buildSconsCommand(quiet, options):
    """ Build the scons command to run.

    The options are a dictionary to be passed to scons as a command line,
    and other scons stuff is set.
    """

    scons_command = _getSconsBinaryCall()

    if quiet:
        scons_command.append("--quiet")

    scons_command += [
        # The scons file
        "-f",
        os.path.join(
            getSconsDataPath(),
            "SingleExe.scons"
        ),

        # Parallel compilation.
        "--jobs",
        str(Options.getJobLimit()),

        # Do not warn about deprecation from Scons
        "--warn=no-deprecated",

        # Don't load "site_scons" at all.
        "--no-site-dir",
    ]

    if Options.isShowScons():
        scons_command.append("--debug=explain")

    # Python2, encoding unicode values
    def encode(value):
        if str is bytes and type(value) is unicode:
            return value.encode("utf8")
        else:
            return value

    # Option values to provide to scons. Find these in the caller.
    for key, value in options.items():
        scons_command.append(
            key + '=' + encode(value)
        )

    # Python2, make argument encoding recognizable.
    if str is bytes:
        scons_command.append(
            "arg_encoding=utf8"
        )

    return scons_command

def runScons(options, quiet):
    with _setupSconsEnvironment():
        scons_command = _buildSconsCommand(quiet, options)

        if Options.isShowScons():
            Tracing.printLine("Scons command:", ' '.join(scons_command))

        Tracing.flushStdout()
        return subprocess.call(scons_command, shell = False) == 0
<commit_msg>Python3.7: Allow using it for Scons<commit_after>#     Copyright 2018, Kay Hayen, mailto:kay.hayen@gmail.com
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
""" Scons interface.

Interaction with scons. Find the binary, and run it with a set of given
options.

"""

import contextlib
import os
import subprocess
import sys

from nuitka import Options, Tracing
from nuitka.__past__ import unicode  # pylint: disable=I0021,redefined-builtin
from nuitka.PythonVersions import getTargetPythonDLLPath, python_version
from nuitka.utils import Execution, Utils

def getSconsDataPath():
    """ Return path to where data for scons lives, e.g. static C source files.

    """

    return os.path.dirname(__file__)

def _getSconsInlinePath():
    """ Return path to inline copy of scons. """

    return os.path.join(
        getSconsDataPath(),
        "inline_copy"
    )

def _getSconsBinaryCall():
    """ Return a way to execute Scons.

    Using potentially in-line copy if no system Scons is available
    or if we are on Windows, there it is mandatory.
    """
    if Utils.getOS() != "Windows":
        scons_path = Execution.getExecutablePath("scons")

        if scons_path is not None:
            return [
                scons_path
            ]

    return [
        _getPythonForSconsExePath(),
        os.path.join(
            _getSconsInlinePath(),
            "bin",
            "scons.py"
        )
    ]

def _getPythonSconsExePathWindows():
    """ Find Python2 on Windows.

    First try a few guesses, the look into registry for user or system wide
    installations of Python2. Both Python 2.6 and 2.7, and 3.5 or higher
    will do.
    """

    # Ordered in the list of preference.
    scons_supported = ("2.7", "2.6", "3.5", "3.6", "3.7")

    # Shortcuts for the default installation directories, to avoid going to
    # registry at all unless necessary. Any Python2 will do for Scons, so it
    # might be avoided entirely.
    for search in scons_supported:
        candidate = r"c:\Python%s\python.exe" % search.replace(".", "")

        if os.path.isfile(candidate):
            return candidate

    # Windows only code, pylint: disable=I0021,import-error,undefined-variable
    if python_version < 300:
        import _winreg as winreg # @UnresolvedImport pylint: disable=I0021,import-error,no-name-in-module
    else:
        import winreg  # @Reimport @UnresolvedImport pylint: disable=I0021,import-error,no-name-in-module

    for search in scons_supported:
        for hkey_branch in (winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER):
            for arch_key in 0, winreg.KEY_WOW64_32KEY, winreg.KEY_WOW64_64KEY:
                try:
                    key = winreg.OpenKey(
                        hkey_branch,
                        r"SOFTWARE\Python\PythonCore\%s\InstallPath" % search,
                        0,
                        winreg.KEY_READ | arch_key
                    )

                    return os.path.join(
                        winreg.QueryValue(key, ""),
                        "python.exe"
                    )
                except WindowsError:  # @UndefinedVariable
                    pass

def _getPythonForSconsExePath():
    """ Find a way to call any Python2.

    Scons needs it as it doesn't support Python3.
    """
    python_exe = Options.getPythonPathForScons()

    if python_exe is not None:
        return python_exe

    if python_version < 300 or python_version >= 350:
        return sys.executable
    elif Utils.getOS() == "Windows":
        python_exe = _getPythonSconsExePathWindows()

        if python_exe is not None:
            return python_exe
        else:
            sys.exit("""\
Error, while Nuitka works with Python 3.2 to 3.4, scons does not, and Nuitka
needs to find a Python executable 2.6/2.7 or 3.5 or higher. Simply under the
C:\\PythonXY, e.g. C:\\Python27 to execute the scons utility which is used
to build the C files to binary.

You may provide it using option "--python-for-scons=path_to_python.exe"
in case it is not visible in registry, e.g. due to using uninstalled
AnaConda Python.
""")

    for version_candidate in ("2.7", "2.6", "3.5", "3.6"):
        candidate = Execution.getExecutablePath("python" + version_candidate)

        if candidate is not None:
            return candidate

    # Lets be optimistic, this is most often going to be new enough or a
    # Python2 variant.
    return "python"

@contextlib.contextmanager
def _setupSconsEnvironment():
    """ Setup the scons execution environment.

    For the scons inline copy on Windows needs to find the library, using
    the "SCONS_LIB_DIR" environment variable "NUITKA_SCONS". And for the
    target Python we provide "NUITKA_PYTHON_DLL_PATH" to see where the
    Python DLL lives, in case it needs to be copied, and then the
    "NUITKA_PYTHON_EXE_PATH" to find the Python binary itself.
    """

    if Utils.getOS() == "Windows":
        # On Windows this Scons variable must be set by us.
        os.environ["SCONS_LIB_DIR"] = os.path.join(
            _getSconsInlinePath(),
            "lib",
            "scons-2.3.2"
        )

        # On Windows, we use the Python.DLL path for some things. We pass it
        # via environment variable
        os.environ["NUITKA_PYTHON_DLL_PATH"] = getTargetPythonDLLPath()

    os.environ["NUITKA_PYTHON_EXE_PATH"] = sys.executable

    # Remove environment variables that can only harm if we have to switch
    # major Python versions, these cannot help Python2 to execute scons, this
    # is a bit of noise, but helpful.
    if python_version >= 300:
        if "PYTHONPATH" in os.environ:
            old_pythonpath = os.environ["PYTHONPATH"]
            del os.environ["PYTHONPATH"]
        else:
            old_pythonpath = None

        if "PYTHONHOME" in os.environ:
            old_pythonhome = os.environ["PYTHONHOME"]
            del os.environ["PYTHONHOME"]
        else:
            old_pythonhome = None

    yield

    if python_version >= 300:
        if old_pythonpath is not None:
            os.environ["PYTHONPATH"] = old_pythonpath

        if old_pythonhome is not None:
            os.environ["PYTHONHOME"] = old_pythonhome

    if Utils.getOS() == "Windows":
        del os.environ["NUITKA_PYTHON_DLL_PATH"]
        del os.environ["NUITKA_PYTHON_EXE_PATH"]

def _buildSconsCommand(quiet, options):
    """ Build the scons command to run.

    The options are a dictionary to be passed to scons as a command line,
    and other scons stuff is set.
    """

    scons_command = _getSconsBinaryCall()

    if quiet:
        scons_command.append("--quiet")

    scons_command += [
        # The scons file
        "-f",
        os.path.join(
            getSconsDataPath(),
            "SingleExe.scons"
        ),

        # Parallel compilation.
        "--jobs",
        str(Options.getJobLimit()),

        # Do not warn about deprecation from Scons
        "--warn=no-deprecated",

        # Don't load "site_scons" at all.
        "--no-site-dir",
    ]

    if Options.isShowScons():
        scons_command.append("--debug=explain")

    # Python2, encoding unicode values
    def encode(value):
        if str is bytes and type(value) is unicode:
            return value.encode("utf8")
        else:
            return value

    # Option values to provide to scons. Find these in the caller.
    for key, value in options.items():
        scons_command.append(
            key + '=' + encode(value)
        )

    # Python2, make argument encoding recognizable.
    if str is bytes:
        scons_command.append(
            "arg_encoding=utf8"
        )

    return scons_command

def runScons(options, quiet):
    with _setupSconsEnvironment():
        scons_command = _buildSconsCommand(quiet, options)

        if Options.isShowScons():
            Tracing.printLine("Scons command:", ' '.join(scons_command))

        Tracing.flushStdout()
        return subprocess.call(scons_command, shell = False) == 0
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8

""" This is a service node (server) to control motors an the minibot
(https://minibot.me). It controls the motors via I2C on a Raspberry Pi.
I expects the messsages "FORWARD, BACKWARD, LEFT, RIGHT, STOP".

Author:  Markus Knapp, 2018
Website: https://direcs.de
"""

# speed of the motors (0-255)
startSpeed = 75
turnSpeed = 120

# name of the package(!).srv
from minibot.srv import *
import rospy

# Service nodes have to be initialised
rospy.init_node('motor_server', anonymous=False)

# for getting the hostname of the underlying system
import socket
# showing hostname
hostname = socket.gethostname()
if hostname == 'minibot':
    rospy.loginfo("Running on host %s.", hostname)
else:
    rospy.logwarn("Running on host %s!", hostname)

###############################
###### motor stuff
###############################
# run some parts only on the real robot
if hostname == 'minibot':
    rospy.loginfo("Setting up I2C...")

    from Adafruit_MotorHAT import Adafruit_MotorHAT, Adafruit_DCMotor

    # create a default motor object, no changes to I2C address or frequency
    mh = Adafruit_MotorHAT(addr=0x60)

    # using motor 1 and 2 on RasPi hat
    myMotor1 = mh.getMotor(1)
    myMotor2 = mh.getMotor(2)
    myMotor3 = mh.getMotor(3)
    myMotor4 = mh.getMotor(4)

    # recommended for auto-disabling motors on shutdown!
    def turnOffMotors():
        myMotor1.run(Adafruit_MotorHAT.RELEASE);
        myMotor2.run(Adafruit_MotorHAT.RELEASE);
        myMotor3.run(Adafruit_MotorHAT.RELEASE);
        myMotor4.run(Adafruit_MotorHAT.RELEASE);

    # turning off motors NOW
    turnOffMotors();

    # set the speed (from 0 (off) to 255 (max speed))
    myMotor1.setSpeed(startSpeed)
    myMotor2.setSpeed(startSpeed)
    myMotor3.setSpeed(startSpeed)
    myMotor4.setSpeed(startSpeed)
else:
    rospy.logwarn("Skipping I2C setup. This is not the robot.")

# define a clean node exit
def my_exit():
  rospy.loginfo("Shutting down motor service...")
  # run some parts only on the real robot
  if hostname == 'minibot':
      turnOffMotors();
  rospy.loginfo("...shutting down motor service complete.")

# call this method on node exit
rospy.on_shutdown(my_exit)

# handle_motor is called with instances of MotorRequest and returns instances of MotorResponse
# The request name comes directly from the .srv filename
def handle_motor(req):
    """ In this function all the work is done :) """

    # switch xxx to HIGH, if '1' was sent
    if (req.direction == "FORWARD"): # and speed. returns result.
        # drive
        rospy.loginfo("Driving forward.")
        if hostname == 'minibot':
            # resetting speed to default
            myMotor1.setSpeed(startSpeed)
            myMotor2.setSpeed(startSpeed)
            myMotor3.setSpeed(startSpeed)
            myMotor4.setSpeed(startSpeed)
            # @todo: increase speed?
            myMotor1.run(Adafruit_MotorHAT.FORWARD)
            myMotor2.run(Adafruit_MotorHAT.BACKWARD)
            myMotor3.run(Adafruit_MotorHAT.BACKWARD)
            myMotor4.run(Adafruit_MotorHAT.FORWARD)
    elif (req.direction == "BACKWARD"):
        rospy.loginfo("Driving backwards.")
        if hostname == 'minibot':
            # resetting speed to default
            myMotor1.setSpeed(startSpeed)
            myMotor2.setSpeed(startSpeed)
            myMotor3.setSpeed(startSpeed)
            myMotor4.setSpeed(startSpeed)
            # @todo: increase speed?
            myMotor1.run(Adafruit_MotorHAT.BACKWARD)
            myMotor2.run(Adafruit_MotorHAT.FORWARD)
            myMotor3.run(Adafruit_MotorHAT.FORWARD)
            myMotor4.run(Adafruit_MotorHAT.BACKWARD)
    elif (req.direction == "LEFT"):
        rospy.loginfo("Driving left.")
        if hostname == 'minibot':
            # adding more power when turning
            myMotor1.setSpeed(turnSpeed)
            myMotor2.setSpeed(turnSpeed)
            myMotor3.setSpeed(turnSpeed)
            myMotor4.setSpeed(turnSpeed)
            # @todo: increase speed?
            myMotor1.run(Adafruit_MotorHAT.BACKWARD)
            myMotor2.run(Adafruit_MotorHAT.FORWARD)
            myMotor3.run(Adafruit_MotorHAT.BACKWARD)
            myMotor4.run(Adafruit_MotorHAT.FORWARD)
    elif (req.direction == "RIGHT"):
        rospy.loginfo("Driving right.")
        if hostname == 'minibot':
            # adding more power when turning
            myMotor1.setSpeed(turnSpeed)
            myMotor2.setSpeed(turnSpeed)
            myMotor3.setSpeed(turnSpeed)
            myMotor4.setSpeed(turnSpeed)
            # @todo: increase speed?
            myMotor1.run(Adafruit_MotorHAT.FORWARD)
            myMotor2.run(Adafruit_MotorHAT.BACKWARD)
            myMotor3.run(Adafruit_MotorHAT.FORWARD)
            myMotor4.run(Adafruit_MotorHAT.BACKWARD)
    elif (req.direction == "STOP"):
        rospy.loginfo("Stopping.")
        if hostname == 'minibot':
            myMotor1.run(Adafruit_MotorHAT.RELEASE)
            myMotor2.run(Adafruit_MotorHAT.RELEASE)
            myMotor3.run(Adafruit_MotorHAT.RELEASE)
            myMotor4.run(Adafruit_MotorHAT.RELEASE)
        # @todo: also slow down?
    else:
      rospy.logerr("Direction '%s' not implemented.", req.direction)

    # The name of the 'xyzResponse' comes directly from the Xyz.srv filename!
    # we return the speed as "okay"
    return MotorResponse(req.speed)

def motor_server():
    # This declares a new service named 'motor with the Motor service type.
    # All requests are passed to the 'handle_motor' function.
    # 'handle_motor' is called with instances of MotorRequest and returns instances of MotorResponse
    s = rospy.Service('motor', Motor, handle_motor)
    rospy.loginfo("Ready to switch motors.")

    # Keep our code from exiting until this service node is shutdown
    rospy.spin()

if __name__ == "__main__":
    motor_server()

""" motor stuff to be implemented here:

        rospy.loginfo("Speed up...")
        for i in range(startSpeed, 255):
            myMotor1.setSpeed(i)
            myMotor2.setSpeed(i)
            myMotor3.setSpeed(i)
            myMotor4.setSpeed(i)
            time.sleep(0.01)

        rospy.loginfo("Slow down...")
        for i in range(255, startSpeed, -1):
            myMotor1.setSpeed(i)
            myMotor2.setSpeed(i)
            myMotor3.setSpeed(i)
            myMotor4.setSpeed(i)
            time.sleep(0.01)
"""
<commit_msg>Decreased turnSpeed<commit_after>#!/usr/bin/env python
# coding=utf-8

""" This is a service node (server) to control motors an the minibot
(https://minibot.me). It controls the motors via I2C on a Raspberry Pi.
I expects the messsages "FORWARD, BACKWARD, LEFT, RIGHT, STOP".

Author:  Markus Knapp, 2018
Website: https://direcs.de
"""

# speed of the motors (0-255)
startSpeed = 75
# the speed when turning the bot can be higher if needed (higher friction)
turnSpeed = 90

# name of the package(!).srv
from minibot.srv import *
import rospy

# Service nodes have to be initialised
rospy.init_node('motor_server', anonymous=False)

# for getting the hostname of the underlying system
import socket
# showing hostname
hostname = socket.gethostname()
if hostname == 'minibot':
    rospy.loginfo("Running on host %s.", hostname)
else:
    rospy.logwarn("Running on host %s!", hostname)

###############################
###### motor stuff
###############################
# run some parts only on the real robot
if hostname == 'minibot':
    rospy.loginfo("Setting up I2C...")

    from Adafruit_MotorHAT import Adafruit_MotorHAT, Adafruit_DCMotor

    # create a default motor object, no changes to I2C address or frequency
    mh = Adafruit_MotorHAT(addr=0x60)

    # using motor 1 and 2 on RasPi hat
    myMotor1 = mh.getMotor(1)
    myMotor2 = mh.getMotor(2)
    myMotor3 = mh.getMotor(3)
    myMotor4 = mh.getMotor(4)

    # recommended for auto-disabling motors on shutdown!
    def turnOffMotors():
        myMotor1.run(Adafruit_MotorHAT.RELEASE);
        myMotor2.run(Adafruit_MotorHAT.RELEASE);
        myMotor3.run(Adafruit_MotorHAT.RELEASE);
        myMotor4.run(Adafruit_MotorHAT.RELEASE);

    # turning off motors NOW
    turnOffMotors();

    # set the speed (from 0 (off) to 255 (max speed))
    myMotor1.setSpeed(startSpeed)
    myMotor2.setSpeed(startSpeed)
    myMotor3.setSpeed(startSpeed)
    myMotor4.setSpeed(startSpeed)
else:
    rospy.logwarn("Skipping I2C setup. This is not the robot.")

# define a clean node exit
def my_exit():
  rospy.loginfo("Shutting down motor service...")
  # run some parts only on the real robot
  if hostname == 'minibot':
      turnOffMotors();
  rospy.loginfo("...shutting down motor service complete.")

# call this method on node exit
rospy.on_shutdown(my_exit)

# handle_motor is called with instances of MotorRequest and returns instances of MotorResponse
# The request name comes directly from the .srv filename
def handle_motor(req):
    """ In this function all the work is done :) """

    # switch xxx to HIGH, if '1' was sent
    if (req.direction == "FORWARD"): # and speed. returns result.
        # drive
        rospy.loginfo("Driving forward.")
        if hostname == 'minibot':
            # resetting speed to default
            myMotor1.setSpeed(startSpeed)
            myMotor2.setSpeed(startSpeed)
            myMotor3.setSpeed(startSpeed)
            myMotor4.setSpeed(startSpeed)
            # @todo: increase speed?
            myMotor1.run(Adafruit_MotorHAT.FORWARD)
            myMotor2.run(Adafruit_MotorHAT.BACKWARD)
            myMotor3.run(Adafruit_MotorHAT.BACKWARD)
            myMotor4.run(Adafruit_MotorHAT.FORWARD)
    elif (req.direction == "BACKWARD"):
        rospy.loginfo("Driving backwards.")
        if hostname == 'minibot':
            # resetting speed to default
            myMotor1.setSpeed(startSpeed)
            myMotor2.setSpeed(startSpeed)
            myMotor3.setSpeed(startSpeed)
            myMotor4.setSpeed(startSpeed)
            # @todo: increase speed?
            myMotor1.run(Adafruit_MotorHAT.BACKWARD)
            myMotor2.run(Adafruit_MotorHAT.FORWARD)
            myMotor3.run(Adafruit_MotorHAT.FORWARD)
            myMotor4.run(Adafruit_MotorHAT.BACKWARD)
    elif (req.direction == "LEFT"):
        rospy.loginfo("Driving left.")
        if hostname == 'minibot':
            # adding more power when turning
            myMotor1.setSpeed(turnSpeed)
            myMotor2.setSpeed(turnSpeed)
            myMotor3.setSpeed(turnSpeed)
            myMotor4.setSpeed(turnSpeed)
            # @todo: increase speed?
            myMotor1.run(Adafruit_MotorHAT.BACKWARD)
            myMotor2.run(Adafruit_MotorHAT.FORWARD)
            myMotor3.run(Adafruit_MotorHAT.BACKWARD)
            myMotor4.run(Adafruit_MotorHAT.FORWARD)
    elif (req.direction == "RIGHT"):
        rospy.loginfo("Driving right.")
        if hostname == 'minibot':
            # adding more power when turning
            myMotor1.setSpeed(turnSpeed)
            myMotor2.setSpeed(turnSpeed)
            myMotor3.setSpeed(turnSpeed)
            myMotor4.setSpeed(turnSpeed)
            # @todo: increase speed?
            myMotor1.run(Adafruit_MotorHAT.FORWARD)
            myMotor2.run(Adafruit_MotorHAT.BACKWARD)
            myMotor3.run(Adafruit_MotorHAT.FORWARD)
            myMotor4.run(Adafruit_MotorHAT.BACKWARD)
    elif (req.direction == "STOP"):
        rospy.loginfo("Stopping.")
        if hostname == 'minibot':
            myMotor1.run(Adafruit_MotorHAT.RELEASE)
            myMotor2.run(Adafruit_MotorHAT.RELEASE)
            myMotor3.run(Adafruit_MotorHAT.RELEASE)
            myMotor4.run(Adafruit_MotorHAT.RELEASE)
        # @todo: also slow down?
    else:
      rospy.logerr("Direction '%s' not implemented.", req.direction)

    # The name of the 'xyzResponse' comes directly from the Xyz.srv filename!
    # we return the speed as "okay"
    return MotorResponse(req.speed)

def motor_server():
    # This declares a new service named 'motor with the Motor service type.
    # All requests are passed to the 'handle_motor' function.
    # 'handle_motor' is called with instances of MotorRequest and returns instances of MotorResponse
    s = rospy.Service('motor', Motor, handle_motor)
    rospy.loginfo("Ready to switch motors.")

    # Keep our code from exiting until this service node is shutdown
    rospy.spin()

if __name__ == "__main__":
    motor_server()

""" motor stuff to be implemented here:

        rospy.loginfo("Speed up...")
        for i in range(startSpeed, 255):
            myMotor1.setSpeed(i)
            myMotor2.setSpeed(i)
            myMotor3.setSpeed(i)
            myMotor4.setSpeed(i)
            time.sleep(0.01)

        rospy.loginfo("Slow down...")
        for i in range(255, startSpeed, -1):
            myMotor1.setSpeed(i)
            myMotor2.setSpeed(i)
            myMotor3.setSpeed(i)
            myMotor4.setSpeed(i)
            time.sleep(0.01)
"""
<|endoftext|>


<commit_before>"""
Node code for slaves.
"""

import py
from py.__.test.rsession.executor import RunExecutor, BoxExecutor, AsyncExecutor
from py.__.test.rsession.outcome import Outcome
from py.__.test.outcome import Skipped
import thread
import os

class PidInfo(object):
    """ Pure container class to store information of actually running
    pid
    """
    def __init__(self):
        self.pid = 0
        self.lock = thread.allocate_lock()

    def set_pid(self, pid):
        self.lock.acquire()
        try:
            self.pid = pid
        finally:
            self.lock.release()

    def kill(self):
        self.lock.acquire()
        try:
            if self.pid:
                os.kill(self.pid, 15)
                self.pid = 0
        finally:
            self.lock.release()

    def waitandclear(self, pid, num):
        """ This is an obscure hack to keep locking properly, adhere to posix semantics
        and try to clean it as much as possible, not clean at all
        """
        self.lock.acquire()
        try:
            retval = os.waitpid(self.pid, 0)
            self.pid = 0
            return retval
        finally:
            self.lock.release()

class SlaveNode(object):
    def __init__(self, config, pidinfo, executor=AsyncExecutor):
        #self.rootcollector = rootcollector
        self.config = config
        self.executor = executor
        self.pidinfo = pidinfo

    def execute(self, itemspec):
        item = self.config._getcollector(itemspec)
        ex = self.executor(item, config=self.config)
        if self.executor is AsyncExecutor:
            cont, pid = ex.execute()
            self.pidinfo.set_pid(pid)
        else:
            # for tests only
            return ex.execute()
        return cont(self.pidinfo.waitandclear)

    def run(self, itemspec):
        #outcome = self.execute(itemspec)
        #return outcome.make_repr()
        outcome = self.execute(itemspec)
        if self.executor.wraps:
            return outcome
        else:
            return outcome.make_repr(self.config.option.tbstyle)

def slave_main(receive, send, path, config, pidinfo):
    import os
    assert os.path.exists(path)
    path = os.path.abspath(path)
    nodes = {}
    def getnode(item):
        node = nodes.get(item[0], None)
        if node is not None:
            return node
        col = py.test.collect.Directory(str(py.path.local(path).join(item[0])))
        node = nodes[item[0]] = SlaveNode(config, pidinfo)
        return node
    while 1:
        nextitem = receive()
        if nextitem is None:
            break
        try:
            node = getnode(nextitem)
            res = node.run(nextitem)
        except Skipped, s:
            send(Outcome(skipped=str(s)).make_repr())
        except:
            excinfo = py.code.ExceptionInfo()
            send(Outcome(excinfo=excinfo, is_critical=True).make_repr())
        else:
            if not res[0] and not res[3] and config.option.exitfirst:
                # we're finished, but need to eat what we can
                send(res)
                break
            send(res)

    while nextitem is not None:
        nextitem = receive()

defaultconftestnames = ['dist_nicelevel']
def setup_slave(host, config):
    channel = host.gw.remote_exec(str(py.code.Source(setup, "setup()")))
    configrepr = config.make_repr(defaultconftestnames)
    #print "sending configrepr", configrepr
    channel.send(host.gw_remotepath)
    channel.send(configrepr)
    return channel

def setup():
    def callback_gen(channel, queue, info):
        def callback(item):
            if item == 42: # magic call-cleanup
                # XXX should kill a pid here
                info.kill()
                channel.close()
                sys.exit(0)
            queue.put(item)
        return callback
    # our current dir is the topdir
    import os, sys
    basedir = channel.receive()
    config_repr = channel.receive()
    # setup defaults...
    sys.path.insert(0, basedir)
    import py
    config = py.test.config
    config.option.boxed = True
    assert not config._initialized
    config.initdirect(basedir, config_repr)
    if not config.option.nomagic:
        py.magic.invoke(assertion=1)
    from py.__.test.rsession.slave import slave_main, PidInfo
    queue = py.std.Queue.Queue()
    pidinfo = PidInfo()
    channel.setcallback(callback_gen(channel, queue, pidinfo))
    slave_main(queue.get, channel.send, basedir, config, pidinfo)
    if not config.option.nomagic:
        py.magic.revoke(assertion=1)
    channel.close()
<commit_msg>[svn r38101] oops, this will be implied anyway<commit_after>"""
Node code for slaves.
"""

import py
from py.__.test.rsession.executor import RunExecutor, BoxExecutor, AsyncExecutor
from py.__.test.rsession.outcome import Outcome
from py.__.test.outcome import Skipped
import thread
import os

class PidInfo(object):
    """ Pure container class to store information of actually running
    pid
    """
    def __init__(self):
        self.pid = 0
        self.lock = thread.allocate_lock()

    def set_pid(self, pid):
        self.lock.acquire()
        try:
            self.pid = pid
        finally:
            self.lock.release()

    def kill(self):
        self.lock.acquire()
        try:
            if self.pid:
                os.kill(self.pid, 15)
                self.pid = 0
        finally:
            self.lock.release()

    def waitandclear(self, pid, num):
        """ This is an obscure hack to keep locking properly, adhere to posix semantics
        and try to clean it as much as possible, not clean at all
        """
        self.lock.acquire()
        try:
            retval = os.waitpid(self.pid, 0)
            self.pid = 0
            return retval
        finally:
            self.lock.release()

class SlaveNode(object):
    def __init__(self, config, pidinfo, executor=AsyncExecutor):
        #self.rootcollector = rootcollector
        self.config = config
        self.executor = executor
        self.pidinfo = pidinfo

    def execute(self, itemspec):
        item = self.config._getcollector(itemspec)
        ex = self.executor(item, config=self.config)
        if self.executor is AsyncExecutor:
            cont, pid = ex.execute()
            self.pidinfo.set_pid(pid)
        else:
            # for tests only
            return ex.execute()
        return cont(self.pidinfo.waitandclear)

    def run(self, itemspec):
        #outcome = self.execute(itemspec)
        #return outcome.make_repr()
        outcome = self.execute(itemspec)
        if self.executor.wraps:
            return outcome
        else:
            return outcome.make_repr(self.config.option.tbstyle)

def slave_main(receive, send, path, config, pidinfo):
    import os
    assert os.path.exists(path)
    path = os.path.abspath(path)
    nodes = {}
    def getnode(item):
        node = nodes.get(item[0], None)
        if node is not None:
            return node
        col = py.test.collect.Directory(str(py.path.local(path).join(item[0])))
        node = nodes[item[0]] = SlaveNode(config, pidinfo)
        return node
    while 1:
        nextitem = receive()
        if nextitem is None:
            break
        try:
            node = getnode(nextitem)
            res = node.run(nextitem)
        except Skipped, s:
            send(Outcome(skipped=str(s)).make_repr())
        except:
            excinfo = py.code.ExceptionInfo()
            send(Outcome(excinfo=excinfo, is_critical=True).make_repr())
        else:
            if not res[0] and not res[3] and config.option.exitfirst:
                # we're finished, but need to eat what we can
                send(res)
                break
            send(res)

    while nextitem is not None:
        nextitem = receive()

defaultconftestnames = ['dist_nicelevel']
def setup_slave(host, config):
    channel = host.gw.remote_exec(str(py.code.Source(setup, "setup()")))
    configrepr = config.make_repr(defaultconftestnames)
    #print "sending configrepr", configrepr
    channel.send(host.gw_remotepath)
    channel.send(configrepr)
    return channel

def setup():
    def callback_gen(channel, queue, info):
        def callback(item):
            if item == 42: # magic call-cleanup
                # XXX should kill a pid here
                info.kill()
                channel.close()
                sys.exit(0)
            queue.put(item)
        return callback
    # our current dir is the topdir
    import os, sys
    basedir = channel.receive()
    config_repr = channel.receive()
    # setup defaults...
    sys.path.insert(0, basedir)
    import py
    config = py.test.config
    assert not config._initialized
    config.initdirect(basedir, config_repr)
    if not config.option.nomagic:
        py.magic.invoke(assertion=1)
    from py.__.test.rsession.slave import slave_main, PidInfo
    queue = py.std.Queue.Queue()
    pidinfo = PidInfo()
    channel.setcallback(callback_gen(channel, queue, pidinfo))
    slave_main(queue.get, channel.send, basedir, config, pidinfo)
    if not config.option.nomagic:
        py.magic.revoke(assertion=1)
    channel.close()
<|endoftext|>