<commit_before>#!/usr/bin/env python3

import lxc
from random import randint
from subprocess import check_call
from time import sleep
import argparse
import sys

GIT_URL = "https://github.com/wlanslovenija/tunneldigger"
GIT_REV = "4e4f13cdc630c46909d47441093a5bdaffa0d67f"

GIT_URL = "https://github.com/lynxis/tunneldigger"

def setup_template():
    """ all test container are cloned from this one
    it's important that this container is *NOT* running!
    """
    container = lxc.Container("tunneldigger-base")

    if not container.defined:
        if not container.create("download", lxc.LXC_CREATE_QUIET, {"dist": "ubuntu",
                                                                   "release": "trusty",
                                                                   "arch": "amd64"}):
            raise RuntimeError("failed to create container")

    if not container.running:
        if not container.start():
            raise RuntimeError("failed to start container")

    container.attach_wait(lxc.attach_run_command, ["dhclient", "eth0"])
    container.attach_wait(lxc.attach_run_command, ["apt-get", "update"])
    container.attach_wait(lxc.attach_run_command, ["apt-get", "dist-upgrade", "-y"])

    # tunneldigger requirements
    pkg_to_install = [
        "iproute",
        "bridge-utils",
        "libnetfilter-conntrack3",
        "python-dev",
        "libevent-dev",
        "ebtables",
        "python-virtualenv",
        "build-essential",
        "libnl-dev",
        "linux-libc-dev",
        ]
    pkg_to_install += [
        "wget",
        "curl",
        "git",
        "iputils-ping"
        ]
    # for testing the connection
    pkg_to_install += [
        "lighttpd"
        ]

    container.attach_wait(lxc.attach_run_command, ["apt-get", "install", "-y"] + pkg_to_install)
    container.shutdown(30)

def get_random_context():
    """ return a random hex similiar to mktemp, but do not check is already used """
    hexi = randint(0, 2**32)
    hexi = hex(hexi)[2:]
    return hexi

def configure_network(container, bridge, is_server):
    """ configure the container and connect them to the bridge
    container is a lxc container
    hexi is the hex for the bridge """
    config = [
        ('lxc.network.1.type', 'veth'),
        ('lxc.network.1.link', bridge),
        ('lxc.network.1.flags', 'up'),
        ]
    if is_server:
        config.append(
            ('lxc.network.1.ipv4', '172.16.16.1/24'),
            )
    else:
        config.append(
            ('lxc.network.1.ipv4', '172.16.16.2/24'),
            )

    for item in config:
        container.set_config_item(item[0], item[1])

def configure_mounts(container):
    container.append_config_item('lxc.mount.entry', '/usr/src usr/src none bind,ro 0 0')

def create_bridge(name):
    """ setup a linux bridge device """
    check_call(["brctl", "addbr", name], timeout=10)
    check_call(["ip", "link", "set", name, "up"], timeout=10)

    # FIXME: lxc_container: confile.c: network_netdev: 474 no network device defined for 'lxc.network.1.link' = 'br-46723922' option
    sleep(3)

def check_internet(container, tries):
    """ check the internet connectivity inside the container """
    for i in range(0, tries):
        ret = container.attach_wait(lxc.attach_run_command, "ping -c 1 -W 1 8.8.8.8".split())
        if ret == 0:
            return True
        sleep(1)
    return False

def testing(client_rev, server_rev):
    """ this does the real test.
    - cloning containers from tunneldigger-base
    - setup network
    - checkout git repos
    - execute "compiler" steps
    - start services
    - do the real tests
    """
    base = lxc.Container("tunneldigger-base")
    if not base.defined:
        raise RuntimeError("Setup first the base container")

    hexi = get_random_context()
    print("generate a run for %s" % hexi)
    server_name = "%s_server" % hexi
    client_name = "%s_client" % hexi
    bridge_name = "br-%s" % hexi
    server = lxc.Container(server_name)
    client = lxc.Container(client_name)

    if base.running:
        raise RuntimeError("base container %s is still running. Please run lxc-stop --name %s -t 5" % (base.name, base.name))

    if server.defined or client.defined:
        raise RuntimeError("server or client container already exist")

    create_bridge(bridge_name)

    server = base.clone(server_name, None, lxc.LXC_CLONE_SNAPSHOT)
    client = base.clone(client_name, None, lxc.LXC_CLONE_SNAPSHOT)

    if not server:
        if client:
            client.destroy()
        raise RuntimeError("could not create server container %s" % server_name)
    if not client:
        if server:
            server.destroy()
        raise RuntimeError("could not create client container %s" % client_name)

    configure_network(server, bridge_name, True)
    configure_network(client, bridge_name, False)

    configure_mounts(client)

    for cont in [client, server]:
        if not cont.start():
          raise RuntimeError("Can not start container %s" % cont.name)
        sleep(3)
        if not check_internet(cont, 10):
            raise RuntimeError("Container doesn't have an internet connection %s" % cont.name)

    spid = run_server(server, server_rev)
    cpid = run_client(client, client_rev)

    sleep(10)

    run_tests(server, client)

def run_server(server, git_rev):
    """ run_server(server)
    server is a container
    """
    server.attach_wait(lxc.attach_run_command, ["git", "clone", GIT_URL, '/srv/tunneldigger/'])
    server.attach_wait(lxc.attach_run_command, ["git", "--git-dir", "/srv/tunneldigger/.git", "--work-tree",
        "/srv/tunneldigger/", "checkout", git_rev, '/srv/tunneldigger/'])
    spid = server.attach(lxc.attach_run_command, ['/srv/tunneldigger/broker/contrib/testrun'])
    return spid

def run_client(client, git_rev):
    """ run_client(client)
    client is a container
    """
    client.attach_wait(lxc.attach_run_command, ["git", "clone", GIT_URL, '/srv/tunneldigger/'])
    client.attach_wait(lxc.attach_run_command, ["git", "--git-dir", "/srv/tunneldigger/.git", "--work-tree",
        "/srv/tunneldigger/", "checkout", git_rev, '/srv/tunneldigger/'])
    cpid = client.attach(lxc.attach_run_command, ['/srv/tunneldigger/client/contrib/testrun'])
    return cpid

def run_tests(server, client):
    """ the client should be already connect to the server """
    ret = client.attach_wait(lxc.attach_run_command, ["wget", "-t", "2", "-T", "4", "http://192.168.254.1:8080/test-data", '-O', '/dev/null'])
    if ret != 0:
        raise RuntimeError("failed to run the tests")

def clean_up():
    """ clean the up all bridge and containers created by this scripts. It will also abort all running tests."""
    pass

def check_host():
    """ check if the host has all known requirements to run this script """
    have_brctl = False

    try:
        check_call(["brctl", "--version"], timeout=3)
        have_brctl = True
    except Exception:
        pass

    if have_brctl:
        sys.stderr.write("No brctl installed\n")

    if have_brctl:
        print("Everything is installed")
        return True
    raise RuntimeError("Missing dependencies. See stderr for more info")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Test Tunneldigger version against each other")
    # operation on the hosts
    parser.add_argument('--check-host', dest='check_host', action='store_true', default=False,
            help="Check if the host has all requirements installed")
    parser.add_argument('--setup', dest='setup', action='store_true', default=False,
            help="Setup the basic template. Must run once before doing the tests.")
    # testing arguments
    parser.add_argument('-t', '--test', dest='test', action='store_true', default=False,
            help="Do a test run. Server rev and Client rev required. See -s and -c.")
    parser.add_argument('-s', '--server', dest='server', type=str,
            help="The revision used by the server")
    parser.add_argument('-c', '--client', dest='client', type=str,
            help="The revision used by the client")
    # clean up
    parser.add_argument('--clean', action='store_true', default=False,
            help="Clean up (old) containers and bridges. This will kill all running tests!")

    args = parser.parse_args()
    print(args)

    if args.check_host:
        check_host()

    if args.setup:
        setup_template()

    if args.test:
        if not args.server or not args.client:
            raise RuntimeError("No client or server revision given. E.g. --test --server aba123 --client aba123.")
        testing(args.client, args.server)

    if args.clean:
        clean_up()
<commit_msg>add TODO comment to clean_up()<commit_after>#!/usr/bin/env python3

import lxc
from random import randint
from subprocess import check_call
from time import sleep
import argparse
import sys

GIT_URL = "https://github.com/wlanslovenija/tunneldigger"
GIT_REV = "4e4f13cdc630c46909d47441093a5bdaffa0d67f"

GIT_URL = "https://github.com/lynxis/tunneldigger"

def setup_template():
    """ all test container are cloned from this one
    it's important that this container is *NOT* running!
    """
    container = lxc.Container("tunneldigger-base")

    if not container.defined:
        if not container.create("download", lxc.LXC_CREATE_QUIET, {"dist": "ubuntu",
                                                                   "release": "trusty",
                                                                   "arch": "amd64"}):
            raise RuntimeError("failed to create container")

    if not container.running:
        if not container.start():
            raise RuntimeError("failed to start container")

    container.attach_wait(lxc.attach_run_command, ["dhclient", "eth0"])
    container.attach_wait(lxc.attach_run_command, ["apt-get", "update"])
    container.attach_wait(lxc.attach_run_command, ["apt-get", "dist-upgrade", "-y"])

    # tunneldigger requirements
    pkg_to_install = [
        "iproute",
        "bridge-utils",
        "libnetfilter-conntrack3",
        "python-dev",
        "libevent-dev",
        "ebtables",
        "python-virtualenv",
        "build-essential",
        "libnl-dev",
        "linux-libc-dev",
        ]
    pkg_to_install += [
        "wget",
        "curl",
        "git",
        "iputils-ping"
        ]
    # for testing the connection
    pkg_to_install += [
        "lighttpd"
        ]

    container.attach_wait(lxc.attach_run_command, ["apt-get", "install", "-y"] + pkg_to_install)
    container.shutdown(30)

def get_random_context():
    """ return a random hex similiar to mktemp, but do not check is already used """
    hexi = randint(0, 2**32)
    hexi = hex(hexi)[2:]
    return hexi

def configure_network(container, bridge, is_server):
    """ configure the container and connect them to the bridge
    container is a lxc container
    hexi is the hex for the bridge """
    config = [
        ('lxc.network.1.type', 'veth'),
        ('lxc.network.1.link', bridge),
        ('lxc.network.1.flags', 'up'),
        ]
    if is_server:
        config.append(
            ('lxc.network.1.ipv4', '172.16.16.1/24'),
            )
    else:
        config.append(
            ('lxc.network.1.ipv4', '172.16.16.2/24'),
            )

    for item in config:
        container.set_config_item(item[0], item[1])

def configure_mounts(container):
    container.append_config_item('lxc.mount.entry', '/usr/src usr/src none bind,ro 0 0')

def create_bridge(name):
    """ setup a linux bridge device """
    check_call(["brctl", "addbr", name], timeout=10)
    check_call(["ip", "link", "set", name, "up"], timeout=10)

    # FIXME: lxc_container: confile.c: network_netdev: 474 no network device defined for 'lxc.network.1.link' = 'br-46723922' option
    sleep(3)

def check_internet(container, tries):
    """ check the internet connectivity inside the container """
    for i in range(0, tries):
        ret = container.attach_wait(lxc.attach_run_command, "ping -c 1 -W 1 8.8.8.8".split())
        if ret == 0:
            return True
        sleep(1)
    return False

def testing(client_rev, server_rev):
    """ this does the real test.
    - cloning containers from tunneldigger-base
    - setup network
    - checkout git repos
    - execute "compiler" steps
    - start services
    - do the real tests
    """
    base = lxc.Container("tunneldigger-base")
    if not base.defined:
        raise RuntimeError("Setup first the base container")

    hexi = get_random_context()
    print("generate a run for %s" % hexi)
    server_name = "%s_server" % hexi
    client_name = "%s_client" % hexi
    bridge_name = "br-%s" % hexi
    server = lxc.Container(server_name)
    client = lxc.Container(client_name)

    if base.running:
        raise RuntimeError("base container %s is still running. Please run lxc-stop --name %s -t 5" % (base.name, base.name))

    if server.defined or client.defined:
        raise RuntimeError("server or client container already exist")

    create_bridge(bridge_name)

    server = base.clone(server_name, None, lxc.LXC_CLONE_SNAPSHOT)
    client = base.clone(client_name, None, lxc.LXC_CLONE_SNAPSHOT)

    if not server:
        if client:
            client.destroy()
        raise RuntimeError("could not create server container %s" % server_name)
    if not client:
        if server:
            server.destroy()
        raise RuntimeError("could not create client container %s" % client_name)

    configure_network(server, bridge_name, True)
    configure_network(client, bridge_name, False)

    configure_mounts(client)

    for cont in [client, server]:
        if not cont.start():
          raise RuntimeError("Can not start container %s" % cont.name)
        sleep(3)
        if not check_internet(cont, 10):
            raise RuntimeError("Container doesn't have an internet connection %s" % cont.name)

    spid = run_server(server, server_rev)
    cpid = run_client(client, client_rev)

    sleep(10)

    run_tests(server, client)

def run_server(server, git_rev):
    """ run_server(server)
    server is a container
    """
    server.attach_wait(lxc.attach_run_command, ["git", "clone", GIT_URL, '/srv/tunneldigger/'])
    server.attach_wait(lxc.attach_run_command, ["git", "--git-dir", "/srv/tunneldigger/.git", "--work-tree",
        "/srv/tunneldigger/", "checkout", git_rev, '/srv/tunneldigger/'])
    spid = server.attach(lxc.attach_run_command, ['/srv/tunneldigger/broker/contrib/testrun'])
    return spid

def run_client(client, git_rev):
    """ run_client(client)
    client is a container
    """
    client.attach_wait(lxc.attach_run_command, ["git", "clone", GIT_URL, '/srv/tunneldigger/'])
    client.attach_wait(lxc.attach_run_command, ["git", "--git-dir", "/srv/tunneldigger/.git", "--work-tree",
        "/srv/tunneldigger/", "checkout", git_rev, '/srv/tunneldigger/'])
    cpid = client.attach(lxc.attach_run_command, ['/srv/tunneldigger/client/contrib/testrun'])
    return cpid

def run_tests(server, client):
    """ the client should be already connect to the server """
    ret = client.attach_wait(lxc.attach_run_command, ["wget", "-t", "2", "-T", "4", "http://192.168.254.1:8080/test-data", '-O', '/dev/null'])
    if ret != 0:
        raise RuntimeError("failed to run the tests")

def clean_up():
    """ clean the up all bridge and containers created by this scripts. It will also abort all running tests."""
    # TODO: implement this
    pass

def check_host():
    """ check if the host has all known requirements to run this script """
    have_brctl = False

    try:
        check_call(["brctl", "--version"], timeout=3)
        have_brctl = True
    except Exception:
        pass

    if have_brctl:
        sys.stderr.write("No brctl installed\n")

    if have_brctl:
        print("Everything is installed")
        return True
    raise RuntimeError("Missing dependencies. See stderr for more info")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Test Tunneldigger version against each other")
    # operation on the hosts
    parser.add_argument('--check-host', dest='check_host', action='store_true', default=False,
            help="Check if the host has all requirements installed")
    parser.add_argument('--setup', dest='setup', action='store_true', default=False,
            help="Setup the basic template. Must run once before doing the tests.")
    # testing arguments
    parser.add_argument('-t', '--test', dest='test', action='store_true', default=False,
            help="Do a test run. Server rev and Client rev required. See -s and -c.")
    parser.add_argument('-s', '--server', dest='server', type=str,
            help="The revision used by the server")
    parser.add_argument('-c', '--client', dest='client', type=str,
            help="The revision used by the client")
    # clean up
    parser.add_argument('--clean', action='store_true', default=False,
            help="Clean up (old) containers and bridges. This will kill all running tests!")

    args = parser.parse_args()
    print(args)

    if args.check_host:
        check_host()

    if args.setup:
        setup_template()

    if args.test:
        if not args.server or not args.client:
            raise RuntimeError("No client or server revision given. E.g. --test --server aba123 --client aba123.")
        testing(args.client, args.server)

    if args.clean:
        clean_up()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
pgoapi - Pokemon Go API
Copyright (c) 2016 tjado <https://github.com/tejado>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
OR OTHER DEALINGS IN THE SOFTWARE.

Author: tjado <https://github.com/tejado>
"""

import os
import re
import json
import struct
import logging
import requests
import argparse
import collections
import pickle
import random

from pgoapi import PGoApi
from pgoapi.utilities import f2i, h2f

from google.protobuf.internal import encoder
from geopy.geocoders import GoogleV3
from s2sphere import CellId, LatLng

# for parsing the pokemon id to its name
# can be disabled
#import pykemon

# for searching only
import geopy
from geopy.distance import VincentyDistance
from time import sleep

# for saving to database

from Pokemon import Pokemon
from Account import Account
from Location import Location
from EncounterList import EncounterList
from datetime import datetime
from pytz import timezone
import pytz

log = logging.getLogger(__name__)

def get_pos_by_name(location_name):
    geolocator = GoogleV3()
    loc = geolocator.geocode(location_name)

    log.info('Your given location: %s', loc.address.encode('utf-8'))
    log.info('lat/long/alt: %s %s %s', loc.latitude, loc.longitude, loc.altitude)

    return (loc.latitude, loc.longitude, loc.altitude)

def get_cellid(lat, long):
    origin = CellId.from_lat_lng(LatLng.from_degrees(lat, long)).parent(15)
    walk = [origin.id()]

    # 10 before and 10 after
    next = origin.next()
    prev = origin.prev()
    for i in range(10):
        walk.append(prev.id())
        walk.append(next.id())
        next = next.next()
        prev = prev.prev()
    return ''.join(map(encode, sorted(walk)))

def encode(cellid):
    output = []
    encoder._VarintEncoder()(output.append, cellid)
    return ''.join(output)

def init_config():
    account, location = None, None
    account = Account.query.filter(Account.isUsed == 0).first_or_404()
    location = Location.query.filter(Location.isOn == 0).first_or_404()
    account.set()
    account.save()
    location.setOn()
    location.save()
    return account, location

def main(account, location):
    #client = pykemon.V1Client()
    # log settings
    # log format
    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s [%(module)10s] [%(levelname)5s] %(message)s')
    # log level for http request class
    logging.getLogger("requests").setLevel(logging.WARNING)
    # log level for main pgoapi class
    logging.getLogger("pgoapi").setLevel(logging.INFO)
    # log level for internal pgoapi class
    logging.getLogger("rpc_api").setLevel(logging.INFO)

    start_lat, start_lon = location.getLocation()
    username, password, auth = account.getInfo()
    #[username, password, auth, lat, lng]
    position = (start_lat, start_lon, 0.0)
    api = PGoApi()

    # provide player position on the earth
    api.set_position(*position)

    if not api.login(auth, username, password):
        account.disable()
        account.save()
        return

    # chain subrequests (methods) into one RPC call
    # get player profile call
    #api.get_player()

    # get inventory call
    #api.get_inventory()

    # get map objects call
    timestamp = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
    cellid = get_cellid(position[0], position[1])

    # try:
    #     ENCOUNTER_ID_LIST  = load_data('./ENCOUNTER_ID_LIST.data')
    # except:
    #     ENCOUNTER_ID_LIST = []
    # ENCOUNTER_ID_LIST = collections.deque(ENCOUNTER_ID_LIST)
    for i in xrange(36):
        bearing = random.randint(0, 90)
        step_length = random.uniform(0.01, 0.02)
        next_lat, next_lon = move(start_lat, start_lon, step_length, bearing)
        #print next_lat, next_lon
        position = (next_lat, next_lon, 0.0)
        api.set_position(*position)
        api.get_map_objects(latitude=f2i(position[0]), longitude=f2i(position[1]), since_timestamp_ms=timestamp, cell_id=cellid)

        # get download settings call
        #api.download_settings(hash="4a2e9bc330dae60e7b74fc85b98868ab4700802e")

        # execute the RPC call
        try:
            response_dict = api.call()
            map_cells = response_dict['responses']['GET_MAP_OBJECTS']['map_cells']
            for cell in map_cells:
                # if "nearby_pokemons" in cell:
                #     print "nearby_pokemons"
                #     print cell['nearby_pokemons']
                if "catchable_pokemons" in cell:
                    #print "catchable_pokemons"
                    print cell['catchable_pokemons']
                    for pokemon in cell['catchable_pokemons']:
                        pokemon_id = int(pokemon['pokemon_id'])
                        longitude = float(pokemon['longitude'])
                        latitude = float(pokemon['latitude'])
                        encounter_id = str(pokemon['encounter_id'])
                        expiration_timestamp_ms = str(pokemon['expiration_timestamp_ms'])
                        if not EncounterList.query.filter(EncounterList.ID == encounter_id).first():
                            print "GOT ONE!"
                            print "Pokemon ID: ",pokemon_id
                            print longitude
                            print latitude
                            #my_date = datetime.datetime.now(pytz.timezone('US/Eastern'))
                            new_pokemon = Pokemon(pid=pokemon_id, lat=latitude, lng=longitude, encounter_id=encounter_id, report_time=datetime.utcnow())
                            new_pokemon.save()
                            el = EncounterList(ID=encounter_id)
                            el.save()
                            # ENCOUNTER_ID_LIST.append(encounter_id)
                            # if len(ENCOUNTER_ID_LIST) > 5000:
                            #     ENCOUNTER_ID_LIST.popleft()
            start_lat, start_lon = next_lat, next_lon
            sleep(20)
        except Exception, e:
            pass
    # save_data(ENCOUNTER_ID_LIST)

# def save_data(dataset):
#     outputFile = 'ENCOUNTER_ID_LIST.data'
#     fw = open(outputFile, 'w')
#     pickle.dump(dataset, fw)
#     fw.close()

def load_data(filename):
    fr = open(filename)
    dataset = pickle.load(fr)
    return dataset

def move(lat1, lon1, d=0.01, b=0):
    # b = 90 #Bearing is 90 degrees converted to radians.
    # d = 0.005 #Distance in km
    origin = geopy.Point(lat1, lon1)
    destination = VincentyDistance(kilometers=d).destination(origin, b)
    lat2, lon2 = destination.latitude, destination.longitude
    return lat2, lon2

if __name__ == '__main__':

    account, location = init_config()

    try:
        while 1:
            main(account, location)
    except:
        if account.getUsed() != 2:
            account.reset()
            account.save()
        location.setOff()
        location.save()

<commit_msg>prevent infinite loop<commit_after>#!/usr/bin/env python
"""
pgoapi - Pokemon Go API
Copyright (c) 2016 tjado <https://github.com/tejado>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
OR OTHER DEALINGS IN THE SOFTWARE.

Author: tjado <https://github.com/tejado>
"""

import os
import re
import json
import struct
import logging
import requests
import argparse
import collections
import pickle
import random

from pgoapi import PGoApi
from pgoapi.utilities import f2i, h2f

from google.protobuf.internal import encoder
from geopy.geocoders import GoogleV3
from s2sphere import CellId, LatLng

# for parsing the pokemon id to its name
# can be disabled
#import pykemon

# for searching only
import geopy
from geopy.distance import VincentyDistance
from time import sleep

# for saving to database

from Pokemon import Pokemon
from Account import Account
from Location import Location
from EncounterList import EncounterList
from datetime import datetime
from pytz import timezone
import pytz

log = logging.getLogger(__name__)

def get_pos_by_name(location_name):
    geolocator = GoogleV3()
    loc = geolocator.geocode(location_name)

    log.info('Your given location: %s', loc.address.encode('utf-8'))
    log.info('lat/long/alt: %s %s %s', loc.latitude, loc.longitude, loc.altitude)

    return (loc.latitude, loc.longitude, loc.altitude)

def get_cellid(lat, long):
    origin = CellId.from_lat_lng(LatLng.from_degrees(lat, long)).parent(15)
    walk = [origin.id()]

    # 10 before and 10 after
    next = origin.next()
    prev = origin.prev()
    for i in range(10):
        walk.append(prev.id())
        walk.append(next.id())
        next = next.next()
        prev = prev.prev()
    return ''.join(map(encode, sorted(walk)))

def encode(cellid):
    output = []
    encoder._VarintEncoder()(output.append, cellid)
    return ''.join(output)

def init_config():
    account, location = None, None
    account = Account.query.filter(Account.isUsed == 0).first_or_404()
    location = Location.query.filter(Location.isOn == 0).first_or_404()
    account.set()
    account.save()
    location.setOn()
    location.save()
    return account, location

def main(account, location):
    #client = pykemon.V1Client()
    # log settings
    # log format
    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s [%(module)10s] [%(levelname)5s] %(message)s')
    # log level for http request class
    logging.getLogger("requests").setLevel(logging.WARNING)
    # log level for main pgoapi class
    logging.getLogger("pgoapi").setLevel(logging.INFO)
    # log level for internal pgoapi class
    logging.getLogger("rpc_api").setLevel(logging.INFO)

    start_lat, start_lon = location.getLocation()
    username, password, auth = account.getInfo()
    #[username, password, auth, lat, lng]
    position = (start_lat, start_lon, 0.0)
    api = PGoApi()

    # provide player position on the earth
    api.set_position(*position)

    if not api.login(auth, username, password):
        account.disable()
        account.save()
        return

    # chain subrequests (methods) into one RPC call
    # get player profile call
    #api.get_player()

    # get inventory call
    #api.get_inventory()

    # get map objects call
    timestamp = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
    cellid = get_cellid(position[0], position[1])

    # try:
    #     ENCOUNTER_ID_LIST  = load_data('./ENCOUNTER_ID_LIST.data')
    # except:
    #     ENCOUNTER_ID_LIST = []
    # ENCOUNTER_ID_LIST = collections.deque(ENCOUNTER_ID_LIST)
    for i in xrange(36):
        bearing = random.randint(0, 90)
        step_length = random.uniform(0.01, 0.02)
        next_lat, next_lon = move(start_lat, start_lon, step_length, bearing)
        #print next_lat, next_lon
        position = (next_lat, next_lon, 0.0)
        api.set_position(*position)
        api.get_map_objects(latitude=f2i(position[0]), longitude=f2i(position[1]), since_timestamp_ms=timestamp, cell_id=cellid)

        # get download settings call
        #api.download_settings(hash="4a2e9bc330dae60e7b74fc85b98868ab4700802e")

        # execute the RPC call
        try:
            response_dict = api.call()
            map_cells = response_dict['responses']['GET_MAP_OBJECTS']['map_cells']
            for cell in map_cells:
                # if "nearby_pokemons" in cell:
                #     print "nearby_pokemons"
                #     print cell['nearby_pokemons']
                if "catchable_pokemons" in cell:
                    #print "catchable_pokemons"
                    print cell['catchable_pokemons']
                    for pokemon in cell['catchable_pokemons']:
                        pokemon_id = int(pokemon['pokemon_id'])
                        longitude = float(pokemon['longitude'])
                        latitude = float(pokemon['latitude'])
                        encounter_id = str(pokemon['encounter_id'])
                        expiration_timestamp_ms = str(pokemon['expiration_timestamp_ms'])
                        if not EncounterList.query.filter(EncounterList.ID == encounter_id).first():
                            print "GOT ONE!"
                            print "Pokemon ID: ",pokemon_id
                            print longitude
                            print latitude
                            #my_date = datetime.datetime.now(pytz.timezone('US/Eastern'))
                            new_pokemon = Pokemon(pid=pokemon_id, lat=latitude, lng=longitude, encounter_id=encounter_id, report_time=datetime.utcnow())
                            new_pokemon.save()
                            el = EncounterList(ID=encounter_id)
                            el.save()
                            # ENCOUNTER_ID_LIST.append(encounter_id)
                            # if len(ENCOUNTER_ID_LIST) > 5000:
                            #     ENCOUNTER_ID_LIST.popleft()
            start_lat, start_lon = next_lat, next_lon
            sleep(20)
        except Exception, e:
            pass
    # save_data(ENCOUNTER_ID_LIST)

# def save_data(dataset):
#     outputFile = 'ENCOUNTER_ID_LIST.data'
#     fw = open(outputFile, 'w')
#     pickle.dump(dataset, fw)
#     fw.close()

def load_data(filename):
    fr = open(filename)
    dataset = pickle.load(fr)
    return dataset

def move(lat1, lon1, d=0.01, b=0):
    # b = 90 #Bearing is 90 degrees converted to radians.
    # d = 0.005 #Distance in km
    origin = geopy.Point(lat1, lon1)
    destination = VincentyDistance(kilometers=d).destination(origin, b)
    lat2, lon2 = destination.latitude, destination.longitude
    return lat2, lon2

if __name__ == '__main__':

    account, location = init_config()

    try:
        while 1:
            if account.getUsed() == 2:
                break
            main(account, location)
    except:
        if account.getUsed() != 2:
            account.reset()
            account.save()
        location.setOff()
        location.save()

<|endoftext|>


<commit_before>"""Exceptions used throughout package"""

class InstallationError(Exception):
    """General exception during installation"""

class UninstallationError(Exception):
    """General exception during uninstallation"""

class DistributionNotFound(InstallationError):
    """Raised when a distribution cannot be found to satisfy a requirement"""

class BestVersionAlreadyInstalled(Exception):
    """Raised when the most up-to-date version of a package is already
    installed.
    """

class BadCommand(Exception):
    """Raised when virtualenv or a command is not found"""

class CommandError(Exception):
    """Raised when there is an error in command-line arguments"""
<commit_msg>All pip exceptions inherit from PipError<commit_after>"""Exceptions used throughout package"""

class PipError(Exception):
    """Base pip exception"""

class InstallationError(PipError):
    """General exception during installation"""

class UninstallationError(PipError):
    """General exception during uninstallation"""

class DistributionNotFound(InstallationError):
    """Raised when a distribution cannot be found to satisfy a requirement"""

class BestVersionAlreadyInstalled(PipError):
    """Raised when the most up-to-date version of a package is already
    installed.  """

class BadCommand(PipError):
    """Raised when virtualenv or a command is not found"""

class CommandError(PipError):
    """Raised when there is an error in command-line arguments"""
<|endoftext|>


<commit_before>import matplotlib.pyplot as plt

from skimage import data,io
from skimage.color import rgb2hed

#ihc_rgb = data.immunohistochemistry()
ihc_rgb = io.imread('resources/he.jpg')
ihc_hed = rgb2hed(ihc_rgb)

fig, axes = plt.subplots(2, 2, figsize=(7, 6))
ax0, ax1, ax2, ax3 = axes.ravel()

ax0.imshow(ihc_rgb)
ax0.set_title("Original image")

ax1.imshow(ihc_hed[:, :, 0], cmap=plt.cm.gray)
ax1.set_title("Hematoxylin")

ax2.imshow(ihc_hed[:, :, 1], cmap=plt.cm.gray)
ax2.set_title("Eosin")

ax3.imshow(ihc_hed[:, :, 2], cmap=plt.cm.gray)
ax3.set_title("DAB")

for ax in axes.ravel():
    ax.axis('off')

fig.subplots_adjust(hspace=0.3)

import numpy as np

from skimage.exposure import rescale_intensity

# Rescale hematoxylin and DAB signals and give them a fluorescence look
h = rescale_intensity(ihc_hed[:, :, 0], out_range=(0, 1))
d = rescale_intensity(ihc_hed[:, :, 2], out_range=(0, 1))
zdh = np.dstack((np.zeros_like(h), d, h))

fig, ax = plt.subplots()
ax.imshow(zdh)
ax.set_title("Stain separated image (rescaled)")
ax.axis('off')
plt.show()
<commit_msg>Goodbye testin<commit_after><|endoftext|>


<commit_before>import re, sys, threading

from pony import options
from pony.utils import import_module
from pony.sqlsymbols import *

class DBException(Exception):
    def __init__(self, *args, **keyargs):
        exceptions = keyargs.pop('exceptions', [])
        assert not keyargs
        if not args and exceptions:
            if len(exceptions) == 1: args = getattr(exceptions[0], 'args', ())
            else: args = ('Multiple exceptions have occured',)
        Exception.__init__(self, *args)
        self.exceptions = exceptions

class NoDefaultDbException(DBException): pass
class CommitException(DBException): pass
class RollbackException(DBException): pass

class RowNotFound(DBException): pass
class MultipleRowsFound(DBException): pass
class TooManyRowsFound(DBException): pass

##StandardError
##        |__Warning
##        |__Error
##           |__InterfaceError
##           |__DatabaseError
##              |__DataError
##              |__OperationalError
##              |__IntegrityError
##              |__InternalError
##              |__ProgrammingError
##              |__NotSupportedError

class Warning(DBException): pass
class Error(DBException): pass
class   InrefaceError(Error): pass
class   DatabaseError(Error): pass
class     DataError(DatabaseError): pass
class     OperationalError(DatabaseError): pass
class     IntegrityError(DatabaseError): pass
class     InternalError(DatabaseError): pass
class     ProgrammingError(DatabaseError): pass
class     NotSupportedError(DatabaseError): pass

def wrap_dbapi_exceptions(provider, func, *args, **keyargs):
    try: return func(*args, **keyargs)
    except provider.NotSupportedError, e: raise NotSupportedError(exceptions=[e])
    except provider.ProgrammingError, e: raise ProgrammingError(exceptions=[e])
    except provider.InternalError, e: raise InternalError(exceptions=[e])
    except provider.IntegrityError, e: raise IntegrityError(exceptions=[e])
    except provider.OperationalError, e: raise OperationalError(exceptions=[e])
    except provider.DataError, e: raise DataError(exceptions=[e])
    except provider.DatabaseError, e: raise DatabaseError(exceptions=[e])
    except provider.InterfaceError, e: raise InterfaceError(exceptions=[e])
    except provider.Error, e: raise Error(exceptions=[e])
    except provider.Warning, e: raise Warning(exceptions=[e])

class Local(threading.local):
    def __init__(self):
        self.default_db = None
        self.connections = {}

local = Local()

param_re = re.compile(r'[$]([$]|[A-Za-z_]\w*)')

sql_cache = {}

def adapt_sql(sql, paramstyle):
    result = sql_cache.get((sql, paramstyle))
    if result is not None: return result
    args, keyargs = [], {}
    if paramstyle == 'qmark':
        def replace(expr): args.append(expr); return '?'
    elif paramstyle == 'format':
        def replace(expr): args.append(expr); return '%s'
    elif paramstyle == 'numeric':
        def replace(expr): args.append(expr); return ':%d' % len(args)
    elif paramstyle == 'named':
        def replace(expr):
            key = 'param%d' % (len(keyargs) + 1)
            keyargs[key] = expr
            return ':' + key
    elif paramstyle == 'pyformat':
        def replace(expr):
            key = 'param%d' % (len(keyargs) + 1)
            keyargs[key] = expr
            return '%%(%s)s' % key
    else: assert False
    def replace_func(match):
        expr = match.group(1)
        if expr == '$': return '$'
        compile(expr, '<?>', 'eval')
        return replace(expr)
    adapted_sql = param_re.sub(replace_func, sql)
    params = args or keyargs or None
    if args:
        source = '(%s,)' % ', '.join(args)
        code = compile(source, '<?>', 'eval')
    elif keyargs:
        source = '{%s}' % ','.join('%r:%s' % item for item in keyargs.items())
        code = compile(source, '<?>', 'eval')
    else: code = compile('None', '<?>', 'eval')
    result = adapted_sql, code
    sql_cache[(sql, paramstyle)] = result
    return result

select_re = re.compile(r'\s*select\b', re.IGNORECASE)

class Database(object):
    def __init__(self, provider, *args, **keyargs):
        if isinstance(provider, basestring): provider = import_module('pony.dbproviders.' + provider)
        self.provider = provider
        self.args = args
        self.keyargs = keyargs
        self.sql_insert_cache = {}
        self.sql_update_cache = {}
        con = self.get_connection()
        self.release()
    def _get_connection(self):
        x = local.connections.get(self)
        if x is not None: return x[:2]
        provider = self.provider
        con = wrap_dbapi_exceptions(provider, provider.connect, *self.args, **self.keyargs)
        local.connections[self] = con, provider, len(local.connections)
        return con, provider
    def get_connection(self):
        con, provider = self._get_connection()
        return con
    def release(self):
        x = local.connections.pop(self, None)
        if x is None: return
        connection, provider, _ = x
        provider.release(connection)
    def commit(self):
        con, provider = self._get_connection()
        wrap_dbapi_exceptions(provider, con.commit)
    def rollback(self):
        con, provider = self._get_connection()
        wrap_dbapi_exceptions(provider, con.rollback)
    def execute(self, sql, globals=None, locals=None):
        sql = sql[:]  # sql = templating.plainstr(sql)
        if globals is None:
            assert locals is None
            globals = sys._getframe(1).f_globals
            locals = sys._getframe(1).f_locals
        con, provider = self._get_connection()
        adapted_sql, code = adapt_sql(sql, provider.paramstyle)
        values = eval(code, globals, locals)
        cursor = con.cursor()
        if values is None: wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql)
        else: wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql, values)
        return cursor
    def select(self, sql, globals=None, locals=None):
        sql = sql[:]  # sql = templating.plainstr(sql)
        if not select_re.match(sql): sql = 'select ' + sql
        if globals is None:
            assert locals is None
            globals = sys._getframe(1).f_globals
            locals = sys._getframe(1).f_locals
        con, provider = self._get_connection()
        adapted_sql, code = adapt_sql(sql, provider.paramstyle)
        values = eval(code, globals, locals)
        cursor = con.cursor()
        if values is None: wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql)
        else: wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql, values)
        result = cursor.fetchmany(options.MAX_ROWS_COUNT)
        if cursor.fetchone() is not None: raise TooManyRowsFound
        if len(cursor.description) == 1: result = [ row[0] for row in result ]
        elif type(result) != list: result = list(result)
        return result
    def get(self, sql, globals=None, locals=None):
        if globals is None:
            assert locals is None
            globals = sys._getframe(1).f_globals
            locals = sys._getframe(1).f_locals
        rows = self.select(sql, globals, locals)
        if not rows: raise RowNotFound
        if len(rows) > 1: raise MultipleRowsFound
        row = rows[0]
        return row
    def exists(self, sql, globals=None, locals=None):
        sql = sql[:]  # sql = templating.plainstr(sql)
        if not select_re.match(sql): sql = 'select ' + sql
        if globals is None:
            assert locals is None
            globals = sys._getframe(1).f_globals
            locals = sys._getframe(1).f_locals
        con, provider = self._get_connection()
        adapted_sql, code = adapt_sql(sql, provider.paramstyle)
        values = eval(code, globals, locals)
        cursor = con.cursor()
        if values is None: wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql)
        else: wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql, values)
        result = cursor.fetchone()
        return bool(result)
    def insert(self, table_name, **keyargs):
        table_name = table_name[:]  # table_name = templating.plainstr(table_name)
        items = keyargs.items()
        items.sort()
        con, provider = self._get_connection()
        key = table_name, tuple(name for name, value in items)
        x = self.sql_insert_cache.get(key)
        if x is None:
            ast = [ INSERT, table_name, [ name for name, value in items ], [ [PARAM, i] for i in range(len(items)) ] ]
            adapted_sql, params = provider.ast2sql(con, ast)
        else: adapted_sql, params = x
        if not isinstance(params, dict):
            for i, param in enumerate(params): assert param == i
            values = tuple(value for name, value in items)
        else: values = dict((key, items[i]) for key, i in params.items())
        cursor = con.cursor()
        wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql, values)
        return getattr(cursor, 'lastrowid', None)

def release():
    for con, provider, _ in local.connections.values():
        provider.release(con)
    local.connections.clear()

def commit():
    db = _get_database()
    return db.commit()

def rollback():
    db = _get_database()
    return db.rollback()

def auto_commit():
    default_db = local.default_db
    databases = [ (num, db) for db, (con, provider, num) in local.connections.items() if db != default_db ]
    databases.sort()
    databases = [ db for num, db in databases ]
    if default_db is not None: databases.insert(0, default_db)
    if not databases: return
    # ...
    exceptions = []
    try:
        try: databases[0].commit()
        except:
            exceptions.append(sys.exc_info())
            for db in databases[1:]:
                try: db.rollback()
                except: exceptions.append(sys.sys.exc_info())
            raise CommitException(exceptions)
        for db in databases[1:]:
            try: db.commit()
            except: exceptions.append(sys.sys.exc_info())
        # write exceptions to log
    finally:
        del exceptions
        local.connections.clear()

def auto_rollback():
    exceptions = []
    try:
        for db in local.connections:
            try: db.rollback()
            except: exceptions.append(sys.sys.exc_info())
        if exceptions: raise RollbackException(exceptions)
    finally:
        del exceptions
        local.connections.clear()

def with_transaction(func, *args, **keyargs):
    try: result = func(*args, **keyargs)
    except:
        exc_info = sys.exc_info()
        try:
            # write to log
            auto_rollback()
        finally:
            try: raise exc_info[0], exc_info[1], exc_info[2]
            finally: del exc_info
    else: auto_commit()
    return result
<commit_msg>Revert changes from previous revision<commit_after>import re, sys, threading

from pony import options
from pony.utils import import_module
from pony.sqlsymbols import *

class DBException(Exception):
    def __init__(self, *args, **keyargs):
        exceptions = keyargs.pop('exceptions', [])
        assert not keyargs
        if not args and exceptions:
            if len(exceptions) == 1: args = getattr(exceptions[0], 'args', ())
            else: args = ('Multiple exceptions have occured',)
        Exception.__init__(self, *args)
        self.exceptions = exceptions

class NoDefaultDbException(DBException): pass
class CommitException(DBException): pass
class RollbackException(DBException): pass

class RowNotFound(DBException): pass
class MultipleRowsFound(DBException): pass
class TooManyRowsFound(DBException): pass

##StandardError
##        |__Warning
##        |__Error
##           |__InterfaceError
##           |__DatabaseError
##              |__DataError
##              |__OperationalError
##              |__IntegrityError
##              |__InternalError
##              |__ProgrammingError
##              |__NotSupportedError

class Warning(DBException): pass
class Error(DBException): pass
class   InrefaceError(Error): pass
class   DatabaseError(Error): pass
class     DataError(DatabaseError): pass
class     OperationalError(DatabaseError): pass
class     IntegrityError(DatabaseError): pass
class     InternalError(DatabaseError): pass
class     ProgrammingError(DatabaseError): pass
class     NotSupportedError(DatabaseError): pass

def wrap_dbapi_exceptions(provider, func, *args, **keyargs):
    try: return func(*args, **keyargs)
    except provider.NotSupportedError, e: raise NotSupportedError(exceptions=[e])
    except provider.ProgrammingError, e: raise ProgrammingError(exceptions=[e])
    except provider.InternalError, e: raise InternalError(exceptions=[e])
    except provider.IntegrityError, e: raise IntegrityError(exceptions=[e])
    except provider.OperationalError, e: raise OperationalError(exceptions=[e])
    except provider.DataError, e: raise DataError(exceptions=[e])
    except provider.DatabaseError, e: raise DatabaseError(exceptions=[e])
    except provider.InterfaceError, e: raise InterfaceError(exceptions=[e])
    except provider.Error, e: raise Error(exceptions=[e])
    except provider.Warning, e: raise Warning(exceptions=[e])

class Local(threading.local):
    def __init__(self):
        self.default_db = None
        self.connections = {}

local = Local()

param_re = re.compile(r'[$]([$]|[A-Za-z_]\w*)')

sql_cache = {}

def adapt_sql(sql, paramstyle):
    result = sql_cache.get((sql, paramstyle))
    if result is not None: return result
    args, keyargs = [], {}
    if paramstyle == 'qmark':
        def replace(expr): args.append(expr); return '?'
    elif paramstyle == 'format':
        def replace(expr): args.append(expr); return '%s'
    elif paramstyle == 'numeric':
        def replace(expr): args.append(expr); return ':%d' % len(args)
    elif paramstyle == 'named':
        def replace(expr):
            key = 'param%d' % (len(keyargs) + 1)
            keyargs[key] = expr
            return ':' + key
    elif paramstyle == 'pyformat':
        def replace(expr):
            key = 'param%d' % (len(keyargs) + 1)
            keyargs[key] = expr
            return '%%(%s)s' % key
    else: assert False
    def replace_func(match):
        expr = match.group(1)
        if expr == '$': return '$'
        compile(expr, '<?>', 'eval')
        return replace(expr)
    adapted_sql = param_re.sub(replace_func, sql)
    params = args or keyargs or None
    if args:
        source = '(%s,)' % ', '.join(args)
        code = compile(source, '<?>', 'eval')
    elif keyargs:
        source = '{%s}' % ','.join('%r:%s' % item for item in keyargs.items())
        code = compile(source, '<?>', 'eval')
    else: code = compile('None', '<?>', 'eval')
    result = adapted_sql, code
    sql_cache[(sql, paramstyle)] = result
    return result

select_re = re.compile(r'\s*select\b', re.IGNORECASE)

class Database(object):
    def __init__(self, provider, *args, **keyargs):
        if isinstance(provider, basestring): provider = import_module('pony.dbproviders.' + provider)
        self.provider = provider
        self.args = args
        self.keyargs = keyargs
        self.sql_insert_cache = {}
        self.sql_update_cache = {}
        con = self.get_connection()
        self.release()
    def _get_connection(self):
        x = local.connections.get(self)
        if x is not None: return x[:2]
        provider = self.provider
        con = wrap_dbapi_exceptions(provider, provider.connect, *self.args, **self.keyargs)
        local.connections[self] = con, provider, len(local.connections)
        return con, provider
    def get_connection(self):
        con, provider = self._get_connection()
        return con
    def release(self):
        x = local.connections.pop(self, None)
        if x is None: return
        connection, provider, _ = x
        provider.release(connection)
    def commit(self):
        con, provider = self._get_connection()
        wrap_dbapi_exceptions(provider, con.commit)
    def rollback(self):
        con, provider = self._get_connection()
        wrap_dbapi_exceptions(provider, con.rollback)
    def execute(self, sql, globals=None, locals=None):
        sql = sql[:]  # sql = templating.plainstr(sql)
        if globals is None:
            assert locals is None
            globals = sys._getframe(1).f_globals
            locals = sys._getframe(1).f_locals
        con, provider = self._get_connection()
        adapted_sql, code = adapt_sql(sql, provider.paramstyle)
        values = eval(code, globals, locals)
        cursor = con.cursor()
        if values is None: wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql)
        else: wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql, values)
        return cursor
    def select(self, sql, globals=None, locals=None):
        sql = sql[:]  # sql = templating.plainstr(sql)
        if not select_re.match(sql): sql = 'select ' + sql
        if globals is None:
            assert locals is None
            globals = sys._getframe(1).f_globals
            locals = sys._getframe(1).f_locals
        con, provider = self._get_connection()
        adapted_sql, code = adapt_sql(sql, provider.paramstyle)
        values = eval(code, globals, locals)
        cursor = con.cursor()
        if values is None: wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql)
        else: wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql, values)
        result = cursor.fetchmany(options.MAX_ROWS_COUNT)
        if cursor.fetchone() is not None: raise TooManyRowsFound
        if len(cursor.description) == 1: result = [ row[0] for row in result ]
        elif type(result) != list: result = list(result)
        return result
    def get(self, sql, globals=None, locals=None):
        if globals is None:
            assert locals is None
            globals = sys._getframe(1).f_globals
            locals = sys._getframe(1).f_locals
        rows = self.select(sql, globals, locals)
        if not rows: raise RowNotFound
        if len(rows) > 1: raise MultipleRowsFound
        row = rows[0]
        return row
    def exists(self, sql, globals=None, locals=None):
        sql = sql[:]  # sql = templating.plainstr(sql)
        if not select_re.match(sql): sql = 'select ' + sql
        if globals is None:
            assert locals is None
            globals = sys._getframe(1).f_globals
            locals = sys._getframe(1).f_locals
        con, provider = self._get_connection()
        adapted_sql, code = adapt_sql(sql, provider.paramstyle)
        values = eval(code, globals, locals)
        cursor = con.cursor()
        if values is None: wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql)
        else: wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql, values)
        result = cursor.fetchone()
        return bool(result)
    def insert(self, table_name, **keyargs):
        table_name = table_name[:]  # table_name = templating.plainstr(table_name)
        items = keyargs.items()
        items.sort()
        con, provider = self._get_connection()
        key = table_name, tuple(name for name, value in items)
        x = self.sql_insert_cache.get(key)
        if x is None:
            ast = [ INSERT, table_name, [ name for name, value in items ], [ [PARAM, i] for i in range(len(items)) ] ]
            adapted_sql, params = provider.ast2sql(con, ast)
        else: adapted_sql, params = x
        if not isinstance(params, dict):
            for i, param in enumerate(params): assert param == i
            values = tuple(value for name, value in items)
        else: values = dict((key, items[i]) for key, i in params.items())
        cursor = con.cursor()
        wrap_dbapi_exceptions(provider, cursor.execute, adapted_sql, values)
        return getattr(cursor, 'lastrowid', None)

def _get_database():
    db = local.default_db
    if db is not None: return db
    db = sys._getframe(2).f_globals.get('__database__')
    if db is None: raise NoDefaultDbException('There is no default database defined')
    local.default_db = db
    return db

def get_connection():
    db = _get_database()
    return db.get_connection()

def release():
    for con, provider, _ in local.connections.values():
        provider.release(con)
    local.connections.clear()

def execute(sql):
    db = _get_database()
    return db.execute(sql, sys._getframe(1).f_globals, sys._getframe(1).f_locals)

def select(sql):
    db = _get_database()
    return db.select(sql, sys._getframe(1).f_globals, sys._getframe(1).f_locals)

def get(sql):
    db = _get_database()
    return db.get(sql, sys._getframe(1).f_globals, sys._getframe(1).f_locals)

def exists(sql):
    db = _get_database()
    return db.exists(sql, sys._getframe(1).f_globals, sys._getframe(1).f_locals)

def insert(table_name, **keyargs):
    db = _get_database()
    return db.insert(table_name, **keyargs)

def commit():
    db = _get_database()
    return db.commit()

def rollback():
    db = _get_database()
    return db.rollback()

def auto_commit():
    default_db = local.default_db
    databases = [ (num, db) for db, (con, provider, num) in local.connections.items() if db != default_db ]
    databases.sort()
    databases = [ db for num, db in databases ]
    if default_db is not None: databases.insert(0, default_db)
    if not databases: return
    # ...
    exceptions = []
    try:
        try: databases[0].commit()
        except:
            exceptions.append(sys.exc_info())
            for db in databases[1:]:
                try: db.rollback()
                except: exceptions.append(sys.sys.exc_info())
            raise CommitException(exceptions)
        for db in databases[1:]:
            try: db.commit()
            except: exceptions.append(sys.sys.exc_info())
        # write exceptions to log
    finally:
        del exceptions
        local.connections.clear()

def auto_rollback():
    exceptions = []
    try:
        for db in local.connections:
            try: db.rollback()
            except: exceptions.append(sys.sys.exc_info())
        if exceptions: raise RollbackException(exceptions)
    finally:
        del exceptions
        local.connections.clear()

def with_transaction(func, *args, **keyargs):
    try: result = func(*args, **keyargs)
    except:
        exc_info = sys.exc_info()
        try:
            # write to log
            auto_rollback()
        finally:
            try: raise exc_info[0], exc_info[1], exc_info[2]
            finally: del exc_info
    else: auto_commit()
    return result
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''A tiny library for parsing, modifying, and composing SRT files.'''

import functools
import re
from datetime import timedelta
from itertools import groupby

SUBTITLE_PATTERN = r'(\d+)\n(\d+:\d+:\d+,\d+) --> (\d+:\d+:\d+,\d+)\n(.+?)\n\n'
SUBTITLE_REGEX = re.compile(SUBTITLE_PATTERN, re.MULTILINE | re.DOTALL)

@functools.total_ordering  # pylint: disable=too-few-public-methods
class Subtitle(object):
    '''
    The metadata relating to a single subtitle. Subtitles are sorted by start
    time by default.

    :param index: The SRT index for this subtitle
    :param start: A timedelta object representing the time that the subtitle
                  should start being shown
    :param end: A timedelta object representing the time that the subtitle
                should stop being shown
    :param content: The subtitle content
    '''

    def __init__(self, index, start, end, content):
        self.index = index
        self.start = start
        self.end = end
        self.content = content

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

    def __lt__(self, other):
        return self.start < other.start

    def __repr__(self):
        return '<%s:%d>' % (self.__class__.__name__, self.index)

    def to_srt(self):
        '''
        Convert the current subtitle to an SRT block.

        :returns: A string with the metadata of the current Subtitle object as
                  an SRT formatted subtitle block
        '''
        return '%d\n%s --> %s\n%s\n\n' % (
            self.index, timedelta_to_srt_timestamp(self.start),
            timedelta_to_srt_timestamp(self.end), self.content,
        )

def timedelta_to_srt_timestamp(timedelta_timestamp):
    '''
    Convert a timedelta to an SRT timestamp.

    >>> delta = datetime.timedelta(hours=1, minutes=23, seconds=4)
    >>> timedelta_to_srt_timestamp(delta)
    '01:23:04,000

    :param timedelta_timestamp: A timestamp represented as a datetime timedelta
    :returns: An SRT formatted (HH:MM:SS,mmm) string representing the same
              timestamp passed in
    '''
    hrs, remainder = divmod(timedelta_timestamp.seconds, 3600)
    mins, secs = divmod(remainder, 60)
    msecs = timedelta_timestamp.microseconds // 1000
    return '%02d:%02d:%02d,%03d' % (hrs, mins, secs, msecs)

def srt_timestamp_to_timedelta(srt_timestamp):
    '''
    Convert an SRT timestamp to a timedelta.

    :param srt_timestamp: A timestamp in SRT format (HH:MM:SS,mmm)
    :returns: A timedelta object representing the same timestamp passed in
    '''
    hrs, mins, secs, msecs = (int(x) for x in re.split('[,:]', srt_timestamp))
    return timedelta(hours=hrs, minutes=mins, seconds=secs, milliseconds=msecs)

def parse(srt):
    '''
    Convert an SRT formatted string to a generator of Subtitle objects.

    :param srt: A string containing SRT formatted data
    :returns: A generator of the subtitles contained in the SRT file as
              Subtitle objects
    '''
    for match in SUBTITLE_REGEX.finditer(srt):
        raw_index, raw_start, raw_end, content = match.groups()
        yield Subtitle(
            index=int(raw_index), start=srt_timestamp_to_timedelta(raw_start),
            end=srt_timestamp_to_timedelta(raw_end), content=content,
        )

def parse_file(srt):
    '''
    Parse an SRT formatted stream into Subtitle objects.

    :param srt: A stream containing SRT formatted data
    :returns: A generator of the subtitles contained in the SRT file as
              Subtitle objects
    '''
    srt_chomped = (line.rstrip('\n') for line in srt)
    srt_blocks = [
        '\n'.join(srt_lines) + '\n\n'
        for line_has_content, srt_lines in groupby(srt_chomped, bool)
        if line_has_content
    ]

    for srt_block in srt_blocks:
        subtitle, = parse(srt_block)
        yield subtitle

def compose(subtitles):
    '''
    Convert an iterator of Subtitle objects to a string of joined SRT blocks.

    This may be a convienient interface when converting back and forth between
    Python objects and SRT formatted blocks, but you may wish to consider using
    the provided stream interface, `compose_file`, instead, which is
    considerably more memory efficient when dealing with particularly large SRT
    data.

    :param subtitles: An iterator of Subtitle objects, in the order they should
                      be in the output
    :returns: A single SRT formatted string, with each input Subtitle
              represented as an SRT block
    '''
    return ''.join(subtitle.to_srt() for subtitle in subtitles)

def compose_file(subtitles, output):
    '''
    Stream a sequence of Subtitle objects into an SRT formatted stream.

    :param subtitles: An iterator of Subtitle objects, in the order they should
                      be written to the file
    :param output: A stream to write the resulting SRT blocks to

    Returns:
        The number of subtitles that were written to the stream.
    '''
    num_written = 0
    for num_written, subtitle in enumerate(subtitles, start=1):
        output.write(subtitle.to_srt())
    return num_written
<commit_msg>Add doctest examples to all docstrings<commit_after>#!/usr/bin/env python

'''A tiny library for parsing, modifying, and composing SRT files.'''

import functools
import re
from datetime import timedelta
from itertools import groupby

SUBTITLE_PATTERN = r'(\d+)\n(\d+:\d+:\d+,\d+) --> (\d+:\d+:\d+,\d+)\n(.+?)\n\n'
SUBTITLE_REGEX = re.compile(SUBTITLE_PATTERN, re.MULTILINE | re.DOTALL)

@functools.total_ordering  # pylint: disable=too-few-public-methods
class Subtitle(object):
    r'''
    The metadata relating to a single subtitle. Subtitles are sorted by start
    time by default.

    :param index: The SRT index for this subtitle
    :param start: A timedelta object representing the time that the subtitle
                  should start being shown
    :param end: A timedelta object representing the time that the subtitle
                should stop being shown
    :param content: The subtitle content
    '''

    def __init__(self, index, start, end, content):
        self.index = index
        self.start = start
        self.end = end
        self.content = content

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

    def __lt__(self, other):
        return self.start < other.start

    def __repr__(self):
        return '<%s:%d>' % (self.__class__.__name__, self.index)

    def to_srt(self):
        r'''
        Convert the current subtitle to an SRT block.

        >>> sub = Subtitle(
        ...     index=1, start=srt_timestamp_to_timedelta('00:01:02,003'),
        ...     end=srt_timestamp_to_timedelta('00:02:03,004'), content='foo',
        ... )
        >>> sub.to_srt()
        '1\n00:01:02,003 --> 00:02:03,004\nfoo\n\n'

        :returns: A string with the metadata of the current Subtitle object as
                  an SRT formatted subtitle block
        '''
        return '%d\n%s --> %s\n%s\n\n' % (
            self.index, timedelta_to_srt_timestamp(self.start),
            timedelta_to_srt_timestamp(self.end), self.content,
        )

def timedelta_to_srt_timestamp(timedelta_timestamp):
    r'''
    Convert a timedelta to an SRT timestamp.

    >>> delta = datetime.timedelta(hours=1, minutes=23, seconds=4)
    >>> timedelta_to_srt_timestamp(delta)
    '01:23:04,000'

    :param timedelta_timestamp: A timestamp represented as a datetime timedelta
    :returns: An SRT formatted (HH:MM:SS,mmm) string representing the same
              timestamp passed in
    '''
    hrs, remainder = divmod(timedelta_timestamp.seconds, 3600)
    mins, secs = divmod(remainder, 60)
    msecs = timedelta_timestamp.microseconds // 1000
    return '%02d:%02d:%02d,%03d' % (hrs, mins, secs, msecs)

def srt_timestamp_to_timedelta(srt_timestamp):
    r'''
    Convert an SRT timestamp to a timedelta.

    >>> srt_timestamp = '01:23:04,000'
    >>> srt_timestamp_to_timedelta(srt_timestamp)
    datetime.timedelta(0, 4984)

    :param srt_timestamp: A timestamp in SRT format (HH:MM:SS,mmm)
    :returns: A timedelta object representing the same timestamp passed in
    '''
    hrs, mins, secs, msecs = (int(x) for x in re.split('[,:]', srt_timestamp))
    return timedelta(hours=hrs, minutes=mins, seconds=secs, milliseconds=msecs)

def parse(srt):
    r'''
    Convert an SRT formatted string to a generator of Subtitle objects.

    If you are reading from a file, consider using :py:func:`parse_file`
    instead.

    >>> subs = tinysrt.parse("""\
    ... 422
    ... 00:31:39,931 --> 00:31:41,931
    ... Using mainly spoons,
    ...
    ... 423
    ... 00:31:41,933 --> 00:31:43,435
    ... we dig a tunnel under the city and release it into the wild.
    ...
    ... """)
    >>> list(subs)
    [<Subtitle:422>, <Subtitle:423>]

    :param srt: A string containing SRT formatted data
    :returns: A generator of the subtitles contained in the SRT file as
              Subtitle objects
    '''
    for match in SUBTITLE_REGEX.finditer(srt):
        raw_index, raw_start, raw_end, content = match.groups()
        yield Subtitle(
            index=int(raw_index), start=srt_timestamp_to_timedelta(raw_start),
            end=srt_timestamp_to_timedelta(raw_end), content=content,
        )

def parse_file(srt):
    r'''
    Parse an SRT formatted stream into Subtitle objects.

    >>> with open('mwazowski.srt') as srt_f:
    ...     subs = parse_file(srt_f)
    ...
    >>> list(subs)
    [<Subtitle:422>, <Subtitle:423>]

    :param srt: A stream containing SRT formatted data
    :returns: A generator of the subtitles contained in the SRT file as
              Subtitle objects
    '''
    srt_chomped = (line.rstrip('\n') for line in srt)
    srt_blocks = [
        '\n'.join(srt_lines) + '\n\n'
        for line_has_content, srt_lines in groupby(srt_chomped, bool)
        if line_has_content
    ]

    for srt_block in srt_blocks:
        subtitle, = parse(srt_block)
        yield subtitle

def compose(subtitles):
    r'''
    Convert an iterator of Subtitle objects to a string of joined SRT blocks.

    This may be a convienient interface when converting back and forth between
    Python objects and SRT formatted blocks, but you may wish to consider using
    the provided stream interface, :py:func:`compose_file`, instead, which is
    considerably more memory efficient when dealing with particularly large SRT
    data.

    >>> subs
    [<Subtitle:422>, <Subtitle:423>]
    >>> compose(subs)
    '422\n00:31:39,931 --> 00:31:41,931\nUsing mainly[...]'

    :param subtitles: An iterator of Subtitle objects, in the order they should
                      be in the output
    :returns: A single SRT formatted string, with each input Subtitle
              represented as an SRT block
    '''
    return ''.join(subtitle.to_srt() for subtitle in subtitles)

def compose_file(subtitles, output):
    r'''
    Stream a sequence of Subtitle objects into an SRT formatted stream.

    >>> subs
    [<Subtitle:422>, <Subtitle:423>]
    >>> with open('out.srt', 'w') as srt_f:
    ...    compose_file(subs, srt_f)
    ...
    2

    :param subtitles: An iterator of Subtitle objects, in the order they should
                      be written to the file
    :param output: A stream to write the resulting SRT blocks to
    :returns: The number of subtitles that were written to the stream
    '''
    num_written = 0
    for num_written, subtitle in enumerate(subtitles, start=1):
        output.write(subtitle.to_srt())
    return num_written
<|endoftext|>


<commit_before>import os
import sys
import random
import string

tagpath = "/home/shark/.tinytag/"

def listTags():
    return os.listdir(tagpath)

def randomSymbol():
    return random.choice(string.ascii_lowercase +
                         string.ascii_uppercase +
                         string.digits)

def randomTag():
    # this generates (26 + 26 + 10)^2 = 3844 different tags
    return randomSymbol() + randomSymbol()

def generateTag():
    # TODO ...
    existing = listTags()

    tag = randomTag()

    while tag in existing:
        tag = randomTag()

    return tag

def createLink(filename, tag):
    os.symlink(filename, os.path.join(tagpath, tag))

def main(argv):
    if len(argv) != 2:
        print("Usage: tinytag <filename>")
        sys.exit(-1)

    filename = argv[1]
    pathname = os.path.realpath(filename)
    if not os.path.exists(pathname):
        print("File '{}' does not exist.".format(filename))
        sys.exit(-1)
    tag = generateTag()

    createLink(pathname, tag)

    print("The tag for '{}' is: '{}'".format(filename, tag))

if __name__ == "__main__":
    main(sys.argv)
<commit_msg>Basic functionality, command line options, removing and listing tags<commit_after>import os
import sys
import random
import string
import argparse

tagpath = "/home/shark/.tinytag/"

def listTags():
    return os.listdir(tagpath)

def randomSymbol():
    return random.choice(string.ascii_lowercase +
                         string.ascii_uppercase +
                         string.digits)

def randomTag():
    # this generates (26 + 26 + 10)^2 = 3844 different tags
    return randomSymbol() + randomSymbol()

def generateTag():
    # TODO ...
    existing = listTags()

    tag = randomTag()

    while tag in existing:
        tag = randomTag()

    return tag

def getSymlinkPath(tag):
    return os.path.join(tagpath, tag)

def createLink(filename, tag):
    os.symlink(filename, getSymlinkPath(tag))

def getFilename(tag):
    tagfile = getSymlinkPath(tag)
    return os.path.realpath(tagfile)

def main():
    parser = argparse.ArgumentParser(prog="tinytag")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("filename", nargs="?",
                       help="create a new tinytag for the given file",
                       action="store")
    group.add_argument("-l", "--list",
                       help="list all tinytags", action="store_true")
    group.add_argument("-r", "--remove", dest="tag",
                       help="remove a certain tinytag", action="store")
    args = parser.parse_args()

    if args.filename:
        fname = args.filename
        pname = os.path.realpath(fname)
        if not os.path.exists(pname):
            print("File '{}' does not exist.".format(fname))
            sys.exit(-1)
        tag = generateTag()

        createLink(pname, tag)

        print("The tag for '{}' is: '{}'".format(fname, tag))

    elif args.tag:
        tag = args.tag
        fname = getFilename(tag)
        os.unlink(getSymlinkPath(tag))
        print("Removed tag '{}' for file '{}'".format(tag, fname))

    else:
        print("List of tinytags:")
        print("")

        tlist = listTags()
        for tag in tlist:
            fname = getFilename(tag)
            print("{}:  {}".format(tag, fname))

if __name__ == "__main__":
    main()
<|endoftext|>