<commit_before>import main
import robocup
import behavior
import constants
import enum
import standard_play
import evaluation
import tactics.coordinated_pass
import skills.move
import skills.capture
import situational_play_selection

##
#
# A play written to use a distraction to draw the opponents defence off center before taking a shot
#
class Distraction(standard_play.StandardPlay):

    _situationList = [
        situational_play_selection.SituationalPlaySelector.Situation.ATTACK_GOAL
    ] # yapf: disable

    class State(enum.Enum):
        setup = 1, 'capture ball and move distractor and striker into position'
        optional_adjustment = 2, 'capture and setup a pass to the center right'
        center_pass = 3, 'pass to the center right of the field'
        passing = 4, 'pass from first capture to distractor'
        cross = 5, 'pass from distractor to striker'
        shoot = 6, 'shoot at goal'

    def __init__(self):
        super().__init__(continuous=False)

        for s in Distraction.State:
            self.add_state(s, behavior.Behavior.State.running)

        self.ball_is_far = main.ball().pos.y < (0.4 * constants.Field.Length)

        #the first Distraction point
        self.Distraction_point = robocup.Point(0.40 * constants.Field.Width,
                                               0.95 * constants.Field.Length)

        #the second Distraction point
        self.Distraction_recieve_pass_point = robocup.Point(
            0.40 * constants.Field.Width, 0.8 * constants.Field.Length)

        #striker's position
        self.striker_point = robocup.Point(-0.40 * constants.Field.Width,
                                           0.9 * constants.Field.Length)

        #center of field position, used if ball is far
        self.center = robocup.Point(0.5 * constants.Field.Width,
                                    0.5 * constants.Field.Length)

        self.distract_box = robocup.Rect(
            robocup.Point(0.33 * constants.Field.Width / 2,
                          (7 / 9) * constants.Field.Length),
            robocup.Point(constants.Field.Width, constants.Field.Length))

        self.striker_box = robocup.Rect(
            robocup.Point(-0.33 * constants.Field.Width / 2,
                          (7 / 9) * constants.Field.Length),
            robocup.Point(-constants.Field.Width, constants.Field.Length))

        self.distracter_get_close_ball = self.distract_box.contains_point(
            main.ball().pos)
        self.striker_get_close_ball = self.striker_box.contains_point(
            main.ball().pos)

        self.pass_striker_instead = False
        self.dont_shoot = False

        #If ball is not far
        self.add_transition(
            behavior.Behavior.State.start,
            Distraction.State.setup, lambda: not self.ball_is_far,
            'capture ball')

        #If the ball is near the distracter position, then skip steps that get it to that position
        self.add_transition(
            Distraction.State.setup,
            Distraction.State.cross, lambda: self.distracter_get_close_ball and
            self.subbehavior_with_name('capture').is_done_running(),
            'distracter get ball')

        #If the ball is near the strikers position, then skip the steps that get it to that position
        self.add_transition(
            Distraction.State.setup,
            Distraction.State.shoot, lambda: self.striker_get_close_ball and
            self.subbehavior_with_name('capture').is_done_running(),
            'striker get ball')

        #If the ball is far away, then go from capturing the ball to getting robots set up to recieve a closer pass
        self.add_transition(
            behavior.Behavior.State.start,
            Distraction.State.optional_adjustment, lambda: self.ball_is_far,
            'setup close pass')

        #Capture the ball and pass to a distracting robot if the ball is not far away
        self.add_transition(
            Distraction.State.setup, Distraction.State.passing, lambda: self.
            subbehavior_with_name('capture').is_done_running(), 'pass')

        #If the chance to pass and score with the striker is higher than passing to the Distraction recieve pass position
        self.add_transition(
            Distraction.State.setup,
            Distraction.State.cross, lambda: self.distracter_get_close_ball and
            self.subbehavior_with_name('capture').is_done_running(), 'cross')

        #Go from setting up a close pass to making the pass towards the center of the field
        self.add_transition(
            Distraction.State.optional_adjustment,
            Distraction.State.center_pass, lambda: self.subbehavior_with_name(
                'move half').is_done_running() and self.subbehavior_with_name(
                    'capture 2').is_done_running(), 'closer pass')

        #After getting the ball to the center right of the field, pass to the distracting robot
        self.add_transition(
            Distraction.State.center_pass,
            Distraction.State.passing, lambda: self.subbehavior_with_name(
                'center pass').is_done_running(), 'pass upfield')

        #Have the distracrting robot capture the ball and go to passing to the striker
        self.add_transition(
            Distraction.State.passing, Distraction.State.cross, lambda:
            (self.has_subbehavior_with_name('distract pass') and self.
             subbehavior_with_name('distract pass').is_done_running(
             )) or (self.has_subbehavior_with_name('get close ball') and self.
                    subbehavior_with_name('get close ball').is_done_running()),
            'have the distracrting robot capture the ball and go to passing to the striker'
        )

        #The striker gets the ball, either through pass or capturing, and goes to shoot
        self.add_transition(
            Distraction.State.passing, Distraction.State.shoot, lambda: (
                self.has_subbehavior_with_name('striker pass') and self.
                subbehavior_with_name('striker pass').is_done_running()) or
            (self.has_subbehavior_with_name('striker get close ball') and self.
             subbehavior_with_name('striker get close ball').is_done_running(
             )) or (self.has_subbehavior_with_name('get close ball') and self.
                    subbehavior_with_name('get close ball').is_done_running()),
            'stiker get ball')

        #go from the striker receiving the cross to shooting the ball
        self.add_transition(
            Distraction.State.cross,
            Distraction.State.shoot, lambda: self.subbehavior_with_name(
                'pass to striker').is_done_running(), 'cross shoot')

        #Don't shoot if shoot chance is too low, and pass back to distracter
        self.add_transition(Distraction.State.shoot,
                            Distraction.State.passing, lambda: self.dont_shoot,
                            'striker passes ')

        self.add_transition(
            Distraction.State.shoot, Distraction.State.setup, lambda: self.
            subbehavior_with_name('shooting').is_done_running(), 'repeat')

    def on_enter_setup(self):
        self.remove_all_subbehaviors()
        #capture ball and get striker and distractor in position
        self.add_subbehavior(
            skills.capture.Capture(), 'capture', required=True)

        if not self.distracter_get_close_ball:
            self.add_subbehavior(
                skills.move.Move(self.Distraction_recieve_pass_point),
                'distract moves',
                required=True)

        if not self.striker_get_close_ball:
            self.add_subbehavior(
                skills.move.Move(self.striker_point),
                'striker moves',
                required=False,
                priority=10)

    def on_enter_optional_adjustment(self):
        self.remove_all_subbehaviors()
        #if the ball is too far then the distractor moves to the center
        self.add_subbehavior(
            skills.capture.Capture(), 'capture 2', required=True)
        self.add_subbehavior(
            skills.move.Move(self.center), 'move half', required=True)
        self.add_subbehavior(
            skills.move.Move(self.striker_point),
            'make striker stay',
            required=True)

    def on_enter_center_pass(self):
        self.remove_all_subbehaviors()
        #pass the ball to the robot in the center and move a robot to the distract position
        self.add_subbehavior(
            tactics.coordinated_pass.CoordinatedPass(self.center),
            'center pass',
            required=True)
        self.add_subbehavior(
            skills.move.Move(self.Distraction_recieve_pass_point),
            'move back to distract',
            required=False,
            priority=10)

    def on_enter_passing(self):
        self.remove_all_subbehaviors()
        #either pass to striker or distracter depending on shot chance
        pass_to_distract_chance = evaluation.passing.eval_pass(
            main.ball().pos, self.Distraction_recieve_pass_point,
            main.our_robots())
        pass_to_striker_chance = evaluation.passing.eval_pass(
            main.ball().pos, self.striker_point, main.our_robots())
        shot_of_striker_chance = evaluation.shooting.eval_shot(
            self.striker_point, main.our_robots())

        if pass_to_distract_chance <= pass_to_striker_chance * shot_of_striker_chance:
            pass_striker_instead = True

        self.add_subbehavior(
            skills.move.Move(self.striker_point),
            'make striker stay again',
            required=True)
        self.add_subbehavior(
            tactics.coordinated_pass.CoordinatedPass(
                self.Distraction_recieve_pass_point),
            'distract pass',
            required=True)

    def on_enter_cross(self):
        self.remove_all_subbehaviors()
        #if the ball is passed to the distractor the ball is passed to the striker, as the third robot moves to the right to distract more
        #add chip here if pass chance is low
        self.add_subbehavior(
            tactics.coordinated_pass.CoordinatedPass(self.striker_point),
            'pass to striker',
            required=True)
        self.add_subbehavior(
            skills.move.Move(self.Distraction_point),
            'shift right',
            required=False,
            priority=10)

    def on_enter_shoot(self):
        self.remove_all_subbehaviors()
        #Depending on shot and pass chances,
        #the striker will shoot
        #or
        #the striker will pass to the distractor and the distractor will shoot
        pass_striker_to_distractor_chance = evaluation.passing.eval_pass(
            self.striker_point, self.Distraction_recieve_pass_point,
            main.our_robots())
        shot_of_striker_chance = evaluation.shooting.eval_shot(
            self.striker_point, main.our_robots())
        shot_of_distractor_chance = evaluation.shooting.eval_shot(
            self.Distraction_recieve_pass_point, main.our_robots())

        if pass_striker_to_distractor_chance * shot_of_distractor_chance > shot_of_striker_chance:
            dont_shoot = True

        self.add_subbehavior(
            skills.move.Move(self.Distraction_recieve_pass_point),
            'make first distractor stay again',
            required=True)
        self.add_subbehavior(
            skills.move.Move(self.Distraction_point),
            'make distracor stay',
            required=False,
            priority=10)
        self.add_subbehavior(
            skills.pivot_kick.PivotKick(), 'shooting', required=True)

    @classmethod
    def score(cls):
        score = super().score()

        #If the score from the super function is valid, use that with some offset
        if (score != float("inf")):
            scoreOffset = 0
            return score + scoreOffset
        else:
            return 10 if main.game_state().is_playing() else float("inf")
<commit_msg>Changed some points to not be hard codeded and robots will not go to pass points too close to the ball<commit_after>import main
import robocup
import behavior
import constants
import enum
import standard_play
import evaluation
import tactics.coordinated_pass
import skills.move
import skills.capture
import situational_play_selection

##
#
# A play written to use a distraction to draw the opponents defence off center before taking a shot
#
class Distraction(standard_play.StandardPlay):

    _situationList = [
        situational_play_selection.SituationalPlaySelector.Situation.ATTACK_GOAL
    ] # yapf: disable

    class State(enum.Enum):
        setup = 1, 'capture ball and move distractor and striker into position'
        optional_adjustment = 2, 'capture and setup a pass to the center right'
        center_pass = 3, 'pass to the center right of the field'
        passing = 4, 'pass from first capture to distractor'
        cross = 5, 'pass from distractor to striker'
        shoot = 6, 'shoot at goal'

    def __init__(self):
        super().__init__(continuous=False)

        for s in Distraction.State:
            self.add_state(s, behavior.Behavior.State.running)

        self.ball_is_far = main.ball().pos.y < (0.4 * constants.Field.Length)

        #the first Distraction point
        self.Distraction_point = robocup.Point(0.40 * constants.Field.Width,
                                               0.95 * constants.Field.Length)

        #the second Distraction point
        self.Distraction_recieve_pass_point = robocup.Point(
            0.40 * constants.Field.Width, 0.8 * constants.Field.Length)

        #striker's position
        self.striker_point = robocup.Point(-0.40 * constants.Field.Width,
                                           0.9 * constants.Field.Length)

        #center of field position, used if ball is far
        self.center = robocup.Point(0.5 * constants.Field.Width,
                                    0.5 * constants.Field.Length)

        self.distract_box = robocup.Rect(
            robocup.Point(0.33 * constants.Field.Width / 2,
                          (7 / 9) * constants.Field.Length),
            robocup.Point(constants.Field.Width, constants.Field.Length))

        self.striker_box = robocup.Rect(
            robocup.Point(-0.33 * constants.Field.Width / 2,
                          (7 / 9) * constants.Field.Length),
            robocup.Point(-constants.Field.Width, constants.Field.Length))

        self.distracter_get_close_ball = self.distract_box.contains_point(
            main.ball().pos)
        self.striker_get_close_ball = self.striker_box.contains_point(
            main.ball().pos)

        self.pass_striker_instead = False
        self.dont_shoot = False

        #If ball is not far
        self.add_transition(
            behavior.Behavior.State.start,
            Distraction.State.setup, lambda: not self.ball_is_far,
            'capture ball')

        #If the ball is near the distracter position, then skip steps that get it to that position
        self.add_transition(
            Distraction.State.setup,
            Distraction.State.cross, lambda: self.distracter_get_close_ball and
            self.subbehavior_with_name('capture').is_done_running(),
            'distracter get ball')

        #If the ball is near the strikers position, then skip the steps that get it to that position
        self.add_transition(
            Distraction.State.setup,
            Distraction.State.shoot, lambda: self.striker_get_close_ball and
            self.subbehavior_with_name('capture').is_done_running(),
            'striker get ball')

        #If the ball is far away, then go from capturing the ball to getting robots set up to recieve a closer pass
        self.add_transition(
            behavior.Behavior.State.start,
            Distraction.State.optional_adjustment, lambda: self.ball_is_far,
            'setup close pass')

        #Capture the ball and pass to a distracting robot if the ball is not far away
        self.add_transition(
            Distraction.State.setup, Distraction.State.passing, lambda: self.
            subbehavior_with_name('capture').is_done_running(), 'pass')

        #If the chance to pass and score with the striker is higher than passing to the Distraction recieve pass position
        self.add_transition(
            Distraction.State.setup,
            Distraction.State.cross, lambda: self.distracter_get_close_ball and
            self.subbehavior_with_name('capture').is_done_running(), 'cross')

        #Go from setting up a close pass to making the pass towards the center of the field
        self.add_transition(
            Distraction.State.optional_adjustment,
            Distraction.State.center_pass, lambda: self.subbehavior_with_name(
                'move half').is_done_running() and self.subbehavior_with_name(
                    'capture 2').is_done_running(), 'closer pass')

        #After getting the ball to the center right of the field, pass to the distracting robot
        self.add_transition(
            Distraction.State.center_pass,
            Distraction.State.passing, lambda: self.subbehavior_with_name(
                'center pass').is_done_running(), 'pass upfield')

        #Have the distracrting robot capture the ball and go to passing to the striker
        self.add_transition(
            Distraction.State.passing, Distraction.State.cross, lambda:
            (self.has_subbehavior_with_name('distract pass') and self.
             subbehavior_with_name('distract pass').is_done_running(
             )) or (self.has_subbehavior_with_name('get close ball') and self.
                    subbehavior_with_name('get close ball').is_done_running()),
            'have the distracrting robot capture the ball and go to passing to the striker'
        )

        #The striker gets the ball, either through pass or capturing, and goes to shoot
        self.add_transition(
            Distraction.State.passing, Distraction.State.shoot, lambda: (
                self.has_subbehavior_with_name('striker pass') and self.
                subbehavior_with_name('striker pass').is_done_running()) or
            (self.has_subbehavior_with_name('striker get close ball') and self.
             subbehavior_with_name('striker get close ball').is_done_running(
             )) or (self.has_subbehavior_with_name('get close ball') and self.
                    subbehavior_with_name('get close ball').is_done_running()),
            'stiker get ball')

        #go from the striker receiving the cross to shooting the ball
        self.add_transition(
            Distraction.State.cross,
            Distraction.State.shoot, lambda: self.subbehavior_with_name(
                'pass to striker').is_done_running(), 'cross shoot')

        #Don't shoot if shoot chance is too low, and pass back to distracter
        self.add_transition(Distraction.State.shoot,
                            Distraction.State.passing, lambda: self.dont_shoot,
                            'striker passes ')

        self.add_transition(
            Distraction.State.shoot, Distraction.State.setup, lambda: self.
            subbehavior_with_name('shooting').is_done_running(), 'repeat')

    def on_enter_setup(self):
        self.remove_all_subbehaviors()
        #capture ball and get striker and distractor in position
        self.add_subbehavior(
            skills.capture.Capture(), 'capture', required=True)

        if not self.distracter_get_close_ball:
            self.add_subbehavior(
                skills.move.Move(self.Distraction_recieve_pass_point),
                'distract moves',
                required=True)

        if not self.striker_get_close_ball:
            self.add_subbehavior(
                skills.move.Move(self.striker_point),
                'striker moves',
                required=False,
                priority=10)

    def on_enter_optional_adjustment(self):
        self.remove_all_subbehaviors()
        #if the ball is too far then the distractor moves to the center
        self.add_subbehavior(
            skills.capture.Capture(), 'capture 2', required=True)
        self.add_subbehavior(
            skills.move.Move(self.center), 'move half', required=True)
        self.add_subbehavior(
            skills.move.Move(self.striker_point),
            'make striker stay',
            required=True)

    def on_enter_center_pass(self):
        self.remove_all_subbehaviors()
        #pass the ball to the robot in the center and move a robot to the distract position
        self.add_subbehavior(
            tactics.coordinated_pass.CoordinatedPass(self.center),
            'center pass',
            required=True)
        self.add_subbehavior(
            skills.move.Move(self.Distraction_recieve_pass_point),
            'move back to distract',
            required=False,
            priority=10)

    def on_enter_passing(self):
        self.remove_all_subbehaviors()
        #either pass to striker or distracter depending on shot chance
        pass_to_distract_chance = evaluation.passing.eval_pass(
            main.ball().pos, self.Distraction_recieve_pass_point,
            main.our_robots())
        pass_to_striker_chance = evaluation.passing.eval_pass(
            main.ball().pos, self.striker_point, main.our_robots())
        shot_of_striker_chance = evaluation.shooting.eval_shot(
            self.striker_point, main.our_robots())

        if pass_to_distract_chance <= pass_to_striker_chance * shot_of_striker_chance:
            pass_striker_instead = True

        self.add_subbehavior(
            skills.move.Move(self.striker_point),
            'make striker stay again',
            required=True)
        self.add_subbehavior(
            tactics.coordinated_pass.CoordinatedPass(
                self.Distraction_recieve_pass_point),
            'distract pass',
            required=True)

    def on_enter_cross(self):
        self.remove_all_subbehaviors()
        #if the ball is passed to the distractor the ball is passed to the striker, as the third robot moves to the right to distract more
        ball = main.ball().pos
        self.striker_point.x = -(ball.x / 2 + constants.Field.Width / 4)
        self.add_subbehavior(tactics.coordinated_pass.CoordinatedPass(
            self.striker_point, use_chipper=True),
                             'pass to striker',
                             required=True)
        self.add_subbehavior(
            skills.move.Move(self.Distraction_point),
            'shift right',
            required=False,
            priority=10)

    def on_enter_shoot(self):
        self.remove_all_subbehaviors()
        #Depending on shot and pass chances,
        #the striker will shoot
        #or
        #the striker will pass to the distractor and the distractor will shoot
        pass_striker_to_distractor_chance = evaluation.passing.eval_pass(
            self.striker_point, self.Distraction_recieve_pass_point,
            main.our_robots())
        shot_of_striker_chance = evaluation.shooting.eval_shot(
            self.striker_point, main.our_robots())
        shot_of_distractor_chance = evaluation.shooting.eval_shot(
            self.Distraction_recieve_pass_point, main.our_robots())

        if pass_striker_to_distractor_chance * shot_of_distractor_chance > shot_of_striker_chance:
            dont_shoot = True

        self.add_subbehavior(
            skills.move.Move(self.Distraction_recieve_pass_point),
            'make first distractor stay again',
            required=True)
        self.add_subbehavior(
            skills.move.Move(self.Distraction_point),
            'make distracor stay',
            required=False,
            priority=10)
        self.add_subbehavior(
            skills.pivot_kick.PivotKick(), 'shooting', required=True)

    @classmethod
    def score(cls):
        score = super().score()

        #If the score from the super function is valid, use that with some offset
        if (score != float("inf")):
            scoreOffset = 0
            return score + scoreOffset
        else:
            return 10 if main.game_state().is_playing() else float("inf")
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""Data post-processor."""

import configparser
import os
import subprocess
import sys
import syslog
import time
import traceback

import mausy5043funcs.fileops3 as mf
from mausy5043libs.libdaemon3 import Daemon
from mausy5043libs.libgraph3 import Graph
from mausy5043libs.libsqldata3 import SqlDataFetch

# constants
DEBUG       = False
IS_JOURNALD = os.path.isfile('/bin/journalctl')
MYID        = "".join(list(filter(str.isdigit, os.path.realpath(__file__).split('/')[-1])))
MYAPP       = os.path.realpath(__file__).split('/')[-3]
MYAPPDIR    = "/".join(list(filter(str, os.path.realpath(__file__).split('/')[:-2])))
NODE        = os.uname()[1]
GRAPH_UPDATE    = 30   # in minutes
SQL_UPDATE_HOUR = GRAPH_UPDATE  # in minutes (shouldn't be shorter than GRAPH_UPDATE)
SQL_UPDATE_DAY  = 60  # in minutes
SQL_UPDATE_WEEK = 4   # in hours
SQL_UPDATE_YEAR = 8   # in hours

# initialise logging
syslog.openlog(ident=MYAPP, facility=syslog.LOG_LOCAL0)

class MyDaemon(Daemon):
  """Override Daemon-class run() function."""

  @staticmethod
  def run():
    """Execute main loop."""
    iniconf = configparser.ConfigParser()
    iniconf.read('/' + MYAPPDIR + '/config.ini')
    # flock       = iniconf.get(MYID, "lockfile")
    script_name = iniconf.get(MYID, "lftpscript")
    sample_time = iniconf.getint(MYID, "reporttime") / iniconf.getint(MYID, "samplespercycle")
    sqldata.fetch()
    if trendgraph.make() == 0:
      upload_page(script_name)
    while True:
      try:
        start_time   = time.time()

        do_stuff(script_name)

        pause_time    = sample_time - (time.time() - start_time) - (start_time % sample_time)
        if pause_time > 0:
          mf.syslog_trace("Waiting  : {0}s".format(pause_time), False, DEBUG)
          mf.syslog_trace("................................", False, DEBUG)
          time.sleep(pause_time)
      except Exception:
        mf.syslog_trace("Unexpected error in run()", syslog.LOG_CRIT, DEBUG)
        mf.syslog_trace(traceback.format_exc(), syslog.LOG_CRIT, DEBUG)
        raise

def do_stuff(script):
  """Run various scripts."""
  # wait 4 seconds for processes to finish
  # unlock(flock)  # remove stale lock
  time.sleep(4)

  # Retrieve data from MySQL database
  result = sqldata.fetch()
  mf.syslog_trace("...datafetch:  {0}".format(result), False, DEBUG)

  # Create the graphs based on the MySQL data
  result = trendgraph.make()
  mf.syslog_trace("...trendgrph:  {0}".format(result), False, DEBUG)
  if result == 0:
    upload_page(script)

def upload_page(script):
  """Upload the webpage."""
  try:
    # Upload the webpage and graphs
    if os.path.isfile('/tmp/' + MYAPP + '/site/default.md'):
      write_lftp(script)
      cmnd = ['lftp', '-f', script]
      mf.syslog_trace("...:  {0}".format(cmnd), False, DEBUG)
      cmnd = subprocess.check_output(cmnd, timeout=20)
      mf.syslog_trace("...uploadpag:  {0}".format(cmnd), False, DEBUG)
  except subprocess.TimeoutExpired:
    mf.syslog_trace("***TIMEOUT***:  {0}".format(cmnd), syslog.LOG_ERR, DEBUG)
    time.sleep(17 * 60)             # wait 17 minutes for the router to restart.
  except subprocess.CalledProcessError:
    mf.syslog_trace("***ERROR***:    {0}".format(cmnd), syslog.LOG_ERR, DEBUG)
    time.sleep(17 * 60)             # wait 17 minutes for the router to restart.

def write_lftp(script):
  """Output the webpage-upload-script."""
  with open(script, 'w') as file_handle:
    file_handle.write('# DO NOT EDIT\n')
    file_handle.write('# This file is created automatically by ' + MYAPP + '\n\n')
    file_handle.write('# lftp script\n\n')
    file_handle.write('set cmd:fail-exit yes;\n')
    file_handle.write('open hendrixnet.nl;\n')
    file_handle.write('cd 06.ups/;\n')
    file_handle.write('set cmd:fail-exit no;\n')
    file_handle.write('mirror --reverse --delete --verbose=3 -c /tmp/' + MYAPP + '/site/ . ;\n')
    file_handle.write('\n')

if __name__ == "__main__":
  daemon = MyDaemon('/tmp/' + MYAPP + '/' + MYID + '.pid')                                    # pylint: disable=C0103
  trendgraph = Graph(os.environ['HOME'] + '/' + MYAPP + '/mkgraphs.sh', GRAPH_UPDATE)         # pylint: disable=C0103
  sqldata = (SqlDataFetch(os.environ['HOME'] + '/' + MYAPP + '/queries',                      # pylint: disable=C0103
                          '/srv/semaphores',
                          SQL_UPDATE_HOUR, SQL_UPDATE_DAY, SQL_UPDATE_WEEK, SQL_UPDATE_YEAR))

  syslog.openlog(ident=MYAPP, facility=syslog.LOG_LOCAL0)                                     # initialise logging
  if len(sys.argv) == 2:
    if sys.argv[1] == 'start':
      daemon.start()
    elif sys.argv[1] == 'stop':
      daemon.stop()
    elif sys.argv[1] == 'restart':
      daemon.restart()
    elif sys.argv[1] == 'debug':
      # assist with debugging.
      print("Debug-mode started. Use <Ctrl>+C to stop.")
      DEBUG = True
      mf.syslog_trace("Daemon logging is ON", syslog.LOG_DEBUG, DEBUG)
      daemon.run()
    else:
      print("Unknown command")
      sys.exit(2)
    sys.exit(0)
  else:
    print("usage: {0!s} start|stop|restart|foreground".format(sys.argv[0]))
    sys.exit(2)
<commit_msg>Update ups98d.py<commit_after>#!/usr/bin/env python3

"""Data post-processor."""

import configparser
import os
import subprocess
import sys
import syslog
import time
import traceback

import mausy5043funcs.fileops3 as mf
from mausy5043libs.libdaemon3 import Daemon
from mausy5043libs.libgraph3 import Graph
from mausy5043libs.libsqldata3 import SqlDataFetch

# constants
DEBUG       = False
IS_JOURNALD = os.path.isfile('/bin/journalctl')
MYID        = "".join(list(filter(str.isdigit, os.path.realpath(__file__).split('/')[-1])))
MYAPP       = os.path.realpath(__file__).split('/')[-3]
MYAPPDIR    = "/".join(list(filter(str, os.path.realpath(__file__).split('/')[:-2])))
NODE        = os.uname()[1]
GRAPH_UPDATE    = 3    # in minutes
SQL_UPDATE_HOUR = GRAPH_UPDATE  # in minutes (shouldn't be shorter than GRAPH_UPDATE)
SQL_UPDATE_DAY  = 27  # in minutes
SQL_UPDATE_WEEK = 4   # in hours
SQL_UPDATE_YEAR = 8   # in hours

# initialise logging
syslog.openlog(ident=MYAPP, facility=syslog.LOG_LOCAL0)

class MyDaemon(Daemon):
  """Override Daemon-class run() function."""

  @staticmethod
  def run():
    """Execute main loop."""
    iniconf = configparser.ConfigParser()
    iniconf.read('/' + MYAPPDIR + '/config.ini')
    # flock       = iniconf.get(MYID, "lockfile")
    script_name = iniconf.get(MYID, "lftpscript")
    sample_time = iniconf.getint(MYID, "reporttime") / iniconf.getint(MYID, "samplespercycle")
    sqldata.fetch()
    if trendgraph.make() == 0:
      upload_page(script_name)
    while True:
      try:
        start_time   = time.time()

        do_stuff(script_name)

        pause_time    = sample_time - (time.time() - start_time) - (start_time % sample_time)
        if pause_time > 0:
          mf.syslog_trace("Waiting  : {0}s".format(pause_time), False, DEBUG)
          mf.syslog_trace("................................", False, DEBUG)
          time.sleep(pause_time)
      except Exception:
        mf.syslog_trace("Unexpected error in run()", syslog.LOG_CRIT, DEBUG)
        mf.syslog_trace(traceback.format_exc(), syslog.LOG_CRIT, DEBUG)
        raise

def do_stuff(script):
  """Run various scripts."""
  # wait 4 seconds for processes to finish
  # unlock(flock)  # remove stale lock
  time.sleep(4)

  # Retrieve data from MySQL database
  result = sqldata.fetch()
  mf.syslog_trace("...datafetch:  {0}".format(result), False, DEBUG)

  # Create the graphs based on the MySQL data
  result = trendgraph.make()
  mf.syslog_trace("...trendgrph:  {0}".format(result), False, DEBUG)
  if result == 0:
    upload_page(script)

def upload_page(script):
  """Upload the webpage."""
  try:
    # Upload the webpage and graphs
    if os.path.isfile('/tmp/' + MYAPP + '/site/default.md'):
      write_lftp(script)
      cmnd = ['lftp', '-f', script]
      mf.syslog_trace("...:  {0}".format(cmnd), False, DEBUG)
      cmnd = subprocess.check_output(cmnd, timeout=20)
      mf.syslog_trace("...uploadpag:  {0}".format(cmnd), False, DEBUG)
  except subprocess.TimeoutExpired:
    mf.syslog_trace("***TIMEOUT***:  {0}".format(cmnd), syslog.LOG_ERR, DEBUG)
    time.sleep(17 * 60)             # wait 17 minutes for the router to restart.
  except subprocess.CalledProcessError:
    mf.syslog_trace("***ERROR***:    {0}".format(cmnd), syslog.LOG_ERR, DEBUG)
    time.sleep(17 * 60)             # wait 17 minutes for the router to restart.

def write_lftp(script):
  """Output the webpage-upload-script."""
  with open(script, 'w') as file_handle:
    file_handle.write('# DO NOT EDIT\n')
    file_handle.write('# This file is created automatically by ' + MYAPP + '\n\n')
    file_handle.write('# lftp script\n\n')
    file_handle.write('set cmd:fail-exit yes;\n')
    file_handle.write('open hendrixnet.nl;\n')
    file_handle.write('cd 06.ups/;\n')
    file_handle.write('set cmd:fail-exit no;\n')
    file_handle.write('mirror --reverse --delete --verbose=3 -c /tmp/' + MYAPP + '/site/ . ;\n')
    file_handle.write('\n')

if __name__ == "__main__":
  daemon = MyDaemon('/tmp/' + MYAPP + '/' + MYID + '.pid')                                    # pylint: disable=C0103
  trendgraph = Graph(os.environ['HOME'] + '/' + MYAPP + '/mkgraphs.sh', GRAPH_UPDATE)         # pylint: disable=C0103
  sqldata = (SqlDataFetch(os.environ['HOME'] + '/' + MYAPP + '/queries',                      # pylint: disable=C0103
                          '/srv/semaphores',
                          SQL_UPDATE_HOUR, SQL_UPDATE_DAY, SQL_UPDATE_WEEK, SQL_UPDATE_YEAR))

  syslog.openlog(ident=MYAPP, facility=syslog.LOG_LOCAL0)                                     # initialise logging
  if len(sys.argv) == 2:
    if sys.argv[1] == 'start':
      daemon.start()
    elif sys.argv[1] == 'stop':
      daemon.stop()
    elif sys.argv[1] == 'restart':
      daemon.restart()
    elif sys.argv[1] == 'debug':
      # assist with debugging.
      print("Debug-mode started. Use <Ctrl>+C to stop.")
      DEBUG = True
      mf.syslog_trace("Daemon logging is ON", syslog.LOG_DEBUG, DEBUG)
      daemon.run()
    else:
      print("Unknown command")
      sys.exit(2)
    sys.exit(0)
  else:
    print("usage: {0!s} start|stop|restart|foreground".format(sys.argv[0]))
    sys.exit(2)
<|endoftext|>


<commit_before># coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_etree_fromstring,
    compat_parse_qs,
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    unified_strdate,
    int_or_none,
    qualities,
    unescapeHTML,
    urlencode_postdata,
)

class OdnoklassnikiIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                https?://
                    (?:(?:www|m|mobile)\.)?
                    (?:odnoklassniki|ok)\.ru/
                    (?:
                        video(?:embed)?/|
                        web-api/video/moviePlayer/|
                        live/|
                        dk\?.*?st\.mvId=
                    )
                    (?P<id>[\d-]+)
                '''
    _TESTS = [{
        # metadata in JSON
        'url': 'http://ok.ru/video/20079905452',
        'md5': '0b62089b479e06681abaaca9d204f152',
        'info_dict': {
            'id': '20079905452',
            'ext': 'mp4',
            'title': 'Культура меняет нас (прекрасный ролик!))',
            'duration': 100,
            'upload_date': '20141207',
            'uploader_id': '330537914540',
            'uploader': 'Виталий Добровольский',
            'like_count': int,
            'age_limit': 0,
        },
    }, {
        # metadataUrl
        'url': 'http://ok.ru/video/63567059965189-0?fromTime=5',
        'md5': '6ff470ea2dd51d5d18c295a355b0b6bc',
        'info_dict': {
            'id': '63567059965189-0',
            'ext': 'mp4',
            'title': 'Девушка без комплексов ...',
            'duration': 191,
            'upload_date': '20150518',
            'uploader_id': '534380003155',
            'uploader': '☭ Андрей Мещанинов ☭',
            'like_count': int,
            'age_limit': 0,
            'start_time': 5,
        },
    }, {
        # YouTube embed (metadataUrl, provider == USER_YOUTUBE)
        'url': 'http://ok.ru/video/64211978996595-1',
        'md5': '2f206894ffb5dbfcce2c5a14b909eea5',
        'info_dict': {
            'id': 'V_VztHT5BzY',
            'ext': 'mp4',
            'title': 'Космическая среда от 26 августа 2015',
            'description': 'md5:848eb8b85e5e3471a3a803dae1343ed0',
            'duration': 440,
            'upload_date': '20150826',
            'uploader_id': 'tvroscosmos',
            'uploader': 'Телестудия Роскосмоса',
            'age_limit': 0,
        },
    }, {
        # YouTube embed (metadata, provider == USER_YOUTUBE, no metadata.movie.title field)
        'url': 'http://ok.ru/video/62036049272859-0',
        'info_dict': {
            'id': '62036049272859-0',
            'ext': 'mp4',
            'title': 'МУЗЫКА     ДОЖДЯ .',
            'description': 'md5:6f1867132bd96e33bf53eda1091e8ed0',
            'upload_date': '20120106',
            'uploader_id': '473534735899',
            'uploader': 'МARINA D',
            'age_limit': 0,
        },
        'params': {
            'skip_download': True,
        },
        'skip': 'Video has not been found',
    }, {
        'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
        'only_matching': True,
    }, {
        'url': 'http://www.ok.ru/video/20648036891',
        'only_matching': True,
    }, {
        'url': 'http://www.ok.ru/videoembed/20648036891',
        'only_matching': True,
    }, {
        'url': 'http://m.ok.ru/video/20079905452',
        'only_matching': True,
    }, {
        'url': 'http://mobile.ok.ru/video/20079905452',
        'only_matching': True,
    }, {
        'url': 'https://www.ok.ru/live/484531969818',
        'only_matching': True,
    }, {
        'url': 'https://m.ok.ru/dk?st.cmd=movieLayer&st.discId=863789452017&st.retLoc=friend&st.rtu=%2Fdk%3Fst.cmd%3DfriendMovies%26st.mode%3Down%26st.mrkId%3D%257B%2522uploadedMovieMarker%2522%253A%257B%2522marker%2522%253A%25221519410114503%2522%252C%2522hasMore%2522%253Atrue%257D%252C%2522sharedMovieMarker%2522%253A%257B%2522marker%2522%253Anull%252C%2522hasMore%2522%253Afalse%257D%257D%26st.friendId%3D561722190321%26st.frwd%3Don%26_prevCmd%3DfriendMovies%26tkn%3D7257&st.discType=MOVIE&st.mvId=863789452017&_prevCmd=friendMovies&tkn=3648#lst#',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        start_time = int_or_none(compat_parse_qs(
            compat_urllib_parse_urlparse(url).query).get('fromTime', [None])[0])

        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://ok.ru/video/%s' % video_id, video_id)

        error = self._search_regex(
            r'[^>]+class="vp_video_stub_txt"[^>]*>([^<]+)<',
            webpage, 'error', default=None)
        if error:
            raise ExtractorError(error, expected=True)

        player = self._parse_json(
            unescapeHTML(self._search_regex(
                r'data-options=(?P<quote>["\'])(?P<player>{.+?%s.+?})(?P=quote)' % video_id,
                webpage, 'player', group='player')),
            video_id)

        flashvars = player['flashvars']

        metadata = flashvars.get('metadata')
        if metadata:
            metadata = self._parse_json(metadata, video_id)
        else:
            data = {}
            st_location = flashvars.get('location')
            if st_location:
                data['st.location'] = st_location
            metadata = self._download_json(
                compat_urllib_parse_unquote(flashvars['metadataUrl']),
                video_id, 'Downloading metadata JSON',
                data=urlencode_postdata(data))

        movie = metadata['movie']

        # Some embedded videos may not contain title in movie dict (e.g.
        # http://ok.ru/video/62036049272859-0) thus we allow missing title
        # here and it's going to be extracted later by an extractor that
        # will process the actual embed.
        provider = metadata.get('provider')
        title = movie['title'] if provider == 'UPLOADED_ODKL' else movie.get('title')

        thumbnail = movie.get('poster')
        duration = int_or_none(movie.get('duration'))

        author = metadata.get('author', {})
        uploader_id = author.get('id')
        uploader = author.get('name')

        upload_date = unified_strdate(self._html_search_meta(
            'ya:ovs:upload_date', webpage, 'upload date', default=None))

        age_limit = None
        adult = self._html_search_meta(
            'ya:ovs:adult', webpage, 'age limit', default=None)
        if adult:
            age_limit = 18 if adult == 'true' else 0

        like_count = int_or_none(metadata.get('likeCount'))

        info = {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'upload_date': upload_date,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'like_count': like_count,
            'age_limit': age_limit,
            'start_time': start_time,
        }

        if provider == 'USER_YOUTUBE':
            info.update({
                '_type': 'url_transparent',
                'url': movie['contentId'],
            })
            return info

        assert title
        if provider == 'LIVE_TV_APP':
            info['title'] = self._live_title(title)

        quality = qualities(('4', '0', '1', '2', '3', '5'))

        formats = [{
            'url': f['url'],
            'ext': 'mp4',
            'format_id': f['name'],
        } for f in metadata['videos']]

        m3u8_url = metadata.get('hlsManifestUrl')
        if m3u8_url:
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, video_id, 'mp4', 'm3u8_native',
                m3u8_id='hls', fatal=False))

        dash_manifest = metadata.get('metadataEmbedded')
        if dash_manifest:
            formats.extend(self._parse_mpd_formats(
                compat_etree_fromstring(dash_manifest), 'mpd'))

        for fmt in formats:
            fmt_type = self._search_regex(
                r'\btype[/=](\d)', fmt['url'],
                'format type', default=None)
            if fmt_type:
                fmt['quality'] = quality(fmt_type)

        # Live formats
        m3u8_url = metadata.get('hlsMasterPlaylistUrl')
        if m3u8_url:
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, video_id, 'mp4', entry_protocol='m3u8',
                m3u8_id='hls', fatal=False))
        rtmp_url = metadata.get('rtmpUrl')
        if rtmp_url:
            formats.append({
                'url': rtmp_url,
                'format_id': 'rtmp',
                'ext': 'flv',
            })

        self._sort_formats(formats)

        info['formats'] = formats
        return info
<commit_msg>[odnoklassniki] Detect paid videos<commit_after># coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_etree_fromstring,
    compat_parse_qs,
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    unified_strdate,
    int_or_none,
    qualities,
    unescapeHTML,
    urlencode_postdata,
)

class OdnoklassnikiIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                https?://
                    (?:(?:www|m|mobile)\.)?
                    (?:odnoklassniki|ok)\.ru/
                    (?:
                        video(?:embed)?/|
                        web-api/video/moviePlayer/|
                        live/|
                        dk\?.*?st\.mvId=
                    )
                    (?P<id>[\d-]+)
                '''
    _TESTS = [{
        # metadata in JSON
        'url': 'http://ok.ru/video/20079905452',
        'md5': '0b62089b479e06681abaaca9d204f152',
        'info_dict': {
            'id': '20079905452',
            'ext': 'mp4',
            'title': 'Культура меняет нас (прекрасный ролик!))',
            'duration': 100,
            'upload_date': '20141207',
            'uploader_id': '330537914540',
            'uploader': 'Виталий Добровольский',
            'like_count': int,
            'age_limit': 0,
        },
    }, {
        # metadataUrl
        'url': 'http://ok.ru/video/63567059965189-0?fromTime=5',
        'md5': '6ff470ea2dd51d5d18c295a355b0b6bc',
        'info_dict': {
            'id': '63567059965189-0',
            'ext': 'mp4',
            'title': 'Девушка без комплексов ...',
            'duration': 191,
            'upload_date': '20150518',
            'uploader_id': '534380003155',
            'uploader': '☭ Андрей Мещанинов ☭',
            'like_count': int,
            'age_limit': 0,
            'start_time': 5,
        },
    }, {
        # YouTube embed (metadataUrl, provider == USER_YOUTUBE)
        'url': 'http://ok.ru/video/64211978996595-1',
        'md5': '2f206894ffb5dbfcce2c5a14b909eea5',
        'info_dict': {
            'id': 'V_VztHT5BzY',
            'ext': 'mp4',
            'title': 'Космическая среда от 26 августа 2015',
            'description': 'md5:848eb8b85e5e3471a3a803dae1343ed0',
            'duration': 440,
            'upload_date': '20150826',
            'uploader_id': 'tvroscosmos',
            'uploader': 'Телестудия Роскосмоса',
            'age_limit': 0,
        },
    }, {
        # YouTube embed (metadata, provider == USER_YOUTUBE, no metadata.movie.title field)
        'url': 'http://ok.ru/video/62036049272859-0',
        'info_dict': {
            'id': '62036049272859-0',
            'ext': 'mp4',
            'title': 'МУЗЫКА     ДОЖДЯ .',
            'description': 'md5:6f1867132bd96e33bf53eda1091e8ed0',
            'upload_date': '20120106',
            'uploader_id': '473534735899',
            'uploader': 'МARINA D',
            'age_limit': 0,
        },
        'params': {
            'skip_download': True,
        },
        'skip': 'Video has not been found',
    }, {
        'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
        'only_matching': True,
    }, {
        'url': 'http://www.ok.ru/video/20648036891',
        'only_matching': True,
    }, {
        'url': 'http://www.ok.ru/videoembed/20648036891',
        'only_matching': True,
    }, {
        'url': 'http://m.ok.ru/video/20079905452',
        'only_matching': True,
    }, {
        'url': 'http://mobile.ok.ru/video/20079905452',
        'only_matching': True,
    }, {
        'url': 'https://www.ok.ru/live/484531969818',
        'only_matching': True,
    }, {
        'url': 'https://m.ok.ru/dk?st.cmd=movieLayer&st.discId=863789452017&st.retLoc=friend&st.rtu=%2Fdk%3Fst.cmd%3DfriendMovies%26st.mode%3Down%26st.mrkId%3D%257B%2522uploadedMovieMarker%2522%253A%257B%2522marker%2522%253A%25221519410114503%2522%252C%2522hasMore%2522%253Atrue%257D%252C%2522sharedMovieMarker%2522%253A%257B%2522marker%2522%253Anull%252C%2522hasMore%2522%253Afalse%257D%257D%26st.friendId%3D561722190321%26st.frwd%3Don%26_prevCmd%3DfriendMovies%26tkn%3D7257&st.discType=MOVIE&st.mvId=863789452017&_prevCmd=friendMovies&tkn=3648#lst#',
        'only_matching': True,
    }, {
        # Paid video
        'url': 'https://ok.ru/video/954886983203',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        start_time = int_or_none(compat_parse_qs(
            compat_urllib_parse_urlparse(url).query).get('fromTime', [None])[0])

        video_id = self._match_id(url)

        webpage = self._download_webpage(
            'http://ok.ru/video/%s' % video_id, video_id)

        error = self._search_regex(
            r'[^>]+class="vp_video_stub_txt"[^>]*>([^<]+)<',
            webpage, 'error', default=None)
        if error:
            raise ExtractorError(error, expected=True)

        player = self._parse_json(
            unescapeHTML(self._search_regex(
                r'data-options=(?P<quote>["\'])(?P<player>{.+?%s.+?})(?P=quote)' % video_id,
                webpage, 'player', group='player')),
            video_id)

        flashvars = player['flashvars']

        metadata = flashvars.get('metadata')
        if metadata:
            metadata = self._parse_json(metadata, video_id)
        else:
            data = {}
            st_location = flashvars.get('location')
            if st_location:
                data['st.location'] = st_location
            metadata = self._download_json(
                compat_urllib_parse_unquote(flashvars['metadataUrl']),
                video_id, 'Downloading metadata JSON',
                data=urlencode_postdata(data))

        movie = metadata['movie']

        # Some embedded videos may not contain title in movie dict (e.g.
        # http://ok.ru/video/62036049272859-0) thus we allow missing title
        # here and it's going to be extracted later by an extractor that
        # will process the actual embed.
        provider = metadata.get('provider')
        title = movie['title'] if provider == 'UPLOADED_ODKL' else movie.get('title')

        thumbnail = movie.get('poster')
        duration = int_or_none(movie.get('duration'))

        author = metadata.get('author', {})
        uploader_id = author.get('id')
        uploader = author.get('name')

        upload_date = unified_strdate(self._html_search_meta(
            'ya:ovs:upload_date', webpage, 'upload date', default=None))

        age_limit = None
        adult = self._html_search_meta(
            'ya:ovs:adult', webpage, 'age limit', default=None)
        if adult:
            age_limit = 18 if adult == 'true' else 0

        like_count = int_or_none(metadata.get('likeCount'))

        info = {
            'id': video_id,
            'title': title,
            'thumbnail': thumbnail,
            'duration': duration,
            'upload_date': upload_date,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'like_count': like_count,
            'age_limit': age_limit,
            'start_time': start_time,
        }

        if provider == 'USER_YOUTUBE':
            info.update({
                '_type': 'url_transparent',
                'url': movie['contentId'],
            })
            return info

        assert title
        if provider == 'LIVE_TV_APP':
            info['title'] = self._live_title(title)

        quality = qualities(('4', '0', '1', '2', '3', '5'))

        formats = [{
            'url': f['url'],
            'ext': 'mp4',
            'format_id': f['name'],
        } for f in metadata['videos']]

        m3u8_url = metadata.get('hlsManifestUrl')
        if m3u8_url:
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, video_id, 'mp4', 'm3u8_native',
                m3u8_id='hls', fatal=False))

        dash_manifest = metadata.get('metadataEmbedded')
        if dash_manifest:
            formats.extend(self._parse_mpd_formats(
                compat_etree_fromstring(dash_manifest), 'mpd'))

        for fmt in formats:
            fmt_type = self._search_regex(
                r'\btype[/=](\d)', fmt['url'],
                'format type', default=None)
            if fmt_type:
                fmt['quality'] = quality(fmt_type)

        # Live formats
        m3u8_url = metadata.get('hlsMasterPlaylistUrl')
        if m3u8_url:
            formats.extend(self._extract_m3u8_formats(
                m3u8_url, video_id, 'mp4', entry_protocol='m3u8',
                m3u8_id='hls', fatal=False))
        rtmp_url = metadata.get('rtmpUrl')
        if rtmp_url:
            formats.append({
                'url': rtmp_url,
                'format_id': 'rtmp',
                'ext': 'flv',
            })

        if not formats:
            payment_info = metadata.get('paymentInfo')
            if payment_info:
                raise ExtractorError('This video is paid, subscribe to download it', expected=True)

        self._sort_formats(formats)

        info['formats'] = formats
        return info
<|endoftext|>


<commit_before>import json
from functools import reduce
from collections import defaultdict
from node import Node, Interface
from link import Link, LinkConnector

class NodeDB:
  def __init__(self, time=0):
    self.time = time
    self._nodes = []
    self._links = []

  # fetch list of links
  def get_links(self):
    self.update_vpn_links()
    return self.reduce_links()

  # fetch list of nodes
  def get_nodes(self):
    return self._nodes

  # remove all offlines nodes with lastseen < timestamp
  def prune_offline(self, timestamp):
    self._nodes = list(filter(lambda x: x.lastseen >= timestamp, self._nodes))

  # write persistent state to file
  def dump_state(self, filename):
    obj = []

    for node in self._nodes:
      obj.append({ 'id': node.id
                 , 'name': node.name
                 , 'lastseen': node.lastseen
                 , 'firstseen': node.firstseen
                 , 'geo': node.gps
                 })

    with open(filename, "w") as f:
      json.dump(obj, f)

  # load persistent state from file
  def load_state(self, filename):
    try:
      with open(filename, "r") as f:
        obj = json.load(f)
        for n in obj:
          try:
            node = self.maybe_node_by_id(n['id'])
          except:
            node = Node()
            node.id = n['id']
            node.name = n['name']
            node.lastseen = n['lastseen']
            node.gps = n['geo']
            self._nodes.append(node)

          if 'firstseen' in n:
            node.firstseen = n['firstseen']

    except:
      pass

  def maybe_node_by_mac(self, macs):
    for node in self._nodes:
      for mac in macs:
        if mac.lower() in node.macs:
          return node

    raise KeyError

  def maybe_node_by_id(self, mac):
    for node in self._nodes:
      if mac.lower() == node.id:
        return node

    raise KeyError

  def parse_vis_data(self,vis_data):
    for x in vis_data:

      if 'of' in x:
        try:
          node = self.maybe_node_by_mac((x['of'], x['secondary']))
        except:
          node = Node()
          node.lastseen = self.time
          node.firstseen = self.time
          node.flags['online'] = True
          self._nodes.append(node)

        node.add_mac(x['of'])
        node.add_mac(x['secondary'])

    for x in vis_data:
      if 'router' in x:
        # TTs will be processed later
        if x['label'] == "TT":
          continue

        try:
          node = self.maybe_node_by_mac((x['router'], ))
        except:
          node = Node()
          node.lastseen = self.time
          node.firstseen = self.time
          node.flags['online'] = True
          node.add_mac(x['router'])
          self._nodes.append(node)

        try:
          if 'neighbor' in x:
            try:
              node = self.maybe_node_by_mac((x['neighbor']))
            except:
              continue

          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          node = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          node = Node()
          node.lastseen = self.time
          node.firstseen = self.time
          node.flags['online'] = True
          node.add_mac(x['neighbor'])
          self._nodes.append(node)

    for x in vis_data:
      if 'router' in x:
        # TTs will be processed later
        if x['label'] == "TT":
          continue

        try:
          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          router = self.maybe_node_by_mac((x['router'], ))
          neighbor = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          continue

        # filter TT links merged in previous step
        if router == neighbor:
          continue

        link = Link()
        link.source = LinkConnector()
        link.source.interface = x['router']
        link.source.id = self._nodes.index(router)
        link.target = LinkConnector()
        link.target.interface = x['neighbor']
        link.target.id = self._nodes.index(neighbor)
        link.quality = x['label']
        link.id = "-".join(sorted((link.source.interface, link.target.interface)))

        self._links.append(link)

    for x in vis_data:
      if 'primary' in x:
        try:
          node = self.maybe_node_by_mac((x['primary'], ))
        except:
          continue

        node.id = x['primary']

    for x in vis_data:
      if 'router' in x and x['label'] == 'TT':
        try:
          node = self.maybe_node_by_mac((x['router'], ))
          node.add_mac(x['gateway'])
          if not is_similar(x['router'], x['gateway']):
            node.clientcount += 1
        except:
          pass

  def reduce_links(self):
    tmp_links = defaultdict(list)

    for link in self._links:
      tmp_links[link.id].append(link)

    links = []

    def reduce_link(a, b):
      a.id = b.id
      a.source = b.source
      a.target = b.target
      a.type = b.type
      a.quality = ", ".join([x for x in (a.quality, b.quality) if x])

      return a

    for k, v in tmp_links.items():
      new_link = reduce(reduce_link, v, Link())
      links.append(new_link)

    return links

  def import_aliases(self, aliases):
    for mac, alias in aliases.items():
      try:
        node = self.maybe_node_by_mac([mac])
      except:
        # create an offline node
        node = Node()
        node.add_mac(mac)
        self._nodes.append(node)

      if 'name' in alias:
        node.name = alias['name']

      if 'vpn' in alias and alias['vpn'] and mac and node.interfaces and mac in node.interfaces:
        node.interfaces[mac].vpn = True

      if 'gps' in alias:
        node.gps = alias['gps']

      if 'firmware' in alias:
        node.firmware = alias['firmware']

      if 'id' in alias:
        node.id = alias['id']

  def mark_gateway(self, gateway):
    try:
      node = self.maybe_node_by_mac((gateway, ))
      node.flags['gateway'] = True
    except KeyError:
      print("WARNING: did not find gateway ", gateway, " in node list")

  def update_vpn_links(self):
    changes = 1
    while changes > 0:
      changes = 0
      for link in self._links:
        source_interface = self._nodes[link.source.id].interfaces[link.source.interface]
        target_interface = self._nodes[link.target.id].interfaces[link.target.interface]
        if source_interface.vpn or target_interface.vpn:
          source_interface.vpn = True
          target_interface.vpn = True
          if link.type != "vpn":
            changes += 1

          link.type = "vpn"

# compares two MACs and decides whether they are
# similar and could be from the same node
def is_similar(a, b):
  if a == b:
    return True

  try:
    mac_a = list(int(i, 16) for i in a.split(":"))
    mac_b = list(int(i, 16) for i in b.split(":"))
  except ValueError:
    return False

  # first byte must only differ in bit 2
  if mac_a[0] | 2 == mac_b[0] | 2:
    # count different bytes
    c = [x for x in zip(mac_a[1:], mac_b[1:]) if x[0] != x[1]]
  else:
    return False

  # no more than two additional bytes must differ
  if len(c) <= 2:
    delta = 0

  if len(c) > 0:
    delta = sum(abs(i[0] -i[1]) for i in c)

  # These addresses look pretty similar!
  return delta < 8
<commit_msg>simplify clientcount<commit_after>import json
from functools import reduce
from collections import defaultdict
from node import Node, Interface
from link import Link, LinkConnector

class NodeDB:
  def __init__(self, time=0):
    self.time = time
    self._nodes = []
    self._links = []

  # fetch list of links
  def get_links(self):
    self.update_vpn_links()
    return self.reduce_links()

  # fetch list of nodes
  def get_nodes(self):
    return self._nodes

  # remove all offlines nodes with lastseen < timestamp
  def prune_offline(self, timestamp):
    self._nodes = list(filter(lambda x: x.lastseen >= timestamp, self._nodes))

  # write persistent state to file
  def dump_state(self, filename):
    obj = []

    for node in self._nodes:
      obj.append({ 'id': node.id
                 , 'name': node.name
                 , 'lastseen': node.lastseen
                 , 'firstseen': node.firstseen
                 , 'geo': node.gps
                 })

    with open(filename, "w") as f:
      json.dump(obj, f)

  # load persistent state from file
  def load_state(self, filename):
    try:
      with open(filename, "r") as f:
        obj = json.load(f)
        for n in obj:
          try:
            node = self.maybe_node_by_id(n['id'])
          except:
            node = Node()
            node.id = n['id']
            node.name = n['name']
            node.lastseen = n['lastseen']
            node.gps = n['geo']
            self._nodes.append(node)

          if 'firstseen' in n:
            node.firstseen = n['firstseen']

    except:
      pass

  def maybe_node_by_mac(self, macs):
    for node in self._nodes:
      for mac in macs:
        if mac.lower() in node.macs:
          return node

    raise KeyError

  def maybe_node_by_id(self, mac):
    for node in self._nodes:
      if mac.lower() == node.id:
        return node

    raise KeyError

  def parse_vis_data(self,vis_data):
    for x in vis_data:

      if 'of' in x:
        try:
          node = self.maybe_node_by_mac((x['of'], x['secondary']))
        except:
          node = Node()
          node.lastseen = self.time
          node.firstseen = self.time
          node.flags['online'] = True
          self._nodes.append(node)

        node.add_mac(x['of'])
        node.add_mac(x['secondary'])

    for x in vis_data:
      if 'router' in x:
        # TTs will be processed later
        if x['label'] == "TT":
          continue

        try:
          node = self.maybe_node_by_mac((x['router'], ))
        except:
          node = Node()
          node.lastseen = self.time
          node.firstseen = self.time
          node.flags['online'] = True
          node.add_mac(x['router'])
          self._nodes.append(node)

        try:
          if 'neighbor' in x:
            try:
              node = self.maybe_node_by_mac((x['neighbor']))
            except:
              continue

          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          node = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          node = Node()
          node.lastseen = self.time
          node.firstseen = self.time
          node.flags['online'] = True
          node.add_mac(x['neighbor'])
          self._nodes.append(node)

    for x in vis_data:
      if 'router' in x:
        # TTs will be processed later
        if x['label'] == "TT":
          continue

        try:
          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          router = self.maybe_node_by_mac((x['router'], ))
          neighbor = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          continue

        # filter TT links merged in previous step
        if router == neighbor:
          continue

        link = Link()
        link.source = LinkConnector()
        link.source.interface = x['router']
        link.source.id = self._nodes.index(router)
        link.target = LinkConnector()
        link.target.interface = x['neighbor']
        link.target.id = self._nodes.index(neighbor)
        link.quality = x['label']
        link.id = "-".join(sorted((link.source.interface, link.target.interface)))

        self._links.append(link)

    for x in vis_data:
      if 'primary' in x:
        try:
          node = self.maybe_node_by_mac((x['primary'], ))
        except:
          continue

        node.id = x['primary']

    for x in vis_data:
      if 'router' in x and x['label'] == 'TT':
        try:
          node = self.maybe_node_by_mac((x['router'], ))
          node.add_mac(x['gateway'])
          node.clientcount += 1
        except:
          pass

    # don't count node as its own client
    for node in self._nodes:
      if node.clientcount > 0:
        node.clientcount -= 1

  def reduce_links(self):
    tmp_links = defaultdict(list)

    for link in self._links:
      tmp_links[link.id].append(link)

    links = []

    def reduce_link(a, b):
      a.id = b.id
      a.source = b.source
      a.target = b.target
      a.type = b.type
      a.quality = ", ".join([x for x in (a.quality, b.quality) if x])

      return a

    for k, v in tmp_links.items():
      new_link = reduce(reduce_link, v, Link())
      links.append(new_link)

    return links

  def import_aliases(self, aliases):
    for mac, alias in aliases.items():
      try:
        node = self.maybe_node_by_mac([mac])
      except:
        # create an offline node
        node = Node()
        node.add_mac(mac)
        self._nodes.append(node)

      if 'name' in alias:
        node.name = alias['name']

      if 'vpn' in alias and alias['vpn'] and mac and node.interfaces and mac in node.interfaces:
        node.interfaces[mac].vpn = True

      if 'gps' in alias:
        node.gps = alias['gps']

      if 'firmware' in alias:
        node.firmware = alias['firmware']

      if 'id' in alias:
        node.id = alias['id']

  def mark_gateway(self, gateway):
    try:
      node = self.maybe_node_by_mac((gateway, ))
      node.flags['gateway'] = True
    except KeyError:
      print("WARNING: did not find gateway ", gateway, " in node list")

  def update_vpn_links(self):
    changes = 1
    while changes > 0:
      changes = 0
      for link in self._links:
        source_interface = self._nodes[link.source.id].interfaces[link.source.interface]
        target_interface = self._nodes[link.target.id].interfaces[link.target.interface]
        if source_interface.vpn or target_interface.vpn:
          source_interface.vpn = True
          target_interface.vpn = True
          if link.type != "vpn":
            changes += 1

          link.type = "vpn"
<|endoftext|>


<commit_before>"""
sudo raspi-config
enable spi

spi documentation
http://tightdev.net/SpiDev_Doc.pdf
"""

# works with MCP3008, ch0
# returns x/10ths of a volt
import spidev
import time

spi=spidev.SpiDev() # create a spi object

spiBus = 0          # spi port 0
spiDeviceCh = 0       # GPIO CE0
spiDevice = spi.open(spiBus,spiDeviceCh)

# instructions for this value are found in the MCP3008 datasheet
# Table 5-2: Configure bits for the MCP3008
spiStart = 0b00000001
spiControl = 0b00001000 # single end mcp3008 ch0
""" other spi control values to try
0b1000 # single-end ch0
0b1001 # single-end ch1
0b0000 # differential ch0
0b0001 # differential ch1
"""
spiControlList = [0b1000,0b1001,0b0000,0b0001]
spiPlaceholder = 0b00000000

# to_send = [spiControl,0x02]
# to_send = [spiStart,spiControl,spiPlaceholder]

try:
    while True:
        # time.sleep(1)
        # resp = spi.xfer(to_send)
        # print (resp)
        for spiControl in spiControlList:
            to_send = [spiStart,spiControl,spiPlaceholder]
            for counter in range(1,100):
                resp = spi.xfer(to_send)
                print (bin(spiControl) + " - " + str(resp))
except KeyboardInterrupt: #control-c
    spi.close()         # close the spi device
<commit_msg>time sleep<commit_after>"""
sudo raspi-config
enable spi

spi documentation
http://tightdev.net/SpiDev_Doc.pdf
"""

# works with MCP3008, ch0
# returns x/10ths of a volt
import spidev
import time

spi=spidev.SpiDev() # create a spi object

spiBus = 0          # spi port 0
spiDeviceCh = 0       # GPIO CE0
spiDevice = spi.open(spiBus,spiDeviceCh)

# instructions for this value are found in the MCP3008 datasheet
# Table 5-2: Configure bits for the MCP3008
spiStart = 0b00000001
spiControl = 0b00001000 # single end mcp3008 ch0
""" other spi control values to try
0b1000 # single-end ch0
0b1001 # single-end ch1
0b0000 # differential ch0
0b0001 # differential ch1
"""
spiControlList = [0b1000,0b1001,0b0000,0b0001]
spiPlaceholder = 0b00000000

# to_send = [spiControl,0x02]
# to_send = [spiStart,spiControl,spiPlaceholder]

try:
    while True:
        # time.sleep(1)
        # resp = spi.xfer(to_send)
        # print (resp)
        for spiControl in spiControlList:
            to_send = [spiStart,spiControl,spiPlaceholder]
            for counter in range(1,100):
                resp = spi.xfer(to_send)
                print (bin(spiControl) + " - " + str(resp))
                time.sleep(1)
except KeyboardInterrupt: #control-c
    spi.close()         # close the spi device
<|endoftext|>


<commit_before>from rpython.rtyper.lltypesystem import rffi, lltype
from capi import CConfig
from rpython.rlib.rarithmetic import intmask
import sys
import os
import capi

testdb = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test.db")

class Sqlite3(object):
    db = None
    p = None

    def __init__(self, db_name, query):
        self.opendb(db_name)
        self.prepare(query)

    def opendb(self, db_name):
        with rffi.scoped_str2charp(db_name) as db_name, lltype.scoped_alloc(capi.SQLITE3PP.TO, 1) as result:
            errorcode = capi.sqlite3_open(db_name, result)
            assert errorcode == 0
            self.db = rffi.cast(capi.SQLITE3P, result[0])

    def prepare(self, query):
        length = len(query)
        with rffi.scoped_str2charp(query) as query, lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as result, lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as unused_buffer:
            errorcode = capi.sqlite3_prepare(self.db, query, length, result, unused_buffer)
            assert errorcode == 0
            self.p = rffi.cast(capi.VDBEP, result[0])

    def python_OP_Init(self, pc, pOp):
        if pOp.p2:
            pc = pOp.p2 - 1
        return pc

    def python_OP_Rewind(self, pc, pOp):
        with lltype.scoped_alloc(rffi.INTP.TO, 1) as internalPc:
            internalPc[0] = rffi.cast(rffi.INT, pc)
            rc = capi.impl_OP_Rewind(self.p, self.db, internalPc, pOp)
            retPc = internalPc[0]
        return retPc, rc

    def python_OP_Transaction(self, pc, pOp):
        return capi.impl_OP_Transaction(self.p, self.db, pc, pOp)

    def python_OP_TableLock(self, pc, pOp):
        return capi.impl_OP_TableLock(self.p, self.db, pc, pOp)

    def python_OP_Goto(self, pc, pOp):
        pc = pOp.p2 - 1
        return pc

    def python_OP_OpenRead(self, pc, pOp):
        capi.impl_OP_OpenRead(self.p, self.db, pc, pOp)

    def python_OP_Column(self, pc, pOp):
        return capi.impl_OP_Column(self.p, self.db, pc, pOp)

    def python_OP_ResultRow(self, pc, pOp):
        return capi.impl_OP_ResultRow(self.p, self.db, pc, pOp)

    def python_OP_Next(self, pc, pOp):
        with lltype.scoped_alloc(rffi.INTP.TO, 1) as internalPc:
            internalPc[0] = rffi.cast(rffi.INT, pc)
            rc = capi.impl_OP_Next(self.p, self.db, internalPc, pOp)
            retPc = internalPc[0]
        return retPc, rc

    def python_OP_Close(self, pc, pOp):
        capi.impl_OP_Close(self.p, self.db, pc, pOp)

    def python_OP_Halt(self, pc, pOp):
        with lltype.scoped_alloc(rffi.INTP.TO, 1) as internalPc:
            internalPc[0] = rffi.cast(rffi.INT, pc)
            rc = capi.impl_OP_Halt(self.p, self.db, internalPc, pOp)
            retPc = internalPc[0]
        return retPc, rc

    def python_OP_Compare(self, pc, pOp):
        return capi.impl_OP_Compare(self.p, self.db, pc, pOp)

    def python_OP_Integer(self, pc, pOp):
        capi.impl_OP_Integer(self.p, self.db, pc, pOp)

    def python_sqlite3_column_text(self, iCol):
        return capi.sqlite3_column_text(self.p, iCol)
    def python_sqlite3_column_bytes(self, iCol):
        return capi.sqlite3_column_bytes(self.p, iCol)

    def mainloop(self):
        ops = self.p.aOp
        rc = CConfig.SQLITE_OK
        pc = self.p.pc
        if pc < 0:
            pc = 0 # XXX maybe more to too, see vdbeapi.c:418

        while rc == CConfig.SQLITE_OK:
            pOp = ops[pc]

            if pOp.opcode == CConfig.OP_Init:
                print '>>> OP_Init <<<'
                pc = self.python_OP_Init(pc, pOp)
            elif pOp.opcode == CConfig.OP_OpenRead or pOp.opcode == CConfig.OP_OpenWrite:
                print '>>> OP_OpenRead <<<'
                self.python_OP_OpenRead(pc, pOp)
            elif pOp.opcode == CConfig.OP_Rewind:
                print '>>> OP_Rewind <<<'
                pc, rc = self.python_OP_Rewind(pc, pOp)
            elif pOp.opcode == CConfig.OP_Transaction:
                print '>>> OP_Transaction <<<'
                rc = self.python_OP_Transaction(pc, pOp)
                if rc == CConfig.SQLITE_BUSY:
                    print 'ERROR: in OP_Transaction SQLITE_BUSY'
                    # print 'rc = %s\npc = %s' % (rc, pc)
                    return rc
            elif pOp.opcode == CConfig.OP_TableLock:
                print '>>> OP_TableLock <<<'
                rc = self.python_OP_TableLock(pc, pOp)
            elif pOp.opcode == CConfig.OP_Goto:
                print '>>> OP_Goto <<<'
                pc = self.python_OP_Goto(pc, pOp)
            elif pOp.opcode == CConfig.OP_Column:
                print '>>> OP_Column <<<'
                rc = self.python_OP_Column(pc, pOp)
            elif pOp.opcode == CConfig.OP_ResultRow:
                print '>>> OP_ResultRow <<<'
                rc = self.python_OP_ResultRow(pc, pOp)
                if rc == CConfig.SQLITE_ROW:
                    # print 'rc = %s\npc = %s' % (rc, pc)
                    return rc
            elif pOp.opcode == CConfig.OP_Next:
                print '>>> OP_Next <<<'
                pc, rc = self.python_OP_Next(pc, pOp)
            elif pOp.opcode == CConfig.OP_Close:
                print '>>> OP_Close <<<'
                self.python_OP_Close(pc, pOp)
            elif pOp.opcode == CConfig.OP_Halt:
                print '>>> OP_Halt <<<'
                pc, rc = self.python_OP_Halt(pc, pOp)
                return rc
            elif (pOp.opcode == CConfig.OP_Eq or
                  pOp.opcode == CConfig.OP_Ne or
                  pOp.opcode == CConfig.OP_Lt or
                  pOp.opcode == CConfig.OP_Le or
                  pOp.opcode == CConfig.OP_Gt or
                  pOp.opcode == CConfig.OP_Ge):
                print '>>> OP_Compare: %s <<<' % pOp.opcode
                pc = self.python_OP_Compare(pc, pOp)
            elif pOp.opcode == CConfig.OP_Integer:
                print '>>> OP_Integer <<<'
                self.python_OP_Integer(pc, pOp)
            else:
                print 'Opcode %s is not there yet!' % pOp.opcode
                # raise Exception("Unimplemented bytecode %s." % pOp.opcode)
                pass
            # print 'rc = %s\npc = %s' % (rc, pc)
            pc += 1
        return rc

def run():
    sqlite3 = Sqlite3(testdb, 'select name from contacts;')
    rc = sqlite3.mainloop()

def entry_point(argv):
    run()
    return 0

def target(*args):
    return entry_point, None

if __name__ == "__main__":
    entry_point()
<commit_msg>Fixing issues to compile to C.<commit_after>from rpython.rtyper.lltypesystem import rffi, lltype
from capi import CConfig
from rpython.rlib.rarithmetic import intmask
import sys
import os
import capi

testdb = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test.db")

class Sqlite3(object):

    def __init__(self, db_name, query):
        self.opendb(db_name)
        self.prepare(query)

    def opendb(self, db_name):
        with rffi.scoped_str2charp(db_name) as db_name, lltype.scoped_alloc(capi.SQLITE3PP.TO, 1) as result:
            errorcode = capi.sqlite3_open(db_name, result)
            assert(errorcode == 0)
            self.db = rffi.cast(capi.SQLITE3P, result[0])

    def prepare(self, query):
        length = len(query)
        print 'length = %s' % length
        print 'query = %s' % query
        with rffi.scoped_str2charp(query) as query, lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as result, lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as unused_buffer:
            errorcode = capi.sqlite3_prepare(self.db, query, length, result, unused_buffer)
            assert errorcode == 0
            self.p = rffi.cast(capi.VDBEP, result[0])

    def python_OP_Init(self, pc, pOp):
        if pOp.p2:
            pc = pOp.p2 - 1
        return pc

    def python_OP_Rewind(self, pc, pOp):
        with lltype.scoped_alloc(rffi.INTP.TO, 1) as internalPc:
            internalPc[0] = rffi.cast(rffi.INT, pc)
            rc = capi.impl_OP_Rewind(self.p, self.db, internalPc, pOp)
            retPc = internalPc[0]
        return retPc, rc

    def python_OP_Transaction(self, pc, pOp):
        return capi.impl_OP_Transaction(self.p, self.db, pc, pOp)

    def python_OP_TableLock(self, pc, pOp):
        return capi.impl_OP_TableLock(self.p, self.db, pc, pOp)

    def python_OP_Goto(self, pc, pOp):
        pc = pOp.p2 - 1
        return pc

    def python_OP_OpenRead(self, pc, pOp):
        capi.impl_OP_OpenRead(self.p, self.db, pc, pOp)

    def python_OP_Column(self, pc, pOp):
        return capi.impl_OP_Column(self.p, self.db, pc, pOp)

    def python_OP_ResultRow(self, pc, pOp):
        return capi.impl_OP_ResultRow(self.p, self.db, pc, pOp)

    def python_OP_Next(self, pc, pOp):
        with lltype.scoped_alloc(rffi.INTP.TO, 1) as internalPc:
            internalPc[0] = rffi.cast(rffi.INT, pc)
            rc = capi.impl_OP_Next(self.p, self.db, internalPc, pOp)
            retPc = internalPc[0]
        return retPc, rc

    def python_OP_Close(self, pc, pOp):
        capi.impl_OP_Close(self.p, self.db, pc, pOp)

    def python_OP_Halt(self, pc, pOp):
        with lltype.scoped_alloc(rffi.INTP.TO, 1) as internalPc:
            internalPc[0] = rffi.cast(rffi.INT, pc)
            rc = capi.impl_OP_Halt(self.p, self.db, internalPc, pOp)
            retPc = internalPc[0]
        return retPc, rc

    def python_OP_Compare(self, pc, pOp):
        return capi.impl_OP_Compare(self.p, self.db, pc, pOp)

    def python_OP_Integer(self, pc, pOp):
        capi.impl_OP_Integer(self.p, self.db, pc, pOp)

    def python_sqlite3_column_text(self, iCol):
        return capi.sqlite3_column_text(self.p, iCol)
    def python_sqlite3_column_bytes(self, iCol):
        return capi.sqlite3_column_bytes(self.p, iCol)

    def mainloop(self):
        ops = self.p.aOp
        rc = CConfig.SQLITE_OK
        pc = self.p.pc
        if pc < 0:
            pc = 0 # XXX maybe more to too, see vdbeapi.c:418

        while rc == CConfig.SQLITE_OK:
            pOp = ops[pc]

            if pOp.opcode == CConfig.OP_Init:
                print '>>> OP_Init <<<'
                pc = self.python_OP_Init(pc, pOp)
            elif pOp.opcode == CConfig.OP_OpenRead or pOp.opcode == CConfig.OP_OpenWrite:
                print '>>> OP_OpenRead <<<'
                self.python_OP_OpenRead(pc, pOp)
            elif pOp.opcode == CConfig.OP_Rewind:
                print '>>> OP_Rewind <<<'
                pc, rc = self.python_OP_Rewind(pc, pOp)
            elif pOp.opcode == CConfig.OP_Transaction:
                print '>>> OP_Transaction <<<'
                rc = self.python_OP_Transaction(pc, pOp)
                if rc == CConfig.SQLITE_BUSY:
                    print 'ERROR: in OP_Transaction SQLITE_BUSY'
                    # print 'rc = %s\npc = %s' % (rc, pc)
                    return rc
            elif pOp.opcode == CConfig.OP_TableLock:
                print '>>> OP_TableLock <<<'
                rc = self.python_OP_TableLock(pc, pOp)
            elif pOp.opcode == CConfig.OP_Goto:
                print '>>> OP_Goto <<<'
                pc = self.python_OP_Goto(pc, pOp)
            elif pOp.opcode == CConfig.OP_Column:
                print '>>> OP_Column <<<'
                rc = self.python_OP_Column(pc, pOp)
            elif pOp.opcode == CConfig.OP_ResultRow:
                print '>>> OP_ResultRow <<<'
                rc = self.python_OP_ResultRow(pc, pOp)
                if rc == CConfig.SQLITE_ROW:
                    # print 'rc = %s\npc = %s' % (rc, pc)
                    return rc
            elif pOp.opcode == CConfig.OP_Next:
                print '>>> OP_Next <<<'
                pc, rc = self.python_OP_Next(pc, pOp)
            elif pOp.opcode == CConfig.OP_Close:
                print '>>> OP_Close <<<'
                self.python_OP_Close(pc, pOp)
            elif pOp.opcode == CConfig.OP_Halt:
                print '>>> OP_Halt <<<'
                pc, rc = self.python_OP_Halt(pc, pOp)
                return rc
            elif (pOp.opcode == CConfig.OP_Eq or
                  pOp.opcode == CConfig.OP_Ne or
                  pOp.opcode == CConfig.OP_Lt or
                  pOp.opcode == CConfig.OP_Le or
                  pOp.opcode == CConfig.OP_Gt or
                  pOp.opcode == CConfig.OP_Ge):
                print '>>> OP_Compare: %s <<<' % pOp.opcode
                pc = self.python_OP_Compare(pc, pOp)
            elif pOp.opcode == CConfig.OP_Integer:
                print '>>> OP_Integer <<<'
                self.python_OP_Integer(pc, pOp)
            else:
                print 'Opcode %s is not there yet!' % pOp.opcode
                # raise Exception("Unimplemented bytecode %s." % pOp.opcode)
                pass
            # print 'rc = %s\npc = %s' % (rc, pc)
            pc += 1
        return rc

def run():
    sqlite3 = Sqlite3(testdb, 'select name from contacts;')
    rc = sqlite3.mainloop()

def entry_point(argv):
    run()
    return 0

def target(*args):
    return entry_point, None

if __name__ == "__main__":
    entry_point(sys.argv)
<|endoftext|>


<commit_before># Copyright 2011-2013 Colin Scott
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Note: Make sure that this module is loaded after all other modules except
# of_01; the OpenFlow listen socket needs to be mocked.

from server_socket_multiplexer import *
import select
import socket
import os
import signal
from pox.core import core

log = core.getLogger()

def launch(snapshot_address=None):
  '''
  snapshot_address is a path to a unix domain socket to listen on.
  If None, snapshotting is not enabled.
  '''
  # Server side:
  #  - Instantiate ServerMultipexedSelect (this will create a true
  #    socket for the pinger)
  #  - override select.select with ServerMultiplexedSelect
  #  - override socket.socket
  #    - takes two params: protocol, socket type
  #    - if not SOCK_STREAM type, return a normal socket
  #  - we don't know bind address until bind() is called
  #  - after bind(), create true socket, create SocketDemultiplexer
  # All subsequent sockets will be instantiated through accept()

  # N.B. ServerMultiplexedSelect sits underneath POX's select.select calls,
  # and obtains control of the thread whenever POX invokes select.select.
  mux_select = ServerMultiplexedSelect()
  # Monkey patch select.select
  select._old_select = select.select
  select.select = mux_select.select
  # Monkey patch socket.socket
  socket._old_socket = socket.socket
  def socket_patch(protocol, sock_type):
    if sock_type == socket.SOCK_STREAM:
      return ServerMockSocket(protocol, sock_type,
                 set_true_listen_socket=mux_select.set_true_listen_socket)
    else:
      return socket._old_socket(protocol, sock_type)
  socket.socket = socket_patch
  # TODO(cs): will mux_select be GC'ed?

  if snapshot_address:
    # Snapshotting protocol (over a domain socket):

    # In STS:
    #  - set simulation to "record-only" mode
    #  - send "SNAPSHOT" to controller

    # In controller upon receiving "SNAPSHOT":
    #  - fork a child
    #     - in child, set (inherited) domain socket to blocking
    #     - write READY <PID>
    #     - block on recv()
    #  - proceed

    # In STS after peek()ing:
    #  - double check that READY <PID> was received
    #  - kill parent process
    #  - send "PROCEED" to domain socket
    #  - close STSSocketDemultiplexer.true_io_worker.socket
    #  - create a new socket and connect it to the same (address, port) pair
    #    as before
    #  - override STSSocketDemultiplexer.true_io_worker.socket
    #  - unset "record-only" mode
    #  - proceed

    # In child, upon recv()ing "PROCEED":
    #  - create a new blocking listen socket bound to
    #    ServerSocketDemultiplexer.mock_listen_sock.server_info
    #    (old true listen socket should already have been closed)
    #  - wait for ClientSocketDemultiplexer to connect()
    #  - accept(), and close() the previous true socket
    #  - replace true_io_worker.socket with return value from accept()

    # N.B. the synchronization of this snapshot protocol isn't perfect.
    # For example it's possible that the snapshot protocol itself perturbs
    # the behavior of the forked controller.

    # N.B. probably shouldn't run STSSyncProto with snapshotting, since we're
    # messing with sockets.

    def proceed():
      '''
      executed within in the forked controller process after STS has
      finished peek()ing.

      This method blocks until STS connects a new mux socket.
      '''
      log.debug("Received PROCEED command. Wiring new mux socket")
      # Clone the old true listen socket, but make it block
      demuxer = ServerSocketDemultiplexer.instance
      to_clone = demuxer.mock_listen_sock
      try:
        cloned_listen_sock = create_true_listen_socket(to_clone.server_info,
                                                       to_clone.protocol,
                                                       to_clone.sock_type,
                                                       blocking=1)
      except:
        log.critical('''Error cloning mux listen socket. Perhaps the parent process'''
                     ''' hasn't finished the mux handshake yet, and is still '''
                     ''' bound to %s?''' % to_clone.server_info)
        raise

      # block until ClientSocketDemultiplexer connect()s, then replace our mux
      # socket with a new one.
      log.debug("accept()ing new mux socket")
      new_socket = cloned_listen_sock.accept()[0]
      cloned_listen_sock.close()
      demuxer.true_io_worker.socket = new_socket
      log.debug("Done rewiring")

    def snapshot(snapshot_sock):
      log.debug("Received SNAPSHOT signal. fork()ing")

      if mux_select.true_listen_socks != []:
        # snapshotting should only be executed after the socket_mux
        # bind/listen/accept protocol has completed.
        raise RuntimeError("Should not snapshot before mux handshake has completed")

      pid = os.fork()
      if pid == 0: # Child
        snapshot_sock.setblocking(1)
        log.debug("Sending READY")
        snapshot_sock.send("READY %d" % os.getpid())
        command = snapshot_sock.recv(100)
        log.debug("Read command %s" % command)
        snapshot_sock.setblocking(0)
        if command == "PROCEED":
          proceed()
        else:
          raise ValueError("Unknown command %s" % command)

    def read_snapshot_commands(io_worker):
      command = io_worker.peek_receive_buf()
      io_worker.consume_receive_buf(len(command))
      if command == "SNAPSHOT":
        snapshot(snapshot_sock)
      else:
        raise ValueError("Unknown command %s" % command)

    log.debug("Creating snapshot listen socket")
    # Make sure the domain socket does not already exist
    try:
      os.unlink(snapshot_address)
    except OSError:
      if os.path.exists(snapshot_address):
        raise RuntimeError("can't remove snapshot domain socket %s" % str(snapshot_address))

    snapshot_listen_sock = create_true_listen_socket(snapshot_address, socket.AF_UNIX,
                                                     socket.SOCK_STREAM, blocking=1)
    log.debug("accept()ing snapshot socket")
    # N.B. this is a blocking operation. STS needs to connect immediately
    # after booting this controller.
    snapshot_sock = snapshot_listen_sock.accept()[0]
    log.debug("snapshot socket accept()ed")
    snapshot_sock.setblocking(0)
    snapshot_worker = mux_select.create_worker_for_socket(snapshot_sock)
    snapshot_worker.set_receive_handler(read_snapshot_commands)
<commit_msg>Update protocol spec<commit_after># Copyright 2011-2013 Colin Scott
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Note: Make sure that this module is loaded after all other modules except
# of_01; the OpenFlow listen socket needs to be mocked.

from server_socket_multiplexer import *
import select
import socket
import os
import signal
from pox.core import core

log = core.getLogger()

def launch(snapshot_address=None):
  '''
  snapshot_address is a path to a unix domain socket to listen on.
  If None, snapshotting is not enabled.
  '''
  # Server side:
  #  - Instantiate ServerMultipexedSelect (this will create a true
  #    socket for the pinger)
  #  - override select.select with ServerMultiplexedSelect
  #  - override socket.socket
  #    - takes two params: protocol, socket type
  #    - if not SOCK_STREAM type, return a normal socket
  #  - we don't know bind address until bind() is called
  #  - after bind(), create true socket, create SocketDemultiplexer
  # All subsequent sockets will be instantiated through accept()

  # N.B. ServerMultiplexedSelect sits underneath POX's select.select calls,
  # and obtains control of the thread whenever POX invokes select.select.
  mux_select = ServerMultiplexedSelect()
  # Monkey patch select.select
  select._old_select = select.select
  select.select = mux_select.select
  # Monkey patch socket.socket
  socket._old_socket = socket.socket
  def socket_patch(protocol, sock_type):
    if sock_type == socket.SOCK_STREAM:
      return ServerMockSocket(protocol, sock_type,
                 set_true_listen_socket=mux_select.set_true_listen_socket)
    else:
      return socket._old_socket(protocol, sock_type)
  socket.socket = socket_patch
  # TODO(cs): will mux_select be GC'ed?

  if snapshot_address:
    # Snapshotting protocol (over a domain socket):

    # In STS:
    #  - fork() simulation
    #  - send "SNAPSHOT" to controller

    # In controller upon receiving "SNAPSHOT":
    #  - fork a child
    #     - in child, set (inherited) domain socket to blocking
    #     - write READY <PID>
    #     - block on recv()
    #  - proceed

    # In STS after peek()ing:
    #  - double check that READY <PID> was received
    #  - kill parent process
    #  - send "PROCEED" to domain socket
    #  - close STSSocketDemultiplexer.true_io_worker.socket
    #  - create a new socket and connect it to the same (address, port) pair
    #    as before
    #  - override STSSocketDemultiplexer.true_io_worker.socket
    #  - unset "record-only" mode
    #  - proceed

    # In child, upon recv()ing "PROCEED":
    #  - create a new blocking listen socket bound to
    #    ServerSocketDemultiplexer.mock_listen_sock.server_info
    #    (old true listen socket should already have been closed)
    #  - wait for ClientSocketDemultiplexer to connect()
    #  - accept(), and close() the previous true socket
    #  - replace true_io_worker.socket with return value from accept()

    # N.B. the synchronization of this snapshot protocol isn't perfect.
    # For example it's possible that the snapshot protocol itself perturbs
    # the behavior of the forked controller.
    # TODO(cs): set socket buffers to zero?

    # N.B. probably shouldn't run STSSyncProto with snapshotting, since we're
    # messing with sockets.

    def proceed():
      '''
      executed within in the forked controller process after STS has
      finished peek()ing.

      This method blocks until STS connects a new mux socket.
      '''
      log.debug("Received PROCEED command. Wiring new mux socket")
      # Clone the old true listen socket, but make it block
      demuxer = ServerSocketDemultiplexer.instance
      to_clone = demuxer.mock_listen_sock
      try:
        cloned_listen_sock = create_true_listen_socket(to_clone.server_info,
                                                       to_clone.protocol,
                                                       to_clone.sock_type,
                                                       blocking=1)
      except:
        log.critical('''Error cloning mux listen socket. Perhaps the parent process'''
                     ''' hasn't finished the mux handshake yet, and is still '''
                     ''' bound to %s?''' % to_clone.server_info)
        raise

      # block until ClientSocketDemultiplexer connect()s, then replace our mux
      # socket with a new one.
      log.debug("accept()ing new mux socket")
      new_socket = cloned_listen_sock.accept()[0]
      cloned_listen_sock.close()
      demuxer.true_io_worker.socket = new_socket
      log.debug("Done rewiring")

    def snapshot(snapshot_sock):
      log.debug("Received SNAPSHOT signal. fork()ing")

      if mux_select.true_listen_socks != []:
        # snapshotting should only be executed after the socket_mux
        # bind/listen/accept protocol has completed.
        raise RuntimeError("Should not snapshot before mux handshake has completed")

      pid = os.fork()
      if pid == 0: # Child
        snapshot_sock.setblocking(1)
        log.debug("Sending READY")
        snapshot_sock.send("READY %d" % os.getpid())
        command = snapshot_sock.recv(100)
        log.debug("Read command %s" % command)
        snapshot_sock.setblocking(0)
        if command == "PROCEED":
          proceed()
        else:
          raise ValueError("Unknown command %s" % command)

    def read_snapshot_commands(io_worker):
      command = io_worker.peek_receive_buf()
      io_worker.consume_receive_buf(len(command))
      if command == "SNAPSHOT":
        snapshot(snapshot_sock)
      else:
        raise ValueError("Unknown command %s" % command)

    log.debug("Creating snapshot listen socket")
    # Make sure the domain socket does not already exist
    try:
      os.unlink(snapshot_address)
    except OSError:
      if os.path.exists(snapshot_address):
        raise RuntimeError("can't remove snapshot domain socket %s" % str(snapshot_address))

    snapshot_listen_sock = create_true_listen_socket(snapshot_address, socket.AF_UNIX,
                                                     socket.SOCK_STREAM, blocking=1)
    log.debug("accept()ing snapshot socket")
    # N.B. this is a blocking operation. STS needs to connect immediately
    # after booting this controller.
    snapshot_sock = snapshot_listen_sock.accept()[0]
    log.debug("snapshot socket accept()ed")
    snapshot_sock.setblocking(0)
    snapshot_worker = mux_select.create_worker_for_socket(snapshot_sock)
    snapshot_worker.set_receive_handler(read_snapshot_commands)
<|endoftext|>


<commit_before>def while_func(num, mystep):
    i = 0
    numbers = []

    while i < num:
        print ("At the top i is %d" % i)
        numbers.append(i)

        i = i + mystep
        print ("Numbers now: ", numbers)
        print ("At the bottom i is %d" % i)

    print ("The numbers: ")

    for num in numbers:
        print (num)

while_func(10,2)<commit_msg>Submitting lpthw Exercise 33 Q5<commit_after>def while_func(num, mystep=1):
    i = 0
    numbers = []

    for i in range(0,num,mystep):
    # while i < num:
        print ("At the top i is %d" % i)
        numbers.append(i)

        # i = i + mystep
        print ("Numbers now: ", numbers)
        print ("At the bottom i is %d" % i)

    print ("The numbers: ")

    for num in numbers:
        print (num)

while_func(5)

"""
WITH the incrementor in ouput looks like:
At the top i is 0
Numbers now:  [0]
At the bottom i is 1
At the top i is 1
Numbers now:  [0, 1]
At the bottom i is 2
At the top i is 2
Numbers now:  [0, 1, 2]
At the bottom i is 3
At the top i is 3
Numbers now:  [0, 1, 2, 3]
At the bottom i is 4
At the top i is 4
Numbers now:  [0, 1, 2, 3, 4]
At the bottom i is 5     <----- DIFF
The numbers:
0
1
2
3
4

WITHOUT the incrementor in ouput looks like:
At the top i is 0
Numbers now:  [0]
At the bottom i is 0
At the top i is 1
Numbers now:  [0, 1]
At the bottom i is 1
At the top i is 2
Numbers now:  [0, 1, 2]
At the bottom i is 2
At the top i is 3
Numbers now:  [0, 1, 2, 3]
At the bottom i is 3
At the top i is 4
Numbers now:  [0, 1, 2, 3, 4]
At the bottom i is 4     <----- DIFF
The numbers:
0
1
2
3
4
"""<|endoftext|>