<commit_before>#!/usr/bin/python

import pygame
from pygame.locals import *
import sys, os, random, math

from numpy import *

pygame.init()

# Size of the screen
X_SCREEN = 1000
Y_SCREEN = 500

# Padding around the screen
PADDING = 10

# Set offsets of origin of depiction of 1D noise
X_OFFSET_LEFT = 10
X_OFFSET_RIGHT = 400

# Midpoints of the two sets of axis
Y_TOP_OFFSET = Y_SCREEN / 4
Y_BOTTOM_OFFSET = Y_SCREEN / 4 * 3

# Midline of the screen
Y_MIDPOINT = Y_SCREEN / 2

# Set size of the 1D noise output
X_LIMIT = X_SCREEN - X_OFFSET_LEFT - X_OFFSET_RIGHT
Y_LIMIT = Y_SCREEN / 4

D3_WIDTH = 100
D3_HEIGHT = 100
D3_X = 100
D3_Y = 100

D3_OFF_X = X_SCREEN - 60 - D3_WIDTH * 2.5
D3_OFF_Y = Y_SCREEN / 2 - D3_HEIGHT * 2

D3_STRETCH = 2

# Colours
WHITE = (255,255,255)
SILVER = (128,128,128)
BLACK = (0,0,0)
RED = (255,0,0)
GREEN = (0,255,0)
BLUE = (0,0,255)
YELLOW = (255,255,0)
CYAN = (0,255,255)
MAGENTA = (255,0,255)

period = 1
num_octaves = 2
# Octaves from 0 to X
# Octave 1 is period 1, Octave 2 is period 1/2, Octave 3 is period 1/4 etc.

# pixels per period (in x dimension), 20 is smallest which looks good
PPP = 100
# Random seed, this specifies what the output will look like
R = 50
# Persistence, this specifies how much smaller details affect the end result
PERSISTENCE = 0.6
# Octaves, number of frequency ranges
OCTAVES = 6

# Setup display surfaces
screen = pygame.display.set_mode([X_SCREEN, Y_SCREEN])
surface = pygame.Surface([X_SCREEN, Y_SCREEN])

# Random values should always be between 0 and 1
def get_random():
    return random.uniform(-1,1)
    return random.random()

def gen_1D_values(length):
    vals = []
    for l in range(length):
        vals.append(get_random())
    return vals
def gen_1D_value(r, position):
    random.seed(r)
    val = 0
    for l in range(position):
        val = get_random()
    return val

def LinearInterpolate(a, b, x):
    return a*(1-x) + b*x

def CosineInterpolate(a, b, x):
    ft = x * math.pi
    f = (1 - math.cos(ft)) * 0.5
    return a*(1-f) + b*f

def regen_seed():
    random.seed()
    r = random.randint(0,100)
    return r

def regen_seeds(r, octaves):
    random.seed(r)
    randoms = []
    for o in range(octaves):
        randoms.append(random.randint(0,100))
    return randoms

def get_at_point_2D(x, xrandoms, y, yrandoms, ppp, persistence, octaves):
    """Don't smooth on the 1D generation, smooth in 2D"""
    # Returns an array of points representing the raw octave values and resultant value at a point
    # Takes the point, ppp, persistence and octaves values
    amps = []
    yvals = []
    # Divide p by ppp to find how many random points along we are, do this for each octave
    for o in range(octaves):
        pow2o = pow(2,o)
        position, remainder = divmod(x, ppp / pow2o)
        # Convert number of pixels along in a period into a % value for the interpolation function
        if remainder != 0:
            percentalong = float(remainder) / ppp * pow2o
        else:
            percentalong = 0
        yval = gen_1D_value(xrandoms[o], position)
        yvals.append(yval)
        amps.append(pow(persistence, o))

    return reduce(lambda x, y: x+(y[0]*y[1]), zip(yvals, amps), 0) / sum(amps)

def get_at_point_1D(p, randoms, ppp, persistence, octaves):
    # Returns an array of points representing the raw octave values and resultant value at a point
    # Takes the point, ppp, persistence and octaves values
    amps = []
    yvals = []
    # Divide p by ppp to find how many random points along we are, do this for each octave
    for o in range(octaves):
        pow2o = pow(2,o)
        position, remainder = divmod(p, ppp / pow2o)
        # Convert number of pixels along in a period into a % value for the interpolation function
        if remainder != 0:
            percentalong = float(remainder) / ppp * pow2o
        else:
            percentalong = 0
        yval = CosineInterpolate(gen_1D_value(randoms[o],position),
                                 gen_1D_value(randoms[o],position+1),
                                 percentalong)
        yvals.append(yval)
        amps.append(pow(persistence, o))

    # Return yvalues for component points, amplitudes for those yvalues and the overall output for this point
    return (yvals, amps, reduce(lambda x, y: x+(y[0]*y[1]), zip(yvals, amps), 0) / sum(amps))

def generate(ppp, r, persistence, octaves):
    # Generate array of colours, divide 255 by number of octaves
    colours = []
    b = 100.0 / octaves
    c = 200.0 / octaves
    for d in range(octaves):
        colours.append((255 - int(c * d), 100, 155 + int(b * d)))

    surface.fill(BLACK)
    # draw top x axis
    pygame.draw.line(surface, SILVER, (X_OFFSET_LEFT,Y_TOP_OFFSET), (X_OFFSET_LEFT+X_LIMIT,Y_TOP_OFFSET))
    # draw midline
    pygame.draw.line(surface, WHITE, (0,Y_MIDPOINT), (X_OFFSET_LEFT+X_LIMIT,Y_MIDPOINT))
    # draw bottom x axis
    pygame.draw.line(surface, SILVER, (X_OFFSET_LEFT,Y_BOTTOM_OFFSET), (X_OFFSET_LEFT+X_LIMIT,Y_BOTTOM_OFFSET))
    # draw y axis
    pygame.draw.line(surface, WHITE, (X_OFFSET_LEFT,0), (X_OFFSET_LEFT,Y_SCREEN))

    surface.lock()
    randoms = regen_seeds(r, octaves)
    for x in range(X_LIMIT):
        yvals, amps, result = get_at_point_1D(x, randoms, ppp, persistence, octaves)
        for n, amp, y in map(lambda x,y: (x[0],x[1],y), enumerate(amps), yvals):
            surface.set_at((X_OFFSET_LEFT+x,Y_TOP_OFFSET-y*Y_LIMIT*amp), colours[n])
        surface.set_at((X_OFFSET_LEFT+x,Y_BOTTOM_OFFSET-result*Y_LIMIT), RED)
    surface.unlock()

    # Draw the 3D graph representation
    surface.lock()
    # Regenerate the random seeds for each dimension
    randomsX = regen_seeds(r, octaves)
    randomsY = regen_seeds(r+100, octaves)

    for x in range(D3_WIDTH):
        for y in range(D3_HEIGHT):
            xx = (D3_WIDTH/2 + x - y) * 2
            yy = (D3_HEIGHT + x + y)
            # Calculate the height of the map at this point
            zval = get_at_point_2D(x, randomsX, y, randomsY, 300, 0.7, octaves)

            # zval will be in range -1<n<1
            # Multiply this by the graph's height extent
            SCALE = 100.0
            if zval < 0:
                zs = -1
            else:
                zs = 1
            z = zval*SCALE
            if z < 0:
                R = 0
                B = abs(z/SCALE*100+155)
            else:
                R = abs(z/SCALE*100+155)
                B = 0
            # Draw a point to represent this height value
            surface.set_at((D3_OFF_X+xx, D3_OFF_Y+yy-z), (R, 0, B))
    surface.unlock()

def mainloop():
    ppp = PPP
    r = R
    persistence = PERSISTENCE
    octaves = OCTAVES
    generate(ppp, r, persistence, octaves)
    while 1:
        key = pygame.key.get_pressed()
        for event in pygame.event.get():
            if event.type == QUIT or key[K_ESCAPE]:
                pygame.quit(); sys.exit()
            if event.type == KEYDOWN and event.key == K_F12:
                pygame.image.save(surface, "Perlin Noise.png")
            if event.type == KEYDOWN and event.key == K_r:
                r = regen_seed()
                generate(ppp, r, persistence, octaves)
            if event.type == KEYDOWN and event.key == K_p:
                ppp += 10
                generate(ppp, r, persistence, octaves)
            if event.type == KEYDOWN and event.key == K_o:
                ppp -= 10
                if ppp <= 0:
                    ppp = 20
                generate(ppp, r, persistence, octaves)
            if event.type == KEYDOWN and event.key == K_l:
                persistence += 0.05
                if persistence > 1:
                    persistence = 1
                generate(ppp, r, persistence, octaves)
            if event.type == KEYDOWN and event.key == K_k:
                persistence -= 0.05
                if persistence < 0.05:
                    persistence = 0.05
                generate(ppp, r, persistence, octaves)
            if event.type == KEYDOWN and event.key == K_m:
                octaves += 1
                if octaves > 8:
                    octaves = 8
                generate(ppp, r, persistence, octaves)
            if event.type == KEYDOWN and event.key == K_n:
                octaves -= 1
                if octaves < 1:
                    octaves = 1
                generate(ppp, r, persistence, octaves)

        screen.blit(surface,(0,0))
        pygame.display.flip()

mainloop()

<commit_msg>Further work on 2D noise generation, have the basics in place but something is pretty wrong about the output. Looks like it's generating noise on too fine a scale for all of the octaves (e.g. all octaves are on the smallest scale)<commit_after>#!/usr/bin/python

import pygame
from pygame.locals import *
import sys, os, random, math

from numpy import *

pygame.init()

# Size of the screen
X_SCREEN = 1000
Y_SCREEN = 500

# Padding around the screen
PADDING = 10

# Set offsets of origin of depiction of 1D noise
X_OFFSET_LEFT = 10
X_OFFSET_RIGHT = 400

# Midpoints of the two sets of axis
Y_TOP_OFFSET = Y_SCREEN / 4
Y_BOTTOM_OFFSET = Y_SCREEN / 4 * 3

# Midline of the screen
Y_MIDPOINT = Y_SCREEN / 2

# Set size of the 1D noise output
X_LIMIT = X_SCREEN - X_OFFSET_LEFT - X_OFFSET_RIGHT
Y_LIMIT = Y_SCREEN / 4

D3_WIDTH = 100
D3_HEIGHT = 100
D3_X = 100
D3_Y = 100

D3_OFF_X = X_SCREEN - 60 - D3_WIDTH * 2.5
D3_OFF_Y = Y_SCREEN / 2 - D3_HEIGHT * 2

D3_STRETCH = 2

# Colours
WHITE = (255,255,255)
SILVER = (128,128,128)
BLACK = (0,0,0)
RED = (255,0,0)
GREEN = (0,255,0)
BLUE = (0,0,255)
YELLOW = (255,255,0)
CYAN = (0,255,255)
MAGENTA = (255,0,255)

period = 1
num_octaves = 2
# Octaves from 0 to X
# Octave 1 is period 1, Octave 2 is period 1/2, Octave 3 is period 1/4 etc.

# pixels per period (in x dimension), 20 is smallest which looks good
PPP = 100
# Random seed, this specifies what the output will look like
R = 50
# Persistence, this specifies how much smaller details affect the end result
PERSISTENCE = 0.6
# Octaves, number of frequency ranges
OCTAVES = 6

# Setup display surfaces
screen = pygame.display.set_mode([X_SCREEN, Y_SCREEN])
surface = pygame.Surface([X_SCREEN, Y_SCREEN])

# Random values should always be between 0 and 1
def get_random():
    return random.uniform(-1,1)
    return random.random()

def gen_1D_values(length):
    vals = []
    for l in range(length):
        vals.append(get_random())
    return vals
def gen_1D_value(r, position):
    random.seed(r)
    val = 0
    for l in range(position):
        val = get_random()
    return val

def LinearInterpolate(a, b, x):
    return a*(1-x) + b*x

def CosineInterpolate(a, b, x):
    ft = x * math.pi
    f = (1 - math.cos(ft)) * 0.5
    return a*(1-f) + b*f

def regen_seed():
    random.seed()
    r = random.randint(0,100)
    return r

def regen_seeds(random_seed, values):
    random.seed(random_seed)
    randoms = []
    for o in range(values):
        randoms.append(random.randint(0,100))
    return randoms

def pad_array(a):
    """Pad an array around the edges to make it suitable for smoothing"""
    b = insert(a, -1, a[-1,...], axis=0)
    b = insert(b, 0, b[0,...], axis=0)
    b = insert(b, -1, b[...,-1], axis=1)
    b = insert(b, 0, b[...,0], axis=1)
    return b

def get_neighbours(a, x, y):
    """Return array containing height values of the neighbours of a particular value in the input array"""
    b = a[x-1:x+2,y-1:y+2]
    return b

k = fromfunction(lambda x,y: 10*x+y, (5,4), dtype=int)

q = pad_array(k)

print q
print get_neighbours(q, 2, 3)

def gen_2D_noise(xmax, ymax, randoms, octaves):
    """Return a set of arrays representing each octave of noise"""
    octsets = []
    for o in range(octaves):
        # Generate set of X values for generating the set of y values
        xrandoms = regen_seeds(o, xmax)
        a = []
        for x in xrandoms:
            random.seed(x)
            b = []
            for y in range(ymax + 1):
                b.append(get_random())
            a.append(b)
        a = array(a)
        octsets.append(a)
    return octsets

def get_at_point_2D(x, y, octsets, ppp, persistence, octaves):
    """Don't smooth on the 1D generation, smooth in 2D"""
    # Returns an array of points representing the raw octave values and resultant value at a point
    # Takes the point, ppp, persistence and octaves values
    amps = []
    yvals = []
    # Divide p by ppp to find how many random points along we are, do this for each octave
    for o in range(octaves):
        pow2o = pow(2,o)
        position, remainder = divmod(x, ppp / pow2o)
        # Convert number of pixels along in a period into a % value for the interpolation function
        if remainder != 0:
            percentalong = float(remainder) / ppp * pow2o
        else:
            percentalong = 0

        yval = CosineInterpolate(octsets[o][x][y], octsets[o][x][y+1], percentalong)

        yvals.append(yval)
        amps.append(pow(persistence, o))

    return reduce(lambda x, y: x+(y[0]*y[1]), zip(yvals, amps), 0) / sum(amps)

def get_at_point_1D(p, randoms, ppp, persistence, octaves):
    # Returns an array of points representing the raw octave values and resultant value at a point
    # Takes the point, ppp, persistence and octaves values
    amps = []
    yvals = []
    # Divide p by ppp to find how many random points along we are, do this for each octave
    for o in range(octaves):
        pow2o = pow(2,o)
        position, remainder = divmod(p, ppp / pow2o)
        # Convert number of pixels along in a period into a % value for the interpolation function
        if remainder != 0:
            percentalong = float(remainder) / ppp * pow2o
        else:
            percentalong = 0
        yval = CosineInterpolate(gen_1D_value(randoms[o],position),
                                 gen_1D_value(randoms[o],position+1),
                                 percentalong)
        yvals.append(yval)
        amps.append(pow(persistence, o))

    # Return yvalues for component points, amplitudes for those yvalues and the overall output for this point
    return (yvals, amps, reduce(lambda x, y: x+(y[0]*y[1]), zip(yvals, amps), 0) / sum(amps))

def generate(ppp, r, persistence, octaves):
    # Generate array of colours, divide 255 by number of octaves
    colours = []
    b = 100.0 / octaves
    c = 200.0 / octaves
    for d in range(octaves):
        colours.append((255 - int(c * d), 100, 155 + int(b * d)))

    surface.fill(BLACK)
    # draw top x axis
    pygame.draw.line(surface, SILVER, (X_OFFSET_LEFT,Y_TOP_OFFSET), (X_OFFSET_LEFT+X_LIMIT,Y_TOP_OFFSET))
    # draw midline
    pygame.draw.line(surface, WHITE, (0,Y_MIDPOINT), (X_OFFSET_LEFT+X_LIMIT,Y_MIDPOINT))
    # draw bottom x axis
    pygame.draw.line(surface, SILVER, (X_OFFSET_LEFT,Y_BOTTOM_OFFSET), (X_OFFSET_LEFT+X_LIMIT,Y_BOTTOM_OFFSET))
    # draw y axis
    pygame.draw.line(surface, WHITE, (X_OFFSET_LEFT,0), (X_OFFSET_LEFT,Y_SCREEN))

    surface.lock()
    randoms = regen_seeds(r, octaves)
    for x in range(X_LIMIT):
        yvals, amps, result = get_at_point_1D(x, randoms, ppp, persistence, octaves)
        for n, amp, y in map(lambda x,y: (x[0],x[1],y), enumerate(amps), yvals):
            surface.set_at((X_OFFSET_LEFT+x,Y_TOP_OFFSET-y*Y_LIMIT*amp), colours[n])
        surface.set_at((X_OFFSET_LEFT+x,Y_BOTTOM_OFFSET-result*Y_LIMIT), RED)
    surface.unlock()

    # Draw the 3D graph representation
    surface.lock()
    # Regenerate the random seeds for each dimension
    randoms = regen_seeds(r, octaves)

    octsets = gen_2D_noise(D3_WIDTH, D3_HEIGHT, randoms, octaves)

    print octsets
    print octsets[0]
    print octsets[0][0]
    print octsets[0][0][0]

    for x in range(D3_WIDTH):
        for y in range(D3_HEIGHT):
            xx = (D3_WIDTH/2 + x - y) * 2
            yy = (D3_HEIGHT + x + y)
            # Calculate the height of the map at this point
            zval = get_at_point_2D(x, y, octsets, 300, 0.7, octaves)

            # zval will be in range -1<n<1
            # Multiply this by the graph's height extent
            SCALE = 10.0
            if zval < 0:
                zs = -1
            else:
                zs = 1
            z = zval*SCALE
            if z < 0:
                R = 0
                B = abs(z/SCALE*100+155)
            else:
                R = abs(z/SCALE*100+155)
                B = 0
            # Draw a point to represent this height value
            surface.set_at((D3_OFF_X+xx, D3_OFF_Y+yy-z), (R, 0, B))
    surface.unlock()

def mainloop():
    ppp = PPP
    r = R
    persistence = PERSISTENCE
    octaves = OCTAVES
    generate(ppp, r, persistence, octaves)
    while 1:
        key = pygame.key.get_pressed()
        for event in pygame.event.get():
            if event.type == QUIT or key[K_ESCAPE]:
                pygame.quit(); sys.exit()
            if event.type == KEYDOWN and event.key == K_F12:
                pygame.image.save(surface, "Perlin Noise.png")
            if event.type == KEYDOWN and event.key == K_r:
                r = regen_seed()
                generate(ppp, r, persistence, octaves)
            if event.type == KEYDOWN and event.key == K_p:
                ppp += 10
                generate(ppp, r, persistence, octaves)
            if event.type == KEYDOWN and event.key == K_o:
                ppp -= 10
                if ppp <= 0:
                    ppp = 20
                generate(ppp, r, persistence, octaves)
            if event.type == KEYDOWN and event.key == K_l:
                persistence += 0.05
                if persistence > 1:
                    persistence = 1
                generate(ppp, r, persistence, octaves)
            if event.type == KEYDOWN and event.key == K_k:
                persistence -= 0.05
                if persistence < 0.05:
                    persistence = 0.05
                generate(ppp, r, persistence, octaves)
            if event.type == KEYDOWN and event.key == K_m:
                octaves += 1
                if octaves > 8:
                    octaves = 8
                generate(ppp, r, persistence, octaves)
            if event.type == KEYDOWN and event.key == K_n:
                octaves -= 1
                if octaves < 1:
                    octaves = 1
                generate(ppp, r, persistence, octaves)

        screen.blit(surface,(0,0))
        pygame.display.flip()

mainloop()

<|endoftext|>


<commit_before>import datetime as dt
import pandas as pd

from random import Random

from .base import TohuBaseGenerator, SeedGenerator
from .primitive_generators import as_tohu_generator, Constant
from .spawn_mapping import SpawnMapping
from .utils import parse_datetime_string, TohuTimestampError

__all__ = ['Apply', 'Lookup', 'SelectMultiple', 'SelectOne', 'Timestamp']

class DerivedGenerator(TohuBaseGenerator):
    """
    Base class for all derived generators
    """

    def reset_input_generators(self, seed):
        """
        Helper method which explicitly resets all input generators
        to the derived generator. This should only ever be called
        for testing or debugging.
        """
        seed_generator = SeedGenerator().reset(seed=seed)

        for gen in self.input_generators:
            gen.reset(next(seed_generator))

class Apply(DerivedGenerator):

    def __init__(self, callable, *arg_gens, **kwarg_gens):
        super().__init__()
        self.callable = callable
        self.arg_gens_orig = arg_gens
        self.kwarg_gens_orig = kwarg_gens

        self.arg_gens = [g.clone() for g in self.arg_gens_orig]
        self.kwarg_gens = {name: g.clone() for name, g in self.kwarg_gens_orig.items()}
        self.input_generators = [g for g in self.arg_gens_orig] + [g for g in self.kwarg_gens_orig.values()]
        self.constituent_generators = [g for g in self.arg_gens] + [g for g in self.kwarg_gens.values()]
        for gen in self.constituent_generators:
            gen.owner = self

    def __next__(self):
        args = [next(g) for g in self.arg_gens]
        kwargs = {name: next(g) for name, g in self.kwarg_gens.items()}
        return self.callable(*args, **kwargs)

    def reset(self, seed):
        super().reset(seed)

    def spawn(self, spawn_mapping=None):
        spawn_mapping = spawn_mapping or SpawnMapping()
        new_arg_gens_orig = [spawn_mapping[g] for g in self.arg_gens_orig]
        new_kwarg_gens_orig = {name: spawn_mapping[g] for name, g in self.kwarg_gens_orig}
        new_obj = Apply(self.callable, *new_arg_gens_orig, **new_kwarg_gens_orig)
        new_obj._set_random_state_from(self)
        return new_obj

    def _set_random_state_from(self, other):
        for g_self, g_other in zip(self.constituent_generators, other.constituent_generators):
            g_self._set_random_state_from(g_other)

class Lookup(Apply):

    def __init__(self, key, mapping):
        self.key = as_tohu_generator(key)
        self.mapping = as_tohu_generator(mapping)

        def f_lookup(key, mapping):
            return mapping[key]

        super().__init__(f_lookup, self.key, self.mapping)

    def spawn(self, spawn_mapping=None):
        spawn_mapping = spawn_mapping or SpawnMapping()
        new_obj = Lookup(spawn_mapping[self.key], spawn_mapping[self.mapping])
        new_obj._set_random_state_from(self)
        return new_obj

class SelectOne(Apply):

    def __init__(self, values):
        self.values_gen = as_tohu_generator(values)
        self.randgen = Random()
        func = self.randgen.choice
        super().__init__(func, self.values_gen)

    def reset(self, seed):
        super().reset(seed)
        self.randgen.seed(seed)

    def spawn(self, spawn_mapping=None):
        spawn_mapping = spawn_mapping or SpawnMapping()
        new_obj = SelectOne(spawn_mapping[self.values_gen])
        new_obj._set_random_state_from(self)
        return new_obj

    def _set_random_state_from(self, other):
        super()._set_random_state_from(other)
        self.randgen.setstate(other.randgen.getstate())

class SelectMultiple(Apply):

    def __init__(self, values, num):
        self.values_gen = as_tohu_generator(values)
        self.num_gen = as_tohu_generator(num)
        self.randgen = Random()
        func = self.randgen.choices
        super().__init__(func, self.values_gen, k=self.num_gen)

    def reset(self, seed):
        super().reset(seed)
        self.randgen.seed(seed)

    def spawn(self, spawn_mapping=None):
        spawn_mapping = spawn_mapping or SpawnMapping()
        new_obj = SelectMultiple(spawn_mapping[self.values_gen], spawn_mapping[self.num_gen])
        new_obj._set_random_state_from(self)
        return new_obj

    def _set_random_state_from(self, other):
        super()._set_random_state_from(other)
        self.randgen.setstate(other.randgen.getstate())

    def size(self):
        def get_size(x):
            return len(x)
        return Apply(get_size, self)

def as_tohu_timestamp_generator(x, optional_offset=None):
    """
    Helper function which ensures that the returned

    Allowed input types:

      - datetime string of the form "YYYY-MM-DD HH:MM:SS"
      - date string of the form "YYYY-MM-DD"
      - datetime.datetime object
      - pandas.Timestamp object
      - tohu generator which

    The argument `optional_offset`
    """
    if isinstance(x, str):
        ts = parse_datetime_string(x, optional_offset)
        return as_tohu_generator(ts)
    elif isinstance(x, dt.datetime):
        return as_tohu_generator(x)
    elif isinstance(x, pd.Timestamp):
        return as_tohu_generator(x.to_pydatetime())
    elif isinstance(x, TohuBaseGenerator):
        if isinstance(x, Timestamp):
            return x
        elif isinstance(x, Constant):
            if isinstance(x.value, dt.datetime):
               return x
            else:
                raise ValueError(f"If input is a Constant tohu generator, its return value "
                                 f"must be of type datetime.datetime. Got: '{type(x.value)}'")
        else:
            raise TypeError(f"If input is a tohu generator it must be of type 'Timestamp' or 'Constant'. Got: '{type(x)}'")
    else:
        raise ValueError(f"Cannot convert input argument to timestamp: {x}")

def get_earliest_value(g):
    if isinstance(g, Constant):
        return g.value
    elif isinstance(g, Timestamp):
        return get_earliest_value(g.start_gen)
    else:
        raise TypeError("This should not occur.")

def get_latest_value(g):
    if isinstance(g, Constant):
        return g.value
    elif isinstance(g, Timestamp):
        return get_latest_value(g.end_gen)
    else:
        raise TypeError("This should not occur.")

def check_start_before_end(start_gen, end_gen):
    latest_start_value = get_latest_value(start_gen)
    earliest_end_value = get_earliest_value(end_gen)

    if latest_start_value > earliest_end_value:
        error_msg = (
            "Latest start value must be before earliest end value. "
            f"Got: latest start value: {latest_start_value}, earliest end value: {earliest_end_value}"
        )
        raise TohuTimestampError(error_msg)

class Timestamp(Apply):

    def __init__(self, *, start=None, end=None, date=None):
        self.start_gen = as_tohu_timestamp_generator(start)
        self.end_gen = as_tohu_timestamp_generator(end, optional_offset=dt.timedelta(hours=23, minutes=59, seconds=59))
        check_start_before_end(self.start_gen, self.end_gen)
        self.offset_randgen = Random()

        def func(start, end):
            interval = (end - start).total_seconds()
            try:
                offset = self.offset_randgen.randint(0, interval)
            except ValueError:
                raise TohuTimestampError(f"Start generator produced timestamp later than end generator: start={start}, end={end}")
            ts = (start + dt.timedelta(seconds=offset))
            return ts

        super().__init__(func, self.start_gen, self.end_gen)

    def reset(self, seed):
        super().reset(seed)
        self.offset_randgen.seed(next(self.seed_generator))

    def spawn(self, spawn_mapping=None):
        spawn_mapping = spawn_mapping or SpawnMapping()
        new_obj = Timestamp(spawn_mapping[self.start_gen], spawn_mapping[self.end_gen])
        new_obj._set_random_state_from(self)
        return new_obj

    def _set_random_state_from(self, other):
        super()._set_random_state_from(other)
        self.offset_randgen.setstate(other.offset_randgen.getstate())

    def strftime(self, fmt):
        func = lambda x: x.strftime(fmt)
        return Apply(func, self)
<commit_msg>Select elements without replacement<commit_after>import datetime as dt
import pandas as pd

from random import Random

from .base import TohuBaseGenerator, SeedGenerator
from .primitive_generators import as_tohu_generator, Constant
from .spawn_mapping import SpawnMapping
from .utils import parse_datetime_string, TohuTimestampError

__all__ = ['Apply', 'Lookup', 'SelectMultiple', 'SelectOne', 'Timestamp']

class DerivedGenerator(TohuBaseGenerator):
    """
    Base class for all derived generators
    """

    def reset_input_generators(self, seed):
        """
        Helper method which explicitly resets all input generators
        to the derived generator. This should only ever be called
        for testing or debugging.
        """
        seed_generator = SeedGenerator().reset(seed=seed)

        for gen in self.input_generators:
            gen.reset(next(seed_generator))

class Apply(DerivedGenerator):

    def __init__(self, callable, *arg_gens, **kwarg_gens):
        super().__init__()
        self.callable = callable
        self.arg_gens_orig = arg_gens
        self.kwarg_gens_orig = kwarg_gens

        self.arg_gens = [g.clone() for g in self.arg_gens_orig]
        self.kwarg_gens = {name: g.clone() for name, g in self.kwarg_gens_orig.items()}
        self.input_generators = [g for g in self.arg_gens_orig] + [g for g in self.kwarg_gens_orig.values()]
        self.constituent_generators = [g for g in self.arg_gens] + [g for g in self.kwarg_gens.values()]
        for gen in self.constituent_generators:
            gen.owner = self

    def __next__(self):
        args = [next(g) for g in self.arg_gens]
        kwargs = {name: next(g) for name, g in self.kwarg_gens.items()}
        return self.callable(*args, **kwargs)

    def reset(self, seed):
        super().reset(seed)

    def spawn(self, spawn_mapping=None):
        spawn_mapping = spawn_mapping or SpawnMapping()
        new_arg_gens_orig = [spawn_mapping[g] for g in self.arg_gens_orig]
        new_kwarg_gens_orig = {name: spawn_mapping[g] for name, g in self.kwarg_gens_orig}
        new_obj = Apply(self.callable, *new_arg_gens_orig, **new_kwarg_gens_orig)
        new_obj._set_random_state_from(self)
        return new_obj

    def _set_random_state_from(self, other):
        for g_self, g_other in zip(self.constituent_generators, other.constituent_generators):
            g_self._set_random_state_from(g_other)

class Lookup(Apply):

    def __init__(self, key, mapping):
        self.key = as_tohu_generator(key)
        self.mapping = as_tohu_generator(mapping)

        def f_lookup(key, mapping):
            return mapping[key]

        super().__init__(f_lookup, self.key, self.mapping)

    def spawn(self, spawn_mapping=None):
        spawn_mapping = spawn_mapping or SpawnMapping()
        new_obj = Lookup(spawn_mapping[self.key], spawn_mapping[self.mapping])
        new_obj._set_random_state_from(self)
        return new_obj

class SelectOne(Apply):

    def __init__(self, values):
        self.values_gen = as_tohu_generator(values)
        self.randgen = Random()
        func = self.randgen.choice
        super().__init__(func, self.values_gen)

    def reset(self, seed):
        super().reset(seed)
        self.randgen.seed(seed)

    def spawn(self, spawn_mapping=None):
        spawn_mapping = spawn_mapping or SpawnMapping()
        new_obj = SelectOne(spawn_mapping[self.values_gen])
        new_obj._set_random_state_from(self)
        return new_obj

    def _set_random_state_from(self, other):
        super()._set_random_state_from(other)
        self.randgen.setstate(other.randgen.getstate())

class SelectMultiple(Apply):

    def __init__(self, values, num):
        self.values_gen = as_tohu_generator(values)
        self.num_gen = as_tohu_generator(num)
        self.randgen = Random()
        func = self.randgen.sample
        super().__init__(func, self.values_gen, k=self.num_gen)

    def reset(self, seed):
        super().reset(seed)
        self.randgen.seed(seed)

    def spawn(self, spawn_mapping=None):
        spawn_mapping = spawn_mapping or SpawnMapping()
        new_obj = SelectMultiple(spawn_mapping[self.values_gen], spawn_mapping[self.num_gen])
        new_obj._set_random_state_from(self)
        return new_obj

    def _set_random_state_from(self, other):
        super()._set_random_state_from(other)
        self.randgen.setstate(other.randgen.getstate())

    def size(self):
        def get_size(x):
            return len(x)
        return Apply(get_size, self)

def as_tohu_timestamp_generator(x, optional_offset=None):
    """
    Helper function which ensures that the returned

    Allowed input types:

      - datetime string of the form "YYYY-MM-DD HH:MM:SS"
      - date string of the form "YYYY-MM-DD"
      - datetime.datetime object
      - pandas.Timestamp object
      - tohu generator which

    The argument `optional_offset`
    """
    if isinstance(x, str):
        ts = parse_datetime_string(x, optional_offset)
        return as_tohu_generator(ts)
    elif isinstance(x, dt.datetime):
        return as_tohu_generator(x)
    elif isinstance(x, pd.Timestamp):
        return as_tohu_generator(x.to_pydatetime())
    elif isinstance(x, TohuBaseGenerator):
        if isinstance(x, Timestamp):
            return x
        elif isinstance(x, Constant):
            if isinstance(x.value, dt.datetime):
               return x
            else:
                raise ValueError(f"If input is a Constant tohu generator, its return value "
                                 f"must be of type datetime.datetime. Got: '{type(x.value)}'")
        else:
            raise TypeError(f"If input is a tohu generator it must be of type 'Timestamp' or 'Constant'. Got: '{type(x)}'")
    else:
        raise ValueError(f"Cannot convert input argument to timestamp: {x}")

def get_earliest_value(g):
    if isinstance(g, Constant):
        return g.value
    elif isinstance(g, Timestamp):
        return get_earliest_value(g.start_gen)
    else:
        raise TypeError("This should not occur.")

def get_latest_value(g):
    if isinstance(g, Constant):
        return g.value
    elif isinstance(g, Timestamp):
        return get_latest_value(g.end_gen)
    else:
        raise TypeError("This should not occur.")

def check_start_before_end(start_gen, end_gen):
    latest_start_value = get_latest_value(start_gen)
    earliest_end_value = get_earliest_value(end_gen)

    if latest_start_value > earliest_end_value:
        error_msg = (
            "Latest start value must be before earliest end value. "
            f"Got: latest start value: {latest_start_value}, earliest end value: {earliest_end_value}"
        )
        raise TohuTimestampError(error_msg)

class Timestamp(Apply):

    def __init__(self, *, start=None, end=None, date=None):
        self.start_gen = as_tohu_timestamp_generator(start)
        self.end_gen = as_tohu_timestamp_generator(end, optional_offset=dt.timedelta(hours=23, minutes=59, seconds=59))
        check_start_before_end(self.start_gen, self.end_gen)
        self.offset_randgen = Random()

        def func(start, end):
            interval = (end - start).total_seconds()
            try:
                offset = self.offset_randgen.randint(0, interval)
            except ValueError:
                raise TohuTimestampError(f"Start generator produced timestamp later than end generator: start={start}, end={end}")
            ts = (start + dt.timedelta(seconds=offset))
            return ts

        super().__init__(func, self.start_gen, self.end_gen)

    def reset(self, seed):
        super().reset(seed)
        self.offset_randgen.seed(next(self.seed_generator))

    def spawn(self, spawn_mapping=None):
        spawn_mapping = spawn_mapping or SpawnMapping()
        new_obj = Timestamp(spawn_mapping[self.start_gen], spawn_mapping[self.end_gen])
        new_obj._set_random_state_from(self)
        return new_obj

    def _set_random_state_from(self, other):
        super()._set_random_state_from(other)
        self.offset_randgen.setstate(other.offset_randgen.getstate())

    def strftime(self, fmt):
        func = lambda x: x.strftime(fmt)
        return Apply(func, self)
<|endoftext|>


<commit_before># Copyright (C) 2013 by Aivars Kalvans <aivars.kalvans@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import re
import random
import base64
from scrapy import log

class RandomProxy(object):
    def __init__(self, settings):
        self.proxy_list = settings.get('PROXY_LIST')
        fin = open(self.proxy_list)

        self.proxies = {}
        for line in fin.readlines():
            parts = re.match('(\w+://)(\w+:\w+@)?(.+)', line)

            # Cut trailing @
            if parts.group(2):
                user_pass = parts.group(2)[:-1]
            else:
                user_pass = ''

            self.proxies[parts.group(1) + parts.group(3)] = user_pass

        fin.close()

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler.settings)

    def process_request(self, request, spider):
        # Don't overwrite with a random one (server-side state for IP)
        if 'proxy' in request.meta:
            return

        proxy_address = random.choice(self.proxies.keys())
        proxy_user_pass = self.proxies[proxy_address]

        request.meta['proxy'] = proxy_address
        if proxy_user_pass:
            basic_auth = 'Basic ' + base64.encodestring(proxy_user_pass)
            request.headers['Proxy-Authorization'] = basic_auth

    def process_exception(self, request, exception, spider):
        proxy = request.meta['proxy']
        log.msg('Removing failed proxy <%s>, %d proxies left' % (
                    proxy, len(self.proxies)))
        try:
            del self.proxies[proxy]
        except ValueError:
            pass
<commit_msg>Do not fail if file has an empty line in it, fixes #13<commit_after># Copyright (C) 2013 by Aivars Kalvans <aivars.kalvans@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import re
import random
import base64
from scrapy import log

class RandomProxy(object):
    def __init__(self, settings):
        self.proxy_list = settings.get('PROXY_LIST')
        fin = open(self.proxy_list)

        self.proxies = {}
        for line in fin.readlines():
            parts = re.match('(\w+://)(\w+:\w+@)?(.+)', line)
            if not parts:
                continue

            # Cut trailing @
            if parts.group(2):
                user_pass = parts.group(2)[:-1]
            else:
                user_pass = ''

            self.proxies[parts.group(1) + parts.group(3)] = user_pass

        fin.close()

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler.settings)

    def process_request(self, request, spider):
        # Don't overwrite with a random one (server-side state for IP)
        if 'proxy' in request.meta:
            return

        proxy_address = random.choice(self.proxies.keys())
        proxy_user_pass = self.proxies[proxy_address]

        request.meta['proxy'] = proxy_address
        if proxy_user_pass:
            basic_auth = 'Basic ' + base64.encodestring(proxy_user_pass)
            request.headers['Proxy-Authorization'] = basic_auth

    def process_exception(self, request, exception, spider):
        proxy = request.meta['proxy']
        log.msg('Removing failed proxy <%s>, %d proxies left' % (
                    proxy, len(self.proxies)))
        try:
            del self.proxies[proxy]
        except ValueError:
            pass
<|endoftext|>


<commit_before>import os
import smbus
import subprocess

def i2c_bus():
    return smbus.SMBus(1)

def i2c_bus_num():
        return '1'

def serial_bus():
    return "/dev/ttyUSB0"
    #return "/dev/ttyAMA0"<commit_msg>indent<commit_after>import os
import smbus
import subprocess

def i2c_bus():
    return smbus.SMBus(1)

def i2c_bus_num():
    return '1'

def serial_bus():
    return "/dev/ttyUSB0"
    #return "/dev/ttyAMA0"<|endoftext|>


<commit_before>#MAEC Malware Subject Class

#Copyright (c) 2014, The MITRE Corporation
#All rights reserved

#Compatible with MAEC v4.1
#Last updated 08/20/2014

import cybox
from cybox.common import VocabString, PlatformSpecification, ToolInformationList, ToolInformation
from cybox.objects.file_object import File
from cybox.objects.uri_object import URI
from cybox.core import Object

import maec
import maec.bindings.maec_package as package_binding
from maec.bundle.bundle import Bundle
from maec.package.action_equivalence import ActionEquivalenceList
from maec.package.analysis import Analysis
from maec.package.malware_subject_reference import MalwareSubjectReference
from maec.package.object_equivalence import ObjectEquivalenceList

class MinorVariants(maec.EntityList):
    _contained_type = Object
    _binding_class = package_binding.MinorVariantListType
    _binding_var = "Minor_Variant"
    _namespace = maec.package._namespace

class Analyses(maec.EntityList):
    _contained_type = Analysis
    _binding_class = package_binding.AnalysisListType
    _binding_var = "Analysis"
    _namespace = maec.package._namespace

class MalwareSubjectRelationship(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareSubjectRelationshipType
    _namespace = maec.package._namespace

    malware_subject_reference = maec.TypedField("Malware_Subject_Reference", MalwareSubjectReference, multiple = True)
    type = maec.TypedField("type", VocabString)

    def __init__(self):
        super(MalwareSubjectRelationship, self).__init__()

class MalwareSubjectRelationshipList(maec.EntityList):
    _contained_type = MalwareSubjectRelationship
    _binding_class = package_binding.MalwareSubjectRelationshipListType
    _binding_var = "Relationship"
    _namespace = maec.package._namespace

class MetaAnalysis(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MetaAnalysisType
    _namespace = maec.package._namespace

    action_equivalences = maec.TypedField("Action_Equivalences", ActionEquivalenceList)
    object_equivalences = maec.TypedField("Object_Equivalences", ObjectEquivalenceList)

    def __init__(self):
        super(MetaAnalysis, self).__init__()

class FindingsBundleList(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.FindingsBundleListType
    _namespace = maec.package._namespace

    meta_analysis = maec.TypedField("Meta_Analysis", MetaAnalysis)
    bundle = maec.TypedField("Bundle", Bundle, multiple = True)
    bundle_external_reference = maec.TypedField("Bundle_External_Reference", multiple = True)

    def __init__(self):
        super(FindingsBundleList, self).__init__()

    def add_bundle(self, bundle):
        if not self.bundle:
            self.bundle = []
        self.bundle.append(bundle)

    def add_bundle_external_reference(self, bundle_external_reference):
        if not self.bundle_external_reference:
            self.bundle_external_reference = []
        self.bundle_external_reference.append(bundle_external_reference)

class MalwareDevelopmentEnvironment(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareDevelopmentEnvironmentType
    _namespace = maec.package._namespace

    tools = maec.TypedField("Tools", ToolInformation)
    debugging_file = maec.TypedField("Debugging_File", File, multiple = True)

    def __init__(self):
        super(MalwareDevelopmentEnvironment, self).__init__()

class MalwareConfigurationParameter(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareConfigurationParameterType
    _namespace = maec.package._namespace

    name = maec.TypedField("Name", VocabString)
    value = maec.TypedField("Value")

    def __init__(self):
        super(MalwareConfigurationParameter, self).__init__()

class MalwareBinaryConfigurationStorageDetails(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareBinaryConfigurationStorageDetailsType
    _namespace = maec.package._namespace

    file_offset = maec.TypedField("File_Offset")
    section_name = maec.TypedField("Section_Name")
    section_offset = maec.TypedField("Section_Offset")

    def __init__(self):
        super(MalwareBinaryConfigurationStorageDetails, self).__init__()

class MalwareConfigurationStorageDetails(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareConfigurationStorageDetailsType
    _namespace = maec.package._namespace

    malware_binary = maec.TypedField("Malware_Binary", MalwareBinaryConfigurationStorageDetails)
    file = maec.TypedField("File", File)
    url = maec.TypedField("URL", URI, multiple = True)

    def __init__(self):
        super(MalwareConfigurationStorageDetails, self).__init__()

class MalwareConfigurationObfuscationAlgorithm(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareConfigurationObfuscationAlgorithmType
    _namespace = maec.package._namespace

    ordinal_position = maec.TypedField("ordinal_position")
    key = maec.TypedField("Key")
    algorithm_name = maec.TypedField("Algorithm_Name", VocabString)

    def __init__(self):
        super(MalwareConfigurationObfuscationAlgorithm, self).__init__()

class MalwareConfigurationObfuscationDetails(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareConfigurationObfuscationDetailsType
    _namespace = maec.package._namespace

    is_encoded = maec.TypedField("is_encoded")
    is_encrypted = maec.TypedField("is_encrypted")
    algorithm_details = maec.TypedField("Algorithm_Details", MalwareConfigurationObfuscationAlgorithm, multiple = True)

    def __init__(self):
        super(MalwareConfigurationObfuscationDetails, self).__init__()
        self.algorithm_details = []

class MalwareConfigurationDetails(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareConfigurationDetailsType
    _namespace = maec.package._namespace

    storage = maec.TypedField("Storage", MalwareConfigurationStorageDetails)
    obfuscation = maec.TypedField("Obfuscation", MalwareConfigurationObfuscationDetails)
    configuration_parameter = maec.TypedField("Configuration_Parameter", MalwareConfigurationParameter, multiple = True)

    def __init__(self):
        super(MalwareConfigurationDetails, self).__init__()

class MalwareSubject(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareSubjectType
    _namespace = maec.package._namespace

    id_ = maec.TypedField("id")
    malware_instance_object_attributes = maec.TypedField("Malware_Instance_Object_Attributes", Object)
    label = maec.TypedField("Label", VocabString, multiple=True)
    configuration_details = maec.TypedField("Configuration_Details", MalwareConfigurationDetails)
    minor_variants = maec.TypedField("Minor_Variants", MinorVariants)
    development_environment = maec.TypedField("Development_Environment", MalwareDevelopmentEnvironment)
    #field_data = maec.TypedField("field_data") # TODO: support metadata:fieldDataEntry
    analyses = maec.TypedField("Analyses", Analyses)
    findings_bundles = maec.TypedField("Findings_Bundles", FindingsBundleList)
    relationships = maec.TypedField("Relationships", MalwareSubjectRelationshipList)
    compatible_platform = maec.TypedField("Compatible_Platform", PlatformSpecification, multiple=True)

    def __init__(self, id = None, malware_instance_object_attributes = None):
        super(MalwareSubject, self).__init__()
        if id:
            self.id_ = id
        else:
            self.id_ = maec.utils.idgen.create_id(prefix="malware_subject")
        #Set the Malware Instance Object Attributes (a CybOX object) if they are not none
        self.malware_instance_object_attributes = malware_instance_object_attributes

    #Public methods
    #Set the Malware_Instance_Object_Attributes with a CybOX object
    def set_malware_instance_object_attributes(self, malware_instance_object_attributes):
        self.malware_instance_object_attributes = malware_instance_object_attributes

    #Add an Analysis to the Analyses
    def add_analysis(self, analysis):
        if not self.analyses:
            self.analyses = Analyses()
        self.analyses.append(analysis)

    def get_analyses(self):
        return self.analyses

    #Get all Bundles in the Subject
    def get_all_bundles(self):
        return self.findings_bundles.bundles

    #Add a MAEC Bundle to the Findings Bundles
    def add_findings_bundle(self, bundle):
        if not self.findings_bundles:
            self.findings_bundles = FindingsBundleList()
        self.findings_bundles.add_bundle(bundle)

    def deduplicate_bundles(self):
        """DeDuplicate all Findings Bundles in the Malware Subject. For now, only handles Objects"""
        for findings_bundle in self.findings_bundles.bundles:
            findings_bundle.deduplicate()

    def dereference_bundles(self):
        """Dereference all Findings Bundles in the Malware Subject. For now, only handles Objects"""
        all_bundles = self.get_all_bundles()
        for bundle in all_bundles:
            bundle.dereference_objects([self.malware_instance_object_attributes])

    def normalize_bundles(self):
        """Normalize all Findings Bundles in the Malware Subject. For now, only handles Objects"""
        all_bundles = self.get_all_bundles()
        for bundle in all_bundles:
            bundle.normalize_objects()

class MalwareSubjectList(maec.EntityList):
    _contained_type = MalwareSubject
    _binding_class = package_binding.MalwareSubjectListType
    _binding_var = "Malware_Subject"
    _namespace = maec.package._namespace<commit_msg>Updated deduplicate_bundles to use get_all_bundles, like other _bundles methods<commit_after>#MAEC Malware Subject Class

#Copyright (c) 2014, The MITRE Corporation
#All rights reserved

#Compatible with MAEC v4.1
#Last updated 08/20/2014

import cybox
from cybox.common import VocabString, PlatformSpecification, ToolInformationList, ToolInformation
from cybox.objects.file_object import File
from cybox.objects.uri_object import URI
from cybox.core import Object

import maec
import maec.bindings.maec_package as package_binding
from maec.bundle.bundle import Bundle
from maec.package.action_equivalence import ActionEquivalenceList
from maec.package.analysis import Analysis
from maec.package.malware_subject_reference import MalwareSubjectReference
from maec.package.object_equivalence import ObjectEquivalenceList

class MinorVariants(maec.EntityList):
    _contained_type = Object
    _binding_class = package_binding.MinorVariantListType
    _binding_var = "Minor_Variant"
    _namespace = maec.package._namespace

class Analyses(maec.EntityList):
    _contained_type = Analysis
    _binding_class = package_binding.AnalysisListType
    _binding_var = "Analysis"
    _namespace = maec.package._namespace

class MalwareSubjectRelationship(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareSubjectRelationshipType
    _namespace = maec.package._namespace

    malware_subject_reference = maec.TypedField("Malware_Subject_Reference", MalwareSubjectReference, multiple = True)
    type = maec.TypedField("type", VocabString)

    def __init__(self):
        super(MalwareSubjectRelationship, self).__init__()

class MalwareSubjectRelationshipList(maec.EntityList):
    _contained_type = MalwareSubjectRelationship
    _binding_class = package_binding.MalwareSubjectRelationshipListType
    _binding_var = "Relationship"
    _namespace = maec.package._namespace

class MetaAnalysis(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MetaAnalysisType
    _namespace = maec.package._namespace

    action_equivalences = maec.TypedField("Action_Equivalences", ActionEquivalenceList)
    object_equivalences = maec.TypedField("Object_Equivalences", ObjectEquivalenceList)

    def __init__(self):
        super(MetaAnalysis, self).__init__()

class FindingsBundleList(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.FindingsBundleListType
    _namespace = maec.package._namespace

    meta_analysis = maec.TypedField("Meta_Analysis", MetaAnalysis)
    bundle = maec.TypedField("Bundle", Bundle, multiple = True)
    bundle_external_reference = maec.TypedField("Bundle_External_Reference", multiple = True)

    def __init__(self):
        super(FindingsBundleList, self).__init__()

    def add_bundle(self, bundle):
        if not self.bundle:
            self.bundle = []
        self.bundle.append(bundle)

    def add_bundle_external_reference(self, bundle_external_reference):
        if not self.bundle_external_reference:
            self.bundle_external_reference = []
        self.bundle_external_reference.append(bundle_external_reference)

class MalwareDevelopmentEnvironment(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareDevelopmentEnvironmentType
    _namespace = maec.package._namespace

    tools = maec.TypedField("Tools", ToolInformation)
    debugging_file = maec.TypedField("Debugging_File", File, multiple = True)

    def __init__(self):
        super(MalwareDevelopmentEnvironment, self).__init__()

class MalwareConfigurationParameter(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareConfigurationParameterType
    _namespace = maec.package._namespace

    name = maec.TypedField("Name", VocabString)
    value = maec.TypedField("Value")

    def __init__(self):
        super(MalwareConfigurationParameter, self).__init__()

class MalwareBinaryConfigurationStorageDetails(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareBinaryConfigurationStorageDetailsType
    _namespace = maec.package._namespace

    file_offset = maec.TypedField("File_Offset")
    section_name = maec.TypedField("Section_Name")
    section_offset = maec.TypedField("Section_Offset")

    def __init__(self):
        super(MalwareBinaryConfigurationStorageDetails, self).__init__()

class MalwareConfigurationStorageDetails(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareConfigurationStorageDetailsType
    _namespace = maec.package._namespace

    malware_binary = maec.TypedField("Malware_Binary", MalwareBinaryConfigurationStorageDetails)
    file = maec.TypedField("File", File)
    url = maec.TypedField("URL", URI, multiple = True)

    def __init__(self):
        super(MalwareConfigurationStorageDetails, self).__init__()

class MalwareConfigurationObfuscationAlgorithm(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareConfigurationObfuscationAlgorithmType
    _namespace = maec.package._namespace

    ordinal_position = maec.TypedField("ordinal_position")
    key = maec.TypedField("Key")
    algorithm_name = maec.TypedField("Algorithm_Name", VocabString)

    def __init__(self):
        super(MalwareConfigurationObfuscationAlgorithm, self).__init__()

class MalwareConfigurationObfuscationDetails(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareConfigurationObfuscationDetailsType
    _namespace = maec.package._namespace

    is_encoded = maec.TypedField("is_encoded")
    is_encrypted = maec.TypedField("is_encrypted")
    algorithm_details = maec.TypedField("Algorithm_Details", MalwareConfigurationObfuscationAlgorithm, multiple = True)

    def __init__(self):
        super(MalwareConfigurationObfuscationDetails, self).__init__()
        self.algorithm_details = []

class MalwareConfigurationDetails(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareConfigurationDetailsType
    _namespace = maec.package._namespace

    storage = maec.TypedField("Storage", MalwareConfigurationStorageDetails)
    obfuscation = maec.TypedField("Obfuscation", MalwareConfigurationObfuscationDetails)
    configuration_parameter = maec.TypedField("Configuration_Parameter", MalwareConfigurationParameter, multiple = True)

    def __init__(self):
        super(MalwareConfigurationDetails, self).__init__()

class MalwareSubject(maec.Entity):
    _binding = package_binding
    _binding_class = package_binding.MalwareSubjectType
    _namespace = maec.package._namespace

    id_ = maec.TypedField("id")
    malware_instance_object_attributes = maec.TypedField("Malware_Instance_Object_Attributes", Object)
    label = maec.TypedField("Label", VocabString, multiple=True)
    configuration_details = maec.TypedField("Configuration_Details", MalwareConfigurationDetails)
    minor_variants = maec.TypedField("Minor_Variants", MinorVariants)
    development_environment = maec.TypedField("Development_Environment", MalwareDevelopmentEnvironment)
    #field_data = maec.TypedField("field_data") # TODO: support metadata:fieldDataEntry
    analyses = maec.TypedField("Analyses", Analyses)
    findings_bundles = maec.TypedField("Findings_Bundles", FindingsBundleList)
    relationships = maec.TypedField("Relationships", MalwareSubjectRelationshipList)
    compatible_platform = maec.TypedField("Compatible_Platform", PlatformSpecification, multiple=True)

    def __init__(self, id = None, malware_instance_object_attributes = None):
        super(MalwareSubject, self).__init__()
        if id:
            self.id_ = id
        else:
            self.id_ = maec.utils.idgen.create_id(prefix="malware_subject")
        #Set the Malware Instance Object Attributes (a CybOX object) if they are not none
        self.malware_instance_object_attributes = malware_instance_object_attributes

    #Public methods
    #Set the Malware_Instance_Object_Attributes with a CybOX object
    def set_malware_instance_object_attributes(self, malware_instance_object_attributes):
        self.malware_instance_object_attributes = malware_instance_object_attributes

    #Add an Analysis to the Analyses
    def add_analysis(self, analysis):
        if not self.analyses:
            self.analyses = Analyses()
        self.analyses.append(analysis)

    def get_analyses(self):
        return self.analyses

    #Get all Bundles in the Subject
    def get_all_bundles(self):
        return self.findings_bundles.bundles

    #Add a MAEC Bundle to the Findings Bundles
    def add_findings_bundle(self, bundle):
        if not self.findings_bundles:
            self.findings_bundles = FindingsBundleList()
        self.findings_bundles.add_bundle(bundle)

    def deduplicate_bundles(self):
        """DeDuplicate all Findings Bundles in the Malware Subject. For now, only handles Objects"""
        all_bundles = self.get_all_bundles()
        for bundle in all_bundles:
            bundle.deduplicate()

    def dereference_bundles(self):
        """Dereference all Findings Bundles in the Malware Subject. For now, only handles Objects"""
        all_bundles = self.get_all_bundles()
        for bundle in all_bundles:
            bundle.dereference_objects([self.malware_instance_object_attributes])

    def normalize_bundles(self):
        """Normalize all Findings Bundles in the Malware Subject. For now, only handles Objects"""
        all_bundles = self.get_all_bundles()
        for bundle in all_bundles:
            bundle.normalize_objects()

class MalwareSubjectList(maec.EntityList):
    _contained_type = MalwareSubject
    _binding_class = package_binding.MalwareSubjectListType
    _binding_var = "Malware_Subject"
    _namespace = maec.package._namespace<|endoftext|>


<commit_before>import sys
sys.path.append('PythonTools')
import unittest
from InterfaceMode import *
from random import randint, uniform
from Constants import *

test_runs = 20

min_rand = 0
max_rand = 150

chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!£$%^&*()-+=@?/\.,#~;:][{}<>|"

rand_str_length = 10

# Create a list of single commands that can be matched with integer arguments
int_args = [START_TIME, FINAL_TIME, OUTER_RES, INNER_RES, KAPPA, DRIVE_ALPHA]

# Create a list of single commands that can ONLY be matched with integer arguments (not converted to float)
only_int_args = [OUTER_RES, INNER_RES]

# Create a list of single commands that can be matched with float arguments
float_args = [START_TIME, FINAL_TIME, KAPPA, DRIVE_ALPHA]

# Create a list of single commands that can be matched with 'boolean' (y/n) arguments
bool_args = [SAVE_PLOTS, SAVE_DATA, COMPILE]

# Create a list 'multiple' commands (i.e. take the form PARAM VAL1 VAL2) that accept int values
mult_int_args = [MULTIPLE_RES,MULTIPLE_TIMES]

# Create a list 'multiple' commands (i.e. take the form PARAM VAL1 VAL2) that accept float values
mult_float_args = [MULTIPLE_TIMES]

# Create a list 'multiple' commands (i.e. take the form PARAM VAL1 VAL2) that only accept int values
mult_only_int_args = [MULTIPLE_RES]

class InterfaceModeTests(unittest.TestCase):

    def valid_yes(self,first_arg):
        return first_arg + " y"

    def valid_no(self,first_arg):
        return first_arg + " n"

    def valid_int_single(self,first_arg):
        return first_arg + " " + str(randint(min_rand,max_rand))

    def valid_int_multiple(self,first_arg):
        return " ".join([first_arg, str(randint(min_rand,max_rand)), str(randint(min_rand,max_rand))])

    def random_string(self,first_arg):
        return first_arg + " " + "".join([chars[randint(min_rand,max_rand) % len(chars)] for _ in range(rand_str_length)])

    def valid_float_single(self,first_arg):
        return first_arg + " " + str(uniform(min_rand,max_rand))

    def valid_float_multiple(self,first_arg):
        return " ".join([first_arg, str(uniform(min_rand,max_rand)), str(uniform(min_rand,max_rand))])

    def test_change_multiple_parameters(self):

        # Create InterfaceMode objects
        single_mode = SingleTrace()
        forsin_mode = ForwardSingle()
        invsin_mode = InverseSingle()

        # Repeat the tests a set number of times:
        for _ in range(test_runs):

            for command in mult_int_args:

                # Check that the int commands accept ints
                arg = self.valid_int_multiple(command)
                self.assertEqual(single_mode.change_multiple_parameters(*arg.split()),True,arg)
                self.assertEqual(forsin_mode.change_multiple_parameters(*arg.split()),True,arg)
                self.assertEqual(invsin_mode.change_multiple_parameters(*arg.split()),True,arg)

                # Check that the int commands reject strings
                arg = self.random_string(command) + " " + self.random_string("")
                self.assertEqual(single_mode.change_multiple_parameters(*arg.split()),False,arg)
                self.assertEqual(forsin_mode.change_multiple_parameters(*arg.split()),False,arg)
                self.assertEqual(invsin_mode.change_multiple_parameters(*arg.split()),False,arg)

            for command in mult_float_args:

                # Check that the float commands accept floats
                arg = self.valid_float_multiple(command)
                self.assertEqual(single_mode.change_multiple_parameters(*arg.split()),True,arg)
                self.assertEqual(forsin_mode.change_multiple_parameters(*arg.split()),True,arg)
                self.assertEqual(invsin_mode.change_multiple_parameters(*arg.split()),True,arg)

                # Check that the float commands reject strings
                arg = self.random_string(command) + " " + self.random_string("")
                self.assertEqual(single_mode.change_multiple_parameters(*arg.split()),False,arg)
                self.assertEqual(forsin_mode.change_multiple_parameters(*arg.split()),False,arg)
                self.assertEqual(invsin_mode.change_multiple_parameters(*arg.split()),False,arg)

            for command in mult_only_int_args:

                # Check that the int only commands reject floats
                arg = self.valid_float_multiple(command)
                self.assertEqual(single_mode.change_multiple_parameters(*arg.split()),False,arg)
                self.assertEqual(forsin_mode.change_multiple_parameters(*arg.split()),False,arg)
                self.assertEqual(invsin_mode.change_multiple_parameters(*arg.split()),False,arg)

    def test_change_single_parameter(self):

        # Create InterfaceMode objects
        single_mode = SingleTrace()
        forsin_mode = ForwardSingle()
        invsin_mode = InverseSingle()

        # Repeat the tests a set number of times:
        for _ in range(test_runs):

            for command in int_args:

                # Check that the int commands accept ints
                arg = self.valid_int_single(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),True,arg)
                self.assertEqual(forsin_mode.change_single_parameter(*arg.split()),True,arg)
                self.assertEqual(invsin_mode.change_single_parameter(*arg.split()),True,arg)

                # Check that the int commands reject strings
                arg = self.random_string(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),False,arg)
                self.assertEqual(forsin_mode.change_single_parameter(*arg.split()),False,arg)
                self.assertEqual(invsin_mode.change_single_parameter(*arg.split()),False,arg)

            for command in float_args:

                # Check that the float commands accept floats
                arg = self.valid_float_single(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),True,arg)
                self.assertEqual(forsin_mode.change_single_parameter(*arg.split()),True,arg)
                self.assertEqual(invsin_mode.change_single_parameter(*arg.split()),True,arg)

                # Check that the float commands reject strings
                arg = self.random_string(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),False,arg)
                self.assertEqual(forsin_mode.change_single_parameter(*arg.split()),False,arg)
                self.assertEqual(invsin_mode.change_single_parameter(*arg.split()),False,arg)

            for command in only_int_args:

                # Check that the int only commands reject floats
                arg = self.valid_float_single(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),False,arg)
                self.assertEqual(forsin_mode.change_single_parameter(*arg.split()),False,arg)
                self.assertEqual(invsin_mode.change_single_parameter(*arg.split()),False,arg)

            for command in bool_args:

                # Check that the bool commands accept a 'true' response
                arg = self.valid_yes(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),True,arg)
                self.assertEqual(forsin_mode.change_single_parameter(*arg.split()),True,arg)
                self.assertEqual(invsin_mode.change_single_parameter(*arg.split()),True,arg)

                # Check that the bool commands accept a 'false' response
                arg = self.valid_no(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),True,arg)
                self.assertEqual(forsin_mode.change_single_parameter(*arg.split()),True,arg)
                self.assertEqual(invsin_mode.change_single_parameter(*arg.split()),True,arg)

                # Check that the bool commands reject random strings
                arg = self.random_string(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),False,arg)
                self.assertEqual(forsin_mode.change_single_parameter(*arg.split()),False,arg)
                self.assertEqual(invsin_mode.change_single_parameter(*arg.split()),False,arg)

                # Check that the bool commands reject ints
                arg = self.valid_int_single(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),False,arg)
                self.assertEqual(forsin_mode.change_single_parameter(*arg.split()),False,arg)
                self.assertEqual(invsin_mode.change_single_parameter(*arg.split()),False,arg)

                # Check that the bool commands reject floats
                arg = self.valid_float_single(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),False,arg)
                self.assertEqual(forsin_mode.change_single_parameter(*arg.split()),False,arg)
                self.assertEqual(invsin_mode.change_single_parameter(*arg.split()),False,arg)

    def test_change_parameters(self):

        # Create an InterfaceMode object
        single_mode = SingleTrace()
        forsin_mode = ForwardSingle()
        invsin_mode = InverseSingle()

        # Repeat the tests a set number of times:
        for _ in range(test_runs):

            # Check that the int commands accept ints and reject strings
            for command in int_args:

                arg = self.valid_int_single(command)
                self.assertEqual(single_mode.change_parameters(arg),True,arg)
                self.assertEqual(forsin_mode.change_parameters(arg),True,arg)
                self.assertEqual(invsin_mode.change_parameters(arg),True,arg)

                arg = self.random_string(command)
                self.assertEqual(single_mode.change_parameters(arg),False,arg)
                self.assertEqual(forsin_mode.change_parameters(arg),False,arg)
                self.assertEqual(invsin_mode.change_parameters(arg),False,arg)

            # Check that the float commands accept floats and reject strings
            for command in float_args:

                arg = self.valid_float_single(command)
                self.assertEqual(single_mode.change_parameters(arg),True,arg)
                self.assertEqual(forsin_mode.change_parameters(arg),True,arg)
                self.assertEqual(invsin_mode.change_parameters(arg),True,arg)

                arg = self.random_string(command)
                self.assertEqual(single_mode.change_parameters(arg),False,arg)
                self.assertEqual(forsin_mode.change_parameters(arg),False,arg)
                self.assertEqual(invsin_mode.change_parameters(arg),False,arg)

            # Check that the int only commands reject floats
            for command in only_int_args:

                arg = self.valid_float_single(command)
                self.assertEqual(single_mode.change_parameters(arg),False,arg)
                self.assertEqual(forsin_mode.change_parameters(arg),False,arg)
                self.assertEqual(invsin_mode.change_parameters(arg),False,arg)

            # Check that the int commands accept ints and reject strings
            for command in mult_int_args:

                arg = self.valid_int_multiple(command)
                self.assertEqual(single_mode.change_parameters(arg),True,arg)
                self.assertEqual(forsin_mode.change_parameters(arg),True,arg)
                self.assertEqual(invsin_mode.change_parameters(arg),True,arg)

                arg = self.random_string(command)
                self.assertEqual(single_mode.change_parameters(arg),False,arg)
                self.assertEqual(forsin_mode.change_parameters(arg),False,arg)
                self.assertEqual(invsin_mode.change_parameters(arg),False,arg)

            # Check that the float commands accept floats and reject strings
            for command in mult_float_args:

                arg = self.valid_float_multiple(command)
                self.assertEqual(single_mode.change_parameters(arg),True,arg)
                self.assertEqual(forsin_mode.change_parameters(arg),True,arg)
                self.assertEqual(invsin_mode.change_parameters(arg),True,arg)

                arg = self.random_string(command)
                self.assertEqual(single_mode.change_parameters(arg),False,arg)
                self.assertEqual(forsin_mode.change_parameters(arg),False,arg)
                self.assertEqual(invsin_mode.change_parameters(arg),False,arg)

            # Check that the int only commands reject floats
            for command in mult_only_int_args:

                arg = self.valid_float_multiple(command)
                self.assertEqual(single_mode.change_parameters(arg),False,arg)
                self.assertEqual(forsin_mode.change_parameters(arg),False,arg)
                self.assertEqual(invsin_mode.change_parameters(arg),False,arg)

    def test_exactinverse_mode(self):

        # Create an InterfaceMode object
        inverse_mode = ExactInverse()

        # Create a list of good arguments
        good_args = ["starttime 0","finaltime 3","outerres 3","saveplots y","saveplots n","savedata y", \
                "savedata n","kappa 3.5","drivealpha 0.2"]

        # Create a list of good three-argument commands
        good_three = ["times 0 10", "times 4.8 10", "times 0.5 10.5", "times 2 10.5"]

        # Create a list of bad three-argument commands - resolution pairs not used for exact inverse
        bad_three = ["times a b", "res 100 2","res !! -", "times 4.8 b", "times a 10.5", "res 10.2 1", "res 10.2 a", "res a 1", "res 1 a", "res 1 1"]

        # Create a list of bad arguments - compile and innerres are not used in exact inverse mode
        bad_args = ["starttime x","compile y","compile n","finaltime -","outerres 3.2","innerres 5","innerres 0.1","saveplots x","saveplots 2.5","savedata 1"]

        # Check that InterfaceMode returns True for the correct arguments
        for arg in good_args:
            self.assertEqual(inverse_mode.change_single_parameter(*arg.split()),True,arg)
            self.assertEqual(inverse_mode.change_parameters(arg),True,arg)
        for arg in good_three:
            self.assertEqual(inverse_mode.change_multiple_parameters(*arg.split()),True,arg)
            self.assertEqual(inverse_mode.change_parameters(arg),True,arg)

        # Check that InterfaceMode returns False for the bad arguments
        for arg in bad_args:
            self.assertEqual(inverse_mode.change_single_parameter(*arg.split()),False,arg)
            self.assertEqual(inverse_mode.change_parameters(arg),False,arg)
        for arg in bad_three:
            self.assertEqual(inverse_mode.change_multiple_parameters(*arg.split()),False,arg)
            self.assertEqual(inverse_mode.change_parameters(arg),False,arg)

    def test_kappa(self):

        # Create a Kappa InterfaceMode object
        kappa_forsin_mode = ForSinKappa()

        # Create a list of good arguments
        good_notkappa_args = ["starttime 0","finaltime 3","outerres 3","innerres 5","saveplots y","saveplots n","savedata y", \
                "savedata n","compile y","compile n"]

        # Create a list of good arguments
        good_kappa_args = ["kappalist first 0 last 5 step 0.5", "kappalist first 0 last 5 num 12"]

        # Create a list of good three-argument commands
        good_three = ["times 0 10", "res 100 2", "times 4.8 10", "times 0.5 10.5", "times 2 10.5", "res 1 1"]

        # Create a list of bad three-argument commands
        bad_three = ["times a b", "res !! -", "times 4.8 b", "times a 10.5", "res 10.2 1", "res 10.2 a", "res a 1", "res 1 a"]

        # Create a list of bad arguments
        bad_args = ["calphalist first 0 last 5 step 0.5", "kappalist last 0 last 5 num 12", "kappalist first -1 last 5 step 0.5" \
                "calphalist first 2 last 0 step 0.5", "kappalist first 1 last 1.5 step 0.6", "kappalist first 1 last 1 step 0.5" \
                "kappalist first 0 last 1 num 0", "kappalist first 1 last 1.5 num 1", "kappalist first 1 last 1.5 num 2"]

        # Check that InterfaceMode returns True for the correct arguments
        for arg in good_notkappa_args:
            self.assertEqual(kappa_forsin_mode.change_single_parameter(*arg.split()),True,arg)
            self.assertEqual(kappa_forsin_mode.change_parameters(arg),True,arg)
        for arg in good_three:
            self.assertEqual(kappa_forsin_mode.change_multiple_parameters(*arg.split()),True,arg)
            self.assertEqual(kappa_forsin_mode.change_parameters(arg),True,arg)
        for arg in good_kappa_args:
            self.assertEqual(kappa_forsin_mode.change_parameters(arg),True,arg)

        # Check that InterfaceMode returns False for the bad arguments
        for arg in bad_args:
            self.assertEqual(kappa_forsin_mode.change_parameters(arg),False,arg)
        for arg in bad_three:
            self.assertEqual(kappa_forsin_mode.change_multiple_parameters(*arg.split()),False,arg)
            self.assertEqual(kappa_forsin_mode.change_parameters(arg),False,arg)

<commit_msg>testing assignments<commit_after>import sys
sys.path.append('PythonTools')
import unittest
from InterfaceMode import *
from random import randint, uniform
from Constants import *

test_runs = 20

min_rand = 0
max_rand = 150

chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!£$%^&*()-+=@?/\.,#~;:][{}<>|"

rand_str_length = 10

# Create a list of time-related commands for the InterfaceMode object
time_args = [START_TIME, FINAL_TIME]

# Create a list of single commands that can be matched with integer arguments
int_args = [START_TIME, FINAL_TIME, OUTER_RES, INNER_RES, KAPPA, DRIVE_ALPHA]

# Create a list of single commands that can ONLY be matched with integer arguments (not converted to float)
only_int_args = [OUTER_RES, INNER_RES]

# Create a list of single commands that can be matched with float arguments
float_args = [START_TIME, FINAL_TIME, KAPPA, DRIVE_ALPHA]

# Create a list of single commands that can be matched with 'boolean' (y/n) arguments
bool_args = [SAVE_PLOTS, SAVE_DATA, COMPILE]

# Create a list 'multiple' commands (i.e. take the form PARAM VAL1 VAL2) that accept int values
mult_int_args = [MULTIPLE_RES,MULTIPLE_TIMES]

# Create a list 'multiple' commands (i.e. take the form PARAM VAL1 VAL2) that accept float values
mult_float_args = [MULTIPLE_TIMES]

# Create a list 'multiple' commands (i.e. take the form PARAM VAL1 VAL2) that only accept int values
mult_only_int_args = [MULTIPLE_RES]

class InterfaceModeTests(unittest.TestCase):

    # Create an valid y boolean argument
    def valid_yes(self,first_arg):
        return first_arg + " y"

    # Create an valid n boolean argument
    def valid_no(self,first_arg):
        return first_arg + " n"

    # Create an valid int argument
    def valid_int_single(self,first_arg):
        return first_arg + " " + str(randint(min_rand,max_rand))

    # Create an valid multiple int argument
    def valid_int_multiple(self,first_arg):
        return " ".join([first_arg, str(randint(min_rand,max_rand)), str(randint(min_rand,max_rand))])

    # Create an invalid argument in the form of a random string
    def random_string(self,first_arg):
        return first_arg + " " + "".join([chars[randint(min_rand,max_rand) % len(chars)] for _ in range(rand_str_length)])

    # Create a valid float argument
    def valid_float_single(self,first_arg):
        return first_arg + " " + str(uniform(min_rand,max_rand))

    # Create a valid multiple float argument
    def valid_float_multiple(self,first_arg):
        return " ".join([first_arg, str(uniform(min_rand,max_rand)), str(uniform(min_rand,max_rand))])

    def two_command_string(self,param,value):
        return param + " " + str(value)

    def test_single_trace(self):

        # Create an InterfaceMode object
        single_mode = SingleTrace()

        for _ in range(test_runs):

            for command in float_args:

                # Check that the float commands accept floats
                arg = self.valid_float_single(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),True,arg)

                # Check that the float commands reject strings
                arg = self.random_string(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),False,arg)

                # Check that the float commands accept floats
                arg = self.valid_float_single(command)
                self.assertEqual(single_mode.change_parameters(arg),True,arg)

                # Check that the float commands reject strings
                arg = self.random_string(command)
                self.assertEqual(single_mode.change_parameters(arg),False,arg)

            for command in only_int_args:

                # Check that the int only commands reject floats
                arg = self.valid_float_single(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),False,arg)

                # Check that the int only commands reject floats
                arg = self.valid_float_single(command)
                self.assertEqual(single_mode.change_parameters(arg),False,arg)

            for command in bool_args:

                # Check that the bool commands accept a 'true' response
                arg = self.valid_yes(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),True,arg)

                # Check that the bool commands accept a 'false' response
                arg = self.valid_no(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),True,arg)

                # Check that the bool commands reject random strings
                arg = self.random_string(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),False,arg)

                # Check that the bool commands reject ints
                arg = self.valid_int_single(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),False,arg)

                # Check that the bool commands reject floats
                arg = self.valid_float_single(command)
                self.assertEqual(single_mode.change_single_parameter(*arg.split()),False,arg)

            for command in mult_int_args:

                # Check that the int commands accept ints
                arg = self.valid_int_multiple(command)
                self.assertEqual(single_mode.change_multiple_parameters(*arg.split()),True,arg)

                # Check that the int commands reject strings
                arg = self.random_string(command) + " " + self.random_string("")
                self.assertEqual(single_mode.change_multiple_parameters(*arg.split()),False,arg)

                # Check that the int commands accept ints
                arg = self.valid_int_multiple(command)
                self.assertEqual(single_mode.change_parameters(arg),True,arg)

                # Check that the int commands reject strings
                arg = self.random_string(command)
                self.assertEqual(single_mode.change_parameters(arg),False,arg)

            for command in mult_float_args:

                # Check that the float commands accept floats
                arg = self.valid_float_multiple(command)
                self.assertEqual(single_mode.change_multiple_parameters(*arg.split()),True,arg)

                # Check that the float commands reject strings
                arg = self.random_string(command) + " " + self.random_string("")
                self.assertEqual(single_mode.change_multiple_parameters(*arg.split()),False,arg)

                # Check that the float commands accept floats
                arg = self.valid_float_multiple(command)
                self.assertEqual(single_mode.change_parameters(arg),True,arg)

                # Check that the float commands reject strings
                arg = self.random_string(command)
                self.assertEqual(single_mode.change_parameters(arg),False,arg)

            for command in mult_only_int_args:

                # Check that the int only commands reject floats
                arg = self.valid_float_multiple(command)
                self.assertEqual(single_mode.change_multiple_parameters(*arg.split()),False,arg)

                # Check that the int only commands reject floats
                arg = self.valid_float_multiple(command)
                self.assertEqual(single_mode.change_parameters(arg),False,arg)

            for command in time_args:

                # Set the time to a random int with change_single_parameter
                rand_int = randint(min_rand,max_rand)
                arg = self.two_command_string(command,rand_int)
                single_mode.change_single_parameter(*arg.split())

                # Check that the assignment worked
                self.assertEqual(single_mode.time_settings[command],rand_int,single_mode.time_settings[command])

                # Set the time to a random int with change_parameters
                rand_int = randint(min_rand,max_rand)
                arg = self.two_command_string(command,rand_int)
                single_mode.change_parameters(arg)

                # Check that the assignment worked
                self.assertEqual(single_mode.time_settings[command],rand_int,single_mode.time_settings[command])

    def test_exactinverse_mode(self):

        # Create an InterfaceMode object
        inverse_mode = ExactInverse()

        # Create a list of good arguments
        good_args = ["starttime 0","finaltime 3","outerres 3","saveplots y","saveplots n","savedata y", \
                "savedata n","kappa 3.5","drivealpha 0.2"]

        # Create a list of good three-argument commands
        good_three = ["times 0 10", "times 4.8 10", "times 0.5 10.5", "times 2 10.5"]

        # Create a list of bad three-argument commands - resolution pairs not used for exact inverse
        bad_three = ["times a b", "res 100 2","res !! -", "times 4.8 b", "times a 10.5", "res 10.2 1", "res 10.2 a", "res a 1", "res 1 a", "res 1 1"]

        # Create a list of bad arguments - compile and innerres are not used in exact inverse mode
        bad_args = ["starttime x","compile y","compile n","finaltime -","outerres 3.2","innerres 5","innerres 0.1","saveplots x","saveplots 2.5","savedata 1"]

        # Check that InterfaceMode returns True for the correct arguments
        for arg in good_args:
            self.assertEqual(inverse_mode.change_single_parameter(*arg.split()),True,arg)
            self.assertEqual(inverse_mode.change_parameters(arg),True,arg)
        for arg in good_three:
            self.assertEqual(inverse_mode.change_multiple_parameters(*arg.split()),True,arg)
            self.assertEqual(inverse_mode.change_parameters(arg),True,arg)

        # Check that InterfaceMode returns False for the bad arguments
        for arg in bad_args:
            self.assertEqual(inverse_mode.change_single_parameter(*arg.split()),False,arg)
            self.assertEqual(inverse_mode.change_parameters(arg),False,arg)
        for arg in bad_three:
            self.assertEqual(inverse_mode.change_multiple_parameters(*arg.split()),False,arg)
            self.assertEqual(inverse_mode.change_parameters(arg),False,arg)

    def test_kappa(self):

        # Create a Kappa InterfaceMode object
        kappa_forsin_mode = ForSinKappa()

        # Create a list of good arguments
        good_notkappa_args = ["starttime 0","finaltime 3","outerres 3","innerres 5","saveplots y","saveplots n","savedata y", \
                "savedata n","compile y","compile n"]

        # Create a list of good arguments
        good_kappa_args = ["kappalist first 0 last 5 step 0.5", "kappalist first 0 last 5 num 12"]

        # Create a list of good three-argument commands
        good_three = ["times 0 10", "res 100 2", "times 4.8 10", "times 0.5 10.5", "times 2 10.5", "res 1 1"]

        # Create a list of bad three-argument commands
        bad_three = ["times a b", "res !! -", "times 4.8 b", "times a 10.5", "res 10.2 1", "res 10.2 a", "res a 1", "res 1 a"]

        # Create a list of bad arguments
        bad_args = ["calphalist first 0 last 5 step 0.5", "kappalist last 0 last 5 num 12", "kappalist first -1 last 5 step 0.5" \
                "calphalist first 2 last 0 step 0.5", "kappalist first 1 last 1.5 step 0.6", "kappalist first 1 last 1 step 0.5" \
                "kappalist first 0 last 1 num 0", "kappalist first 1 last 1.5 num 1", "kappalist first 1 last 1.5 num 2"]

        # Check that InterfaceMode returns True for the correct arguments
        for arg in good_notkappa_args:
            self.assertEqual(kappa_forsin_mode.change_single_parameter(*arg.split()),True,arg)
            self.assertEqual(kappa_forsin_mode.change_parameters(arg),True,arg)
        for arg in good_three:
            self.assertEqual(kappa_forsin_mode.change_multiple_parameters(*arg.split()),True,arg)
            self.assertEqual(kappa_forsin_mode.change_parameters(arg),True,arg)
        for arg in good_kappa_args:
            self.assertEqual(kappa_forsin_mode.change_parameters(arg),True,arg)

        # Check that InterfaceMode returns False for the bad arguments
        for arg in bad_args:
            self.assertEqual(kappa_forsin_mode.change_parameters(arg),False,arg)
        for arg in bad_three:
            self.assertEqual(kappa_forsin_mode.change_multiple_parameters(*arg.split()),False,arg)
            self.assertEqual(kappa_forsin_mode.change_parameters(arg),False,arg)

<|endoftext|>


<commit_before># python script to rip subject data from a sql database from the fossilfinder data
# NOTES
# raw data should have a reference_key that is clean - starting at 1 and incrementing by 1 for each row - not doing this will throw an error

import MySQLdb
conn = MySQLdb.connect("server","username","password","database")
c = conn.cursor()

# very unclean way of counting the number of rows in the raw data table to enable a while loop to count rows as it rips and parses each one
numrows = c.execute("select count(*) from raw_data")
rows = c.fetchall()
rows1 = list(rows[0])
numrows_raw_data = (rows1[0])

#this while loop takes one line at a time from the raw data table - code within parses the row and puts it into an existing table called click_data
while numrows_raw_data > 0:
  #  print (numrows_raw_data)
    numrows_raw_data_str = str(numrows_raw_data)
    #sql query to pull row from database
    numrows = c.execute("select subject_data from raw_data where reference_key= %s",(numrows_raw_data_str,))

    #turns output from query into useful form - a list
    rows = c.fetchall()
    rows1 = list(rows[0])
    rows1 = rows1[0]
    rows1 = rows1.decode('utf-8')
    #print (rows1)

    sub_id = rows1[rows1.find("subject_id")+12:]
    sub_id = sub_id[:sub_id.find(",")]
    #print (sub_id)

    test1 = rows1.find("Filename")
    if test1 > 0:
        img_name = rows1[rows1.find("Filename")+11:]
        img_name = img_name[:img_name.find("\"")]
        date = ""
        time = ""
    else:
        img_name = rows1[rows1.find("Image_Name")+13:]
        img_name = img_name[:img_name.find("\"")]
        date = rows1[rows1.find("Date")+7:]
        date = date[:date.find("\"")]
        time = rows1[rows1.find("Time")+7:]
        time = time[:time.find("\"")]
        date = date.replace(":", "-")
        date_space = " "
        date_time = date+date_space+time

    c.execute("insert into subjects (subject_ids, image_name, taken_timestamp, reference_key) values (%s, %s, %s, NULL)",(sub_id, img_name, date_time))
    conn.commit()

    numrows_raw_data = numrows_raw_data - 1

#end of code
<commit_msg>Update raw2subject.py<commit_after># python script to rip subject data from a sql database from the fossilfinder data
# NOTES
# raw data should have a reference_key that is clean - starting at 1 and incrementing by 1 for each row - not doing this will throw an error

import MySQLdb
conn = MySQLdb.connect("server","username","password","database")
c = conn.cursor()

# very unclean way of counting the number of rows in the raw data table to enable a while loop to count rows as it rips and parses each one
numrows = c.execute("select count(*) from raw_data")
rows = c.fetchall()
rows1 = list(rows[0])
numrows_raw_data = (rows1[0])

#this while loop takes one line at a time from the raw data table - code within parses the row and puts it into an existing table called click_data
while numrows_raw_data > 0:
  #  print (numrows_raw_data)
    numrows_raw_data_str = str(numrows_raw_data)
    #sql query to pull row from database
    numrows = c.execute("select subject_data from raw_data where reference_key= %s",(numrows_raw_data_str,))

    #turns output from query into useful form - a list
    rows = c.fetchall()
    rows1 = list(rows[0])
    rows1 = rows1[0]
    rows1 = rows1.decode('utf-8')
    #print (rows1)

    sub_id = rows1[2:]
    sub_id = sub_id[:sub_id.find("\"")]
    #print (sub_id)

    test1 = rows1.find("Filename")
    if test1 > 0:
        img_name = rows1[rows1.find("Filename")+11:]
        img_name = img_name[:img_name.find("\"")]
        date = ""
        time = ""
    else:
        img_name = rows1[rows1.find("Image_Name")+13:]
        img_name = img_name[:img_name.find("\"")]
        date = rows1[rows1.find("Date")+7:]
        date = date[:date.find("\"")]
        time = rows1[rows1.find("Time")+7:]
        time = time[:time.find("\"")]
        date = date.replace(":", "-")
        date_space = " "
        date_time = date+date_space+time

    c.execute("insert into subjects (subject_ids, image_name, taken_timestamp, reference_key) values (%s, %s, %s, NULL)",(sub_id, img_name, date_time))
    conn.commit()

    numrows_raw_data = numrows_raw_data - 1

#end of code
<|endoftext|>