<commit_before>import pickle
import hashlib
import datetime
import logging
from django.db import models
from django.db.models import Q
from django.db.models import Max, Min
from django_pgjson.fields import JsonField
from core.utilities import check_domain_valid, get_base_domain

class IndicatorManager(models.Manager):

    LOGGER = logging.getLogger(__name__)

    def host_records(self, indicator):
        record_type = 'HR'

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info__at_domain__endswith=indicator) |
                                             Q(info__at_ip__endswith=indicator))
        return records

    def recent_tc(self, indicator):
        record_type = 'TR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_domain__exact=indicator) |
                                             Q(info__at_ip__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        IndicatorManager.LOGGER.info("Failed to retrieve ThreatCrowd data for indicator %s" % indicator)
        return records

    def recent_hosts(self, indicator):
        record_type = 'HR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_domain__endswith=indicator) |
                                             Q(info__at_ip__endswith=indicator))
        return records

    def historical_hosts(self, indicator, request):
        record_type = 'HR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if request.user.is_staff:
            records = self.get_queryset().filter(Q(record_type=record_type),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_domain__endswith=indicator) |
                                                 Q(info__at_ip__endswith=indicator))

        else:
            records = self.get_queryset().filter(~Q(info_source="PTO"),
                                                 ~Q(info_source="IID"),
                                                 Q(record_type=record_type),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_domain__endswith=indicator) |
                                                 Q(info__at_ip__endswith=indicator))
        return records

    def malware_records(self, indicator):
        record_type = 'MR'

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info__contains=indicator))
        return records

    def recent_malware(self, indicator):
        record_type = 'MR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__contains=indicator))
        return records

    def historical_malware(self, indicator):
        record_type = 'MR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__lt=time_frame),
                                             Q(info__contains=indicator))
        return records

    def whois_records(self, indicator):
        record_type = 'WR'

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                            Q(info__at_query__endswith=indicator) |
                                            Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')
        return records

    def recent_whois(self, indicator):
        record_type = 'WR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        record = self.get_queryset().filter(Q(record_type=record_type),
                                            Q(info_date__gte=time_frame),
                                            Q(info__at_query__endswith=indicator) |
                                            Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')

        if record:
            return record.latest('info_date')

        return record

    def historical_whois(self, indicator):
        record_type = 'WR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        raw_records = self.get_queryset().filter(Q(record_type=record_type),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_query__endswith=indicator) |
                                                 Q(info__at_domain_name__endswith=indicator)).values('info_hash',
                                                                                'info_date')

        tracking = []
        unique_records = []
        annotated_records = raw_records.annotate(latest=Max('info_date')).annotate(earliest=Min('info_date'))

        for record in annotated_records:
            hash_value = record['info_hash']

            if hash_value not in tracking:
                record_info = self.get_queryset().filter(info_hash=hash_value).values('info')[0]['info']
                new_record = {'latest': record['latest'], 'earliest': record['earliest'], 'info': record_info}
                unique_records.append(new_record)
                tracking.append(hash_value)

        return unique_records

    def safebrowsing_record(self, indicator):
        record_type = 'SB'
        # time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)
        if check_domain_valid(indicator):
            records = self.get_queryset().filter(Q(record_type=record_type),
                                                 # Q(info_date__lt=time_frame),
                                                 Q(info__contains=indicator))
                                                     # Q(info__at_domain_name__endswith=indicator)).values('info_hash',
                                                     #                                                    'info_date')
        # records = 'foobar-records'
        return records

class IndicatorRecord(models.Model):

    record_choices = (
        ('HR', 'Host Record'),
        ('MR', 'Malware Record'),
        ('WR', 'Whois Record'),
        ('TR', 'ThreatCrowd Record'),
        ('SB', 'SafeBrowsing Record'),
    )

    source_choices = (
        ('VTO', 'Virus Total'),
        ('TEX', 'Threat Expert'),
        ('IID', 'Internet Identity'),
        ('PTO', 'Passive Total'),
        ('DNS', 'DNS Query'),
        ('REX', 'Robtex'),
        ('WIS', 'WHOIS'),
        ('THR', 'ThreatCrowd'),
        ('GSB', 'Google Safe Browsing'),
    )

    record_type = models.CharField(max_length=2, choices=record_choices)
    created = models.DateTimeField(auto_now_add=True, editable=False)
    modified = models.DateTimeField(auto_now=True)

    info = JsonField()
    info_source = models.CharField(max_length=3, choices=source_choices)
    info_hash = models.CharField(max_length=40)
    info_date = models.DateTimeField()

    objects = IndicatorManager()

    class Meta:
        unique_together = (("info_hash", "info_source", "info_date"),)

    def generate_hash(self):
        info_pickle = pickle.dumps(self.info)
        info_sha1 = hashlib.sha1(info_pickle).hexdigest()
        return info_sha1

    def save(self, *args, **kwargs):

        if not self.info_hash:
            self.info_hash = self.generate_hash()

        super(IndicatorRecord, self).save(*args, **kwargs)

class TaskTracker(models.Model):
    """ Tracker for identifying and resuming tasks """
    keyword = models.CharField(max_length=253)
    group_id = models.CharField(max_length=50)
    type = models.CharField(max_length=50)
    date = models.DateTimeField()

class ExternalSessions(models.Model):
    """ External cookie sessions for scrapers """

    service_choices = (('IID', 'Internet Identity'),)

    service = models.CharField(max_length=3, choices=service_choices)
    cookie = JsonField()

<commit_msg>Better query for indicator record in SafeBrowsing lookup.<commit_after>import pickle
import hashlib
import datetime
import logging
from django.db import models
from django.db.models import Q
from django.db.models import Max, Min
from django_pgjson.fields import JsonField
from core.utilities import check_domain_valid, get_base_domain

class IndicatorManager(models.Manager):

    LOGGER = logging.getLogger(__name__)

    def host_records(self, indicator):
        record_type = 'HR'

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info__at_domain__endswith=indicator) |
                                             Q(info__at_ip__endswith=indicator))
        return records

    def recent_tc(self, indicator):
        record_type = 'TR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_domain__exact=indicator) |
                                             Q(info__at_ip__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        IndicatorManager.LOGGER.info("Failed to retrieve ThreatCrowd data for indicator %s" % indicator)
        return records

    def recent_hosts(self, indicator):
        record_type = 'HR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_domain__endswith=indicator) |
                                             Q(info__at_ip__endswith=indicator))
        return records

    def historical_hosts(self, indicator, request):
        record_type = 'HR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if request.user.is_staff:
            records = self.get_queryset().filter(Q(record_type=record_type),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_domain__endswith=indicator) |
                                                 Q(info__at_ip__endswith=indicator))

        else:
            records = self.get_queryset().filter(~Q(info_source="PTO"),
                                                 ~Q(info_source="IID"),
                                                 Q(record_type=record_type),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_domain__endswith=indicator) |
                                                 Q(info__at_ip__endswith=indicator))
        return records

    def malware_records(self, indicator):
        record_type = 'MR'

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info__contains=indicator))
        return records

    def recent_malware(self, indicator):
        record_type = 'MR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__contains=indicator))
        return records

    def historical_malware(self, indicator):
        record_type = 'MR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__lt=time_frame),
                                             Q(info__contains=indicator))
        return records

    def whois_records(self, indicator):
        record_type = 'WR'

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                            Q(info__at_query__endswith=indicator) |
                                            Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')
        return records

    def recent_whois(self, indicator):
        record_type = 'WR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        record = self.get_queryset().filter(Q(record_type=record_type),
                                            Q(info_date__gte=time_frame),
                                            Q(info__at_query__endswith=indicator) |
                                            Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')

        if record:
            return record.latest('info_date')

        return record

    def historical_whois(self, indicator):
        record_type = 'WR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        raw_records = self.get_queryset().filter(Q(record_type=record_type),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_query__endswith=indicator) |
                                                 Q(info__at_domain_name__endswith=indicator)).values('info_hash',
                                                                                'info_date')

        tracking = []
        unique_records = []
        annotated_records = raw_records.annotate(latest=Max('info_date')).annotate(earliest=Min('info_date'))

        for record in annotated_records:
            hash_value = record['info_hash']

            if hash_value not in tracking:
                record_info = self.get_queryset().filter(info_hash=hash_value).values('info')[0]['info']
                new_record = {'latest': record['latest'], 'earliest': record['earliest'], 'info': record_info}
                unique_records.append(new_record)
                tracking.append(hash_value)

        return unique_records

    def safebrowsing_record(self, indicator):
        record_type = 'SB'
        # time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)
        if check_domain_valid(indicator):
            records = self.get_queryset().filter(Q(record_type=record_type),
                                                 # Q(info_date__lt=time_frame),
                                                 Q(info__at_indicator__exact=indicator))
                                                     # Q(info__at_domain_name__endswith=indicator)).values('info_hash',
                                                     #                                                    'info_date')
        # records = 'foobar-records'
        return records

class IndicatorRecord(models.Model):

    record_choices = (
        ('HR', 'Host Record'),
        ('MR', 'Malware Record'),
        ('WR', 'Whois Record'),
        ('TR', 'ThreatCrowd Record'),
        ('SB', 'SafeBrowsing Record'),
    )

    source_choices = (
        ('VTO', 'Virus Total'),
        ('TEX', 'Threat Expert'),
        ('IID', 'Internet Identity'),
        ('PTO', 'Passive Total'),
        ('DNS', 'DNS Query'),
        ('REX', 'Robtex'),
        ('WIS', 'WHOIS'),
        ('THR', 'ThreatCrowd'),
        ('GSB', 'Google Safe Browsing'),
    )

    record_type = models.CharField(max_length=2, choices=record_choices)
    created = models.DateTimeField(auto_now_add=True, editable=False)
    modified = models.DateTimeField(auto_now=True)

    info = JsonField()
    info_source = models.CharField(max_length=3, choices=source_choices)
    info_hash = models.CharField(max_length=40)
    info_date = models.DateTimeField()

    objects = IndicatorManager()

    class Meta:
        unique_together = (("info_hash", "info_source", "info_date"),)

    def generate_hash(self):
        info_pickle = pickle.dumps(self.info)
        info_sha1 = hashlib.sha1(info_pickle).hexdigest()
        return info_sha1

    def save(self, *args, **kwargs):

        if not self.info_hash:
            self.info_hash = self.generate_hash()

        super(IndicatorRecord, self).save(*args, **kwargs)

class TaskTracker(models.Model):
    """ Tracker for identifying and resuming tasks """
    keyword = models.CharField(max_length=253)
    group_id = models.CharField(max_length=50)
    type = models.CharField(max_length=50)
    date = models.DateTimeField()

class ExternalSessions(models.Model):
    """ External cookie sessions for scrapers """

    service_choices = (('IID', 'Internet Identity'),)

    service = models.CharField(max_length=3, choices=service_choices)
    cookie = JsonField()

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# coding: utf8
# encoding=utf8
from vapory import *
from moviepy.editor import VideoClip
import json
import tweepy
import numpy as np
import math, os, time, sys
from math import pi
from credentials import *
from convertBytes import convert_bytes

#sys.setdefaultencoding('utf8')

def backend(len, dep):
    os.chdir('../HcPlot-hs')
    os.system('stack build')
    #os.system('stack exec create ' + str(len) + ' ' + str(dep) + ' > data')
    # data = os.popen('stack exec create ' + str(len) + ' ' + str(dep)).read()
    os.chdir('../python')
    return json.loads(data)

def nameToJSON(name):
    os.chdir('../HcPlot-hs')
    os.system('stack build')
    HSdata = os.popen("stack exec create n '" + str(name) + "'").read()
    print("HS Data: " + str(HSdata))
    if(HSdata == ""):
        print("Fehlerhafter Name")
        return False
    os.chdir('../python')
    try:
        return json.loads(HSdata)
    except TypeError:
        print("Fehlerhafte JSON")
        return False

#data = backend(5, 2)

lange = 3 #sek. Länge der gif !!! must be int !!!
obj = []
x_cam = []
y_cam = []
z_cam = []
x_max = [0]
x_min = [0]
y_max = [0]
y_min = [0]

def make_scene(t):
    #camera = Camera( 'location', [t*50,20,-((x_max[0]+y_max[0])/1.25)], 'look_at', [x_max[0]/2,0,0])
    t = int(t*25)
    camera = Camera( 'location', [x_cam[t],y_cam[t],z_cam[t]], 'look_at', center)
    return Scene( camera, objects= obj, included=["glass.inc"])

def make_frame(t):
    #return scene.render("test.png", width=1024, height=512, antialiasing = 0.01, quality=100)
    return make_scene(t).render(width=1024, height=512, antialiasing = 0.01, quality=100)

def generate_circle(center=(0,0,0), r=50, n=100):
    return [
        (
            center[0] + (math.cos(2 * pi / n * x) * r),  # x
            center[1] + (math.sin(2 * pi / n * x) * r),  # y
            center[2] + (math.cos(2 * pi / n * x) * r)   # z

        ) for x in range(0, n + 1)]

def render(data, length, shrink, empty, x, y, z):
    if isinstance(data, list):
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.1,0.1,0.1])))) # Create C node
        for i,item in enumerate(data): # Create New nodes around C
            [x_new, y_new, len_new, empty_new] = direct(len(data), length, shrink, empty, i, x, y, z)
            if(x_new > x_max[0]):
                x_max[0] = x_new
            if(x_new < x_min[0]):
                x_min[0] = x_new
            if(y_new > y_max[0]):
                y_max[0] = y_new
            if(y_new < y_min[0]):
                y_min[0] = y_new
            obj.append(Cylinder([x,y,z], render(item, len_new, shrink, empty_new, x_new, y_new, z), 0.3, Finish('ambient', 0.1, 'diffuse', 0.7), Pigment('color', [0.4,0.4,0.4])))
    elif data == 'h': # Create new h node
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.9,0.9,0.9]))))#White color
    return [x,y,z]

# Empty: 0 = left; 1 = up; 2 = right; 3 = down
def direct(datasize, length, shrink, empty, i, x, y, z):
    if datasize != 0:
        if (i + empty) % 4 == 2:
            x_new = x + length; y_new = y
            len_new = length
            empty = 0 #Tested
        elif (i + empty) % 4 == 1:
            x_new = x; y_new = y + length
            len_new = length - shrink
            empty = 1 #Tested
        elif (i + empty) % 4 == 0: # Evtl swap with under
            x_new = x; y_new = y - length
            len_new = length - shrink
            empty = 2 #Tested
        elif (i + empty) % 4 == 3:
            x_new = x - length; y_new = y
            len_new = -length
            empty = 0 #2
    return [x_new, y_new, len_new, empty]

def plot(data):
    print("Data: " + str(data))
    render(data, 15, 10, -1, 0, 0, 0)
    print("X max: " + str(x_max[0]) + ", Y max: " + str(y_max[0]))
    print("Kreispunkte: " + str(lange*25+1))
    center = ((x_max[0]+x_min[0])/2, (y_max[0]+y_min[0])/2, 0)

    print("Kreismitte: " + str(center))
    points = generate_circle(center=center, r=(x_max[0]+y_max[0]), n=lange*25+1)
    for x,y,z in points:
        x_cam.append(x)
        y_cam.append(z)
        z_cam.append(y)

    obj.append(LightSource( [10, 120, -40], 'color', [1.3, 1.3, 1.3]))
    obj.append(Background("color", [1,1,1]))

    VideoClip(make_frame, duration=lange).write_gif("animation.gif",fps=25)
    size = convert_bytes(os.stat("animation.gif").st_size)
    print("Filesize: " + size)

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)

running = True

while running:
    print("Listening Tweets...")
    tweets = api.search(q="@AlkanPlotter")
    for tweet in tweets:
        if "@AlkanPlotter" in tweet.text:
            user = tweet.user.screen_name
            #Entferne Username und Leerzeichen
            name = tweet.text.replace("@AlkanPlotter", "").replace(" ", "")
            print("Name von Twitter: " + str(name))
            data = nameToJSON(name)
            if(data):
                plot(data)
                api.update_with_media("animation.gif", name)
                running = False
            else:
                print("Tweet ungultig!")
                running = False
<commit_msg>Debug<commit_after>#!/usr/bin/env python3
# coding: utf8
# encoding=utf8
from vapory import *
from moviepy.editor import VideoClip
import json
import tweepy
import numpy as np
import math, os, time, sys
from math import pi
from credentials import *
from convertBytes import convert_bytes

#sys.setdefaultencoding('utf8')

def backend(len, dep):
    os.chdir('../HcPlot-hs')
    os.system('stack build')
    #os.system('stack exec create ' + str(len) + ' ' + str(dep) + ' > data')
    # data = os.popen('stack exec create ' + str(len) + ' ' + str(dep)).read()
    os.chdir('../python')
    return json.loads(data)

def nameToJSON(name):
    os.chdir('../HcPlot-hs')
    os.system('stack build')
    HSdata = os.popen("stack exec create n '" + str(name) + "'").read()
    print("HS Data: " + str(HSdata))
    if(HSdata == ""):
        print("Fehlerhafter Name")
        return False
    os.chdir('../python')
    try:
        return json.loads(HSdata)
    except TypeError:
        print("Fehlerhafte JSON")
        return False

#data = backend(5, 2)

lange = 3 #sek. Länge der gif !!! must be int !!!
obj = []
x_cam = []
y_cam = []
z_cam = []
x_max = [0]
x_min = [0]
y_max = [0]
y_min = [0]

def make_scene(t):
    #camera = Camera( 'location', [t*50,20,-((x_max[0]+y_max[0])/1.25)], 'look_at', [x_max[0]/2,0,0])
    t = int(t*25)
    camera = Camera( 'location', [x_cam[t],y_cam[t],z_cam[t]], 'look_at', center)
    return Scene( camera, objects= obj, included=["glass.inc"])

def make_frame(t):
    #return scene.render("test.png", width=1024, height=512, antialiasing = 0.01, quality=100)
    center = ((x_max[0]+x_min[0])/2, (y_max[0]+y_min[0])/2, 0)
    return make_scene(t, center).render(width=1024, height=512, antialiasing = 0.01, quality=100)

def generate_circle(center=(0,0,0), r=50, n=100):
    return [
        (
            center[0] + (math.cos(2 * pi / n * x) * r),  # x
            center[1] + (math.sin(2 * pi / n * x) * r),  # y
            center[2] + (math.cos(2 * pi / n * x) * r)   # z

        ) for x in range(0, n + 1)]

def render(data, length, shrink, empty, x, y, z):
    if isinstance(data, list):
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.1,0.1,0.1])))) # Create C node
        for i,item in enumerate(data): # Create New nodes around C
            [x_new, y_new, len_new, empty_new] = direct(len(data), length, shrink, empty, i, x, y, z)
            if(x_new > x_max[0]):
                x_max[0] = x_new
            if(x_new < x_min[0]):
                x_min[0] = x_new
            if(y_new > y_max[0]):
                y_max[0] = y_new
            if(y_new < y_min[0]):
                y_min[0] = y_new
            obj.append(Cylinder([x,y,z], render(item, len_new, shrink, empty_new, x_new, y_new, z), 0.3, Finish('ambient', 0.1, 'diffuse', 0.7), Pigment('color', [0.4,0.4,0.4])))
    elif data == 'h': # Create new h node
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.9,0.9,0.9]))))#White color
    return [x,y,z]

# Empty: 0 = left; 1 = up; 2 = right; 3 = down
def direct(datasize, length, shrink, empty, i, x, y, z):
    if datasize != 0:
        if (i + empty) % 4 == 2:
            x_new = x + length; y_new = y
            len_new = length
            empty = 0 #Tested
        elif (i + empty) % 4 == 1:
            x_new = x; y_new = y + length
            len_new = length - shrink
            empty = 1 #Tested
        elif (i + empty) % 4 == 0: # Evtl swap with under
            x_new = x; y_new = y - length
            len_new = length - shrink
            empty = 2 #Tested
        elif (i + empty) % 4 == 3:
            x_new = x - length; y_new = y
            len_new = -length
            empty = 0 #2
    return [x_new, y_new, len_new, empty]

def plot(data):
    print("Data: " + str(data))
    render(data, 15, 10, -1, 0, 0, 0)
    print("X max: " + str(x_max[0]) + ", Y max: " + str(y_max[0]))
    print("Kreispunkte: " + str(lange*25+1))
    center = ((x_max[0]+x_min[0])/2, (y_max[0]+y_min[0])/2, 0)

    print("Kreismitte: " + str(center))
    points = generate_circle(center=center, r=(x_max[0]+y_max[0]), n=lange*25+1)
    for x,y,z in points:
        x_cam.append(x)
        y_cam.append(z)
        z_cam.append(y)

    obj.append(LightSource( [10, 120, -40], 'color', [1.3, 1.3, 1.3]))
    obj.append(Background("color", [1,1,1]))

    VideoClip(make_frame, duration=lange).write_gif("animation.gif",fps=25)
    size = convert_bytes(os.stat("animation.gif").st_size)
    print("Filesize: " + size)

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)

running = True

while running:
    print("Listening Tweets...")
    tweets = api.search(q="@AlkanPlotter")
    for tweet in tweets:
        if "@AlkanPlotter" in tweet.text:
            user = tweet.user.screen_name
            #Entferne Username und Leerzeichen
            name = tweet.text.replace("@AlkanPlotter", "").replace(" ", "")
            print("Name von Twitter: " + str(name))
            data = nameToJSON(name)
            if(data):
                plot(data)
                api.update_with_media("animation.gif", name)
                running = False
            else:
                print("Tweet ungultig!")
                running = False
<|endoftext|>


<commit_before>import pickle
import hashlib
import datetime
import logging
from django.db import models
from django.db.models import Q
from django.db.models import Max, Min
from django_pgjson.fields import JsonField
from core.utilities import check_domain_valid, get_base_domain
from pivoteer.records import RecordType, RecordSource

LOGGER = logging.getLogger(__name__)

class IndicatorManager(models.Manager):

    def host_records(self, indicator):
        record_type = RecordType.HR

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__at_domain__iendswith=indicator) |
                                             Q(info__at_ip__endswith=indicator))
        return records

    def recent_cert(self, indicator):
        """Retrieve the most recent censys.io certificate result for the provided indicator
        # Updated by LNguyen
        # Date: 05May2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains

            Args:
                indicator (str): The indicator to search for

            Returns (IndicatorRecord): The IndicatorRecord for the most recently saved
                result for the provided indicator or an empty query set if no record was found.
        """

        # TODO: Why are we returning empty query sets hiversus None when there are no results?

        record_type = RecordType.CE
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__icontains=indicator)).values('info', 'info_date')

        # records = self.get_queryset().filter(Q(record_type=record_type.name),
        #                                      Q(info_date__gte=time_frame),
        #                                      Q(info__at_indicator__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        LOGGER.info("Failed to retrieve certificate data for indicator %s" % indicator)
        return records

    def recent_tc(self, indicator):
        """Retrieve the most recent ThreatCrowd record for the provided indicator
        # Updated by LNguyen
        # Date: 05May2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains

            Args:
                indicator (str): The indicator to search for

            Returns (IndicatorRecord): The indicator record for the most recently saved
                result for the provided indicator.
        """
        record_type = RecordType.TR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__icontains=indicator)).values('info', 'info_date')
        # records = self.get_queryset().filter(Q(record_type=record_type.name),
        #                                      Q(info_date__gte=time_frame),
        #                                      Q(info__at_domain__exact=indicator) |
        #                                      Q(info__at_ip__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        LOGGER.info("Failed to retrieve ThreatCrowd data for indicator %s" % indicator)
        return records

    def recent_hosts(self, indicator):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains
        # Date: 1Aug2017
        # Description: Update query to exclude PDNS Data since it only contains passive DNS info
        record_type = RecordType.HR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(~Q(info_source=RecordSource.PDS.name),
                                             Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__icontains=indicator))

        # records = self.get_queryset().filter(Q(record_type=record_type.name),
        #                                      Q(info_date__gte=time_frame),
        #                                      Q(info__at_domain__endswith=indicator) |
        #                                      Q(info__at_ip__endswith=indicator))
        return records

    def historical_hosts(self, indicator, request):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains
        # Date: 1Aug2017
        # Description: Update to include PDNS Data into Historical dataset
        record_type = RecordType.HR
        record_source = RecordSource.PDS
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if request.user.is_staff:
            records = self.get_queryset().filter(~Q(info_source=RecordSource.PDS.name),
                                                 Q(record_type=record_type.name),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__icontains=indicator)) | \
                      self.get_queryset().filter(Q(info_source=RecordSource.PDS.name),
                                                 Q(info__icontains=indicator))

        else:
            records = self.get_queryset().filter(~Q(info_source=RecordSource.PTO.name),
                                                 ~Q(info_source=RecordSource.IID.name),
                                                 ~Q(info_source=RecordSource.PDS.name),
                                                 Q(record_type=record_type.name),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__icontains=indicator)) | \
                      self.get_queryset().filter(Q(info_source=record_source.PDS.name),
                                                 Q(info__icontains=indicator))

        return records

    def malware_records(self, indicator):
        # Updated by LNguyen
        # Date: 12Mayl2017
        # Description: Update to store dataset in array variable because dates were not being handled as datetime and date sorting was not working
        record_type = RecordType.MR

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__icontains=indicator))
        records_complete = []
        for record in records:
            new_record = {
                'info': record.info,
                'info_date': record.info_date,
                'info_hash':record.info_hash,
                'get_info_source_display':record.get_info_source_display()
            }
            records_complete.append(new_record)

        return records

    def recent_malware(self, indicator):
        record_type = RecordType.MR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__icontains=indicator))
        return records

    def historical_malware(self, indicator):
        record_type = RecordType.MR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__lt=time_frame),
                                             Q(info__icontains=indicator))
        return records

    def whois_records(self, indicator):
        record_type = RecordType.WR

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                            Q(info__at_query__iendswith=indicator) |
                                            Q(info__at_domain_name__iendswith=indicator)).values('info', 'info_date')
        return records

    def recent_whois(self, indicator):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains

        record_type = RecordType.WR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)
        record = self.get_queryset().filter(Q(record_type=record_type.name),
                                            Q(info_date__gte=time_frame),
                                            Q(info__at_query__iendswith=indicator) |
                                            Q(info__at_domain_name__iendswith=indicator)).values('info', 'info_date')

        if record:
            return record.latest('info_date')

        return record

    def historical_whois(self, indicator):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains
        # Updated by LNguyen
        # Date: 12Mayl2017
        # Description: Update to store dataset in array variable because dates were not being handled as datetime and date sorting was not working

        record_type = RecordType.WR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        raw_records = self.get_queryset().filter(Q(record_type=record_type.name),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__icontains=indicator)).values('info_hash', 'info_date')

        # raw_records = self.get_queryset().filter(Q(record_type=record_type.name),
        #                                          Q(info_date__lt=time_frame),
        #                                          Q(info__at_query__endswith=indicator) |
        #                                          Q(info__at_domain_name__endswith=indicator)).values('info_hash',
        #                                                                                'info_date')

        tracking = []
        unique_records = []
        annotated_records = raw_records.annotate(latest=Max('info_date')).annotate(earliest=Min('info_date'))

        for record in annotated_records:
            hash_value = record['info_hash']

            if hash_value not in tracking:
                record_info = self.get_queryset().filter(info_hash=hash_value).values('info')[0]['info']
                span = str(record['earliest']) + " / " + str(record['latest'])
                new_record = {'latest': record['latest'],
                              'earliest': record['earliest'],
                              'info_date': span,
                              'info': record_info}
                unique_records.append(new_record)
                tracking.append(hash_value)

        return unique_records

    def get_threatlab_record(self, indicator):
        """
        Retrieve DNSTwist records for an indicator from the database.

        :param indicator: The indicator value
        :return:  The DNSTwist records for the indicator
        """
        import dateutil.parser

        record_type = RecordType.TL

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__at_indicator__iexact=indicator)).values('info', 'info_date')

        # records_complete = []
        #
        # for record in records:
        #
        #     for result in record['info']['results']:
        #
        #         new_record = {
        #             'domain': result['domain'],
        #             'ip': result['rdata'][0],
        #             'timefirst': dateutil.parser.parse(result['timefirst']),
        #             'timelast': dateutil.parser.parse(result['timelast']),
        #             'type': result['type'],
        #             'info_date':record['info_date']
        #         }
        #         records_complete.append(new_record)

        # for record in records_complete:
        #     print(record.info_date)
        #     print(record.domain)

        return records

    def safebrowsing_record(self, indicator):
        record_type = RecordType.SB
        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__at_indicator__iexact=indicator))
        return records

    def get_search_records(self, indicator):
        """
        Retrieve any search records from within the last 24 hours for an indicator from the database.

        :param indicator: The indicator value
        :return:  The search records for the indicator
        """
        record_type = RecordType.SR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)
        value = indicator
        if check_domain_valid(indicator):
            value = get_base_domain(indicator)
        LOGGER.debug("Using search value: %s", value)
        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_indicator__iexact=value)).values('info', 'info_date')
        if LOGGER.isEnabledFor(logging.INFO):
            rank = 0
            msg = "Found %d search record(s):" % len(records)
            for record in records:
                info = record['info']
                results = info['results']
                for result in results:
                    rank += 1
                    url = result['url']
                    msg += "\n\t%d - %s" % (rank, url)
            LOGGER.info(msg)
        return records

    def get_dnstwist_record(self, indicator):
        """
        Retrieve DNSTwist records for an indicator from the database.

        :param indicator: The indicator value
        :return:  The DNSTwist records for the indicator
        """

        record_type = RecordType.DR

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__at_indicator__iexact=indicator)).values('info', 'info_date')

        return records

class IndicatorRecord(models.Model):

    record_choices = tuple((rt.name, rt.title) for rt in RecordType)
    source_choices = tuple((rs.name, rs.title) for rs in RecordSource)

    record_type = models.CharField(max_length=2, choices=record_choices)
    created = models.DateTimeField(auto_now_add=True, editable=False)
    modified = models.DateTimeField(auto_now=True)

    info = JsonField()
    info_source = models.CharField(max_length=3, choices=source_choices)
    info_hash = models.CharField(max_length=40)
    info_date = models.DateTimeField()

    objects = IndicatorManager()

    class Meta:
        unique_together = (("info_hash", "info_source", "info_date"),)

    def generate_hash(self):
        info_pickle = pickle.dumps(self.info)
        info_sha1 = hashlib.sha1(info_pickle).hexdigest()
        return info_sha1

    def save(self, *args, **kwargs):

        if not self.info_hash:
            self.info_hash = self.generate_hash()

        super(IndicatorRecord, self).save(*args, **kwargs)

class TaskTracker(models.Model):
    """ Tracker for identifying and resuming tasks """
    keyword = models.CharField(max_length=253)
    group_id = models.CharField(max_length=50)
    type = models.CharField(max_length=50)
    date = models.DateTimeField()

class ExternalSessions(models.Model):
    """ External cookie sessions for scrapers """

    # Note: Yes, this syntax is mildly awkward, but it allows for very easy addition of additional sources in the list
    # at the end of the line
    service_choices = tuple((rs.name, rs.title) for rs in RecordSource if rs in [RecordSource.IID])

    service = models.CharField(max_length=3, choices=service_choices)
    cookie = JsonField()

<commit_msg>Limit Historical data by 100<commit_after>import pickle
import hashlib
import datetime
import logging
from django.db import models
from django.db.models import Q
from django.db.models import Max, Min
from django_pgjson.fields import JsonField
from core.utilities import check_domain_valid, get_base_domain
from pivoteer.records import RecordType, RecordSource

LOGGER = logging.getLogger(__name__)

class IndicatorManager(models.Manager):

    def host_records(self, indicator):
        record_type = RecordType.HR

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__at_domain__iendswith=indicator) |
                                             Q(info__at_ip__endswith=indicator))
        return records

    def recent_cert(self, indicator):
        """Retrieve the most recent censys.io certificate result for the provided indicator
        # Updated by LNguyen
        # Date: 05May2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains

            Args:
                indicator (str): The indicator to search for

            Returns (IndicatorRecord): The IndicatorRecord for the most recently saved
                result for the provided indicator or an empty query set if no record was found.
        """

        # TODO: Why are we returning empty query sets hiversus None when there are no results?

        record_type = RecordType.CE
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__icontains=indicator)).values('info', 'info_date')

        # records = self.get_queryset().filter(Q(record_type=record_type.name),
        #                                      Q(info_date__gte=time_frame),
        #                                      Q(info__at_indicator__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        LOGGER.info("Failed to retrieve certificate data for indicator %s" % indicator)
        return records

    def recent_tc(self, indicator):
        """Retrieve the most recent ThreatCrowd record for the provided indicator
        # Updated by LNguyen
        # Date: 05May2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains

            Args:
                indicator (str): The indicator to search for

            Returns (IndicatorRecord): The indicator record for the most recently saved
                result for the provided indicator.
        """
        record_type = RecordType.TR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__icontains=indicator)).values('info', 'info_date')
        # records = self.get_queryset().filter(Q(record_type=record_type.name),
        #                                      Q(info_date__gte=time_frame),
        #                                      Q(info__at_domain__exact=indicator) |
        #                                      Q(info__at_ip__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        LOGGER.info("Failed to retrieve ThreatCrowd data for indicator %s" % indicator)
        return records

    def recent_hosts(self, indicator):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains
        # Date: 1Aug2017
        # Description: Update query to exclude PDNS Data since it only contains passive DNS info
        record_type = RecordType.HR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(~Q(info_source=RecordSource.PDS.name),
                                             Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__icontains=indicator))

        # records = self.get_queryset().filter(Q(record_type=record_type.name),
        #                                      Q(info_date__gte=time_frame),
        #                                      Q(info__at_domain__endswith=indicator) |
        #                                      Q(info__at_ip__endswith=indicator))
        return records

    def historical_hosts(self, indicator, request):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains
        # Date: 1Aug2017
        # Description: Update to include PDNS Data into Historical dataset
        record_type = RecordType.HR
        record_source = RecordSource.PDS
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if request.user.is_staff:
            records = ( self.get_queryset().filter(~Q(info_source=RecordSource.PDS.name),
                                                 Q(record_type=record_type.name),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__icontains=indicator)) | \
                      self.get_queryset().filter(Q(info_source=RecordSource.PDS.name),
                                                 Q(info__icontains=indicator))).order_by('created')[:100]

        else:
            records = ( self.get_queryset().filter(~Q(info_source=RecordSource.PTO.name),
                                                 ~Q(info_source=RecordSource.IID.name),
                                                 ~Q(info_source=RecordSource.PDS.name),
                                                 Q(record_type=record_type.name),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__icontains=indicator)) | \
                      self.get_queryset().filter(Q(info_source=record_source.PDS.name),
                                                 Q(info__icontains=indicator))).order_by('created')[:100]

        return records

    def malware_records(self, indicator):
        # Updated by LNguyen
        # Date: 12Mayl2017
        # Description: Update to store dataset in array variable because dates were not being handled as datetime and date sorting was not working
        record_type = RecordType.MR

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__icontains=indicator))
        records_complete = []
        for record in records:
            new_record = {
                'info': record.info,
                'info_date': record.info_date,
                'info_hash':record.info_hash,
                'get_info_source_display':record.get_info_source_display()
            }
            records_complete.append(new_record)

        return records

    def recent_malware(self, indicator):
        record_type = RecordType.MR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__icontains=indicator))
        return records

    def historical_malware(self, indicator):
        record_type = RecordType.MR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__lt=time_frame),
                                             Q(info__icontains=indicator))
        return records

    def whois_records(self, indicator):
        record_type = RecordType.WR

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                            Q(info__at_query__iendswith=indicator) |
                                            Q(info__at_domain_name__iendswith=indicator)).values('info', 'info_date')
        return records

    def recent_whois(self, indicator):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains

        record_type = RecordType.WR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)
        record = self.get_queryset().filter(Q(record_type=record_type.name),
                                            Q(info_date__gte=time_frame),
                                            Q(info__at_query__iendswith=indicator) |
                                            Q(info__at_domain_name__iendswith=indicator)).values('info', 'info_date')

        if record:
            return record.latest('info_date')

        return record

    def historical_whois(self, indicator):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains
        # Updated by LNguyen
        # Date: 12Mayl2017
        # Description: Update to store dataset in array variable because dates were not being handled as datetime and date sorting was not working

        record_type = RecordType.WR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        raw_records = self.get_queryset().filter(Q(record_type=record_type.name),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__icontains=indicator)).values('info_hash', 'info_date')

        # raw_records = self.get_queryset().filter(Q(record_type=record_type.name),
        #                                          Q(info_date__lt=time_frame),
        #                                          Q(info__at_query__endswith=indicator) |
        #                                          Q(info__at_domain_name__endswith=indicator)).values('info_hash',
        #                                                                                'info_date')

        tracking = []
        unique_records = []
        annotated_records = raw_records.annotate(latest=Max('info_date')).annotate(earliest=Min('info_date'))

        for record in annotated_records:
            hash_value = record['info_hash']

            if hash_value not in tracking:
                record_info = self.get_queryset().filter(info_hash=hash_value).values('info')[0]['info']
                span = str(record['earliest']) + " / " + str(record['latest'])
                new_record = {'latest': record['latest'],
                              'earliest': record['earliest'],
                              'info_date': span,
                              'info': record_info}
                unique_records.append(new_record)
                tracking.append(hash_value)

        return unique_records

    def get_threatlab_record(self, indicator):
        """
        Retrieve DNSTwist records for an indicator from the database.

        :param indicator: The indicator value
        :return:  The DNSTwist records for the indicator
        """
        import dateutil.parser

        record_type = RecordType.TL

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__at_indicator__iexact=indicator)).values('info', 'info_date')

        # records_complete = []
        #
        # for record in records:
        #
        #     for result in record['info']['results']:
        #
        #         new_record = {
        #             'domain': result['domain'],
        #             'ip': result['rdata'][0],
        #             'timefirst': dateutil.parser.parse(result['timefirst']),
        #             'timelast': dateutil.parser.parse(result['timelast']),
        #             'type': result['type'],
        #             'info_date':record['info_date']
        #         }
        #         records_complete.append(new_record)

        # for record in records_complete:
        #     print(record.info_date)
        #     print(record.domain)

        return records

    def safebrowsing_record(self, indicator):
        record_type = RecordType.SB
        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__at_indicator__iexact=indicator))
        return records

    def get_search_records(self, indicator):
        """
        Retrieve any search records from within the last 24 hours for an indicator from the database.

        :param indicator: The indicator value
        :return:  The search records for the indicator
        """
        record_type = RecordType.SR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)
        value = indicator
        if check_domain_valid(indicator):
            value = get_base_domain(indicator)
        LOGGER.debug("Using search value: %s", value)
        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_indicator__iexact=value)).values('info', 'info_date')
        if LOGGER.isEnabledFor(logging.INFO):
            rank = 0
            msg = "Found %d search record(s):" % len(records)
            for record in records:
                info = record['info']
                results = info['results']
                for result in results:
                    rank += 1
                    url = result['url']
                    msg += "\n\t%d - %s" % (rank, url)
            LOGGER.info(msg)
        return records

    def get_dnstwist_record(self, indicator):
        """
        Retrieve DNSTwist records for an indicator from the database.

        :param indicator: The indicator value
        :return:  The DNSTwist records for the indicator
        """

        record_type = RecordType.DR

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__at_indicator__iexact=indicator)).values('info', 'info_date')

        return records

class IndicatorRecord(models.Model):

    record_choices = tuple((rt.name, rt.title) for rt in RecordType)
    source_choices = tuple((rs.name, rs.title) for rs in RecordSource)

    record_type = models.CharField(max_length=2, choices=record_choices)
    created = models.DateTimeField(auto_now_add=True, editable=False)
    modified = models.DateTimeField(auto_now=True)

    info = JsonField()
    info_source = models.CharField(max_length=3, choices=source_choices)
    info_hash = models.CharField(max_length=40)
    info_date = models.DateTimeField()

    objects = IndicatorManager()

    class Meta:
        unique_together = (("info_hash", "info_source", "info_date"),)

    def generate_hash(self):
        info_pickle = pickle.dumps(self.info)
        info_sha1 = hashlib.sha1(info_pickle).hexdigest()
        return info_sha1

    def save(self, *args, **kwargs):

        if not self.info_hash:
            self.info_hash = self.generate_hash()

        super(IndicatorRecord, self).save(*args, **kwargs)

class TaskTracker(models.Model):
    """ Tracker for identifying and resuming tasks """
    keyword = models.CharField(max_length=253)
    group_id = models.CharField(max_length=50)
    type = models.CharField(max_length=50)
    date = models.DateTimeField()

class ExternalSessions(models.Model):
    """ External cookie sessions for scrapers """

    # Note: Yes, this syntax is mildly awkward, but it allows for very easy addition of additional sources in the list
    # at the end of the line
    service_choices = tuple((rs.name, rs.title) for rs in RecordSource if rs in [RecordSource.IID])

    service = models.CharField(max_length=3, choices=service_choices)
    cookie = JsonField()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (C) 2010, 2011 Sebastian Wiesner <lunaryorn@googlemail.com>

# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation; either version 2.1 of the License, or (at your
# option) any later version.

# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
# for more details.

# You should have received a copy of the GNU Lesser General Public License
# along with this library; if not, write to the Free Software Foundation,
# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA

"""
    pyudev.core
    ===========

    Core types and functions of :mod:`pyudev`.

    .. moduleauthor::  Sebastian Wiesner  <lunaryorn@googlemail.com>
"""

from __future__ import (print_function, division, unicode_literals,
                        absolute_import)

try:
    from subprocess import check_output
except ImportError:
    from pyudev._compat import check_output

from pyudev.device import Device
from pyudev._libudev import libudev
from pyudev._util import (ensure_unicode_string, ensure_byte_string,
                          udev_list_iterate, property_value_to_bytes)

__all__ = ['udev_version', 'Context', 'Enumerator']

def udev_version():
    """
    Get the version of the underlying udev library.

    udev doesn't use a standard major-minor versioning scheme, but instead
    labels releases with a single consecutive number.  Consequently, the
    version number returned by this function is a single integer, and not a
    tuple (like for instance the interpreter version in
    :data:`sys.version_info`).

    As libudev itself does not provide a function to query the version number,
    this function calls the ``udevadm`` utilitiy, so be prepared to catch
    :exc:`~exceptions.EnvironmentError` and
    :exc:`~subprocess.CalledProcessError` if you call this function.

    Return the version number as single integer.  Raise
    :exc:`~exceptions.ValueError`, if the version number retrieved from udev
    could not be converted to an integer.  Raise
    :exc:`~exceptions.EnvironmentError`, if ``udevadm`` was not found, or could
    not be executed.  Raise :exc:`subprocess.CalledProcessError`, if
    ``udevadm`` returned a non-zero exit code.  On Python 2.7 or newer, the
    ``output`` attribute of this exception is correctly set.

    .. versionadded:: 0.8
    """
    output = ensure_unicode_string(check_output(['udevadm', '--version']))
    return int(output.strip())

class Context(object):
    """
    The udev context.

    This is *the* central object to access udev.  An instance of this class
    must be created before anything else can be done.  It holds the udev
    configuration and provides the interface to list devices (see
    :meth:`list_devices`).

    Instances of this class can directly be given as ``udev *`` to functions
    wrapped through :mod:`ctypes`.
    """

    def __init__(self):
        """
        Create a new context.
        """
        self._as_parameter_ = libudev.udev_new()

    def __del__(self):
        libudev.udev_unref(self)

    @property
    def sys_path(self):
        """
        The ``sysfs`` mount point defaulting to ``/sys'`` as unicode string.

        The mount point can be overwritten using the environment variable
        :envvar:`SYSFS_PATH`.  Use this for testing purposes.
        """
        return ensure_unicode_string(libudev.udev_get_sys_path(self))

    @property
    def device_path(self):
        """
        The device directory path defaulting to ``/dev`` as unicode string.

        This can be overridden in the udev configuration.
        """
        return ensure_unicode_string(libudev.udev_get_dev_path(self))

    @property
    def log_priority(self):
        """
        The logging priority of the interal logging facitility of udev as
        integer with a standard :mod:`syslog` priority.  Assign to this
        property to change the logging priority.

        UDev uses the standard :mod:`syslog` priorities.  Constants for these
        priorities are defined in the :mod:`syslog` module in the standard
        library:

        >>> import syslog
        >>> context = pyudev.Context()
        >>> context.log_priority = syslog.LOG_DEBUG
        """
        return libudev.udev_get_log_priority(self)

    @log_priority.setter
    def log_priority(self, value):
        libudev.udev_set_log_priority(self, value)

    def list_devices(self, **kwargs):
        """
        List all available devices.

        The arguments of this method are the same as for
        :meth:`Enumerator.match()`.  In fact, the arguments are simply passed
        straight to method :meth:`~Enumerator.match()`.

        This function creates and returns an :class:`Enumerator` object,
        that can be used to filter the list of devices, and eventually
        retrieve :class:`Device` objects representing matching devices.

        .. versionchanged:: 0.8
           Accept keyword arguments now for easy matching
        """
        return Enumerator(self).match(**kwargs)

class Enumerator(object):
    """
    Enumerate all available devices.

    To retrieve devices, simply iterate over an instance of this class.
    This operation yields :class:`Device` objects representing the available
    devices.

    Before iteration the device list can be filtered by subsystem or by
    property values using :meth:`match_subsystem` and
    :meth:`match_property`.  Multiple subsystem (property) filters are
    combined using a logical OR, filters of different types are combined
    using a logical AND.  The following filter for instance::

        devices.match_subsystem('block').match_property(
            'ID_TYPE', 'disk').match_property('DEVTYPE', 'disk')

    means the following::

        subsystem == 'block' and (ID_TYPE == 'disk' or DEVTYPE == 'disk')

    Once added, a filter cannot be removed anymore.  Create a new object
    instead.

    Instances of this class can directly be given as given ``udev_enumerate *``
    to functions wrapped through :mod:`ctypes`.
    """

    def __init__(self, context):
        """
        Create a new enumerator with the given ``context`` (a
        :class:`Context` instance).

        While you can create objects of this class directly, this is not
        recommended.  Call :method:`Context.list_devices()` instead.
        """
        if not isinstance(context, Context):
            raise TypeError('Invalid context object')
        self.context = context
        self._as_parameter_ = libudev.udev_enumerate_new(context)
        self._parents = []

    def __del__(self):
        libudev.udev_enumerate_unref(self)

    def match(self, **kwargs):
        """
        Include devices according to the rules defined by the keyword
        arguments.  These keyword arguments are interpreted as follows:

        - The value for the keyword argument ``subsystem`` is forwarded to
          :meth:`match_subsystem()`.
        - The value for the keyword argument ``sys_name`` is forwared to
          :meth:`match_sys_name()`.
        - The value for the keyword argument ``tag`` is forwared to
          :meth:`match_tag()`
        - All other keyword arguments are forwareded one by one to
          :meth:`match_property()`.  The keyword argument itself is interpreted
          as property name, the value of the keyword argument as the property
          value.

        All keyword arguments are optional, calling this method without no
        arguments at all is simply a noop.

        Return the instance again.

        .. versionadded:: 0.8
        """
        subsystem = kwargs.pop('subsystem', None)
        if subsystem is not None:
            self.match_subsystem(subsystem)
        sys_name = kwargs.pop('sys_name', None)
        if sys_name is not None:
            self.match_sys_name(sys_name)
        tag = kwargs.pop('tag', None)
        if tag is not None:
            self.match_tag(tag)
        for property, value in kwargs.iteritems():
            self.match_property(property, value)
        return self

    def match_subsystem(self, subsystem):
        """
        Include all devices, which are part of the given ``subsystem``.

        ``subsystem`` is either a unicode string or a byte string,
        containing the name of the subsystem.

        Return the instance again.
        """
        libudev.udev_enumerate_add_match_subsystem(
            self, ensure_byte_string(subsystem))
        return self

    def match_sys_name(self, sys_name):
        """
        Include all devices with the given name.

        ``sys_name`` is a byte or unicode string containing the device name.

        Return the instance again.

        .. versionadded:: 0.8
        """
        libudev.udev_enumerate_add_match_sysname(
            self, ensure_byte_string(sys_name))
        return self

    def match_property(self, property, value):
        """
        Include all devices, whose ``property`` has the given ``value``.

        ``property`` is either a unicode string or a byte string, containing
        the name of the property to match.  ``value`` is a property value,
        being one of the following types:

        - :func:`int`
        - :func:`bool`
        - A unicode or byte string
        - Anything convertable to a byte string

        Return the instance again.
        """
        libudev.udev_enumerate_add_match_property(
            self, ensure_byte_string(property), property_value_to_bytes(value))
        return self

    def match_tag(self, tag):
        """
        Include all devices, which have the given ``tag`` attached.

        ``tag`` is a byte or unicode string containing the tag name.

        Return the instance again.

        .. versionadded:: 0.6

        .. udevminversion:: 154
        """
        libudev.udev_enumerate_add_match_tag(self, ensure_byte_string(tag))
        return self

    def match_is_initialized(self):
        """
        Include only devices, which are initialized.

        Initialized devices have properly set device node permissions and
        context, and are (in case of network devices) fully renamed.

        Currently this will not affect devices which do not have device nodes
        and are not network interfaces.

        Return the instance again.

        .. seealso:: :attr:`Device.is_initialized`

        .. versionadded:: 0.8

        .. udevminversion:: 165
        """
        libudev.udev_enumerate_add_match_is_initialized(self)
        return self

    def match_children(self, device):
        """
        Include all *direct* children of the given ``device``.  A child is a
        device, whose :attr:`Device.parent` points to ``device``.

        Return the instance again.
        """
        self._parents.append(device)
        return self

    def __iter__(self):
        """
        Iterate over all matching devices.

        Yield :class:`Device` objects.
        """
        libudev.udev_enumerate_scan_devices(self)
        entry = libudev.udev_enumerate_get_list_entry(self)
        for name in udev_list_iterate(entry):
            device = Device.from_sys_path(self.context, name)
            if (not self._parents) or any(device.parent == p for p
                                          in self._parents):
                yield device
<commit_msg>Little documentation fix<commit_after># -*- coding: utf-8 -*-
# Copyright (C) 2010, 2011 Sebastian Wiesner <lunaryorn@googlemail.com>

# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation; either version 2.1 of the License, or (at your
# option) any later version.

# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
# for more details.

# You should have received a copy of the GNU Lesser General Public License
# along with this library; if not, write to the Free Software Foundation,
# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA

"""
    pyudev.core
    ===========

    Core types and functions of :mod:`pyudev`.

    .. moduleauthor::  Sebastian Wiesner  <lunaryorn@googlemail.com>
"""

from __future__ import (print_function, division, unicode_literals,
                        absolute_import)

try:
    from subprocess import check_output
except ImportError:
    from pyudev._compat import check_output

from pyudev.device import Device
from pyudev._libudev import libudev
from pyudev._util import (ensure_unicode_string, ensure_byte_string,
                          udev_list_iterate, property_value_to_bytes)

__all__ = ['udev_version', 'Context', 'Enumerator']

def udev_version():
    """
    Get the version of the underlying udev library.

    udev doesn't use a standard major-minor versioning scheme, but instead
    labels releases with a single consecutive number.  Consequently, the
    version number returned by this function is a single integer, and not a
    tuple (like for instance the interpreter version in
    :data:`sys.version_info`).

    As libudev itself does not provide a function to query the version number,
    this function calls the ``udevadm`` utilitiy, so be prepared to catch
    :exc:`~exceptions.EnvironmentError` and
    :exc:`~subprocess.CalledProcessError` if you call this function.

    Return the version number as single integer.  Raise
    :exc:`~exceptions.ValueError`, if the version number retrieved from udev
    could not be converted to an integer.  Raise
    :exc:`~exceptions.EnvironmentError`, if ``udevadm`` was not found, or could
    not be executed.  Raise :exc:`subprocess.CalledProcessError`, if
    ``udevadm`` returned a non-zero exit code.  On Python 2.7 or newer, the
    ``output`` attribute of this exception is correctly set.

    .. versionadded:: 0.8
    """
    output = ensure_unicode_string(check_output(['udevadm', '--version']))
    return int(output.strip())

class Context(object):
    """
    The udev context.

    This is *the* central object to access udev.  An instance of this class
    must be created before anything else can be done.  It holds the udev
    configuration and provides the interface to list devices (see
    :meth:`list_devices`).

    Instances of this class can directly be given as ``udev *`` to functions
    wrapped through :mod:`ctypes`.
    """

    def __init__(self):
        """
        Create a new context.
        """
        self._as_parameter_ = libudev.udev_new()

    def __del__(self):
        libudev.udev_unref(self)

    @property
    def sys_path(self):
        """
        The ``sysfs`` mount point defaulting to ``/sys'`` as unicode string.

        The mount point can be overwritten using the environment variable
        :envvar:`SYSFS_PATH`.  Use this for testing purposes.
        """
        return ensure_unicode_string(libudev.udev_get_sys_path(self))

    @property
    def device_path(self):
        """
        The device directory path defaulting to ``/dev`` as unicode string.

        This can be overridden in the udev configuration.
        """
        return ensure_unicode_string(libudev.udev_get_dev_path(self))

    @property
    def log_priority(self):
        """
        The logging priority of the interal logging facitility of udev as
        integer with a standard :mod:`syslog` priority.  Assign to this
        property to change the logging priority.

        UDev uses the standard :mod:`syslog` priorities.  Constants for these
        priorities are defined in the :mod:`syslog` module in the standard
        library:

        >>> import syslog
        >>> context = pyudev.Context()
        >>> context.log_priority = syslog.LOG_DEBUG
        """
        return libudev.udev_get_log_priority(self)

    @log_priority.setter
    def log_priority(self, value):
        libudev.udev_set_log_priority(self, value)

    def list_devices(self, **kwargs):
        """
        List all available devices.

        The arguments of this method are the same as for
        :meth:`Enumerator.match()`.  In fact, the arguments are simply passed
        straight to method :meth:`~Enumerator.match()`.

        This function creates and returns an :class:`Enumerator` object,
        that can be used to filter the list of devices, and eventually
        retrieve :class:`Device` objects representing matching devices.

        .. versionchanged:: 0.8
           Accept keyword arguments now for easy matching
        """
        return Enumerator(self).match(**kwargs)

class Enumerator(object):
    """
    Enumerate all available devices.

    To retrieve devices, simply iterate over an instance of this class.
    This operation yields :class:`Device` objects representing the available
    devices.

    Before iteration the device list can be filtered by subsystem or by
    property values using :meth:`match_subsystem` and
    :meth:`match_property`.  Multiple subsystem (property) filters are
    combined using a logical OR, filters of different types are combined
    using a logical AND.  The following filter for instance::

        devices.match_subsystem('block').match_property(
            'ID_TYPE', 'disk').match_property('DEVTYPE', 'disk')

    means the following::

        subsystem == 'block' and (ID_TYPE == 'disk' or DEVTYPE == 'disk')

    Once added, a filter cannot be removed anymore.  Create a new object
    instead.

    Instances of this class can directly be given as given ``udev_enumerate *``
    to functions wrapped through :mod:`ctypes`.
    """

    def __init__(self, context):
        """
        Create a new enumerator with the given ``context`` (a
        :class:`Context` instance).

        While you can create objects of this class directly, this is not
        recommended.  Call :method:`Context.list_devices()` instead.
        """
        if not isinstance(context, Context):
            raise TypeError('Invalid context object')
        self.context = context
        self._as_parameter_ = libudev.udev_enumerate_new(context)
        self._parents = []

    def __del__(self):
        libudev.udev_enumerate_unref(self)

    def match(self, **kwargs):
        """
        Include devices according to the rules defined by the keyword
        arguments.  These keyword arguments are interpreted as follows:

        - The value for the keyword argument ``subsystem`` is forwarded to
          :meth:`match_subsystem()`.
        - The value for the keyword argument ``sys_name`` is forwared to
          :meth:`match_sys_name()`.
        - The value for the keyword argument ``tag`` is forwared to
          :meth:`match_tag()`
        - All other keyword arguments are forwareded one by one to
          :meth:`match_property()`.  The keyword argument itself is interpreted
          as property name, the value of the keyword argument as the property
          value.

        All keyword arguments are optional, calling this method without no
        arguments at all is simply a noop.

        Return the instance again.

        .. versionadded:: 0.8
        """
        subsystem = kwargs.pop('subsystem', None)
        if subsystem is not None:
            self.match_subsystem(subsystem)
        sys_name = kwargs.pop('sys_name', None)
        if sys_name is not None:
            self.match_sys_name(sys_name)
        tag = kwargs.pop('tag', None)
        if tag is not None:
            self.match_tag(tag)
        for property, value in kwargs.iteritems():
            self.match_property(property, value)
        return self

    def match_subsystem(self, subsystem):
        """
        Include all devices, which are part of the given ``subsystem``.

        ``subsystem`` is either a unicode string or a byte string,
        containing the name of the subsystem.

        Return the instance again.
        """
        libudev.udev_enumerate_add_match_subsystem(
            self, ensure_byte_string(subsystem))
        return self

    def match_sys_name(self, sys_name):
        """
        Include all devices with the given name.

        ``sys_name`` is a byte or unicode string containing the device name.

        Return the instance again.

        .. versionadded:: 0.8
        """
        libudev.udev_enumerate_add_match_sysname(
            self, ensure_byte_string(sys_name))
        return self

    def match_property(self, property, value):
        """
        Include all devices, whose ``property`` has the given ``value``.

        ``property`` is either a unicode string or a byte string, containing
        the name of the property to match.  ``value`` is a property value,
        being one of the following types:

        - :func:`int`
        - :func:`bool`
        - A byte string
        - Anything convertable to a unicode string (including a unicode string
          itself :)

        Return the instance again.
        """
        libudev.udev_enumerate_add_match_property(
            self, ensure_byte_string(property), property_value_to_bytes(value))
        return self

    def match_tag(self, tag):
        """
        Include all devices, which have the given ``tag`` attached.

        ``tag`` is a byte or unicode string containing the tag name.

        Return the instance again.

        .. versionadded:: 0.6

        .. udevminversion:: 154
        """
        libudev.udev_enumerate_add_match_tag(self, ensure_byte_string(tag))
        return self

    def match_is_initialized(self):
        """
        Include only devices, which are initialized.

        Initialized devices have properly set device node permissions and
        context, and are (in case of network devices) fully renamed.

        Currently this will not affect devices which do not have device nodes
        and are not network interfaces.

        Return the instance again.

        .. seealso:: :attr:`Device.is_initialized`

        .. versionadded:: 0.8

        .. udevminversion:: 165
        """
        libudev.udev_enumerate_add_match_is_initialized(self)
        return self

    def match_children(self, device):
        """
        Include all *direct* children of the given ``device``.  A child is a
        device, whose :attr:`Device.parent` points to ``device``.

        Return the instance again.
        """
        self._parents.append(device)
        return self

    def __iter__(self):
        """
        Iterate over all matching devices.

        Yield :class:`Device` objects.
        """
        libudev.udev_enumerate_scan_devices(self)
        entry = libudev.udev_enumerate_get_list_entry(self)
        for name in udev_list_iterate(entry):
            device = Device.from_sys_path(self.context, name)
            if (not self._parents) or any(device.parent == p for p
                                          in self._parents):
                yield device
<|endoftext|>