<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from datetime import date
from unittest import TestCase

from pycoda.records import (InitialRecord, OldBalanceRecord, TransactionRecord,
                            TransactionPurposeRecord, TransactionDetailRecord,
                            InformationRecord, InformationPurposeRecord,
                            InformationDetailRecord, NewBalanceRecord,
                            ExtraMessageRecord, FinalRecord)

class InitialRecordTest(TestCase):
    RAW = ('0000019091672505        00417969  VIKINGCO NV               KRED'
           'BEBB   00886946917 00000                                       2')

    def setUp(self):
        self.record = InitialRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_raw_record(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

    def test_creation_date(self):
        self.record = InitialRecord(creation_date=date(2017, 3, 28))
        assert self.record.creation_date() == date(2017, 3, 28)

    def test_bank_identification_number(self):
        self.record = InitialRecord(bank_identification_number=725)
        assert self.record.bank_identification_number() == 725

    def test_duplicate_code_true(self):
        self.record = InitialRecord(duplicate_code='D')
        assert self.record.is_duplicate()

    def test_duplicate_code_false(self):
        self.record = InitialRecord(duplicate_code=' ')
        assert not self.record.is_duplicate()

    def test_reference(self):
        self.record = InitialRecord(reference='00417969')
        assert self.record.reference() == '00417969'

    def test_addressee(self):
        self.record = InitialRecord(addressee='Unleashed NV')
        assert self.record.addressee() == 'Unleashed NV'

    def test_bic(self):
        self.record = InitialRecord(bic='KREDBEBB')
        assert self.record.bic() == 'KREDBEBB'

    def test_account_holder_reference(self):
        self.record = InitialRecord(account_holder_reference='00886946917')
        assert self.record.account_holder_reference() == '00886946917'

    def test_free(self):
        self.record = InitialRecord(free='12345')
        assert self.record.free() == '12345'

    def test_transaction_reference(self):
        self.record = InitialRecord(transaction_reference='abc123xyz')
        assert self.record.transaction_reference() == 'abc123xyz'

    def test_related_reference(self):
        self.record = InitialRecord(related_reference='qwerty')
        assert self.record.related_reference() == 'qwerty'

class OldBalanceRecordTest(TestCase):
    RAW = ('12256BE02737026917240                  EUR0000005020346650150916'
           'VIKINGCO NV               KBC-Bedrijfsrekening               119')

    def setUp(self):
        self.record = OldBalanceRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_raw_record(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

class TransactionRecordTest(TestCase):
    RAW = ('2100220000AQQE12627 BHKDGLGTESC0000000000000460140916105500000  '
           '                                                   14091625611 0')

    def setUp(self):
        self.record = TransactionRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_raw_record(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

class TransactionPurposeRecordTest(TestCase):
    RAW = ('2200220000                                                     C'
           '20160903040112-0001F                                         0 0')

    def setUp(self):
        self.record = TransactionPurposeRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_success_payment(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

class TransactionDetailRecordTest(TestCase):
    RAW = ('2300230105BEBEBEBEBEBEBEBE                     mvstagingpos20iii'
           'iiiiiiiii Test    1-trtrtrtr                            0    0 1')

    def setUp(self):
        self.record = TransactionDetailRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_success_payment(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

class InformationRecordTest(TestCase):
    RAW = ('3100230106AQPJ06455 SDDBCDBCRFN505500001001mvstagingpos20iiiiiii'
           'iiiii Test                                                   0 0')

    def setUp(self):
        self.record = InformationRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_success_payment(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

class InformationPurposeRecordTest(TestCase):
    def setUp(self):
        self.record = InformationPurposeRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

class InformationDetailRecordTest(TestCase):
    def setUp(self):
        self.record = InformationDetailRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

class NewBalanceRecordTest(TestCase):
    RAW = ('8231BE02737026917240                  EUR0000005973199110180816 '
           '                                                               0')

    def setUp(self):
        self.record = NewBalanceRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_success_payment(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

class ExtraMessageRecordTest(TestCase):
    def setUp(self):
        self.record = ExtraMessageRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

class FinalRecordTest(TestCase):
    RAW = ('9               027972000000000322000000000142995170            '
           '                                                               2')

    def setUp(self):
        self.record = FinalRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_raw_record(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW
<commit_msg>Renames some of test record cases<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from datetime import date
from unittest import TestCase

from pycoda.records import (InitialRecord, OldBalanceRecord, TransactionRecord,
                            TransactionPurposeRecord, TransactionDetailRecord,
                            InformationRecord, InformationPurposeRecord,
                            InformationDetailRecord, NewBalanceRecord,
                            ExtraMessageRecord, FinalRecord)

class InitialRecordTest(TestCase):
    RAW = ('0000019091672505        00417969  VIKINGCO NV               KRED'
           'BEBB   00886946917 00000                                       2')

    def setUp(self):
        self.record = InitialRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_raw_record(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

    def test_creation_date(self):
        self.record = InitialRecord(creation_date=date(2017, 3, 28))
        assert self.record.creation_date() == date(2017, 3, 28)

    def test_bank_identification_number(self):
        self.record = InitialRecord(bank_identification_number=725)
        assert self.record.bank_identification_number() == 725

    def test_duplicate_code_true(self):
        self.record = InitialRecord(duplicate_code='D')
        assert self.record.is_duplicate()

    def test_duplicate_code_false(self):
        self.record = InitialRecord(duplicate_code=' ')
        assert not self.record.is_duplicate()

    def test_reference(self):
        self.record = InitialRecord(reference='00417969')
        assert self.record.reference() == '00417969'

    def test_addressee(self):
        self.record = InitialRecord(addressee='Unleashed NV')
        assert self.record.addressee() == 'Unleashed NV'

    def test_bic(self):
        self.record = InitialRecord(bic='KREDBEBB')
        assert self.record.bic() == 'KREDBEBB'

    def test_account_holder_reference(self):
        self.record = InitialRecord(account_holder_reference='00886946917')
        assert self.record.account_holder_reference() == '00886946917'

    def test_free(self):
        self.record = InitialRecord(free='12345')
        assert self.record.free() == '12345'

    def test_transaction_reference(self):
        self.record = InitialRecord(transaction_reference='abc123xyz')
        assert self.record.transaction_reference() == 'abc123xyz'

    def test_related_reference(self):
        self.record = InitialRecord(related_reference='qwerty')
        assert self.record.related_reference() == 'qwerty'

class OldBalanceRecordTest(TestCase):
    RAW = ('12256BE02737026917240                  EUR0000005020346650150916'
           'VIKINGCO NV               KBC-Bedrijfsrekening               119')

    def setUp(self):
        self.record = OldBalanceRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_raw_record(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

class TransactionRecordTest(TestCase):
    RAW = ('2100220000AQQE12627 BHKDGLGTESC0000000000000460140916105500000  '
           '                                                   14091625611 0')

    def setUp(self):
        self.record = TransactionRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_raw_record(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

class TransactionPurposeRecordTest(TestCase):
    RAW = ('2200220000                                                     C'
           '20160903040112-0001F                                         0 0')

    def setUp(self):
        self.record = TransactionPurposeRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_raw_record(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

class TransactionDetailRecordTest(TestCase):
    RAW = ('2300230105BEBEBEBEBEBEBEBE                     mvstagingpos20iii'
           'iiiiiiiii Test    1-trtrtrtr                            0    0 1')

    def setUp(self):
        self.record = TransactionDetailRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_raw_record(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

class InformationRecordTest(TestCase):
    RAW = ('3100230106AQPJ06455 SDDBCDBCRFN505500001001mvstagingpos20iiiiiii'
           'iiiii Test                                                   0 0')

    def setUp(self):
        self.record = InformationRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_raw_record(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

class InformationPurposeRecordTest(TestCase):
    def setUp(self):
        self.record = InformationPurposeRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

class InformationDetailRecordTest(TestCase):
    def setUp(self):
        self.record = InformationDetailRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

class NewBalanceRecordTest(TestCase):
    RAW = ('8231BE02737026917240                  EUR0000005973199110180816 '
           '                                                               0')

    def setUp(self):
        self.record = NewBalanceRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_raw_record(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW

class ExtraMessageRecordTest(TestCase):
    def setUp(self):
        self.record = ExtraMessageRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

class FinalRecordTest(TestCase):
    RAW = ('9               027972000000000322000000000142995170            '
           '                                                               2')

    def setUp(self):
        self.record = FinalRecord()

    def test_field_positions_are_consecutive(self):
        field_iterator = zip(self.record._fields[:-1], self.record._fields[1:])
        for field, next_field in field_iterator:
            assert field.position + field.length == next_field.position

    def test_example_loads_dumps_raw_record(self):
        self.record.loads(self.RAW)
        assert self.record.dumps() == self.RAW
<|endoftext|>


<commit_before>from urllib.parse import urlencode
from modules.base.command import Command

class LetMeGoogleThatForYou(Command):
    def __init__(self, bot):
        super().__init__(bot)
        self.name = ['lmgtfy', 'google']
        self.help = 'Te ayuda a buscar algo en google'
        self.owner_only = True

    async def handle(self, message, cmd):
        if len(cmd.args) < 1:
            await cmd.answer('Formato: !lmgtfy <texto>')
            return

        texto = ' '.join(cmd.args)
        url = 'http://lmgtfy.com/?' + urlencode({'q': texto})

        await cmd.answer(url)
<commit_msg>!lmgtfy, agregar 's'<commit_after>from urllib.parse import urlencode
from modules.base.command import Command

class LetMeGoogleThatForYou(Command):
    def __init__(self, bot):
        super().__init__(bot)
        self.name = ['lmgtfy', 'google']
        self.help = 'Te ayuda a buscar algo en google'
        self.owner_only = True

    async def handle(self, message, cmd):
        if len(cmd.args) < 1:
            await cmd.answer('Formato: !lmgtfy <texto>')
            return

        texto = ' '.join(cmd.args)
        url = 'https://lmgtfy.com/?' + urlencode({'q': texto})

        await cmd.answer(url)
<|endoftext|>


<commit_before>"""
fizzbuzz.py
Author: Avery Wallis
Credit: None, so far

Assignment:

Write a program that prints the numbers from 1 to 100. But for
multiples of three print “Fizz” instead of the number and for
the multiples of five print “Buzz”. For numbers which are multiples
of both three and five print “FizzBuzz”.

We will use a variation of this test in which the last number of
the series isn't necessarily 100, and the two numbers being tested
for multiples aren't necessarily three and five. For example, your
program should behave just like this:

How many numbers shall we print? 25
For multiples of what number shall we print 'Fizz'? 3
For multiples of what number shall we print 'Buzz'? 5
1
2
Fizz
4
Buzz
Fizz
7
8
Fizz
Buzz
11
Fizz
13
14
FizzBuzz
16
17
Fizz
19
Buzz
Fizz
22
23
Buzz
"""
n=int(input('How many numbers shall we print? '))
f= int(input("For multiples of what number shall we print 'Fizz'? "))
b= int(input("For multiples of what number shall we print 'Buzz'? "))

for x in range(1, n+1):
    if x%f==0 or x%b==0:
        print('FizzBuzz')
    if x%f==0:
        print('Fizz')
    if x%b==0:
        print('Buzz')
    else:
        print(x)

<commit_msg>Updated from Brython Server: 2/5/2016 2:40:41 PM<commit_after>"""
fizzbuzz.py
Author: Avery Wallis
Credit: None, so far

Assignment:

Write a program that prints the numbers from 1 to 100. But for
multiples of three print “Fizz” instead of the number and for
the multiples of five print “Buzz”. For numbers which are multiples
of both three and five print “FizzBuzz”.

We will use a variation of this test in which the last number of
the series isn't necessarily 100, and the two numbers being tested
for multiples aren't necessarily three and five. For example, your
program should behave just like this:

How many numbers shall we print? 25
For multiples of what number shall we print 'Fizz'? 3
For multiples of what number shall we print 'Buzz'? 5
1
2
Fizz
4
Buzz
Fizz
7
8
Fizz
Buzz
11
Fizz
13
14
FizzBuzz
16
17
Fizz
19
Buzz
Fizz
22
23
Buzz
"""
n=int(input('How many numbers shall we print? '))
f= int(input("For multiples of what number shall we print 'Fizz'? "))
b= int(input("For multiples of what number shall we print 'Buzz'? "))

for x in range(1, n+1):
    if x%f==0 and x%b==0:
        print('FizzBuzz')
    if x%f==0:
        print('Fizz')
    if x%b==0:
        print('Buzz')
    else:
        print(x)

<|endoftext|>


<commit_before># Copyright (C) 2010-2014 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import sys
import re

from django.conf import settings
from django.template import RequestContext
from django.http import HttpResponse
from django.shortcuts import render_to_response
from django.views.decorators.http import require_safe

import pymongo
from bson.objectid import ObjectId
from django.core.exceptions import PermissionDenied
from gridfs import GridFS

sys.path.append(settings.CUCKOO_PATH)

from lib.cuckoo.core.database import Database, TASK_PENDING

results_db = pymongo.connection.Connection(settings.MONGO_HOST, settings.MONGO_PORT).cuckoo
fs = GridFS(results_db)

@require_safe
def index(request):
    db = Database()
    tasks_files = db.list_tasks(limit=50, category="file", not_status=TASK_PENDING)
    tasks_urls = db.list_tasks(limit=50, category="url", not_status=TASK_PENDING)

    analyses_files = []
    analyses_urls = []

    if tasks_files:
        for task in tasks_files:
            new = task.to_dict()
            new["sample"] = db.view_sample(new["sample_id"]).to_dict()
            if db.view_errors(task.id):
                new["errors"] = True

            analyses_files.append(new)

    if tasks_urls:
        for task in tasks_urls:
            new = task.to_dict()

            if db.view_errors(task.id):
                new["errors"] = True

            analyses_urls.append(new)

    return render_to_response("analysis/index.html",
                              {"files": analyses_files, "urls": analyses_urls},
                              context_instance=RequestContext(request))

@require_safe
def pending(request):
    db = Database()
    tasks = db.list_tasks(status=TASK_PENDING)

    pending = []
    for task in tasks:
        pending.append(task.to_dict())

    return render_to_response("analysis/pending.html",
                              {"tasks" : pending},
                              context_instance=RequestContext(request))

@require_safe
def chunk(request, task_id, pid, pagenum):
    try:
        pid, pagenum = int(pid), int(pagenum)-1
    except:
        raise PermissionDenied

    if request.is_ajax():
        record = results_db.analysis.find_one(
            {
                "info.id": int(task_id),
                "behavior.processes.process_id": pid
            },
            {
                "behavior.processes.process_id": 1,
                "behavior.processes.calls": 1
            }
        )

        if not record:
            raise PermissionDenied

        process = None
        for pdict in record["behavior"]["processes"]:
            if pdict["process_id"] == pid:
                process = pdict

        if not process:
            raise PermissionDenied

        objectid = process["calls"][pagenum]
        chunk = results_db.calls.find_one({"_id": ObjectId(objectid)})

        return render_to_response("analysis/behavior/_chunk.html",
                                  {"chunk": chunk},
                                  context_instance=RequestContext(request))
    else:
        raise PermissionDenied

@require_safe
def filtred_chunk(request, task_id, pid, filter_crit):
    try:
        pid = int(pid)
    except:
        raise PermissionDenied

    if request.is_ajax():
        record = results_db.analysis.find_one( { "info.id": int(task_id), "behavior.processes.process_id": pid }, { "behavior.processes.process_id": 1, "behavior.processes.calls": 1 } )

        if not record:
            raise PermissionDenied

        process = None
        for pdict in record["behavior"]["processes"]:
            if pdict["process_id"] == pid:
                process = pdict

        if not process:
            raise PermissionDenied

	filter_process = { 'process_id': pid, 'calls': [] }

	for call in process['calls']:
	    chunk = results_db.calls.find_one({"_id": call})
	    for call in chunk['calls']:
		if call['category'] == filter_crit:
		    filter_process['calls'].append(call)

        return render_to_response("analysis/behavior/_chunk.html",
                                  {"chunk": filter_process},
                                  context_instance=RequestContext(request))
    else:
        raise PermissionDenied

@require_safe
def report(request, task_id):
    report = results_db.analysis.find_one({"info.id": int(task_id)}, sort=[("_id", pymongo.DESCENDING)])

    if not report:
        return render_to_response("error.html",
                                  {"error" : "The specified analysis does not exist"},
                                  context_instance=RequestContext(request))

    return render_to_response("analysis/report.html",
                              {"analysis": report},
                              context_instance=RequestContext(request))

@require_safe
def file(request, category, object_id):
    file_object = results_db.fs.files.find_one({"_id": ObjectId(object_id)})

    if file_object:
        content_type = file_object.get("contentType", "application/octet-stream")
        file_item = fs.get(ObjectId(file_object["_id"]))

        file_name = file_item.sha256
        if category == "pcap":
            file_name += ".pcap"
        elif category == "screenshot":
            file_name += ".jpg"
        else:
            file_name += ".bin"

        response = HttpResponse(file_item.read(), content_type=content_type)
        response["Content-Disposition"] = "attachment; filename={0}".format(file_name)

        return response
    else:
        return render_to_response("error.html",
                                  {"error": "File not found"},
                                  context_instance=RequestContext(request))

def search(request):
    if "search" in request.POST:
        error = None

        try:
            term, value = request.POST["search"].strip().split(":", 1)
        except ValueError:
            term = ""
            value = request.POST["search"].strip()

        if term:
            # Check on search size.
            if len(value) < 3:
                return render_to_response("analysis/search.html",
                                          {"analyses": None,
                                           "term": request.POST["search"],
                                           "error": "Search term too short, minimum 3 characters required"},
                                          context_instance=RequestContext(request))

            # Search logic.
            if term == "name":
                records = results_db.analysis.find({"target.file.name": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "type":
                records = results_db.analysis.find({"target.file.type": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "ssdeep":
                records = results_db.analysis.find({"target.file.ssdeep": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "crc32":
                records = results_db.analysis.find({"target.file.crc32": value}).sort([["_id", -1]])
            elif term == "file":
                records = results_db.analysis.find({"behavior.summary.files": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "key":
                records = results_db.analysis.find({"behavior.summary.keys": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "mutex":
                records = results_db.analysis.find({"behavior.summary.mutexes": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "domain":
                records = results_db.analysis.find({"network.domains.domain": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "ip":
                records = results_db.analysis.find({"network.hosts": value}).sort([["_id", -1]])
            elif term == "signature":
                records = results_db.analysis.find({"signatures.description": {"$regex" : value, "$options" : "-1"}}).sort([["_id", -1]])
            else:
                return render_to_response("analysis/search.html",
                                          {"analyses": None,
                                           "term": request.POST["search"],
                                           "error": "Invalid search term: %s" % term},
                                          context_instance=RequestContext(request))
        else:
            if re.match(r"^([a-fA-F\d]{32})$", value):
                records = results_db.analysis.find({"target.file.md5": value}).sort([["_id", -1]])
            elif re.match(r"^([a-fA-F\d]{40})$", value):
                records = results_db.analysis.find({"target.file.sha1": value}).sort([["_id", -1]])
            elif re.match(r"^([a-fA-F\d]{64})$", value):
                records = results_db.analysis.find({"target.file.sha256": value}).sort([["_id", -1]])
            elif re.match(r"^([a-fA-F\d]{128})$", value):
                records = results_db.analysis.find({"target.file.sha512": value}).sort([["_id", -1]])
            else:
                return render_to_response("analysis/search.html",
                                          {"analyses": None,
                                           "term": None,
                                           "error": "Unable to recognize the search syntax"},
                                          context_instance=RequestContext(request))

        # Get data from cuckoo db.
        db = Database()
        analyses = []

        for result in records:
            new = db.view_task(result["info"]["id"])

            if not new:
                continue

            new = new.to_dict()

            if result["info"]["category"] == "file":
                if new["sample_id"]:
                    sample = db.view_sample(new["sample_id"])
                    if sample:
                        new["sample"] = sample.to_dict()

            analyses.append(new)

        return render_to_response("analysis/search.html",
                                  {"analyses": analyses,
                                   "term": request.POST["search"],
                                   "error": None},
                                  context_instance=RequestContext(request))
    else:
        return render_to_response("analysis/search.html",
                                  {"analyses": None,
                                   "term": None,
                                   "error": None},
                                  context_instance=RequestContext(request))
<commit_msg>Update views.py<commit_after># Copyright (C) 2010-2014 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import sys
import re

from django.conf import settings
from django.template import RequestContext
from django.http import HttpResponse
from django.shortcuts import render_to_response
from django.views.decorators.http import require_safe

import pymongo
from bson.objectid import ObjectId
from django.core.exceptions import PermissionDenied
from gridfs import GridFS

sys.path.append(settings.CUCKOO_PATH)

from lib.cuckoo.core.database import Database, TASK_PENDING

results_db = pymongo.connection.Connection(settings.MONGO_HOST, settings.MONGO_PORT).cuckoo
fs = GridFS(results_db)

@require_safe
def index(request):
    db = Database()
    tasks_files = db.list_tasks(limit=50, category="file", not_status=TASK_PENDING)
    tasks_urls = db.list_tasks(limit=50, category="url", not_status=TASK_PENDING)

    analyses_files = []
    analyses_urls = []

    if tasks_files:
        for task in tasks_files:
            new = task.to_dict()
            new["sample"] = db.view_sample(new["sample_id"]).to_dict()
            if db.view_errors(task.id):
                new["errors"] = True

            analyses_files.append(new)

    if tasks_urls:
        for task in tasks_urls:
            new = task.to_dict()

            if db.view_errors(task.id):
                new["errors"] = True

            analyses_urls.append(new)

    return render_to_response("analysis/index.html",
                              {"files": analyses_files, "urls": analyses_urls},
                              context_instance=RequestContext(request))

@require_safe
def pending(request):
    db = Database()
    tasks = db.list_tasks(status=TASK_PENDING)

    pending = []
    for task in tasks:
        pending.append(task.to_dict())

    return render_to_response("analysis/pending.html",
                              {"tasks" : pending},
                              context_instance=RequestContext(request))

@require_safe
def chunk(request, task_id, pid, pagenum):
    try:
        pid, pagenum = int(pid), int(pagenum)-1
    except:
        raise PermissionDenied

    if request.is_ajax():
        record = results_db.analysis.find_one(
            {
                "info.id": int(task_id),
                "behavior.processes.process_id": pid
            },
            {
                "behavior.processes.process_id": 1,
                "behavior.processes.calls": 1
            }
        )

        if not record:
            raise PermissionDenied

        process = None
        for pdict in record["behavior"]["processes"]:
            if pdict["process_id"] == pid:
                process = pdict

        if not process:
            raise PermissionDenied

        objectid = process["calls"][pagenum]
        chunk = results_db.calls.find_one({"_id": ObjectId(objectid)})

        return render_to_response("analysis/behavior/_chunk.html",
                                  {"chunk": chunk},
                                  context_instance=RequestContext(request))
    else:
        raise PermissionDenied

@require_safe
def filtred_chunk(request, task_id, pid, filter_crit):
    try:
        pid = int(pid)
    except:
        raise PermissionDenied

    if request.is_ajax():
        record = results_db.analysis.find_one( { "info.id": int(task_id), "behavior.processes.process_id": pid }, { "behavior.processes.process_id": 1, "behavior.processes.calls": 1 } )

        if not record:
            raise PermissionDenied

        process = None
        for pdict in record["behavior"]["processes"]:
            if pdict["process_id"] == pid:
                process = pdict

        if not process:
            raise PermissionDenied

	filter_process = { 'process_id': pid, 'calls': [] }

	for call in process['calls']:
	    chunk = results_db.calls.find_one({"_id": call})
	    for call in chunk['calls']:
		if call['category'] == filter_crit:
		    filter_process['calls'].append(call)

        return render_to_response("analysis/behavior/_chunk.html",
                                  {"chunk": filter_process},
                                  context_instance=RequestContext(request))
    else:
        raise PermissionDenied

@require_safe
def report(request, task_id):
    report = results_db.analysis.find_one({"info.id": int(task_id)}, sort=[("_id", pymongo.DESCENDING)])

    if not report:
        return render_to_response("error.html",
                                  {"error" : "The specified analysis does not exist"},
                                  context_instance=RequestContext(request))

    return render_to_response("analysis/report.html",
                              {"analysis": report},
                              context_instance=RequestContext(request))

@require_safe
def file(request, category, object_id):
    file_object = results_db.fs.files.find_one({"_id": ObjectId(object_id)})

    if file_object:
        content_type = file_object.get("contentType", "application/octet-stream")
        file_item = fs.get(ObjectId(file_object["_id"]))

        file_name = file_item.sha256
        if category == "pcap":
            file_name += ".pcap"
        elif category == "screenshot":
            file_name += ".jpg"
        else:
            file_name += ".bin"

        response = HttpResponse(file_item.read(), content_type=content_type)
        response["Content-Disposition"] = "attachment; filename={0}".format(file_name)

        return response
    else:
        return render_to_response("error.html",
                                  {"error": "File not found"},
                                  context_instance=RequestContext(request))

def search(request):
    if "search" in request.POST:
        error = None

        try:
            term, value = request.POST["search"].strip().split(":", 1)
        except ValueError:
            term = ""
            value = request.POST["search"].strip()

        if term:
            # Check on search size.
            if len(value) < 3:
                return render_to_response("analysis/search.html",
                                          {"analyses": None,
                                           "term": request.POST["search"],
                                           "error": "Search term too short, minimum 3 characters required"},
                                          context_instance=RequestContext(request))

            # Search logic.
            if term == "name":
                records = results_db.analysis.find({"target.file.name": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "type":
                records = results_db.analysis.find({"target.file.type": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "ssdeep":
                records = results_db.analysis.find({"target.file.ssdeep": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "crc32":
                records = results_db.analysis.find({"target.file.crc32": value}).sort([["_id", -1]])
            elif term == "file":
                records = results_db.analysis.find({"behavior.summary.files": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "key":
                records = results_db.analysis.find({"behavior.summary.keys": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "mutex":
                records = results_db.analysis.find({"behavior.summary.mutexes": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "domain":
                records = results_db.analysis.find({"network.domains.domain": {"$regex": value, "$options": "-i"}}).sort([["_id", -1]])
            elif term == "ip":
                records = results_db.analysis.find({"network.hosts": value}).sort([["_id", -1]])
            elif term == "signature":
                records = results_db.analysis.find({"signatures.description": {"$regex" : value, "$options" : "-1"}}).sort([["_id", -1]])
            else:
                return render_to_response("analysis/search.html",
                                          {"analyses": None,
                                           "term": request.POST["search"],
                                           "error": "Invalid search term: %s" % term},
                                          context_instance=RequestContext(request))
        else:
            if re.match(r"^([a-fA-F\d]{32})$", value):
                records = results_db.analysis.find({"target.file.md5": value}).sort([["_id", -1]])
            elif re.match(r"^([a-fA-F\d]{40})$", value):
                records = results_db.analysis.find({"target.file.sha1": value}).sort([["_id", -1]])
            elif re.match(r"^([a-fA-F\d]{64})$", value):
                records = results_db.analysis.find({"target.file.sha256": value}).sort([["_id", -1]])
            elif re.match(r"^([a-fA-F\d]{128})$", value):
                records = results_db.analysis.find({"target.file.sha512": value}).sort([["_id", -1]])
            else:
                return render_to_response("analysis/search.html",
                                          {"analyses": None,
                                           "term": None,
                                           "error": "Unable to recognize the search syntax"},
                                          context_instance=RequestContext(request))

        # Get data from cuckoo db.
        db = Database()
        analyses = []

        for result in records:
            new = db.view_task(result["info"]["id"])

            if not new:
                continue

            new = new.to_dict()

            if result["info"]["category"] == "file":
                if new["sample_id"]:
                    sample = db.view_sample(new["sample_id"])
                    if sample:
                        new["sample"] = sample.to_dict()

            analyses.append(new)

        return render_to_response("analysis/search.html",
                                  {"analyses": analyses,
                                   "term": request.POST["search"],
                                   "error": None},
                                  context_instance=RequestContext(request))
    else:
        return render_to_response("analysis/search.html",
                                  {"analyses": None,
                                   "term": None,
                                   "error": None},
                                  context_instance=RequestContext(request))
<|endoftext|>


<commit_before>
"""
Implementation of the command-line I{pyflakes} tool.
"""

import sys
import os
import _ast
import getopt
import textwrap

checker = __import__('pyflakes.checker').checker

def check(codeString, filename):
    """
    Check the Python source given by C{codeString} for flakes.

    @param codeString: The Python source to check.
    @type codeString: C{str}

    @param filename: The name of the file the source came from, used to report
        errors.
    @type filename: C{str}

    @return: The number of warnings emitted.
    @rtype: C{int}
    """
    # First, compile into an AST and handle syntax errors.
    try:
        tree = compile(codeString, filename, "exec", _ast.PyCF_ONLY_AST)
    except SyntaxError, value:
        msg = value.args[0]

        (lineno, offset, text) = value.lineno, value.offset, value.text

        # If there's an encoding problem with the file, the text is None.
        if text is None:
            # Avoid using msg, since for the only known case, it contains a
            # bogus message that claims the encoding the file declared was
            # unknown.
            print >> sys.stderr, "%s: problem decoding source" % (filename, )
        else:
            line = text.splitlines()[-1]

            if offset is not None:
                offset = offset - (len(text) - len(line))

            print >> sys.stderr, '%s:%d: %s' % (filename, lineno, msg)
            print >> sys.stderr, line

            if offset is not None:
                print >> sys.stderr, " " * offset, "^"

        return 1
    else:
        # Okay, it's syntactically valid.  Now check it.
        w = checker.Checker(tree, filename)
        w.messages.sort(lambda a, b: cmp(a.lineno, b.lineno))
        for warning in w.messages:
            print warning
        return len(w.messages)

def checkPath(filename):
    """
    Check the given path, printing out any warnings detected.

    @return: the number of warnings printed
    """
    try:
        return check(file(filename, 'U').read() + '\n', filename)
    except IOError, msg:
        print >> sys.stderr, "%s: %s" % (filename, msg.args[1])
        return 1

def usage():
    print textwrap.dedent("""\
        USAGE: pyflakes [PATH [PATH]]

        Checks for pyflakes in python source in each PATH.  If no paths are specified,
        checks source provided on stdin.
        """)

def main():
    warnings = 0
    args = sys.argv[1:]
    optlist, args = getopt.getopt(args, 'h', [ '--help' ])

    for opt, optval in optlist:
        if opt in ('-h', '--help'):
            return usage()

    if args:
        for arg in args:
            if os.path.isdir(arg):
                for dirpath, dirnames, filenames in os.walk(arg):
                    for filename in filenames:
                        if filename.endswith('.py'):
                            warnings += checkPath(os.path.join(dirpath, filename))
            else:
                warnings += checkPath(arg)
    else:
        warnings += check(sys.stdin.read(), '<stdin>')

    raise SystemExit(warnings > 0)
<commit_msg>Update pyflakes.py<commit_after>
"""
Implementation of the command-line I{pyflakes} tool.
"""

#import sys
#import os
#import _ast
#import getopt
#import textwrap

checker = __import__('pyflakes.checker').checker

def check(codeString, filename):
    """
    Check the Python source given by C{codeString} for flakes.

    @param codeString: The Python source to check.
    @type codeString: C{str}

    @param filename: The name of the file the source came from, used to report
        errors.
    @type filename: C{str}

    @return: The number of warnings emitted.
    @rtype: C{int}
    """
    # First, compile into an AST and handle syntax errors.
    try:
        tree = compile(codeString, filename, "exec", _ast.PyCF_ONLY_AST)
    except SyntaxError, value:
        msg = value.args[0]

        (lineno, offset, text) = value.lineno, value.offset, value.text

        # If there's an encoding problem with the file, the text is None.
        if text is None:
            # Avoid using msg, since for the only known case, it contains a
            # bogus message that claims the encoding the file declared was
            # unknown.
            print >> sys.stderr, "%s: problem decoding source" % (filename, )
        else:
            line = text.splitlines()[-1]

            if offset is not None:
                offset = offset - (len(text) - len(line))

            print >> sys.stderr, '%s:%d: %s' % (filename, lineno, msg)
            print >> sys.stderr, line

            if offset is not None:
                print >> sys.stderr, " " * offset, "^"

        return 1
    else:
        # Okay, it's syntactically valid.  Now check it.
        w = checker.Checker(tree, filename)
        w.messages.sort(lambda a, b: cmp(a.lineno, b.lineno))
        for warning in w.messages:
            print warning
        return len(w.messages)

def checkPath(filename):
    """
    Check the given path, printing out any warnings detected.

    @return: the number of warnings printed
    """
    try:
        return check(file(filename, 'U').read() + '\n', filename)
    except IOError, msg:
        print >> sys.stderr, "%s: %s" % (filename, msg.args[1])
        return 1

def usage():
    print textwrap.dedent("""\
        USAGE: pyflakes [PATH [PATH]]

        Checks for pyflakes in python source in each PATH.  If no paths are specified,
        checks source provided on stdin.
        """)

def main():
    warnings = 0
    args = sys.argv[1:]
    optlist, args = getopt.getopt(args, 'h', [ '--help' ])

    for opt, optval in optlist:
        if opt in ('-h', '--help'):
            return usage()

    if args:
        for arg in args:
            if os.path.isdir(arg):
                for dirpath, dirnames, filenames in os.walk(arg):
                    for filename in filenames:
                        if filename.endswith('.py'):
                            warnings += checkPath(os.path.join(dirpath, filename))
            else:
                warnings += checkPath(arg)
    else:
        warnings += check(sys.stdin.read(), '<stdin>')

    raise SystemExit(warnings > 0)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from pycuda.compiler import SourceModule

from pyfr.backends.cuda.provider import CudaKernelProvider
from pyfr.backends.cuda.queue import CudaComputeKernel

from pyfr.util import npdtype_to_ctype

class CudaSampleKernels(CudaKernelProvider):
    def __init__(self, backend):
        pass

    def square(self, mat):
        fn = self._get_function('square', 'square', 'Piii',
                                dict(mat_order=mat.order,
                                     mat_ctype=npdtype_to_ctype(mat.dtype)))

        grid, block = self._get_2d_grid_block(fn, mat.nrow, mat.ncol)

        class SquareKernel(CudaComputeKernel):
            def __call__(self, stream):
                fn.prepared_async_call(grid, block, stream, mat.data,
                                       mat.nrow, mat.ncol, mat.leaddim)

        return SquareKernel()
<commit_msg>Remove the unused CUDA sample kernels.<commit_after><|endoftext|>


<commit_before>'''
Resources needed by pkg providers
'''

# Import python libs
import distutils.version  # pylint: disable=E0611
import fnmatch
import logging
import os
import pprint
import re
import sys
import yaml

# Import salt libs
import salt.utils

log = logging.getLogger(__name__)

def _parse_pkg_meta(path):
    '''
    Parse metadata from a binary package and return the package's name and
    version number.
    '''
    def parse_rpm(path):
        try:
            from salt.modules.yumpkg5 import __QUERYFORMAT, _parse_pkginfo
            from salt.utils import namespaced_function
            _parse_pkginfo = namespaced_function(_parse_pkginfo, globals())
        except ImportError:
            log.critical('Error importing helper functions. This is almost '
                         'certainly a bug.')
            return '', ''
        pkginfo = __salt__['cmd.run_all'](
            'rpm -qp --queryformat {0!r} {1!r}'.format(__QUERYFORMAT, path)
        ).get('stdout', '').strip()
        pkginfo = _parse_pkginfo(pkginfo)
        if pkginfo is None:
            return '', ''
        else:
            return pkginfo.name, pkginfo.version

    def parse_pacman(path):
        name = ''
        version = ''
        result = __salt__['cmd.run_all']('pacman -Qpi "{0}"'.format(path))
        if result['retcode'] == 0:
            for line in result['stdout'].splitlines():
                if not name:
                    match = re.match(r'^Name\s*:\s*(\S+)', line)
                    if match:
                        name = match.group(1)
                        continue
                if not version:
                    match = re.match(r'^Version\s*:\s*(\S+)', line)
                    if match:
                        version = match.group(1)
                        continue
        return name, version

    def parse_deb(path):
        name = ''
        version = ''
        arch = ''
        # This is ugly, will have to try to find a better way of accessing the
        # __grains__ global.
        cpuarch = sys.modules[
            __salt__['test.ping'].__module__
        ].__grains__.get('cpuarch', '')

        result = __salt__['cmd.run_all']('dpkg-deb -I "{0}"'.format(path))
        if result['retcode'] == 0:
            for line in result['stdout'].splitlines():
                if not name:
                    try:
                        name = re.match(
                            r'^\s*Package\s*:\s*(\S+)',
                            line
                        ).group(1)
                    except AttributeError:
                        continue
                if not version:
                    try:
                        version = re.match(
                            r'^\s*Version\s*:\s*(\S+)',
                            line
                        ).group(1)
                    except AttributeError:
                        continue
                if cpuarch == 'x86_64' and not arch:
                    try:
                        arch = re.match(
                            r'^\s*Architecture\s*:\s*(\S+)',
                            line
                        ).group(1)
                    except AttributeError:
                        continue
        if arch:
            name += ':{0}'.format(arch)
        return name, version

    if __grains__['os_family'] in ('Suse', 'RedHat', 'Mandriva'):
        metaparser = parse_rpm
    elif __grains__['os_family'] in ('Arch',):
        metaparser = parse_pacman
    elif __grains__['os_family'] in ('Debian',):
        metaparser = parse_deb
    else:
        log.error('No metadata parser found for {0}'.format(path))
        return '', ''

    return metaparser(path)

def pack_pkgs(pkgs):
    '''
    Accepts a list of packages or package/version pairs (or a string
    representing said list) and returns a dict of name/version pairs. For a
    given package, if no version was specified (i.e. the value is a string and
    not a dict, then the dict returned will use None as the value for that
    package.

    Example: '["foo", {"bar": 1.2}, "baz"]' would become
             {'foo': None, 'bar': 1.2, 'baz': None}

    CLI Example::

        salt '*' pkg_resource.pack_pkgs '["foo", {"bar": 1.2}, "baz"]'
    '''
    if isinstance(pkgs, basestring):
        try:
            pkgs = yaml.safe_load(pkgs)
        except yaml.parser.ParserError as err:
            log.error(err)
            return {}
    if not isinstance(pkgs, list) \
            or [x for x in pkgs if not isinstance(x, (basestring, int,
                                                      float, dict))]:
        log.error('Invalid input: {0}'.format(pprint.pformat(pkgs)))
        log.error('Input must be a list of strings/dicts')
        return {}
    ret = {}
    for pkg in pkgs:
        if isinstance(pkg, (basestring, int, float)):
            ret[pkg] = None
        else:
            if len(pkg) != 1:
                log.error('Invalid input: package name/version pairs must '
                          'contain only one element (data passed: '
                          '{0}).'.format(pkg))
                return {}
            ret.update(pkg)
    return dict([(str(x), str(y) if y is not None else y)
                 for x, y in ret.iteritems()])

def pack_sources(sources):
    '''
    Accepts list of dicts (or a string representing a list of dicts) and packs
    the key/value pairs into a single dict.

    Example: '[{"foo": "salt://foo.rpm"}, {"bar": "salt://bar.rpm"}]' would
    become {"foo": "salt://foo.rpm", "bar": "salt://bar.rpm"}

    CLI Example::

        salt '*' pkg_resource.pack_sources '[{"foo": "salt://foo.rpm"}, {"bar": "salt://bar.rpm"}]'
    '''
    if isinstance(sources, basestring):
        try:
            sources = yaml.safe_load(sources)
        except yaml.parser.ParserError as err:
            log.error(err)
            return {}
    ret = {}
    for source in sources:
        if (not isinstance(source, dict)) or len(source) != 1:
            log.error('Invalid input: {0}'.format(pprint.pformat(sources)))
            log.error('Input must be a list of 1-element dicts')
            return {}
        else:
            ret.update(source)
    return ret

def _verify_binary_pkg(srcinfo):
    '''
    Compares package files (s) against the metadata to confirm that they match
    what is expected.
    '''
    problems = []
    for pkg_name, pkg_uri, pkg_path, pkg_type in srcinfo:
        pkgmeta_name, pkgmeta_version = _parse_pkg_meta(pkg_path)
        if not pkgmeta_name:
            if pkg_type == 'remote':
                problems.append('Failed to cache {0}. Are you sure this '
                                'path is correct?'.format(pkg_uri))
            elif pkg_type == 'local':
                if not os.path.isfile(pkg_path):
                    problems.append('Package file {0} not found. Are '
                                    'you sure this path is '
                                    'correct?'.format(pkg_path))
                else:
                    problems.append('Unable to parse package metadata for '
                                    '{0}.'.format(pkg_path))
        elif pkg_name != pkgmeta_name:
            problems.append('Package file {0} (Name: {1}) does not '
                            'match the specified package name '
                            '({2}).'.format(pkg_uri, pkgmeta_name, pkg_name))
    return problems

def check_desired(desired=None):
    '''
    Examines desired package names to make sure they were formatted properly.
    Returns a list of problems encountered.

    CLI Examples::

        salt '*' pkg_resource.check_desired
    '''
    problems = []

    # If minion is Gentoo-based, ensure packages are properly submitted as
    # category/pkgname. For any package that does not follow this format, offer
    # matches from the portage tree.
    if __grains__['os_family'] == 'Gentoo':
        for pkg in (desired or []):
            if '/' not in pkg:
                matches = __salt__['pkg.porttree_matches'](pkg)
                if matches:
                    msg = 'Package category missing for "{0}" (possible ' \
                          'matches: {1}).'.format(pkg, ', '.join(matches))
                else:
                    msg = 'Package category missing for "{0}" and no match ' \
                          'found in portage tree.'.format(pkg)
                log.error(msg)
                problems.append(msg)

    return problems

def parse_targets(name=None, pkgs=None, sources=None, **kwargs):
    '''
    Parses the input to pkg.install and returns back the package(s) to be
    installed. Returns a list of packages, as well as a string noting whether
    the packages are to come from a repository or a binary package.

    CLI Example::

        salt '*' pkg_resource.parse_targets
    '''
    if __grains__['os'] == 'MacOS' and sources:
        log.warning('Parameter "sources" ignored on MacOS hosts.')

    if pkgs and sources:
        log.error('Only one of "pkgs" and "sources" can be used.')
        return None, None

    elif pkgs:
        pkgs = pack_pkgs(pkgs)
        if not pkgs:
            return None, None
        else:
            return pkgs, 'repository'

    elif sources and __grains__['os'] != 'MacOS':
        sources = pack_sources(sources)
        if not sources:
            return None, None

        srcinfo = []
        for pkg_name, pkg_src in sources.iteritems():
            if __salt__['config.valid_fileproto'](pkg_src):
                # Cache package from remote source (salt master, HTTP, FTP)
                srcinfo.append((pkg_name,
                                pkg_src,
                               __salt__['cp.cache_file'](pkg_src,
                                                         kwargs.get('__env__',
                                                                    'base')),
                               'remote'))
            else:
                # Package file local to the minion
                srcinfo.append((pkg_name, pkg_src, pkg_src, 'local'))

        # Check metadata to make sure the name passed matches the source
        if __grains__['os_family'] not in ('Solaris',) \
                and __grains__['os'] not in ('Gentoo', 'OpenBSD', 'FreeBSD'):
            problems = _verify_binary_pkg(srcinfo)
            # If any problems are found in the caching or metadata parsing done
            # in the above for loop, log each problem and return None,None,
            # which will keep package installation from proceeding.
            if problems:
                for problem in problems:
                    log.error(problem)
                return None, None

        # srcinfo is a 4-tuple (pkg_name,pkg_uri,pkg_path,pkg_type), so grab
        # the package path (3rd element of tuple).
        return [x[2] for x in srcinfo], 'file'

    elif name:
        return dict([(x, None) for x in name.split(',')]), 'repository'

    else:
        log.error('No package sources passed to pkg.install.')
        return None, None

def version(*names, **kwargs):
    '''
    Common interface for obtaining the version of installed packages.

    CLI Example::

        salt '*' pkg_resource.version vim
        salt '*' pkg_resource.version foo bar baz
        salt '*' pkg_resource.version 'python*'
    '''
    ret = {}
    versions_as_list = \
        salt.utils.is_true(kwargs.get('versions_as_list'))
    pkg_glob = False
    if len(names) != 0:
        pkgs = __salt__['pkg.list_pkgs'](versions_as_list=True)
        for name in names:
            if '*' in name:
                pkg_glob = True
                for match in fnmatch.filter(pkgs.keys(), name):
                    ret[match] = pkgs.get(match, [])
            else:
                ret[name] = pkgs.get(name, [])
    if not versions_as_list:
        __salt__['pkg_resource.stringify'](ret)
    # Return a string if no globbing is used, and there is one item in the
    # return dict
    if len(ret) == 1 and not pkg_glob:
        try:
            return ret.values()[0]
        except IndexError:
            return ''
    return ret

def add_pkg(pkgs, name, version):
    '''
    Add a package to a dict of installed packages.

    CLI Example::

        salt '*' pkg_resource.add_pkg '{}' bind 9
    '''
    try:
        pkgs.setdefault(name, []).append(version)
    except AttributeError as e:
        log.exception(e)

def sort_pkglist(pkgs):
    '''
    Accepts a dict obtained from pkg.list_pkgs() and sorts in place the list of
    versions for any packages that have multiple versions installed, so that
    two package lists can be compared to one another.

    CLI Example::

        salt '*' pkg_resource.sort_pkglist '["3.45", "2.13"]'
    '''
    # It doesn't matter that ['4.9','4.10'] would be sorted to ['4.10','4.9'],
    # so long as the sorting is consistent.
    try:
        for key in pkgs.keys():
            pkgs[key].sort()
    except AttributeError as e:
        log.exception(e)

def stringify(pkgs):
    '''
    Takes a dict of package name/version information and joins each list of
    installed versions into a string.

    CLI Example::

        salt '*' pkg_resource.stringify 'vim: 7.127'
    '''
    try:
        for key in pkgs.keys():
            pkgs[key] = ','.join(pkgs[key])
    except AttributeError as e:
        log.exception(e)

def find_changes(old=None, new=None):
    '''
    Compare before and after results from pkg.list_pkgs() to determine what
    changes were made to the packages installed on the minion.

    CLI Example::

        salt '*' pkg_resource.find_changes
    '''
    pkgs = {}
    for npkg in set((new or {}).keys()).union((old or {}).keys()):
        if npkg not in old:
            # the package is freshly installed
            pkgs[npkg] = {'old': '',
                          'new': new[npkg]}
        elif npkg not in new:
            # the package is removed
            pkgs[npkg] = {'new': '',
                          'old': old[npkg]}
        elif new[npkg] != old[npkg]:
            # the package was here before and the version has changed
            pkgs[npkg] = {'old': old[npkg],
                          'new': new[npkg]}
    return pkgs

def perform_cmp(pkg1='', pkg2=''):
    '''
    Compares two version strings using distutils.version.LooseVersion. This is
    a fallback for providers which don't have a version comparison utility
    built into them.  Return -1 if version1 < version2, 0 if version1 ==
    version2, and 1 if version1 > version2. Return None if there was a problem
    making the comparison.

    CLI Example::

        salt '*' pkg_resource.perform_cmp
    '''
    try:
        if distutils.version.LooseVersion(pkg1) < \
                distutils.version.LooseVersion(pkg2):
            return -1
        elif distutils.version.LooseVersion(pkg1) == \
                distutils.version.LooseVersion(pkg2):
            return 0
        elif distutils.version.LooseVersion(pkg1) > \
                distutils.version.LooseVersion(pkg2):
            return 1
    except Exception as e:
        log.exception(e)
    return None

def compare(pkg1='', oper='==', pkg2=''):
    '''
    Package version comparison function.

    CLI Example::

        salt '*' pkg_resource.compare
    '''
    cmp_map = {'<': (-1,), '<=': (-1, 0), '==': (0,),
               '>=': (0, 1), '>': (1,)}
    if oper not in ['!='] + cmp_map.keys():
        log.error('Invalid operator "{0}" for package '
                  'comparison'.format(oper))
        return False

    cmp_result = __salt__['pkg.perform_cmp'](pkg1, pkg2)
    if cmp_result is None:
        return False

    if oper == '!=':
        return cmp_result not in cmp_map['==']
    else:
        return cmp_result in cmp_map[oper]

def version_clean(version):
    '''
    Clean the version string removing extra data.
    This function will simply try to call "pkg.version_clean".

    CLI Example::

        salt '*' pkg_resource.version_clean <version_string>
    '''
    if version and 'pkg.version_clean' in __salt__:
        return __salt__['pkg.version_clean'](version)

    return version

def check_extra_requirements(pkgname, pkgver):
    '''
    Check if the installed package already has the given requirements.
    This function will simply try to call "pkg.check_extra_requirements".

    CLI Example::

        salt '*' pkg_resource.check_extra_requirements <pkgname> <extra_requirements>
    '''
    if pkgver and 'pkg.check_extra_requirements' in __salt__:
        return __salt__['pkg.check_extra_requirements'](pkgname, pkgver)

    return True
<commit_msg>Remove version comparison code from pkg_resource module<commit_after>'''
Resources needed by pkg providers
'''

# Import python libs
import fnmatch
import logging
import os
import pprint
import re
import sys
import yaml

# Import salt libs
import salt.utils

log = logging.getLogger(__name__)

def _parse_pkg_meta(path):
    '''
    Parse metadata from a binary package and return the package's name and
    version number.
    '''
    def parse_rpm(path):
        try:
            from salt.modules.yumpkg5 import __QUERYFORMAT, _parse_pkginfo
            from salt.utils import namespaced_function
            _parse_pkginfo = namespaced_function(_parse_pkginfo, globals())
        except ImportError:
            log.critical('Error importing helper functions. This is almost '
                         'certainly a bug.')
            return '', ''
        pkginfo = __salt__['cmd.run_all'](
            'rpm -qp --queryformat {0!r} {1!r}'.format(__QUERYFORMAT, path)
        ).get('stdout', '').strip()
        pkginfo = _parse_pkginfo(pkginfo)
        if pkginfo is None:
            return '', ''
        else:
            return pkginfo.name, pkginfo.version

    def parse_pacman(path):
        name = ''
        version = ''
        result = __salt__['cmd.run_all']('pacman -Qpi "{0}"'.format(path))
        if result['retcode'] == 0:
            for line in result['stdout'].splitlines():
                if not name:
                    match = re.match(r'^Name\s*:\s*(\S+)', line)
                    if match:
                        name = match.group(1)
                        continue
                if not version:
                    match = re.match(r'^Version\s*:\s*(\S+)', line)
                    if match:
                        version = match.group(1)
                        continue
        return name, version

    def parse_deb(path):
        name = ''
        version = ''
        arch = ''
        # This is ugly, will have to try to find a better way of accessing the
        # __grains__ global.
        cpuarch = sys.modules[
            __salt__['test.ping'].__module__
        ].__grains__.get('cpuarch', '')

        result = __salt__['cmd.run_all']('dpkg-deb -I "{0}"'.format(path))
        if result['retcode'] == 0:
            for line in result['stdout'].splitlines():
                if not name:
                    try:
                        name = re.match(
                            r'^\s*Package\s*:\s*(\S+)',
                            line
                        ).group(1)
                    except AttributeError:
                        continue
                if not version:
                    try:
                        version = re.match(
                            r'^\s*Version\s*:\s*(\S+)',
                            line
                        ).group(1)
                    except AttributeError:
                        continue
                if cpuarch == 'x86_64' and not arch:
                    try:
                        arch = re.match(
                            r'^\s*Architecture\s*:\s*(\S+)',
                            line
                        ).group(1)
                    except AttributeError:
                        continue
        if arch:
            name += ':{0}'.format(arch)
        return name, version

    if __grains__['os_family'] in ('Suse', 'RedHat', 'Mandriva'):
        metaparser = parse_rpm
    elif __grains__['os_family'] in ('Arch',):
        metaparser = parse_pacman
    elif __grains__['os_family'] in ('Debian',):
        metaparser = parse_deb
    else:
        log.error('No metadata parser found for {0}'.format(path))
        return '', ''

    return metaparser(path)

def pack_pkgs(pkgs):
    '''
    Accepts a list of packages or package/version pairs (or a string
    representing said list) and returns a dict of name/version pairs. For a
    given package, if no version was specified (i.e. the value is a string and
    not a dict, then the dict returned will use None as the value for that
    package.

    Example: '["foo", {"bar": 1.2}, "baz"]' would become
             {'foo': None, 'bar': 1.2, 'baz': None}

    CLI Example::

        salt '*' pkg_resource.pack_pkgs '["foo", {"bar": 1.2}, "baz"]'
    '''
    if isinstance(pkgs, basestring):
        try:
            pkgs = yaml.safe_load(pkgs)
        except yaml.parser.ParserError as err:
            log.error(err)
            return {}
    if not isinstance(pkgs, list) \
            or [x for x in pkgs if not isinstance(x, (basestring, int,
                                                      float, dict))]:
        log.error('Invalid input: {0}'.format(pprint.pformat(pkgs)))
        log.error('Input must be a list of strings/dicts')
        return {}
    ret = {}
    for pkg in pkgs:
        if isinstance(pkg, (basestring, int, float)):
            ret[pkg] = None
        else:
            if len(pkg) != 1:
                log.error('Invalid input: package name/version pairs must '
                          'contain only one element (data passed: '
                          '{0}).'.format(pkg))
                return {}
            ret.update(pkg)
    return dict([(str(x), str(y) if y is not None else y)
                 for x, y in ret.iteritems()])

def pack_sources(sources):
    '''
    Accepts list of dicts (or a string representing a list of dicts) and packs
    the key/value pairs into a single dict.

    Example: '[{"foo": "salt://foo.rpm"}, {"bar": "salt://bar.rpm"}]' would
    become {"foo": "salt://foo.rpm", "bar": "salt://bar.rpm"}

    CLI Example::

        salt '*' pkg_resource.pack_sources '[{"foo": "salt://foo.rpm"}, {"bar": "salt://bar.rpm"}]'
    '''
    if isinstance(sources, basestring):
        try:
            sources = yaml.safe_load(sources)
        except yaml.parser.ParserError as err:
            log.error(err)
            return {}
    ret = {}
    for source in sources:
        if (not isinstance(source, dict)) or len(source) != 1:
            log.error('Invalid input: {0}'.format(pprint.pformat(sources)))
            log.error('Input must be a list of 1-element dicts')
            return {}
        else:
            ret.update(source)
    return ret

def _verify_binary_pkg(srcinfo):
    '''
    Compares package files (s) against the metadata to confirm that they match
    what is expected.
    '''
    problems = []
    for pkg_name, pkg_uri, pkg_path, pkg_type in srcinfo:
        pkgmeta_name, pkgmeta_version = _parse_pkg_meta(pkg_path)
        if not pkgmeta_name:
            if pkg_type == 'remote':
                problems.append('Failed to cache {0}. Are you sure this '
                                'path is correct?'.format(pkg_uri))
            elif pkg_type == 'local':
                if not os.path.isfile(pkg_path):
                    problems.append('Package file {0} not found. Are '
                                    'you sure this path is '
                                    'correct?'.format(pkg_path))
                else:
                    problems.append('Unable to parse package metadata for '
                                    '{0}.'.format(pkg_path))
        elif pkg_name != pkgmeta_name:
            problems.append('Package file {0} (Name: {1}) does not '
                            'match the specified package name '
                            '({2}).'.format(pkg_uri, pkgmeta_name, pkg_name))
    return problems

def check_desired(desired=None):
    '''
    Examines desired package names to make sure they were formatted properly.
    Returns a list of problems encountered.

    CLI Examples::

        salt '*' pkg_resource.check_desired
    '''
    problems = []

    # If minion is Gentoo-based, ensure packages are properly submitted as
    # category/pkgname. For any package that does not follow this format, offer
    # matches from the portage tree.
    if __grains__['os_family'] == 'Gentoo':
        for pkg in (desired or []):
            if '/' not in pkg:
                matches = __salt__['pkg.porttree_matches'](pkg)
                if matches:
                    msg = 'Package category missing for "{0}" (possible ' \
                          'matches: {1}).'.format(pkg, ', '.join(matches))
                else:
                    msg = 'Package category missing for "{0}" and no match ' \
                          'found in portage tree.'.format(pkg)
                log.error(msg)
                problems.append(msg)

    return problems

def parse_targets(name=None, pkgs=None, sources=None, **kwargs):
    '''
    Parses the input to pkg.install and returns back the package(s) to be
    installed. Returns a list of packages, as well as a string noting whether
    the packages are to come from a repository or a binary package.

    CLI Example::

        salt '*' pkg_resource.parse_targets
    '''
    if __grains__['os'] == 'MacOS' and sources:
        log.warning('Parameter "sources" ignored on MacOS hosts.')

    if pkgs and sources:
        log.error('Only one of "pkgs" and "sources" can be used.')
        return None, None

    elif pkgs:
        pkgs = pack_pkgs(pkgs)
        if not pkgs:
            return None, None
        else:
            return pkgs, 'repository'

    elif sources and __grains__['os'] != 'MacOS':
        sources = pack_sources(sources)
        if not sources:
            return None, None

        srcinfo = []
        for pkg_name, pkg_src in sources.iteritems():
            if __salt__['config.valid_fileproto'](pkg_src):
                # Cache package from remote source (salt master, HTTP, FTP)
                srcinfo.append((pkg_name,
                                pkg_src,
                               __salt__['cp.cache_file'](pkg_src,
                                                         kwargs.get('__env__',
                                                                    'base')),
                               'remote'))
            else:
                # Package file local to the minion
                srcinfo.append((pkg_name, pkg_src, pkg_src, 'local'))

        # Check metadata to make sure the name passed matches the source
        if __grains__['os_family'] not in ('Solaris',) \
                and __grains__['os'] not in ('Gentoo', 'OpenBSD', 'FreeBSD'):
            problems = _verify_binary_pkg(srcinfo)
            # If any problems are found in the caching or metadata parsing done
            # in the above for loop, log each problem and return None,None,
            # which will keep package installation from proceeding.
            if problems:
                for problem in problems:
                    log.error(problem)
                return None, None

        # srcinfo is a 4-tuple (pkg_name,pkg_uri,pkg_path,pkg_type), so grab
        # the package path (3rd element of tuple).
        return [x[2] for x in srcinfo], 'file'

    elif name:
        return dict([(x, None) for x in name.split(',')]), 'repository'

    else:
        log.error('No package sources passed to pkg.install.')
        return None, None

def version(*names, **kwargs):
    '''
    Common interface for obtaining the version of installed packages.

    CLI Example::

        salt '*' pkg_resource.version vim
        salt '*' pkg_resource.version foo bar baz
        salt '*' pkg_resource.version 'python*'
    '''
    ret = {}
    versions_as_list = \
        salt.utils.is_true(kwargs.get('versions_as_list'))
    pkg_glob = False
    if len(names) != 0:
        pkgs = __salt__['pkg.list_pkgs'](versions_as_list=True)
        for name in names:
            if '*' in name:
                pkg_glob = True
                for match in fnmatch.filter(pkgs.keys(), name):
                    ret[match] = pkgs.get(match, [])
            else:
                ret[name] = pkgs.get(name, [])
    if not versions_as_list:
        __salt__['pkg_resource.stringify'](ret)
    # Return a string if no globbing is used, and there is one item in the
    # return dict
    if len(ret) == 1 and not pkg_glob:
        try:
            return ret.values()[0]
        except IndexError:
            return ''
    return ret

def add_pkg(pkgs, name, version):
    '''
    Add a package to a dict of installed packages.

    CLI Example::

        salt '*' pkg_resource.add_pkg '{}' bind 9
    '''
    try:
        pkgs.setdefault(name, []).append(version)
    except AttributeError as e:
        log.exception(e)

def sort_pkglist(pkgs):
    '''
    Accepts a dict obtained from pkg.list_pkgs() and sorts in place the list of
    versions for any packages that have multiple versions installed, so that
    two package lists can be compared to one another.

    CLI Example::

        salt '*' pkg_resource.sort_pkglist '["3.45", "2.13"]'
    '''
    # It doesn't matter that ['4.9','4.10'] would be sorted to ['4.10','4.9'],
    # so long as the sorting is consistent.
    try:
        for key in pkgs.keys():
            pkgs[key].sort()
    except AttributeError as e:
        log.exception(e)

def stringify(pkgs):
    '''
    Takes a dict of package name/version information and joins each list of
    installed versions into a string.

    CLI Example::

        salt '*' pkg_resource.stringify 'vim: 7.127'
    '''
    try:
        for key in pkgs.keys():
            pkgs[key] = ','.join(pkgs[key])
    except AttributeError as e:
        log.exception(e)

def find_changes(old=None, new=None):
    '''
    Compare before and after results from pkg.list_pkgs() to determine what
    changes were made to the packages installed on the minion.

    CLI Example::

        salt '*' pkg_resource.find_changes
    '''
    pkgs = {}
    for npkg in set((new or {}).keys()).union((old or {}).keys()):
        if npkg not in old:
            # the package is freshly installed
            pkgs[npkg] = {'old': '',
                          'new': new[npkg]}
        elif npkg not in new:
            # the package is removed
            pkgs[npkg] = {'new': '',
                          'old': old[npkg]}
        elif new[npkg] != old[npkg]:
            # the package was here before and the version has changed
            pkgs[npkg] = {'old': old[npkg],
                          'new': new[npkg]}
    return pkgs

def version_clean(version):
    '''
    Clean the version string removing extra data.
    This function will simply try to call ``pkg.version_clean``.

    CLI Example::

        salt '*' pkg_resource.version_clean <version_string>
    '''
    if version and 'pkg.version_clean' in __salt__:
        return __salt__['pkg.version_clean'](version)

    return version

def check_extra_requirements(pkgname, pkgver):
    '''
    Check if the installed package already has the given requirements.
    This function will simply try to call "pkg.check_extra_requirements".

    CLI Example::

        salt '*' pkg_resource.check_extra_requirements <pkgname> <extra_requirements>
    '''
    if pkgver and 'pkg.check_extra_requirements' in __salt__:
        return __salt__['pkg.check_extra_requirements'](pkgname, pkgver)

    return True
<|endoftext|>