<commit_before># This file is part of Viper - https://github.com/viper-framework/viper
# See the file 'LICENSE' for copying permission.

import os
import re
from socket import inet_pton, AF_INET6, error as socket_error

from viper.common.abstracts import Module
from viper.core.session import __sessions__

DOMAIN_REGEX = re.compile('([a-z0-9][a-z0-9\-]{0,61}[a-z0-9]\.)+[a-z0-9][a-z0-9\-]*[a-z0-9]', re.IGNORECASE)
IPV4_REGEX = re.compile('[1-2]?[0-9]?[0-9]\.[1-2]?[0-9]?[0-9]\.[1-2]?[0-9]?[0-9]\.[1-2]?[0-9]?[0-9]')
IPV6_REGEX = re.compile('((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}'
                        '|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9'
                        'A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25['
                        '0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3'
                        '})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|['
                        '1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,'
                        '4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:'
                        '))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-'
                        '5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]'
                        '{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d'
                        '\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7}'
                        ')|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d'
                        '\d|[1-9]?\d)){3}))|:)))(%.+)?', re.IGNORECASE | re.S)
TLD = [
    'AC', 'ACADEMY', 'ACTOR', 'AD', 'AE', 'AERO', 'AF', 'AG', 'AGENCY', 'AI', 'AL', 'AM', 'AN', 'AO', 'AQ', 'AR',
    'ARPA', 'AS', 'ASIA', 'AT', 'AU', 'AW', 'AX', 'AZ', 'BA', 'BAR', 'BARGAINS', 'BB', 'BD', 'BE', 'BERLIN', 'BEST',
    'BF', 'BG', 'BH', 'BI', 'BID', 'BIKE', 'BIZ', 'BJ', 'BLUE', 'BM', 'BN', 'BO', 'BOUTIQUE', 'BR', 'BS', 'BT',
    'BUILD', 'BUILDERS', 'BUZZ', 'BV', 'BW', 'BY', 'BZ', 'CA', 'CAB', 'CAMERA', 'CAMP', 'CARDS', 'CAREERS', 'CAT',
    'CATERING', 'CC', 'CD', 'CENTER', 'CEO', 'CF', 'CG', 'CH', 'CHEAP', 'CHRISTMAS', 'CI', 'CK', 'CL', 'CLEANING',
    'CLOTHING', 'CLUB', 'CM', 'CN', 'CO', 'CODES', 'COFFEE', 'COM', 'COMMUNITY', 'COMPANY', 'COMPUTER', 'CONDOS',
    'CONSTRUCTION', 'CONTRACTORS', 'COOL', 'COOP', 'CR', 'CRUISES', 'CU', 'CV', 'CW', 'CX', 'CY', 'CZ', 'DANCE',
    'DATING', 'DE', 'DEMOCRAT', 'DIAMONDS', 'DIRECTORY', 'DJ', 'DK', 'DM', 'DNP', 'DO', 'DOMAINS', 'DZ', 'EC',
    'EDU', 'EDUCATION', 'EE', 'EG', 'EMAIL', 'ENTERPRISES', 'EQUIPMENT', 'ER', 'ES', 'ESTATE', 'ET', 'EU', 'EVENTS',
    'EXPERT', 'EXPOSED', 'FARM', 'FI', 'FISH', 'FJ', 'FK', 'FLIGHTS', 'FLORIST', 'FM', 'FO', 'FOUNDATION', 'FR',
    'FUTBOL', 'GA', 'GALLERY', 'GB', 'GD', 'GE', 'GF', 'GG', 'GH', 'GI', 'GIFT', 'GL', 'GLASS', 'GM', 'GN', 'GOV',
    'GP', 'GQ', 'GR', 'GRAPHICS', 'GS', 'GT', 'GU', 'GUITARS', 'GURU', 'GW', 'GY', 'HK', 'HM', 'HN', 'HOLDINGS',
    'HOLIDAY', 'HOUSE', 'HR', 'HT', 'HU', 'ID', 'IE', 'IL', 'IM', 'IMMOBILIEN', 'IN', 'INDUSTRIES', 'INFO', 'INK',
    'INSTITUTE', 'INT', 'INTERNATIONAL', 'IO', 'IQ', 'IR', 'IS', 'IT', 'JE', 'JM', 'JO', 'JOBS', 'JP', 'KAUFEN',
    'KE', 'KG', 'KH', 'KI', 'KIM', 'KITCHEN', 'KIWI', 'KM', 'KN', 'KOELN', 'KP', 'KR', 'KRED', 'KW', 'KY', 'KZ',
    'LA', 'LAND', 'LB', 'LC', 'LI', 'LIGHTING', 'LIMO', 'LINK', 'LK', 'LR', 'LS', 'LT', 'LU', 'LUXURY', 'LV', 'LY',
    'MA', 'MAISON', 'MANAGEMENT', 'MANGO', 'MARKETING', 'MC', 'MD', 'ME', 'MENU', 'MG', 'MH', 'MIL', 'MK', 'ML',
    'MM', 'MN', 'MO', 'MOBI', 'MODA', 'MONASH', 'MP', 'MQ', 'MR', 'MS', 'MT', 'MU', 'MUSEUM', 'MV', 'MW', 'MX',
    'MY', 'MZ', 'NA', 'NAGOYA', 'NAME', 'NC', 'NE', 'NET', 'NEUSTAR', 'NF', 'NG', 'NI', 'NINJA', 'NL', 'NO', 'NP',
    'NR', 'NU', 'NZ', 'OKINAWA', 'OM', 'ONION', 'ONL', 'ORG', 'PA', 'PARTNERS', 'PARTS', 'PE', 'PF', 'PG', 'PH',
    'PHOTO', 'PHOTOGRAPHY', 'PHOTOS', 'PICS', 'PINK', 'PK', 'PL', 'PLUMBING', 'PM', 'PN', 'POST', 'PR', 'PRO',
    'PRODUCTIONS', 'PROPERTIES', 'PS', 'PT', 'PUB', 'PW', 'PY', 'QA', 'QPON', 'RE', 'RECIPES', 'RED', 'RENTALS',
    'REPAIR', 'REPORT', 'REVIEWS', 'RICH', 'RO', 'RS', 'RU', 'RUHR', 'RW', 'SA', 'SB', 'SC', 'SD', 'SE', 'SEXY',
    'SG', 'SH', 'SHIKSHA', 'SHOES', 'SI', 'SINGLES', 'SJ', 'SK', 'SL', 'SM', 'SN', 'SO', 'SOCIAL', 'SOLAR',
    'SOLUTIONS', 'SR', 'ST', 'SU', 'SUPPLIES', 'SUPPLY', 'SUPPORT', 'SV', 'SX', 'SY', 'SYSTEMS', 'SZ', 'TATTOO',
    'TC', 'TD', 'TECHNOLOGY', 'TEL', 'TF', 'TG', 'TH', 'TIENDA', 'TIPS', 'TJ', 'TK', 'TL', 'TM', 'TN', 'TO',
    'TODAY', 'TOKYO', 'TOOLS', 'TP', 'TR', 'TRAINING', 'TRAVEL', 'TT', 'TV', 'TW', 'TZ', 'UA', 'UG', 'UK', 'UNO',
    'US', 'UY', 'UZ', 'VA', 'VACATIONS', 'VC', 'VE', 'VENTURES', 'VG', 'VI', 'VIAJES', 'VILLAS', 'VISION', 'VN',
    'VOTE', 'VOTING', 'VOTO', 'VOYAGE', 'VU', 'WANG', 'WATCH', 'WED', 'WF', 'WIEN', 'WIKI', 'WORKS', 'WS',
    'XN--3BST00M', 'XN--3DS443G', 'XN--3E0B707E', 'XN--45BRJ9C', 'XN--55QW42G', 'XN--55QX5D', 'XN--6FRZ82G',
    'XN--6QQ986B3XL', 'XN--80AO21A', 'XN--80ASEHDB', 'XN--80ASWG', 'XN--90A3AC', 'XN--C1AVG', 'XN--CG4BKI',
    'XN--CLCHC0EA0B2G2A9GCD', 'XN--D1ACJ3B', 'XN--FIQ228C5HS', 'XN--FIQ64B', 'XN--FIQS8S', 'XN--FIQZ9S',
    'XN--FPCRJ9C3D', 'XN--FZC2C9E2C', 'XN--GECRJ9C', 'XN--H2BRJ9C', 'XN--I1B6B1A6A2E', 'XN--IO0A7I', 'XN--J1AMH',
    'XN--J6W193G', 'XN--KPRW13D', 'XN--KPRY57D', 'XN--L1ACC', 'XN--LGBBAT1AD8J', 'XN--MGB9AWBF', 'XN--MGBA3A4F16A',
    'XN--MGBAAM7A8H', 'XN--MGBAB2BD', 'XN--MGBAYH7GPA', 'XN--MGBBH1A71E', 'XN--MGBC0A9AZCG', 'XN--MGBERP4A5D4AR',
    'XN--MGBX4CD0AB', 'XN--NGBC5AZD', 'XN--NQV7F', 'XN--NQV7FS00EMA', 'XN--O3CW4H', 'XN--OGBPF8FL', 'XN--P1AI',
    'XN--PGBS0DH', 'XN--Q9JYB4C', 'XN--RHQV96G', 'XN--S9BRJ9C', 'XN--UNUP4Y', 'XN--WGBH1C', 'XN--WGBL6A',
    'XN--XKC2AL3HYE2A', 'XN--XKC2DL3A5EE0H', 'XN--YFRO4I67O', 'XN--YGBI2AMMX', 'XN--ZFR164B', 'XXX', 'XYZ', 'YE',
    'YT', 'ZA', 'ZM', 'ZONE', 'ZW']

class Strings(Module):
    cmd = 'strings'
    description = 'Extract strings from file'
    authors = ['nex', 'Brian Wallace']

    def __init__(self):
        super(Strings, self).__init__()
        self.parser.add_argument('-a', '--all', action='store_true', help='Print all strings')
        self.parser.add_argument('-H', '--hosts', action='store_true', help='Extract IP addresses and domains from strings')

    def extract_hosts(self, strings):
        results = []
        for entry in strings:
            to_add = False
            if DOMAIN_REGEX.search(entry) and not IPV4_REGEX.search(entry):
                if entry[entry.rfind('.') + 1:].upper() in TLD:
                    to_add = True
            elif IPV4_REGEX.search(entry):
                to_add = True
            elif IPV6_REGEX.search(entry):
                try:
                    inet_pton(AF_INET6, entry)
                except socket_error:
                    continue
                else:
                    to_add = True

            if to_add:
                if entry not in results:
                    results.append(entry)

        for result in results:
            self.log('item', result)

    def run(self):
        super(Strings, self).run()
        if self.args is None:
            return

        arg_all = self.args.all
        arg_hosts = self.args.hosts

        if not __sessions__.is_set():
            self.log('error', "No open session")
            return

        if os.path.exists(__sessions__.current.file.path):
            regexp = '[\x20\x30-\x39\x41-\x5a\x61-\x7a\-\.:]{4,}'
            strings = re.findall(regexp, __sessions__.current.file.data)

        if arg_all:
            for entry in strings:
                self.log('', entry)
        elif arg_hosts:
            self.extract_hosts(strings)
        else:
            self.log('error', 'At least one of the parameters is required')
            self.usage()
<commit_msg>Added ability to scan the entire repo for interesting strings<commit_after># This file is part of Viper - https://github.com/viper-framework/viper
# See the file 'LICENSE' for copying permission.

import os
import re
from socket import inet_pton, AF_INET6, error as socket_error

from viper.common.abstracts import Module
from viper.common.objects import File
from viper.core.session import __sessions__
from viper.core.database import Database
from viper.core.storage import get_sample_path

DOMAIN_REGEX = re.compile('([a-z0-9][a-z0-9\-]{0,61}[a-z0-9]\.)+[a-z0-9][a-z0-9\-]*[a-z0-9]', re.IGNORECASE)
IPV4_REGEX = re.compile('[1-2]?[0-9]?[0-9]\.[1-2]?[0-9]?[0-9]\.[1-2]?[0-9]?[0-9]\.[1-2]?[0-9]?[0-9]')
IPV6_REGEX = re.compile('((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}'
                        '|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9'
                        'A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25['
                        '0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3'
                        '})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|['
                        '1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,'
                        '4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:'
                        '))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-'
                        '5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]'
                        '{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d'
                        '\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7}'
                        ')|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d'
                        '\d|[1-9]?\d)){3}))|:)))(%.+)?', re.IGNORECASE | re.S)
TLD = [
    'AC', 'ACADEMY', 'ACTOR', 'AD', 'AE', 'AERO', 'AF', 'AG', 'AGENCY', 'AI', 'AL', 'AM', 'AN', 'AO', 'AQ', 'AR',
    'ARPA', 'AS', 'ASIA', 'AT', 'AU', 'AW', 'AX', 'AZ', 'BA', 'BAR', 'BARGAINS', 'BB', 'BD', 'BE', 'BERLIN', 'BEST',
    'BF', 'BG', 'BH', 'BI', 'BID', 'BIKE', 'BIZ', 'BJ', 'BLUE', 'BM', 'BN', 'BO', 'BOUTIQUE', 'BR', 'BS', 'BT',
    'BUILD', 'BUILDERS', 'BUZZ', 'BV', 'BW', 'BY', 'BZ', 'CA', 'CAB', 'CAMERA', 'CAMP', 'CARDS', 'CAREERS', 'CAT',
    'CATERING', 'CC', 'CD', 'CENTER', 'CEO', 'CF', 'CG', 'CH', 'CHEAP', 'CHRISTMAS', 'CI', 'CK', 'CL', 'CLEANING',
    'CLOTHING', 'CLUB', 'CM', 'CN', 'CO', 'CODES', 'COFFEE', 'COM', 'COMMUNITY', 'COMPANY', 'COMPUTER', 'CONDOS',
    'CONSTRUCTION', 'CONTRACTORS', 'COOL', 'COOP', 'CR', 'CRUISES', 'CU', 'CV', 'CW', 'CX', 'CY', 'CZ', 'DANCE',
    'DATING', 'DE', 'DEMOCRAT', 'DIAMONDS', 'DIRECTORY', 'DJ', 'DK', 'DM', 'DNP', 'DO', 'DOMAINS', 'DZ', 'EC',
    'EDU', 'EDUCATION', 'EE', 'EG', 'EMAIL', 'ENTERPRISES', 'EQUIPMENT', 'ER', 'ES', 'ESTATE', 'ET', 'EU', 'EVENTS',
    'EXPERT', 'EXPOSED', 'FARM', 'FI', 'FISH', 'FJ', 'FK', 'FLIGHTS', 'FLORIST', 'FM', 'FO', 'FOUNDATION', 'FR',
    'FUTBOL', 'GA', 'GALLERY', 'GB', 'GD', 'GE', 'GF', 'GG', 'GH', 'GI', 'GIFT', 'GL', 'GLASS', 'GM', 'GN', 'GOV',
    'GP', 'GQ', 'GR', 'GRAPHICS', 'GS', 'GT', 'GU', 'GUITARS', 'GURU', 'GW', 'GY', 'HK', 'HM', 'HN', 'HOLDINGS',
    'HOLIDAY', 'HOUSE', 'HR', 'HT', 'HU', 'ID', 'IE', 'IL', 'IM', 'IMMOBILIEN', 'IN', 'INDUSTRIES', 'INFO', 'INK',
    'INSTITUTE', 'INT', 'INTERNATIONAL', 'IO', 'IQ', 'IR', 'IS', 'IT', 'JE', 'JM', 'JO', 'JOBS', 'JP', 'KAUFEN',
    'KE', 'KG', 'KH', 'KI', 'KIM', 'KITCHEN', 'KIWI', 'KM', 'KN', 'KOELN', 'KP', 'KR', 'KRED', 'KW', 'KY', 'KZ',
    'LA', 'LAND', 'LB', 'LC', 'LI', 'LIGHTING', 'LIMO', 'LINK', 'LK', 'LR', 'LS', 'LT', 'LU', 'LUXURY', 'LV', 'LY',
    'MA', 'MAISON', 'MANAGEMENT', 'MANGO', 'MARKETING', 'MC', 'MD', 'ME', 'MENU', 'MG', 'MH', 'MIL', 'MK', 'ML',
    'MM', 'MN', 'MO', 'MOBI', 'MODA', 'MONASH', 'MP', 'MQ', 'MR', 'MS', 'MT', 'MU', 'MUSEUM', 'MV', 'MW', 'MX',
    'MY', 'MZ', 'NA', 'NAGOYA', 'NAME', 'NC', 'NE', 'NET', 'NEUSTAR', 'NF', 'NG', 'NI', 'NINJA', 'NL', 'NO', 'NP',
    'NR', 'NU', 'NZ', 'OKINAWA', 'OM', 'ONION', 'ONL', 'ORG', 'PA', 'PARTNERS', 'PARTS', 'PE', 'PF', 'PG', 'PH',
    'PHOTO', 'PHOTOGRAPHY', 'PHOTOS', 'PICS', 'PINK', 'PK', 'PL', 'PLUMBING', 'PM', 'PN', 'POST', 'PR', 'PRO',
    'PRODUCTIONS', 'PROPERTIES', 'PS', 'PT', 'PUB', 'PW', 'PY', 'QA', 'QPON', 'RE', 'RECIPES', 'RED', 'RENTALS',
    'REPAIR', 'REPORT', 'REVIEWS', 'RICH', 'RO', 'RS', 'RU', 'RUHR', 'RW', 'SA', 'SB', 'SC', 'SD', 'SE', 'SEXY',
    'SG', 'SH', 'SHIKSHA', 'SHOES', 'SI', 'SINGLES', 'SJ', 'SK', 'SL', 'SM', 'SN', 'SO', 'SOCIAL', 'SOLAR',
    'SOLUTIONS', 'SR', 'ST', 'SU', 'SUPPLIES', 'SUPPLY', 'SUPPORT', 'SV', 'SX', 'SY', 'SYSTEMS', 'SZ', 'TATTOO',
    'TC', 'TD', 'TECHNOLOGY', 'TEL', 'TF', 'TG', 'TH', 'TIENDA', 'TIPS', 'TJ', 'TK', 'TL', 'TM', 'TN', 'TO',
    'TODAY', 'TOKYO', 'TOOLS', 'TP', 'TR', 'TRAINING', 'TRAVEL', 'TT', 'TV', 'TW', 'TZ', 'UA', 'UG', 'UK', 'UNO',
    'US', 'UY', 'UZ', 'VA', 'VACATIONS', 'VC', 'VE', 'VENTURES', 'VG', 'VI', 'VIAJES', 'VILLAS', 'VISION', 'VN',
    'VOTE', 'VOTING', 'VOTO', 'VOYAGE', 'VU', 'WANG', 'WATCH', 'WED', 'WF', 'WIEN', 'WIKI', 'WORKS', 'WS',
    'XN--3BST00M', 'XN--3DS443G', 'XN--3E0B707E', 'XN--45BRJ9C', 'XN--55QW42G', 'XN--55QX5D', 'XN--6FRZ82G',
    'XN--6QQ986B3XL', 'XN--80AO21A', 'XN--80ASEHDB', 'XN--80ASWG', 'XN--90A3AC', 'XN--C1AVG', 'XN--CG4BKI',
    'XN--CLCHC0EA0B2G2A9GCD', 'XN--D1ACJ3B', 'XN--FIQ228C5HS', 'XN--FIQ64B', 'XN--FIQS8S', 'XN--FIQZ9S',
    'XN--FPCRJ9C3D', 'XN--FZC2C9E2C', 'XN--GECRJ9C', 'XN--H2BRJ9C', 'XN--I1B6B1A6A2E', 'XN--IO0A7I', 'XN--J1AMH',
    'XN--J6W193G', 'XN--KPRW13D', 'XN--KPRY57D', 'XN--L1ACC', 'XN--LGBBAT1AD8J', 'XN--MGB9AWBF', 'XN--MGBA3A4F16A',
    'XN--MGBAAM7A8H', 'XN--MGBAB2BD', 'XN--MGBAYH7GPA', 'XN--MGBBH1A71E', 'XN--MGBC0A9AZCG', 'XN--MGBERP4A5D4AR',
    'XN--MGBX4CD0AB', 'XN--NGBC5AZD', 'XN--NQV7F', 'XN--NQV7FS00EMA', 'XN--O3CW4H', 'XN--OGBPF8FL', 'XN--P1AI',
    'XN--PGBS0DH', 'XN--Q9JYB4C', 'XN--RHQV96G', 'XN--S9BRJ9C', 'XN--UNUP4Y', 'XN--WGBH1C', 'XN--WGBL6A',
    'XN--XKC2AL3HYE2A', 'XN--XKC2DL3A5EE0H', 'XN--YFRO4I67O', 'XN--YGBI2AMMX', 'XN--ZFR164B', 'XXX', 'XYZ', 'YE',
    'YT', 'ZA', 'ZM', 'ZONE', 'ZW']

class Strings(Module):
    cmd = 'strings'
    description = 'Extract strings from file'
    authors = ['nex', 'Brian Wallace']

    def __init__(self):
        super(Strings, self).__init__()
        self.parser.add_argument('-a', '--all', action='store_true', help='Print all strings')
        self.parser.add_argument('-H', '--hosts', action='store_true', help='Extract IP addresses and domains from strings')
        self.parser.add_argument('-s', '--scan', action='store_true', help='Scan all files in the project')

    def extract_hosts(self, strings):
        results = []
        for entry in strings:
            to_add = False
            if DOMAIN_REGEX.search(entry) and not IPV4_REGEX.search(entry):
                if entry[entry.rfind('.') + 1:].upper() in TLD:
                    to_add = True
            elif IPV4_REGEX.search(entry):
                to_add = True
            elif IPV6_REGEX.search(entry):
                try:
                    inet_pton(AF_INET6, entry)
                except socket_error:
                    continue
                else:
                    to_add = True

            if to_add:
                if entry not in results:
                    results.append(entry)

        return results

    def run(self):
        # TODO: this function needs to be refactored.

        super(Strings, self).run()
        if self.args is None:
            return

        arg_all = self.args.all
        arg_hosts = self.args.hosts
        arg_scan = self.args.scan

        regexp = '[\x20\x30-\x39\x41-\x5a\x61-\x7a\-\.:]{4,}'

        if arg_scan:
            db = Database()
            samples = db.find(key='all')

            rows = []
            for sample in samples:
                sample_path = get_sample_path(sample.sha256)

                strings = re.findall(regexp, File(sample_path).data)
                results = self.extract_hosts(strings)

                if results:
                    self.log('info', sample.name)

                    for result in results:
                        self.log('item', result)
        else:
            if not __sessions__.is_set():
                self.log('error', "No open session")
                return

            if os.path.exists(__sessions__.current.file.path):
                strings = re.findall(regexp, __sessions__.current.file.data)

            if arg_all:
                for entry in strings:
                    self.log('', entry)
            elif arg_hosts:
                results = self.extract_hosts(strings)
                for result in results:
                    self.log('item', result)

        if not arg_all and not arg_hosts and not arg_scan:
            self.log('error', 'At least one of the parameters is required')
            self.usage()
<|endoftext|>


<commit_before>from indicator import Indicator
import logging
import pandas
import utils

# Load logging configuration
log = logging.getLogger(__name__)

class Validity(Indicator):
    """Class used to compute indicators of type validity."""

    def __init__(self):
        pass

    def execute(self, session):
        """Execute indicator of type validity."""
        # Update session status to running
        session_id = session['id']
        indicator_id = session['indicatorId']
        log.info('Start execution of session Id {session_id} for indicator Id {indicator_id}.'.format(session_id=session_id, indicator_id=indicator_id))
        log.debug('Update session status to Running.')
        utils.update_session_status(session_id, 'Running')

        # Verify if the list of indicator parameters is valid
        indicaor_type_id = session['indicatorByIndicatorId']['indicatorTypeId']
        parameters = session['indicatorByIndicatorId']['parametersByIndicatorId']['nodes']
        parameters = utils.verify_indicator_parameters(indicaor_type_id, parameters)

        # Get target data
        dimensions = parameters[4]
        measures = parameters[5]
        target = parameters[8]
        target_request = parameters[9]
        target_data = super().get_data_frame(target, target_request, dimensions, measures)

        # Evaluate completeness
        alert_operator = parameters[1]  # Alert operator
        alert_threshold = parameters[2]  # Alert threshold
        log.info('Evaluate validity of target data source.')
        result_data = self.evaluate_validity(target_data, measures, alert_operator, alert_threshold)

        # Compute session result
        log.info('Compute session results.')
        nb_records_alert = super().compute_session_result(session_id, alert_operator, alert_threshold, result_data)

        # Send e-mail alert
        if nb_records_alert != 0:
            indicator_name = session['indicatorByIndicatorId']['name']
            distribution_list = parameters[3]  # Distribution list
            super().send_alert(indicator_id, indicator_name, session_id, distribution_list, alert_operator, alert_threshold, nb_records_alert, result_data)

        # Update session status to succeeded
        log.debug('Update session status to Succeeded.')
        utils.update_session_status(session_id, 'Succeeded')
        log.info('Session Id {session_id} for indicator Id {indicator_id} completed successfully.'.format(session_id=session_id, indicator_id=indicator_id))

    def evaluate_validity(self, target_data, measures, alert_operator, alert_threshold):
        """Compute specificities of validity indicator and return results in a data frame."""
        # No tranformation needed for this data frame
        result_data = target_data

        # Formatting data to improve readability
        for measure in measures:
            result_data[measure] = round(result_data[measure], 2).astype(float)

        # For each record and measure in data frame test if alert must be sent and update alert column
        result_data['Alert'] = False
        for measure in measures:
            for row_num in result_data.index:
                measure_value = result_data.loc[row_num, measure]
                if self.is_alert(measure_value, alert_operator, alert_threshold):
                    result_data.loc[row_num, 'Alert'] = True

        return result_data
<commit_msg>Manage NaN values in target data set<commit_after>from indicator import Indicator
import logging
import pandas
import utils

# Load logging configuration
log = logging.getLogger(__name__)

class Validity(Indicator):
    """Class used to compute indicators of type validity."""

    def __init__(self):
        pass

    def execute(self, session):
        """Execute indicator of type validity."""
        # Update session status to running
        session_id = session['id']
        indicator_id = session['indicatorId']
        log.info('Start execution of session Id {session_id} for indicator Id {indicator_id}.'.format(session_id=session_id, indicator_id=indicator_id))
        log.debug('Update session status to Running.')
        utils.update_session_status(session_id, 'Running')

        # Verify if the list of indicator parameters is valid
        indicaor_type_id = session['indicatorByIndicatorId']['indicatorTypeId']
        parameters = session['indicatorByIndicatorId']['parametersByIndicatorId']['nodes']
        parameters = utils.verify_indicator_parameters(indicaor_type_id, parameters)

        # Get target data
        dimensions = parameters[4]
        measures = parameters[5]
        target = parameters[8]
        target_request = parameters[9]
        target_data = super().get_data_frame(target, target_request, dimensions, measures)

        # Evaluate completeness
        alert_operator = parameters[1]  # Alert operator
        alert_threshold = parameters[2]  # Alert threshold
        log.info('Evaluate validity of target data source.')
        result_data = self.evaluate_validity(target_data, measures, alert_operator, alert_threshold)

        # Compute session result
        log.info('Compute session results.')
        nb_records_alert = super().compute_session_result(session_id, alert_operator, alert_threshold, result_data)

        # Send e-mail alert
        if nb_records_alert != 0:
            indicator_name = session['indicatorByIndicatorId']['name']
            distribution_list = parameters[3]  # Distribution list
            super().send_alert(indicator_id, indicator_name, session_id, distribution_list, alert_operator, alert_threshold, nb_records_alert, result_data)

        # Update session status to succeeded
        log.debug('Update session status to Succeeded.')
        utils.update_session_status(session_id, 'Succeeded')
        log.info('Session Id {session_id} for indicator Id {indicator_id} completed successfully.'.format(session_id=session_id, indicator_id=indicator_id))

    def evaluate_validity(self, target_data, measures, alert_operator, alert_threshold):
        """Compute specificities of validity indicator and return results in a data frame."""
        # No tranformation needed for this data frame
        result_data = target_data
        result_data = result_data.fillna(value=0)  # Replace NaN values per 0

        # Formatting data to improve readability
        for measure in measures:
            result_data[measure] = round(result_data[measure], 2).astype(float)

        # For each record and measure in data frame test if alert must be sent and update alert column
        result_data['Alert'] = False
        for measure in measures:
            for row_num in result_data.index:
                measure_value = result_data.loc[row_num, measure]
                if self.is_alert(measure_value, alert_operator, alert_threshold):
                    result_data.loc[row_num, 'Alert'] = True

        return result_data
<|endoftext|>


<commit_before>"""
Python wrapper for Virtuoso shell.

To be used in conjunction with IPython/Jupyter.
"""
import zmq
import json
from jupyter_core.paths import jupyter_data_dir
import re
import colorama

class VirtuosoExceptions(Exception):
    """
    To handle errors throws by the virtuoso shell
    """
    def __init__(self, value):
        self.value = value
        super(VirtuosoExceptions, self).__init__(value)

    def __str__(self):
        return repr(self.value)

class VirtuosoShellClient(object):
    """
    This is the client that talks to dfII's python server
    """
    def __init__(self, port=None):
        super(VirtuosoShellClient, self).__init__()
        self.port = None
        self.host = None
        self.context = None
        self.socket = None
        self.init()

    def init(self):
        # Get connection info from the PyLL JSON file
        CONN_FILE = jupyter_data_dir() + "/runtime/" + "virtuoso-pyll.json"
        with open(CONN_FILE, "r") as COF:
            self.host, self.port = json.load(COF)
        # Connection info will come from a JSON file generated by the dfII/PyLL
        # server. So, read JSON to figure out the connection info.
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.REQ)
        self.socket.connect("tcp://%s:%d" % (self.host, self.port))

    def write(self, payload):
        #TODO: make sure the payload type is correct
        self.socket.send_string(payload)

    def read(self):
        return self.socket.recv().decode()

    def read_parsed(self):
        return json.loads(self.socket.recv().decode())

    def close(self):
        # Close socket
        self.socket.close()

class VirtuosoShell(object):
    """
    This class gives a python interface to the Virtuoso shell.
    """
    _banner = None
    _version_re = None
    _output = ""

    @property
    def banner(self):
        """
        Virtuoso shell's banner
        """
        self._shell.write("getVersion()")
        self._banner = self._shell.read_parsed()['result']
        return self._banner

    @property
    def language_version(self):
        """
        Language version
        """
        __match__ = self._version_re.search(self.banner)
        return __match__.group(1)

    @property
    def output(self):
        """
        Last output returned by the shell
        """
        return self._output

    def __init__(self, *args, **kwargs):
        super(VirtuosoShell, self).__init__(*args, **kwargs)
        self.prompt = [re.compile(r'\r\n<<pyvi>> $'),
                       re.compile(r'^<<pyvi>> $')]
        self._shell_available_re = re.compile(r'"__jupyter_kernel_ready__"'
                                              r'[\s\S]+')
        self._version_re = re.compile(r'version (\d+(\.\d+)+)')
        self._multiline_re = re.compile(r'[^\\]\n')
        self._error_re = re.compile(r'^([\s\S]*?)\*Error\*'
                                    r'(.+)(\s*)([\s\S]*)')
        self._open_braces_re = re.compile(r'\{')
        self._close_braces_re = re.compile(r'\}')
        self._open_paren_re = re.compile(r'\(')
        self._close_paren_re = re.compile(r'\)')
        self._dbl_quote_re = re.compile(r'"')
        self._output_prompt_re = re.compile(r'<<pyvi>> ')
        self._object_prop_re = re.compile(r'(\w+?)\s*([-~]>)(\w*?)$')
        self._object_prop_list_re = re.compile(r'\((\w+?)\)\s*([-~]>)(\w*?)$')
        self._var_name_re = re.compile(r'\s*(\w+)$')
        self.match_dict = {self._object_prop_re: lambda _match: '%s%s?' %
                           (_match.group(1), _match.group(2)),
                           self._object_prop_list_re: lambda _match:
                           'car(%s)%s?' % (_match.group(1), _match.group(2)),
                           self._var_name_re: lambda _match:
                           'listFunctions("^%s" t)\r\nlistVariables("^%s")' %
                           (_match.group(1), _match.group(1))}
        self._start_virtuoso()

    def _start_virtuoso(self):
        """
        Connect to the virtuoso shell
        """
        self._shell = VirtuosoShellClient()

    def _parse_output(self):
        """
        Parse the virtuoso shell's output and handle error.

        #TODO: Can I use the skill debugger somehow?

        In case of error, set status to a tuple of the form :
            (etype, evalue, tb)
        else, set to None
        """
        err_out = self._output['error']
        warn_out = self._output['warning']
        info_out = self._output['info']
        res_out = self._output['result']
        full_out = ""
        _err_match = None

        if err_out is not None:
            full_out += ("%s%s%s%s%s\n" % (colorama.Style.BRIGHT,
                         colorama.Fore.RED, err_out, colorama.Fore.RESET,
                         colorama.Style.NORMAL))
            _err_match = self._error_re.search(err_out)
        if warn_out is not None:
            full_out += ("%s%s%s%s%s\n" % (colorama.Style.BRIGHT,
                         colorama.Fore.YELLOW, warn_out, colorama.Fore.RESET,
                         colorama.Style.NORMAL))
        if info_out is not None:
            full_out += ("%s\n" % info_out)
        if res_out is not None:
            full_out += res_out

        self._output = full_out

        # If the shell reported any errors, throw exception
        if _err_match is not None:
            _exec_error = ("Error", 1, _err_match.group(2))
            raise VirtuosoExceptions(_exec_error)

        #self._exec_error = None
        #_err_match = self._error_re.search(self._output)
        #if _err_match is not None:
        #    self._exec_error = ("Error", 1, _err_match.group(2))

        ## number the output line
        #_output_list = [_line.rstrip() for _line in
        #                self._output_prompt_re.split(self._output) if
        #                _line.rstrip() != '']
        #_out_num = 1
        #_color = colorama.Fore.YELLOW
        #self._output = ''
        #_single_line = (len(_output_list) == 1)
        #for _oline in _output_list:
        #    if self._error_re.search(_oline) is not None:
        #        _color = colorama.Fore.RED
        #        if(not _single_line):
        #            self._output += ('%s%s%d> %s%s%s\n' %
        #                             (colorama.Style.BRIGHT, _color, _out_num,
        #                              _oline, colorama.Fore.RESET,
        #                              colorama.Style.NORMAL))
        #        else:
        #            self._output += ('%s%s%s%s%s\n' % (colorama.Style.BRIGHT,
        #                                               _color, _oline,
        #                                               colorama.Fore.RESET,
        #                                               colorama.Style.NORMAL))
        #    else:
        #        _color = colorama.Fore.YELLOW
        #        if(not _single_line):
        #            self._output += '%s%d>%s %s\n' % (_color, _out_num,
        #                                              colorama.Fore.RESET,
        #                                              _oline)
        #        else:
        #            self._output += '%s\n' % (_oline)
        #    _out_num += 1
        #self._output = self.output.rstrip()
        ## If the shell reported any errors, throw exception
        #if self._exec_error is not None:
        #    raise VirtuosoExceptions(self._exec_error)

    def _pretty_introspection(self, info, keyword):
        import re
        # Optional keywords
        info = re.sub(r'(\?\w+)', r'%s\1%s' % (colorama.Fore.YELLOW,
                                               colorama.Fore.RESET), info,
                      count=0)
        # Required arguments
        info = re.sub(r'(\s*)(%s\()([\r\n]*)([\w\s]+)([\s\S]+)' % keyword,
                      r'\1\2\3%s\4%s\5' % (colorama.Fore.GREEN,
                                           colorama.Fore.RESET), info)
        # Function name
        info = re.sub(r'(%s)(\()' % keyword,
                      r'%s\1%s\2' % (colorama.Fore.BLUE,
                                     colorama.Fore.RESET),
                      info, count=0)
        return info[:-1]

    def run_raw(self, code):
        """
        Send the code as it is.

        This is useful for executing single functions. Output is not
        post-processed in this function.
        """
        self._shell.write(code)
        self._output = self._shell.read()

    def run_cell(self, code):
        """
        Executes the 'code'.

        We need to wrap a block of SKILL code in `prog` or equivalently in
        `{...}` for `evalstring` to work correctly on the dfII side.
        """

        if self._multiline_re.search(code):
            self._shell.write("{" + code + "}")
        else:
            self._shell.write(code)
        self.wait_ready()

        # Check the output and throw exception in case of error
        self._parse_output()

        return self.output

    def get_matches(self, code_line):
        """
        Return a list of functions and variables matching the line's end or
        in case of '->' or '~>', a list of attributes.
        """
        _cmd = None
        _token = ''
        _match_list = []
        _match = self._object_prop_re.search(code_line)
        _raw_mode = True
        if(_match is None):
            _match = self._object_prop_list_re.search(code_line)
        if(_match is None):
            _match = self._var_name_re.search(code_line)
            _raw_mode = False
        if(_match is not None):
            _cmd = self.match_dict[_match.re](_match)
            _token = _match.group(1)
            self.run_raw(_cmd)
            _output = json.loads(self._output)['result']
            if _output != u'nil':
                _output = _output.replace('(', ' ')
                _output = _output.replace(')', ' ')
                _output = _output.replace('nil', ' ')
                _match_list = _output.split()
                if(len(_match.groups()) == 3):
                    # when there is part of an attr.
                    _token = _match.group(3)
                    if(_token != ''):
                        _match_list = [_match for _match in _match_list if
                                       _match.startswith(_token)]

        return((_match_list, _token))

    def get_info(self, token):
        """
        Returns info on the requested object

        """
        # TODO: get info on variables also
        # Make sure that only valid function/variable names are used
        if token.rstrip() != '':
            token = re.match(r'(\S+?)\s*$', token).group(1)
        _cmd = 'help(%s)' % token
        self.run_raw(_cmd)
        _info = json.loads(self._output)['info']
        #_info = ''
        ##if self._shell.before != 'nil':
        ##    _info = self._shell.before
        #if self._output != 'nil':
        #    _info = self._output
        return (self._pretty_introspection(_info, token))

    def interrupt(self):
        """
        Send an interrupt to the virtuoso shell
        """
        #self._shell.sendintr()
        #TODO: finish this
        pass

    def wait_ready(self):
        """
        Wait for the dfII to reply to the previously submitted code
        """
        self._output = self._shell.read_parsed()

    def shutdown(self, restart):
        """
        Shutdown the shell; restart if requested
        """
        if restart:
            self._shell.close()
            self._shell.init()
        else:
            self.run_raw('exit()')
            self._shell.close()
<commit_msg>-For get_info(), handle cases with no available help<commit_after>"""
Python wrapper for Virtuoso shell.

To be used in conjunction with IPython/Jupyter.
"""
import zmq
import json
from jupyter_core.paths import jupyter_data_dir
import re
import colorama

class VirtuosoExceptions(Exception):
    """
    To handle errors throws by the virtuoso shell
    """
    def __init__(self, value):
        self.value = value
        super(VirtuosoExceptions, self).__init__(value)

    def __str__(self):
        return repr(self.value)

class VirtuosoShellClient(object):
    """
    This is the client that talks to dfII's python server
    """
    def __init__(self, port=None):
        super(VirtuosoShellClient, self).__init__()
        self.port = None
        self.host = None
        self.context = None
        self.socket = None
        self.init()

    def init(self):
        # Get connection info from the PyLL JSON file
        CONN_FILE = jupyter_data_dir() + "/runtime/" + "virtuoso-pyll.json"
        with open(CONN_FILE, "r") as COF:
            self.host, self.port = json.load(COF)
        # Connection info will come from a JSON file generated by the dfII/PyLL
        # server. So, read JSON to figure out the connection info.
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.REQ)
        self.socket.connect("tcp://%s:%d" % (self.host, self.port))

    def write(self, payload):
        #TODO: make sure the payload type is correct
        self.socket.send_string(payload)

    def read(self):
        return self.socket.recv().decode()

    def read_parsed(self):
        return json.loads(self.socket.recv().decode())

    def close(self):
        # Close socket
        self.socket.close()

class VirtuosoShell(object):
    """
    This class gives a python interface to the Virtuoso shell.
    """
    _banner = None
    _version_re = None
    _output = ""

    @property
    def banner(self):
        """
        Virtuoso shell's banner
        """
        self._shell.write("getVersion()")
        self._banner = self._shell.read_parsed()['result']
        return self._banner

    @property
    def language_version(self):
        """
        Language version
        """
        __match__ = self._version_re.search(self.banner)
        return __match__.group(1)

    @property
    def output(self):
        """
        Last output returned by the shell
        """
        return self._output

    def __init__(self, *args, **kwargs):
        super(VirtuosoShell, self).__init__(*args, **kwargs)
        self.prompt = [re.compile(r'\r\n<<pyvi>> $'),
                       re.compile(r'^<<pyvi>> $')]
        self._shell_available_re = re.compile(r'"__jupyter_kernel_ready__"'
                                              r'[\s\S]+')
        self._version_re = re.compile(r'version (\d+(\.\d+)+)')
        self._multiline_re = re.compile(r'[^\\]\n')
        self._error_re = re.compile(r'^([\s\S]*?)\*Error\*'
                                    r'(.+)(\s*)([\s\S]*)')
        self._open_braces_re = re.compile(r'\{')
        self._close_braces_re = re.compile(r'\}')
        self._open_paren_re = re.compile(r'\(')
        self._close_paren_re = re.compile(r'\)')
        self._dbl_quote_re = re.compile(r'"')
        self._output_prompt_re = re.compile(r'<<pyvi>> ')
        self._object_prop_re = re.compile(r'(\w+?)\s*([-~]>)(\w*?)$')
        self._object_prop_list_re = re.compile(r'\((\w+?)\)\s*([-~]>)(\w*?)$')
        self._var_name_re = re.compile(r'\s*(\w+)$')
        self.match_dict = {self._object_prop_re: lambda _match: '%s%s?' %
                           (_match.group(1), _match.group(2)),
                           self._object_prop_list_re: lambda _match:
                           'car(%s)%s?' % (_match.group(1), _match.group(2)),
                           self._var_name_re: lambda _match:
                           'listFunctions("^%s" t)\r\nlistVariables("^%s")' %
                           (_match.group(1), _match.group(1))}
        self._start_virtuoso()

    def _start_virtuoso(self):
        """
        Connect to the virtuoso shell
        """
        self._shell = VirtuosoShellClient()

    def _parse_output(self):
        """
        Parse the virtuoso shell's output and handle error.

        #TODO: Can I use the skill debugger somehow?

        In case of error, set status to a tuple of the form :
            (etype, evalue, tb)
        else, set to None
        """
        err_out = self._output['error']
        warn_out = self._output['warning']
        info_out = self._output['info']
        res_out = self._output['result']
        full_out = ""
        _err_match = None

        if err_out is not None:
            full_out += ("%s%s%s%s%s\n" % (colorama.Style.BRIGHT,
                         colorama.Fore.RED, err_out, colorama.Fore.RESET,
                         colorama.Style.NORMAL))
            _err_match = self._error_re.search(err_out)
        if warn_out is not None:
            full_out += ("%s%s%s%s%s\n" % (colorama.Style.BRIGHT,
                         colorama.Fore.YELLOW, warn_out, colorama.Fore.RESET,
                         colorama.Style.NORMAL))
        if info_out is not None:
            full_out += ("%s\n" % info_out)
        if res_out is not None:
            full_out += res_out

        self._output = full_out

        # If the shell reported any errors, throw exception
        if _err_match is not None:
            _exec_error = ("Error", 1, _err_match.group(2))
            raise VirtuosoExceptions(_exec_error)

        #self._exec_error = None
        #_err_match = self._error_re.search(self._output)
        #if _err_match is not None:
        #    self._exec_error = ("Error", 1, _err_match.group(2))

        ## number the output line
        #_output_list = [_line.rstrip() for _line in
        #                self._output_prompt_re.split(self._output) if
        #                _line.rstrip() != '']
        #_out_num = 1
        #_color = colorama.Fore.YELLOW
        #self._output = ''
        #_single_line = (len(_output_list) == 1)
        #for _oline in _output_list:
        #    if self._error_re.search(_oline) is not None:
        #        _color = colorama.Fore.RED
        #        if(not _single_line):
        #            self._output += ('%s%s%d> %s%s%s\n' %
        #                             (colorama.Style.BRIGHT, _color, _out_num,
        #                              _oline, colorama.Fore.RESET,
        #                              colorama.Style.NORMAL))
        #        else:
        #            self._output += ('%s%s%s%s%s\n' % (colorama.Style.BRIGHT,
        #                                               _color, _oline,
        #                                               colorama.Fore.RESET,
        #                                               colorama.Style.NORMAL))
        #    else:
        #        _color = colorama.Fore.YELLOW
        #        if(not _single_line):
        #            self._output += '%s%d>%s %s\n' % (_color, _out_num,
        #                                              colorama.Fore.RESET,
        #                                              _oline)
        #        else:
        #            self._output += '%s\n' % (_oline)
        #    _out_num += 1
        #self._output = self.output.rstrip()
        ## If the shell reported any errors, throw exception
        #if self._exec_error is not None:
        #    raise VirtuosoExceptions(self._exec_error)

    def _pretty_introspection(self, info, keyword):
        import re
        # Optional keywords
        info = re.sub(r'(\?\w+)', r'%s\1%s' % (colorama.Fore.YELLOW,
                                               colorama.Fore.RESET), info,
                      count=0)
        # Required arguments
        info = re.sub(r'(\s*)(%s\()([\r\n]*)([\w\s]+)([\s\S]+)' % keyword,
                      r'\1\2\3%s\4%s\5' % (colorama.Fore.GREEN,
                                           colorama.Fore.RESET), info)
        # Function name
        info = re.sub(r'(%s)(\()' % keyword,
                      r'%s\1%s\2' % (colorama.Fore.BLUE,
                                     colorama.Fore.RESET),
                      info, count=0)
        return info[:-1]

    def run_raw(self, code):
        """
        Send the code as it is.

        This is useful for executing single functions. Output is not
        post-processed in this function.
        """
        self._shell.write(code)
        self._output = self._shell.read()

    def run_cell(self, code):
        """
        Executes the 'code'.

        We need to wrap a block of SKILL code in `prog` or equivalently in
        `{...}` for `evalstring` to work correctly on the dfII side.
        """

        if self._multiline_re.search(code):
            self._shell.write("{" + code + "}")
        else:
            self._shell.write(code)
        self.wait_ready()

        # Check the output and throw exception in case of error
        self._parse_output()

        return self.output

    def get_matches(self, code_line):
        """
        Return a list of functions and variables matching the line's end or
        in case of '->' or '~>', a list of attributes.
        """
        _cmd = None
        _token = ''
        _match_list = []
        _match = self._object_prop_re.search(code_line)
        _raw_mode = True
        if(_match is None):
            _match = self._object_prop_list_re.search(code_line)
        if(_match is None):
            _match = self._var_name_re.search(code_line)
            _raw_mode = False
        if(_match is not None):
            _cmd = self.match_dict[_match.re](_match)
            _token = _match.group(1)
            self.run_raw(_cmd)
            _output = json.loads(self._output)['result']
            if _output != u'nil':
                _output = _output.replace('(', ' ')
                _output = _output.replace(')', ' ')
                _output = _output.replace('nil', ' ')
                _match_list = _output.split()
                if(len(_match.groups()) == 3):
                    # when there is part of an attr.
                    _token = _match.group(3)
                    if(_token != ''):
                        _match_list = [_match for _match in _match_list if
                                       _match.startswith(_token)]

        return((_match_list, _token))

    def get_info(self, token):
        """
        Returns info on the requested object

        """
        # TODO: get info on variables also
        # Make sure that only valid function/variable names are used
        if token.rstrip() != '':
            token = re.match(r'(\S+?)\s*$', token).group(1)
        _cmd = 'help(%s)' % token
        self.run_raw(_cmd)
        _pay = json.loads(self._output)
        # Handle cases where no help is available
        if (_pay['info'] is None) or (_pay['result'] == "nil"):
            return ""
        else:
            return (self._pretty_introspection(_info, token))

    def interrupt(self):
        """
        Send an interrupt to the virtuoso shell
        """
        #self._shell.sendintr()
        #TODO: finish this
        pass

    def wait_ready(self):
        """
        Wait for the dfII to reply to the previously submitted code
        """
        self._output = self._shell.read_parsed()

    def shutdown(self, restart):
        """
        Shutdown the shell; restart if requested
        """
        if restart:
            self._shell.close()
            self._shell.init()
        else:
            self.run_raw('exit()')
            self._shell.close()
<|endoftext|>


<commit_before>import json
from flask import request
from sqlalchemy import text
from sqlalchemy.exc import DatabaseError
from webargs import fields, missing
from webargs.flaskparser import parser, use_kwargs
from scuevals_api.models import Course, Quarter, Department, School
from scuevals_api import api, db
from flask_restful import Resource, abort

class Departments(Resource):
    args = {'university_id': fields.Integer()}

    @use_kwargs(args)
    def get(self, university_id):
        if university_id is missing:
            return {'error': 'missing university_id parameter'}

        departments = Department.query.join(Department.school).filter(School.university_id == university_id).all()

        return [
            {
                'id': department.id,
                'abbr': department.abbreviation,
                'school': department.school.abbreviation
            }
            for department in departments
        ]

    @use_kwargs(args)
    def post(self, university_id):
        if request.headers['Content-Type'] != 'application/json':
            return {'error': 'wrong mime type'}

        if 'departments' not in request.json or not isinstance(request.json['departments'], list):
            return {'error': 'invalid json format'}

        params = {'u_id': university_id, 'data': json.dumps(request.json)}

        try:
            result = db.engine.execute(text('select update_departments(:u_id, (:data)::jsonb)'), params)
        except DatabaseError:
            return {'error': 'invalid json format'}

        return {'result': 'success', 'updated_count': int(result.first()[0])}

class Courses(Resource):
    args = {'quarter_id': fields.Integer()}

    @use_kwargs(args)
    def get(self, quarter_id):

        if quarter_id is missing:
            courses = Course.query.all()
        else:
            courses = Course.query.join(Course.quarters).filter(Quarter.id == quarter_id).all()

        return [
            {
                'id': course.id,
                'department': course.department.abbreviation,
                'number': course.number,
                'title': course.title,
                'quarters': [quarter.id for quarter in course.quarters]
            }
            for course in courses
        ]

class Quarters(Resource):
    def get(self):
        quarters = Quarter.query.all()

        return [
            {
                'id': quarter.id,
                'year': quarter.year,
                'name': quarter.name,
                'current': quarter.current
            }
            for quarter in quarters
        ]

@parser.error_handler
def handle_request_parsing_error(err):
    """webargs error handler that uses Flask-RESTful's abort function to return
    a JSON error response to the client.
    """
    abort(422, errors=err.messages)

api.add_resource(Departments, '/departments')
api.add_resource(Courses, '/courses')
api.add_resource(Quarters, '/quarters')
<commit_msg>Fixed bug with departments api<commit_after>import json
import logging
from flask import request
from sqlalchemy import text
from sqlalchemy.exc import DatabaseError
from webargs import fields, missing
from webargs.flaskparser import parser, use_kwargs
from scuevals_api.models import Course, Quarter, Department, School
from scuevals_api import api, db
from flask_restful import Resource, abort

class Departments(Resource):
    args = {'university_id': fields.Integer()}

    @use_kwargs(args)
    def get(self, university_id):
        if university_id is missing:
            return {'error': 'missing university_id parameter'}

        departments = Department.query.join(Department.school).filter(School.university_id == university_id).all()

        return [
            {
                'id': department.id,
                'abbr': department.abbreviation,
                'school': department.school.abbreviation
            }
            for department in departments
        ]

    @use_kwargs(args)
    def post(self, university_id):
        if request.headers['Content-Type'] != 'application/json':
            return {'error': 'wrong mime type'}

        if 'departments' not in request.json or not isinstance(request.json['departments'], list):
            return {'error': 'invalid json format'}

        if university_id is missing:
            return {'error': 'missing university_id parameter'}

        params = {'u_id': university_id, 'data': json.dumps(request.json)}

        try:
            result = db.engine.execute(text('select update_departments(:u_id, (:data)::jsonb)'), params)
        except DatabaseError as e:
            logging.error('failed to update departments: ' + str(e))
            return {'error': 'database error'}

        return {'result': 'success', 'updated_count': int(result.first()[0])}

class Courses(Resource):
    args = {'quarter_id': fields.Integer()}

    @use_kwargs(args)
    def get(self, quarter_id):

        if quarter_id is missing:
            courses = Course.query.all()
        else:
            courses = Course.query.join(Course.quarters).filter(Quarter.id == quarter_id).all()

        return [
            {
                'id': course.id,
                'department': course.department.abbreviation,
                'number': course.number,
                'title': course.title,
                'quarters': [quarter.id for quarter in course.quarters]
            }
            for course in courses
        ]

class Quarters(Resource):
    def get(self):
        quarters = Quarter.query.all()

        return [
            {
                'id': quarter.id,
                'year': quarter.year,
                'name': quarter.name,
                'current': quarter.current
            }
            for quarter in quarters
        ]

@parser.error_handler
def handle_request_parsing_error(err):
    """webargs error handler that uses Flask-RESTful's abort function to return
    a JSON error response to the client.
    """
    abort(422, errors=err.messages)

api.add_resource(Departments, '/departments')
api.add_resource(Courses, '/courses')
api.add_resource(Quarters, '/quarters')
<|endoftext|>


<commit_before>"""Module for ticket-related utilities."""

from vodreassembler import util
from vodreassembler import protocol

class Ticket:
  def __init__(self, ticket_data):
    self._ticket_data = ticket_data

  @property
  def ticket_id(self):
    return self._ticket_data.id

  @property
  def collision(self):
    return self._ticket_data.collision

  @property
  def random_number(self):
    return self._ticket_data.rn

  @property
  def request_length(self):
    if (self._ticket_data.request_length is None
        and self._ticket_data.request_data is not None):
      return self._ticket_data.request_data.length
    return self._ticket_data.request_length

  @property
  def request_data(self):
    if self._ticket_data.request_data is None:
      return
    return self._ticket_data.request_data.getbytes()

  @property
  def response_length(self):
    if (self._ticket_data.response_length is None
        and self._ticket_data.response_data is not None):
      return self._ticket_data.response_data.length
    return self._ticket_data.response_length

  @property
  def response_data(self):
    if self._ticket_data.response_data is None:
      return
    return self._ticket_data.response_data.getbytes()

class _TicketData:
  def __init__(self, ticket_id):
    self.id = ticket_id
    self.collision = False
    self.rn = None
    self.request_length = None
    self.request_data = None
    self.response_length = None
    self.response_data = None

  def update(self, query):
    assert query.error is None
    if query.type is protocol.QueryType.open_ticket:
      self._update_rn(query.variables['rn'])
      self._update_request_length(query.variables['sz'])
    elif query.type is protocol.QueryType.request_data:
      self._update_request_data(query.variables['bf'], query.variables['wr'])
    elif query.type is protocol.QueryType.check_request:
      if len(query.payload) == 4 and query.payload.startswith(b'L'):
        self._update_response_length(int.from_bytes(query.payload[1:],
                                                    byte_order='big'))
    elif query.type is protocol.QueryType.fetch_response:
      self._update_response_data(query.variables['ln'], query.variables['rd'],
                                 query.payload)
    elif query.type is protocol.QueryType.close_ticket:
      # We don't really care about close_ticket at this point.
      pass

  def _update_rn(self, rn):
    if self.rn is not None and self.rn != rn:
      self.collision = True
      return
    self.rn = rn

  def _update_request_length(self, length):
    if self.request_length is not None and self.request_length != length:
      self.collision = True
      return
    self.request_length = length

  def _update_request_data(self, data, offset):
    if self.request_data is None:
      self.request_data = util.DataAssembler(30, length=self.request_length)
    try:
      self.request_data.add(data, offset)
    except util.UnexpectedChunkError:
      self.collision = True
      return
    if self.request_data.length is not None:
      self._update_request_length(self.request_data.length)

  def _update_response_length(self, length):
    if self.response_length is not None and self.response_length != length:
      self.collision = True
      return
    self.response_length = length

  def _update_response_data(self, segment_length, segment_offset, chunk):
    if self.response_data is None:
      self.response_data = util.DataAssembler(3, length=self.response_length)
    # Response data is queried by split segments. chunk is a piece of data
    # within one of the segments. In order to reassemble the entire data, we
    # must first construct each segment from chunks, then assemble segments into
    # final data. Fortunately, the known implementation uses segment_length=48,
    # which is a multiple of 3; therefore a single DataAssembler can be used.
    absolute_offset = segment_offset + chunk.offset
    try:
      self.response_data.add(chunk.data, absolute_offset)
    except util.UnexpectedChunkError:
      self.collision = True
      return

    if segment_length < 48:
      try:
        self.response_data.length = segment_offset + segment_length
      except ValueError:
        self.collision = True
    if self.response_data.length is not None:
      self._update_response_length(self.response_data.length)

class TicketDatabase:
  def __init__(self, fqdn_suffix=None):
    self._tickets = {}
    self._ticket_data = {}
    self._fqdn_suffix = fqdn_suffix

  def __getitem__(self, ticket_id):
    return self._tickets[ticket_id]

  def __contains__(self, ticket_id):
    return ticket_id in self._tickets

  def __len__(self):
    return len(self._tickets)

  def __iter__(self):
    # Use values, as keys are redundant.
    return iter(self._tickets.values())

  def build_from_records(self, records):
    parser = protocol.QueryParser(self._fqdn_suffix)
    for r in records:
      try:
        query = parser.parse(r)
      except ValueError:
        # Just ignore unparseable record.
        continue

      if query.error:
        # ignore error
        continue

      if 'rn' in query.variables:
        ticket_id = int.from_bytes(query.payload.data, byteorder='big')
      elif 'id' in query.variables:
        ticket_id = query.variables['id']
      else:
        # Cannot map the record with any tickets; ignore
        continue

      ticket_data = self._get_or_create_ticket_data(ticket_id)
      ticket_data.update(query)

  def _get_or_create_ticket_data(self, ticket_id):
    if ticket_id not in self._tickets:
      data = _TicketData(ticket_id)
      self._tickets[ticket_id] = Ticket(data)
      self._ticket_data[ticket_id] = data
    return self._ticket_data[ticket_id]
<commit_msg>Add __repr__ to ticket.Ticket and ticket.TicketDatabase.<commit_after>"""Module for ticket-related utilities."""

from vodreassembler import util
from vodreassembler import protocol

class Ticket:
  def __init__(self, ticket_data):
    self._ticket_data = ticket_data

  @property
  def ticket_id(self):
    return self._ticket_data.id

  @property
  def collision(self):
    return self._ticket_data.collision

  @property
  def random_number(self):
    return self._ticket_data.rn

  @property
  def request_length(self):
    if (self._ticket_data.request_length is None
        and self._ticket_data.request_data is not None):
      return self._ticket_data.request_data.length
    return self._ticket_data.request_length

  @property
  def request_data(self):
    if self._ticket_data.request_data is None:
      return
    return self._ticket_data.request_data.getbytes()

  @property
  def response_length(self):
    if (self._ticket_data.response_length is None
        and self._ticket_data.response_data is not None):
      return self._ticket_data.response_data.length
    return self._ticket_data.response_length

  @property
  def response_data(self):
    if self._ticket_data.response_data is None:
      return
    return self._ticket_data.response_data.getbytes()

  def __repr__(self):
    return ('Ticket(id={!r}, collision={!r}, '
            'request_length={!r}, '
            'response_length={!r})').format(self.ticket_id,
                                            self.collision,
                                            self.request_length,
                                            self.response_length)

class _TicketData:
  def __init__(self, ticket_id):
    self.id = ticket_id
    self.collision = False
    self.rn = None
    self.request_length = None
    self.request_data = None
    self.response_length = None
    self.response_data = None

  def update(self, query):
    assert query.error is None
    if query.type is protocol.QueryType.open_ticket:
      self._update_rn(query.variables['rn'])
      self._update_request_length(query.variables['sz'])
    elif query.type is protocol.QueryType.request_data:
      self._update_request_data(query.variables['bf'], query.variables['wr'])
    elif query.type is protocol.QueryType.check_request:
      if len(query.payload) == 4 and query.payload.startswith(b'L'):
        self._update_response_length(int.from_bytes(query.payload[1:],
                                                    byte_order='big'))
    elif query.type is protocol.QueryType.fetch_response:
      self._update_response_data(query.variables['ln'], query.variables['rd'],
                                 query.payload)
    elif query.type is protocol.QueryType.close_ticket:
      # We don't really care about close_ticket at this point.
      pass

  def _update_rn(self, rn):
    if self.rn is not None and self.rn != rn:
      self.collision = True
      return
    self.rn = rn

  def _update_request_length(self, length):
    if self.request_length is not None and self.request_length != length:
      self.collision = True
      return
    self.request_length = length

  def _update_request_data(self, data, offset):
    if self.request_data is None:
      self.request_data = util.DataAssembler(30, length=self.request_length)
    try:
      self.request_data.add(data, offset)
    except util.UnexpectedChunkError:
      self.collision = True
      return
    if self.request_data.length is not None:
      self._update_request_length(self.request_data.length)

  def _update_response_length(self, length):
    if self.response_length is not None and self.response_length != length:
      self.collision = True
      return
    self.response_length = length

  def _update_response_data(self, segment_length, segment_offset, chunk):
    if self.response_data is None:
      self.response_data = util.DataAssembler(3, length=self.response_length)
    # Response data is queried by split segments. chunk is a piece of data
    # within one of the segments. In order to reassemble the entire data, we
    # must first construct each segment from chunks, then assemble segments into
    # final data. Fortunately, the known implementation uses segment_length=48,
    # which is a multiple of 3; therefore a single DataAssembler can be used.
    absolute_offset = segment_offset + chunk.offset
    try:
      self.response_data.add(chunk.data, absolute_offset)
    except util.UnexpectedChunkError:
      self.collision = True
      return

    if segment_length < 48:
      try:
        self.response_data.length = segment_offset + segment_length
      except ValueError:
        self.collision = True
    if self.response_data.length is not None:
      self._update_response_length(self.response_data.length)

class TicketDatabase:
  def __init__(self, fqdn_suffix=None):
    self._tickets = {}
    self._ticket_data = {}
    self._fqdn_suffix = fqdn_suffix

  def __getitem__(self, ticket_id):
    return self._tickets[ticket_id]

  def __contains__(self, ticket_id):
    return ticket_id in self._tickets

  def __len__(self):
    return len(self._tickets)

  def __repr__(self):
    return '{' + ', '.join(map(repr, self)) + '}'

  def __iter__(self):
    # Use values, as keys are redundant.
    return iter(self._tickets.values())

  def build_from_records(self, records):
    parser = protocol.QueryParser(self._fqdn_suffix)
    for r in records:
      try:
        query = parser.parse(r)
      except ValueError:
        # Just ignore unparseable record.
        continue

      if query.error:
        # ignore error
        continue

      if 'rn' in query.variables:
        ticket_id = int.from_bytes(query.payload.data, byteorder='big')
      elif 'id' in query.variables:
        ticket_id = query.variables['id']
      else:
        # Cannot map the record with any tickets; ignore
        continue

      ticket_data = self._get_or_create_ticket_data(ticket_id)
      ticket_data.update(query)

  def _get_or_create_ticket_data(self, ticket_id):
    if ticket_id not in self._tickets:
      data = _TicketData(ticket_id)
      self._tickets[ticket_id] = Ticket(data)
      self._ticket_data[ticket_id] = data
    return self._ticket_data[ticket_id]
<|endoftext|>


<commit_before># ######################################################################
# Copyright (c) 2014, Brookhaven Science Associates, Brookhaven        #
# National Laboratory. All rights reserved.                            #
#                                                                      #
# Redistribution and use in source and binary forms, with or without   #
# modification, are permitted provided that the following conditions   #
# are met:                                                             #
#                                                                      #
# * Redistributions of source code must retain the above copyright     #
#   notice, this list of conditions and the following disclaimer.      #
#                                                                      #
# * Redistributions in binary form must reproduce the above copyright  #
#   notice this list of conditions and the following disclaimer in     #
#   the documentation and/or other materials provided with the         #
#   distribution.                                                      #
#                                                                      #
# * Neither the name of the Brookhaven Science Associates, Brookhaven  #
#   National Laboratory nor the names of its contributors may be used  #
#   to endorse or promote products derived from this software without  #
#   specific prior written permission.                                 #
#                                                                      #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS  #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT    #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS    #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE       #
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,           #
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES   #
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR   #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)   #
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,  #
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OTHERWISE) ARISING   #
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE   #
# POSSIBILITY OF SUCH DAMAGE.                                          #
########################################################################
'''
Created on Apr 29, 2014
'''
from __future__ import (absolute_import, division, print_function,
                        )
import six
from PyQt4 import QtCore, QtGui
from vistrails.core.modules.vistrails_module import Module, ModuleSettings
from vistrails.core.modules.config import IPort, OPort
import numpy as np
from ..wrap_lib import sig_map, obj_src, docstring_func
import logging
logger = logging.getLogger(__name__)

try:
    from metadataStore.userapi.commands import search
except ImportError:
    def search(*args, **kwargs):
        err_msg = ("search from metadataStore.userapi.commands is not "
                   "importable. Search cannot proceed")
        logger.warning(err_msg)
try:
    from metadataStore.userapi.commands import search_keys_dict
except ImportError:
    search_keys_dict = {"search_keys_dict": "Import Unsuccessful"}

try:
    from metadataStore.utilities.utility import listify, get_calib_dict
except ImportError:
    def listify(*args, **kwargs):
        err_msg = ("listify from metadataStore.analysis.utility is not "
                   "importable. run_header cannot be listified")
        logger.warning(err_msg)

class BrokerQuery(Module):
    _settings = ModuleSettings(namespace="broker")

    _input_ports = [
        IPort(name="unique_query_dict",
              label="guaranteed unique query for the data broker",
              signature="basic:Dictionary"),
        IPort(name="query_dict", label="Query for the data broker",
              signature="basic:Dictionary"),
        IPort(name="is_returning_data", label="Return data with search results",
              signature="basic:Boolean", default=True)
    ]

    _output_ports = [
        OPort(name="query_result", signature="basic:Dictionary"),
    ]

    def compute(self):
        query = None
        if self.has_input("query_dict"):
            query = self.get_input("query_dict")
            return_only_one = False
        if self.has_input("unique_query_dict"):
            query = self.get_input("unique_query_dict")
            return_only_one = True

        if query is None:
            logger.debug("no search dictionary was passed in, search "
                         "cannot proceed")
            return
        logger.debug("broker_query: {0}".format(query))
        data = self.get_input("is_returning_data")
        query["data"] = data
        result = search(**query)
        if return_only_one:
            keys = list(result)
            result = result[keys[0]]
        self.set_output("query_result", result)
        logger.debug("result: {0}".format(list(result)))

class CalibrationParameters(Module):
    _settings = ModuleSettings(namespace="broker")
    _input_ports = [
        IPort(name="run_header",
              label="Run header from the data broker",
              signature="basic:Dictionary"),
    ]
    _output_ports = [
        OPort(name='calib_dict', signature='basic:Dictionary'),
        OPort(name='nested', signature='basic:Boolean'),
    ]

    def compute(self):
        header = self.get_input('run_header')
        calib_dict, nested = get_calib_dict(header)
        self.set_output('calib_dict', calib_dict)
        self.set_output('nested', nested)

class Listify(Module):
    src = "Cannot display wrapped source code as it was not found."
    try:
        src = obj_src(listify)
        doc = ('source for wrapped function '
               'metadataStore.utilities.commands:listify:\n\n' +
               six.text_type(src.__str__()))

    except IOError as ie:
        msg = 'original source cannot be found for Listify'
        logger.debug(msg)
        print(msg)
        src = docstring_func(listify)
        doc = ('docstring for wrapped function '
               'metadataStore.utilities.commands:listify:\n\n' +
               six.text_type(src.__str__()))

    __doc__ = ('Transpose the run header events into data lists\n\n' + doc)

    _settings = ModuleSettings(namespace="broker")

    _input_ports = [
        IPort(name="run_header",
              label="Run header from the data broker",
              signature="basic:Dictionary"),
        IPort(name="data_key",
              label="The data key to turn in to a list",
              signature="basic:String"),
    ]

    _output_ports = [
        OPort(name="listified_data", signature="basic:List"),
        OPort(name="listified_time", signature="basic:List"),
    ]

    def compute(self):
        # gather input
        header = self.get_input("run_header")

        key = None
        if self.has_input("data_key"):
            key = self.get_input("data_key")
        # print('key input: {0}'.format(key))
        data_dict = listify(data_keys=key, run_header=header)
        # print('data_dict: {0}'.format(data_dict))
        # remove time from the dictionary
        time = data_dict.pop('time')
        # stringify the datetime object that gets returned
        time = [t.isoformat() for t in time]
        # get the remaining keys
        key = list(data_dict)
        # verify that there is only one key in the dictionary
        if len(key) != 1:
            raise ValueError("The return value from "
                             "metadataStore.utilities.utility.listify had "
                             "more than one key. Keys: {0}".format(key))
        key = key[0]
        data = data_dict[key]
        # check to see if data is a list of lists
        if len(data) == 1 and isinstance(data[0], list):
            data = data[0]
        # check to see if the data name exists in the sig_map dict. If so, this
        # means that the data elements should be formatted into VisTrails
        # objects
        if key in sig_map:
            sig = sig_map[key].split(':')
            mod = self.registry.get_module_by_name(
                sig[0], sig[1])
            data = [mod(d) for d in data]
        # log the values set to the output ports at a debug level
        logger.debug('data ', data)
        logger.debug('time ', time)
        # set the module's output
        self.set_output("listified_data", data)
        self.set_output("listified_time", time)

def vistrails_modules():
    return [BrokerQuery, Listify, CalibrationParameters]
<commit_msg>MNT : fix code to deal with refactor<commit_after># ######################################################################
# Copyright (c) 2014, Brookhaven Science Associates, Brookhaven        #
# National Laboratory. All rights reserved.                            #
#                                                                      #
# Redistribution and use in source and binary forms, with or without   #
# modification, are permitted provided that the following conditions   #
# are met:                                                             #
#                                                                      #
# * Redistributions of source code must retain the above copyright     #
#   notice, this list of conditions and the following disclaimer.      #
#                                                                      #
# * Redistributions in binary form must reproduce the above copyright  #
#   notice this list of conditions and the following disclaimer in     #
#   the documentation and/or other materials provided with the         #
#   distribution.                                                      #
#                                                                      #
# * Neither the name of the Brookhaven Science Associates, Brookhaven  #
#   National Laboratory nor the names of its contributors may be used  #
#   to endorse or promote products derived from this software without  #
#   specific prior written permission.                                 #
#                                                                      #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS  #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT    #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS    #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE       #
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,           #
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES   #
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR   #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)   #
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,  #
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OTHERWISE) ARISING   #
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE   #
# POSSIBILITY OF SUCH DAMAGE.                                          #
########################################################################
'''
Created on Apr 29, 2014
'''
from __future__ import (absolute_import, division, print_function,
                        )
import six
from PyQt4 import QtCore, QtGui
from vistrails.core.modules.vistrails_module import Module, ModuleSettings
from vistrails.core.modules.config import IPort, OPort
import numpy as np
from ..scrape import sig_map, obj_src, docstring_func
import logging
logger = logging.getLogger(__name__)

try:
    from metadataStore.userapi.commands import search
except ImportError:
    def search(*args, **kwargs):
        err_msg = ("search from metadataStore.userapi.commands is not "
                   "importable. Search cannot proceed")
        logger.warning(err_msg)
try:
    from metadataStore.userapi.commands import search_keys_dict
except ImportError:
    search_keys_dict = {"search_keys_dict": "Import Unsuccessful"}

try:
    from metadataStore.utilities.utility import listify, get_calib_dict
except ImportError:
    def listify(*args, **kwargs):
        err_msg = ("listify from metadataStore.analysis.utility is not "
                   "importable. run_header cannot be listified")
        logger.warning(err_msg)

class BrokerQuery(Module):
    _settings = ModuleSettings(namespace="broker")

    _input_ports = [
        IPort(name="unique_query_dict",
              label="guaranteed unique query for the data broker",
              signature="basic:Dictionary"),
        IPort(name="query_dict", label="Query for the data broker",
              signature="basic:Dictionary"),
        IPort(name="is_returning_data", label="Return data with search results",
              signature="basic:Boolean", default=True)
    ]

    _output_ports = [
        OPort(name="query_result", signature="basic:Dictionary"),
    ]

    def compute(self):
        query = None
        if self.has_input("query_dict"):
            query = self.get_input("query_dict")
            return_only_one = False
        if self.has_input("unique_query_dict"):
            query = self.get_input("unique_query_dict")
            return_only_one = True

        if query is None:
            logger.debug("no search dictionary was passed in, search "
                         "cannot proceed")
            return
        logger.debug("broker_query: {0}".format(query))
        data = self.get_input("is_returning_data")
        query["data"] = data
        result = search(**query)
        if return_only_one:
            keys = list(result)
            result = result[keys[0]]
        self.set_output("query_result", result)
        logger.debug("result: {0}".format(list(result)))

class CalibrationParameters(Module):
    _settings = ModuleSettings(namespace="broker")
    _input_ports = [
        IPort(name="run_header",
              label="Run header from the data broker",
              signature="basic:Dictionary"),
    ]
    _output_ports = [
        OPort(name='calib_dict', signature='basic:Dictionary'),
        OPort(name='nested', signature='basic:Boolean'),
    ]

    def compute(self):
        header = self.get_input('run_header')
        calib_dict, nested = get_calib_dict(header)
        self.set_output('calib_dict', calib_dict)
        self.set_output('nested', nested)

class Listify(Module):
    src = "Cannot display wrapped source code as it was not found."
    try:
        src = obj_src(listify)
        doc = ('source for wrapped function '
               'metadataStore.utilities.commands:listify:\n\n' +
               six.text_type(src.__str__()))

    except IOError as ie:
        msg = 'original source cannot be found for Listify'
        logger.debug(msg)
        print(msg)
        src = docstring_func(listify)
        doc = ('docstring for wrapped function '
               'metadataStore.utilities.commands:listify:\n\n' +
               six.text_type(src.__str__()))

    __doc__ = ('Transpose the run header events into data lists\n\n' + doc)

    _settings = ModuleSettings(namespace="broker")

    _input_ports = [
        IPort(name="run_header",
              label="Run header from the data broker",
              signature="basic:Dictionary"),
        IPort(name="data_key",
              label="The data key to turn in to a list",
              signature="basic:String"),
    ]

    _output_ports = [
        OPort(name="listified_data", signature="basic:List"),
        OPort(name="listified_time", signature="basic:List"),
    ]

    def compute(self):
        # gather input
        header = self.get_input("run_header")

        key = None
        if self.has_input("data_key"):
            key = self.get_input("data_key")
        # print('key input: {0}'.format(key))
        data_dict = listify(data_keys=key, run_header=header)
        # print('data_dict: {0}'.format(data_dict))
        # remove time from the dictionary
        time = data_dict.pop('time')
        # stringify the datetime object that gets returned
        time = [t.isoformat() for t in time]
        # get the remaining keys
        key = list(data_dict)
        # verify that there is only one key in the dictionary
        if len(key) != 1:
            raise ValueError("The return value from "
                             "metadataStore.utilities.utility.listify had "
                             "more than one key. Keys: {0}".format(key))
        key = key[0]
        data = data_dict[key]
        # check to see if data is a list of lists
        if len(data) == 1 and isinstance(data[0], list):
            data = data[0]
        # check to see if the data name exists in the sig_map dict. If so, this
        # means that the data elements should be formatted into VisTrails
        # objects
        if key in sig_map:
            sig = sig_map[key].split(':')
            mod = self.registry.get_module_by_name(
                sig[0], sig[1])
            data = [mod(d) for d in data]
        # log the values set to the output ports at a debug level
        logger.debug('data ', data)
        logger.debug('time ', time)
        # set the module's output
        self.set_output("listified_data", data)
        self.set_output("listified_time", time)

def vistrails_modules():
    return [BrokerQuery, Listify, CalibrationParameters]
<|endoftext|>


<commit_before>#! /usr/bin/env python
# encoding: utf-8
#
# written by Sylvain Rouquette, 2014

'''

This is an extra tool, not bundled with the default waf binary.
To add the cpplint tool to the waf file:
$ ./waf-light --tools=compat15,cpplint
    or, if you have waf >= 1.6.2
$ ./waf update --files=cpplint

this tool also requires cpplint for python.
If you have PIP, you can install it like this: pip install cpplint

But I'd recommend getting the latest version from the SVN,
the PIP version is outdated.
https://code.google.com/p/google-styleguide/source/browse/trunk/cpplint/cpplint.py
Apply this patch if you want to run it with Python 3:
https://code.google.com/p/google-styleguide/issues/detail?id=19

When using this tool, the wscript will look like:

    def options(opt):
        opt.load('compiler_cxx cpplint')

    def configure(conf):
        conf.load('compiler_cxx cpplint')
        # optional, you can also specify them on the command line
        conf.env.CPPLINT_FILTERS = ','.join((
            '-whitespace/newline',      # c++11 lambda
            '-readability/braces',      # c++11 constructor
            '-whitespace/braces',       # c++11 constructor
            '-build/storage_class',     # c++11 for-range
            '-whitespace/blank_line',   # user pref
            '-whitespace/labels'        # user pref
            ))

    def build(bld):
        bld(features='cpplint', source='main.cpp', target='app')
        # add include files, because they aren't usually built
        bld(features='cpplint', source=bld.path.ant_glob('**/*.hpp'))
'''

import sys, re
import logging
import threading
from waflib import Task, Build, TaskGen, Logs, Utils
try:
    from cpplint.cpplint import ProcessFile, _cpplint_state
except ImportError:
    pass

critical_errors = 0
CPPLINT_FORMAT = '[CPPLINT] %(filename)s:\nline %(linenum)s, severity %(confidence)s, category: %(category)s\n%(message)s\n'
RE_EMACS = re.compile('(?P<filename>.*):(?P<linenum>\d+):  (?P<message>.*)  \[(?P<category>.*)\] \[(?P<confidence>\d+)\]');
CPPLINT_RE = {
    'waf': RE_EMACS,
    'emacs': RE_EMACS,
    'vs7': re.compile('(?P<filename>.*)\((?P<linenum>\d+)\):  (?P<message>.*)  \[(?P<category>.*)\] \[(?P<confidence>\d+)\]'),
    'eclipse': re.compile('(?P<filename>.*):(?P<linenum>\d+): warning: (?P<message>.*)  \[(?P<category>.*)\] \[(?P<confidence>\d+)\]'),
}

def init_env_from_options(env):
    from waflib.Options import options
    for key, value in options.__dict__.items():
        if not key.startswith('CPPLINT_') or env[key]:
            continue
        env[key] = value
    if env.CPPLINT_OUTPUT != 'waf':
        _cpplint_state.output_format = env.CPPLINT_OUTPUT

def options(opt):
    opt.add_option('--cpplint-filters', type='string',
                   default='', dest='CPPLINT_FILTERS',
                   help='add filters to cpplint')
    opt.add_option('--cpplint-level', default=1, type='int', dest='CPPLINT_LEVEL',
                   help='specify the log level (default: 1)')
    opt.add_option('--cpplint-break', default=5, type='int', dest='CPPLINT_BREAK',
                   help='break the build if error >= level (default: 5)')
    opt.add_option('--cpplint-skip', action='store_true',
                   default=False, dest='CPPLINT_SKIP',
                   help='skip cpplint during build')
    opt.add_option('--cpplint-output', type='string',
                   default='waf', dest='CPPLINT_OUTPUT',
                   help='select output format (waf, emacs, vs7)')

def configure(conf):
    conf.start_msg('Checking cpplint')
    try:
        import cpplint
        conf.end_msg('ok')
    except ImportError:
        conf.env.CPPLINT_SKIP = True
        conf.end_msg('not found, skipping it.')

class cpplint_formatter(Logs.formatter):
    def __init__(self, fmt):
        logging.Formatter.__init__(self, CPPLINT_FORMAT)
        self.fmt = fmt

    def format(self, rec):
        if self.fmt == 'waf':
            result = CPPLINT_RE[self.fmt].match(rec.msg).groupdict()
            rec.msg = CPPLINT_FORMAT % result
        if rec.levelno <= logging.INFO:
            rec.c1 = Logs.colors.CYAN
        return super(cpplint_formatter, self).format(rec)

class cpplint_handler(Logs.log_handler):
    def __init__(self, stream=sys.stderr, **kw):
        super(cpplint_handler, self).__init__(stream, **kw)
        self.stream = stream

    def emit(self, rec):
        rec.stream = self.stream
        self.emit_override(rec)
        self.flush()

class cpplint_wrapper(object):
    stream = None
    tasks_count = 0
    lock = threading.RLock()

    def __init__(self, logger, threshold, fmt):
        self.logger = logger
        self.threshold = threshold
        self.error_count = 0
        self.fmt = fmt

    def __enter__(self):
        with cpplint_wrapper.lock:
            cpplint_wrapper.tasks_count += 1
            if cpplint_wrapper.tasks_count == 1:
                sys.stderr.flush()
                cpplint_wrapper.stream = sys.stderr
                sys.stderr = self
            return self

    def __exit__(self, exc_type, exc_value, traceback):
        with cpplint_wrapper.lock:
            cpplint_wrapper.tasks_count -= 1
            if cpplint_wrapper.tasks_count == 0:
                sys.stderr = cpplint_wrapper.stream
                sys.stderr.flush()

    def isatty(self):
        return True

    def write(self, message):
        global critical_errors
        result = CPPLINT_RE[self.fmt].match(message)
        if not result:
            return
        level = int(result.groupdict()['confidence'])
        if level >= self.threshold:
            critical_errors += 1
        if level <= 2:
            self.logger.info(message)
        elif level <= 4:
            self.logger.warning(message)
        else:
            self.logger.error(message)

cpplint_logger = None
def get_cpplint_logger(fmt):
    global cpplint_logger
    if cpplint_logger:
        return cpplint_logger
    cpplint_logger = logging.getLogger('cpplint')
    hdlr = cpplint_handler()
    hdlr.setFormatter(cpplint_formatter(fmt))
    cpplint_logger.addHandler(hdlr)
    cpplint_logger.setLevel(logging.DEBUG)
    return cpplint_logger

class cpplint(Task.Task):
    color = 'PINK'

    def __init__(self, *k, **kw):
        super(cpplint, self).__init__(*k, **kw)

    def run(self):
        global critical_errors
        _cpplint_state.SetFilters(self.env.CPPLINT_FILTERS)
        break_level = self.env.CPPLINT_BREAK
        verbosity = self.env.CPPLINT_LEVEL
        with cpplint_wrapper(get_cpplint_logger(self.env.CPPLINT_OUTPUT),
                             break_level, self.env.CPPLINT_OUTPUT):
            ProcessFile(self.inputs[0].abspath(), verbosity)
        return critical_errors

@TaskGen.extension('.h', '.hh', '.hpp', '.hxx')
def cpplint_includes(self, node):
    pass

@TaskGen.feature('cpplint')
@TaskGen.before_method('process_source')
def run_cpplint(self):
    if not self.env.CPPLINT_INITIALIZED:
        self.env.CPPLINT_INITIALIZED = True
        init_env_from_options(self.env)
    if self.env.CPPLINT_SKIP:
        return
    if not self.env.CPPLINT_OUTPUT in CPPLINT_RE:
        return
    for src in self.to_list(getattr(self, 'source', [])):
        if isinstance(src, str):
            self.create_task('cpplint', self.path.find_or_declare(src))
        else:
            self.create_task('cpplint', src)
<commit_msg>cpplint: fixed import. The version on pypi is now up to date and works with Python3.<commit_after>#! /usr/bin/env python
# encoding: utf-8
#
# written by Sylvain Rouquette, 2014

'''

This is an extra tool, not bundled with the default waf binary.
To add the cpplint tool to the waf file:
$ ./waf-light --tools=compat15,cpplint
    or, if you have waf >= 1.6.2
$ ./waf update --files=cpplint

this tool also requires cpplint for python.
If you have PIP, you can install it like this: pip install cpplint

But I'd recommend getting the latest version from the SVN,
the PIP version is outdated.
https://code.google.com/p/google-styleguide/source/browse/trunk/cpplint/cpplint.py
Apply this patch if you want to run it with Python 3:
https://code.google.com/p/google-styleguide/issues/detail?id=19

When using this tool, the wscript will look like:

    def options(opt):
        opt.load('compiler_cxx cpplint')

    def configure(conf):
        conf.load('compiler_cxx cpplint')
        # optional, you can also specify them on the command line
        conf.env.CPPLINT_FILTERS = ','.join((
            '-whitespace/newline',      # c++11 lambda
            '-readability/braces',      # c++11 constructor
            '-whitespace/braces',       # c++11 constructor
            '-build/storage_class',     # c++11 for-range
            '-whitespace/blank_line',   # user pref
            '-whitespace/labels'        # user pref
            ))

    def build(bld):
        bld(features='cpplint', source='main.cpp', target='app')
        # add include files, because they aren't usually built
        bld(features='cpplint', source=bld.path.ant_glob('**/*.hpp'))
'''

import sys, re
import logging
import threading
from waflib import Task, Build, TaskGen, Logs, Utils
try:
    from cpplint import ProcessFile, _cpplint_state
except ImportError:
    pass

critical_errors = 0
CPPLINT_FORMAT = '[CPPLINT] %(filename)s:\nline %(linenum)s, severity %(confidence)s, category: %(category)s\n%(message)s\n'
RE_EMACS = re.compile('(?P<filename>.*):(?P<linenum>\d+):  (?P<message>.*)  \[(?P<category>.*)\] \[(?P<confidence>\d+)\]');
CPPLINT_RE = {
    'waf': RE_EMACS,
    'emacs': RE_EMACS,
    'vs7': re.compile('(?P<filename>.*)\((?P<linenum>\d+)\):  (?P<message>.*)  \[(?P<category>.*)\] \[(?P<confidence>\d+)\]'),
    'eclipse': re.compile('(?P<filename>.*):(?P<linenum>\d+): warning: (?P<message>.*)  \[(?P<category>.*)\] \[(?P<confidence>\d+)\]'),
}

def init_env_from_options(env):
    from waflib.Options import options
    for key, value in options.__dict__.items():
        if not key.startswith('CPPLINT_') or env[key]:
            continue
        env[key] = value
    if env.CPPLINT_OUTPUT != 'waf':
        _cpplint_state.output_format = env.CPPLINT_OUTPUT

def options(opt):
    opt.add_option('--cpplint-filters', type='string',
                   default='', dest='CPPLINT_FILTERS',
                   help='add filters to cpplint')
    opt.add_option('--cpplint-level', default=1, type='int', dest='CPPLINT_LEVEL',
                   help='specify the log level (default: 1)')
    opt.add_option('--cpplint-break', default=5, type='int', dest='CPPLINT_BREAK',
                   help='break the build if error >= level (default: 5)')
    opt.add_option('--cpplint-skip', action='store_true',
                   default=False, dest='CPPLINT_SKIP',
                   help='skip cpplint during build')
    opt.add_option('--cpplint-output', type='string',
                   default='waf', dest='CPPLINT_OUTPUT',
                   help='select output format (waf, emacs, vs7)')

def configure(conf):
    conf.start_msg('Checking cpplint')
    try:
        import cpplint
        conf.end_msg('ok')
    except ImportError:
        conf.env.CPPLINT_SKIP = True
        conf.end_msg('not found, skipping it.')

class cpplint_formatter(Logs.formatter):
    def __init__(self, fmt):
        logging.Formatter.__init__(self, CPPLINT_FORMAT)
        self.fmt = fmt

    def format(self, rec):
        if self.fmt == 'waf':
            result = CPPLINT_RE[self.fmt].match(rec.msg).groupdict()
            rec.msg = CPPLINT_FORMAT % result
        if rec.levelno <= logging.INFO:
            rec.c1 = Logs.colors.CYAN
        return super(cpplint_formatter, self).format(rec)

class cpplint_handler(Logs.log_handler):
    def __init__(self, stream=sys.stderr, **kw):
        super(cpplint_handler, self).__init__(stream, **kw)
        self.stream = stream

    def emit(self, rec):
        rec.stream = self.stream
        self.emit_override(rec)
        self.flush()

class cpplint_wrapper(object):
    stream = None
    tasks_count = 0
    lock = threading.RLock()

    def __init__(self, logger, threshold, fmt):
        self.logger = logger
        self.threshold = threshold
        self.error_count = 0
        self.fmt = fmt

    def __enter__(self):
        with cpplint_wrapper.lock:
            cpplint_wrapper.tasks_count += 1
            if cpplint_wrapper.tasks_count == 1:
                sys.stderr.flush()
                cpplint_wrapper.stream = sys.stderr
                sys.stderr = self
            return self

    def __exit__(self, exc_type, exc_value, traceback):
        with cpplint_wrapper.lock:
            cpplint_wrapper.tasks_count -= 1
            if cpplint_wrapper.tasks_count == 0:
                sys.stderr = cpplint_wrapper.stream
                sys.stderr.flush()

    def isatty(self):
        return True

    def write(self, message):
        global critical_errors
        result = CPPLINT_RE[self.fmt].match(message)
        if not result:
            return
        level = int(result.groupdict()['confidence'])
        if level >= self.threshold:
            critical_errors += 1
        if level <= 2:
            self.logger.info(message)
        elif level <= 4:
            self.logger.warning(message)
        else:
            self.logger.error(message)

cpplint_logger = None
def get_cpplint_logger(fmt):
    global cpplint_logger
    if cpplint_logger:
        return cpplint_logger
    cpplint_logger = logging.getLogger('cpplint')
    hdlr = cpplint_handler()
    hdlr.setFormatter(cpplint_formatter(fmt))
    cpplint_logger.addHandler(hdlr)
    cpplint_logger.setLevel(logging.DEBUG)
    return cpplint_logger

class cpplint(Task.Task):
    color = 'PINK'

    def __init__(self, *k, **kw):
        super(cpplint, self).__init__(*k, **kw)

    def run(self):
        global critical_errors
        _cpplint_state.SetFilters(self.env.CPPLINT_FILTERS)
        break_level = self.env.CPPLINT_BREAK
        verbosity = self.env.CPPLINT_LEVEL
        with cpplint_wrapper(get_cpplint_logger(self.env.CPPLINT_OUTPUT),
                             break_level, self.env.CPPLINT_OUTPUT):
            ProcessFile(self.inputs[0].abspath(), verbosity)
        return critical_errors

@TaskGen.extension('.h', '.hh', '.hpp', '.hxx')
def cpplint_includes(self, node):
    pass

@TaskGen.feature('cpplint')
@TaskGen.before_method('process_source')
def run_cpplint(self):
    if not self.env.CPPLINT_INITIALIZED:
        self.env.CPPLINT_INITIALIZED = True
        init_env_from_options(self.env)
    if self.env.CPPLINT_SKIP:
        return
    if not self.env.CPPLINT_OUTPUT in CPPLINT_RE:
        return
    for src in self.to_list(getattr(self, 'source', [])):
        if isinstance(src, str):
            self.create_task('cpplint', self.path.find_or_declare(src))
        else:
            self.create_task('cpplint', src)
<|endoftext|>